From 6a2078ff1fa4e61b6a071440e3e7a9849f7dfdb6 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 3 Jun 2025 23:47:57 +0700 Subject: [PATCH 01/27] fix: Use outpost.imagePullSecrets value in Helm chart --- .../charts/outpost/templates/outpost-deployment.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deployments/kubernetes/charts/outpost/templates/outpost-deployment.yaml b/deployments/kubernetes/charts/outpost/templates/outpost-deployment.yaml index baecb909..b3f6efc5 100644 --- a/deployments/kubernetes/charts/outpost/templates/outpost-deployment.yaml +++ b/deployments/kubernetes/charts/outpost/templates/outpost-deployment.yaml @@ -16,6 +16,10 @@ spec: app: outpost-api {{- include "outpost.labels" . | nindent 8 }} spec: + {{- with .Values.outpost.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: {{ include "outpost.name" . }}-api image: {{ .Values.outpost.image.repository }}:{{ .Values.outpost.image.tag }} @@ -64,6 +68,10 @@ spec: app: outpost-delivery {{- include "outpost.labels" . | nindent 8 }} spec: + {{- with .Values.outpost.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: {{ include "outpost.name" . }}-delivery image: {{ .Values.outpost.image.repository }}:{{ .Values.outpost.image.tag }} @@ -100,6 +108,10 @@ spec: app: outpost-log {{- include "outpost.labels" . | nindent 8 }} spec: + {{- with .Values.outpost.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: {{ include "outpost.name" . }}-log image: {{ .Values.outpost.image.repository }}:{{ .Values.outpost.image.tag }} From ca4c6da7bee897bdd28d4a8ef1d930e0125852c4 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Wed, 4 Jun 2025 13:23:30 +0100 Subject: [PATCH 02/27] chore(docs): Update alert JSON structure --- docs/pages/features/alerts.mdx | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/pages/features/alerts.mdx b/docs/pages/features/alerts.mdx index ca47e1ed..22cea6a5 100644 --- a/docs/pages/features/alerts.mdx +++ b/docs/pages/features/alerts.mdx @@ -16,18 +16,22 @@ The `ALERT_CONSECUTIVE_FAILURE_COUNT` variable triggers an alert when the consec ```json { - "topic": "alert.consecutive-failure", - "timestamp": "2024-01-01T00:00:00Z", + "topic": "alert.consecutive_failure", + "timestamp": "2025-05-29T05:07:09.269672003Z", "data": { - "max_consecutive_failures": 20, - "consecutive_failures": 5, - "will_disable": true, - "destination": DestinationObject, - "response": { - "status": "500", - "data": { - "some": "value" - } + "event": { + "id": "evt_id", + "topic": "user.created", + "metadata": {}, + "data": {} + }, + "max_consecutive_failures": 3, + "consecutive_failures": 3, + "will_disable": false, + "destination": {}, + "delivery_response": { + "body": "{\"success\":false,\"verified\":false,\"payload\":{\"user_id\":\"userid\"}}", + "status": 400 } } } From b845fd530ade7a072ce31b8996a670d0f2979e93 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 4 Jun 2025 12:24:05 +0000 Subject: [PATCH 03/27] ci: Update outpost version in compose.yml to 0.2.1 --- examples/docker-compose/compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/docker-compose/compose.yml b/examples/docker-compose/compose.yml index fd9baa3c..1a559433 100644 --- a/examples/docker-compose/compose.yml +++ b/examples/docker-compose/compose.yml @@ -1,7 +1,7 @@ name: outpost-example services: api: - image: hookdeck/outpost:v0.2.0 + image: hookdeck/outpost:v0.2.1 env_file: .env depends_on: redis: @@ -14,7 +14,7 @@ services: - 3333:3333 delivery: - image: hookdeck/outpost:v0.2.0 + image: hookdeck/outpost:v0.2.1 env_file: .env depends_on: redis: @@ -25,7 +25,7 @@ services: SERVICE: delivery log: - image: hookdeck/outpost:v0.2.0 + image: hookdeck/outpost:v0.2.1 env_file: .env depends_on: redis: From 82d5d776f1634e36c962b36924b018008ec4affb Mon Sep 17 00:00:00 2001 From: thomasklepacz <120401433+thomasklepacz@users.noreply.github.com> Date: Fri, 6 Jun 2025 12:31:19 -0400 Subject: [PATCH 04/27] chore/swap clickhouse logo with postgres, small text fix in hero --- website/public/images/postgres-icon.svg | 20 ++++++++++++++++++++ website/src/components/Depedencies.astro | 4 ++-- website/src/components/Hero.astro | 6 +++++- 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 website/public/images/postgres-icon.svg diff --git a/website/public/images/postgres-icon.svg b/website/public/images/postgres-icon.svg new file mode 100644 index 00000000..4df481a8 --- /dev/null +++ b/website/public/images/postgres-icon.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/website/src/components/Depedencies.astro b/website/src/components/Depedencies.astro index f77826ab..eddfd5f3 100644 --- a/website/src/components/Depedencies.astro +++ b/website/src/components/Depedencies.astro @@ -19,8 +19,8 @@ REDIS
- CLICKHOUSE - CLICKHOUSE + POSTGRES + POSTGRES
diff --git a/website/src/components/Hero.astro b/website/src/components/Hero.astro index 9cd19fe6..a3618f68 100644 --- a/website/src/components/Hero.astro +++ b/website/src/components/Hero.astro @@ -12,7 +12,7 @@ import { LINKS } from "../links";

Manage and deliver platform events directly to your users' preferred event destinationsEvent Destinations: Webhooks, Hookdeck, AWS SQS, RabbitMQ, GCP Pub/Sub, Amazon EventBridge, and Kafka.

@@ -137,6 +137,10 @@ import { LINKS } from "../links"; p { margin: 0; color: var(--foreground-neutral-2); + + strong { + font-weight: 500; + } } .button-group { From 5556fcb0bd9620bd76733df88ca171e8635d7cf5 Mon Sep 17 00:00:00 2001 From: thomasklepacz <120401433+thomasklepacz@users.noreply.github.com> Date: Fri, 6 Jun 2025 14:23:13 -0400 Subject: [PATCH 05/27] chore/line break fix in features bento (bottom row) --- website/src/components/Features.astro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/components/Features.astro b/website/src/components/Features.astro index 002b901c..9fdca733 100644 --- a/website/src/components/Features.astro +++ b/website/src/components/Features.astro @@ -84,7 +84,7 @@ import Alert from "./Features/Alert.astro";
-

Webhook best practices, by default

+

Default webhook best practices

Opt-out idempotency headers, timestamp and signature, and signature rotation. From 17738fb037a8cdfd3a78fc418bc61c41f626d9e2 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 9 Jun 2025 17:21:54 +0700 Subject: [PATCH 06/27] test: Remove unnecessary test --- internal/mqs/queue_rabbitmq_test.go | 119 ---------------------------- 1 file changed, 119 deletions(-) delete mode 100644 internal/mqs/queue_rabbitmq_test.go diff --git a/internal/mqs/queue_rabbitmq_test.go b/internal/mqs/queue_rabbitmq_test.go deleted file mode 100644 index ad794933..00000000 --- a/internal/mqs/queue_rabbitmq_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package mqs_test - -import ( - "context" - "testing" - "time" - - "github.com/google/uuid" - "github.com/hookdeck/outpost/internal/mqinfra" - "github.com/hookdeck/outpost/internal/mqs" - "github.com/hookdeck/outpost/internal/util/testinfra" - "github.com/hookdeck/outpost/internal/util/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestIntegrationRabbitMQ(t *testing.T) { - t.Parallel() - t.Cleanup(testinfra.Start(t)) - - t.Run("should route messages to correct queue", func(t *testing.T) { - ctx := context.Background() - serverURL := testinfra.EnsureRabbitMQ() - exchange := "test-exchange" - - // Create and declare infrastructure for both queues - config1 := &mqs.QueueConfig{ - RabbitMQ: &mqs.RabbitMQConfig{ - ServerURL: serverURL, - Exchange: exchange, - Queue: "test-queue-1", - }, - } - infra1 := mqinfra.New(config1) - require.NoError(t, infra1.Declare(ctx)) - defer func() { - require.NoError(t, infra1.TearDown(ctx)) - }() - - config2 := &mqs.QueueConfig{ - RabbitMQ: &mqs.RabbitMQConfig{ - ServerURL: serverURL, - Exchange: exchange, - Queue: "test-queue-2", - }, - } - infra2 := mqinfra.New(config2) - require.NoError(t, infra2.Declare(ctx)) - defer func() { - require.NoError(t, infra2.TearDown(ctx)) - }() - - // Create queues - queue1 := mqs.NewQueue(config1) - cleanup1, err := queue1.Init(ctx) - require.NoError(t, err) - defer cleanup1() - - queue2 := mqs.NewQueue(config2) - cleanup2, err := queue2.Init(ctx) - require.NoError(t, err) - defer cleanup2() - - // Subscribe to both queues - sub1, err := queue1.Subscribe(ctx) - require.NoError(t, err) - defer sub1.Shutdown(ctx) - - sub2, err := queue2.Subscribe(ctx) - require.NoError(t, err) - defer sub2.Shutdown(ctx) - - // Publish messages to both queues - msg1 := &testutil.MockMsg{ID: uuid.New().String()} - err = queue1.Publish(ctx, msg1) - assert.NoError(t, err, "failed to publish to queue1") - - msg2 := &testutil.MockMsg{ID: uuid.New().String()} - err = queue2.Publish(ctx, msg2) - assert.NoError(t, err, "failed to publish to queue2") - - // Helper to receive all messages from a subscription with timeout - receiveAllMessages := func(sub mqs.Subscription, timeout time.Duration) []*testutil.MockMsg { - var messages []*testutil.MockMsg - for { - // Create a context with timeout for each receive attempt - receiveCtx, cancel := context.WithTimeout(ctx, timeout) - received, err := sub.Receive(receiveCtx) - cancel() - - if err != nil { - // If we timeout, assume no more messages - break - } - - parsed := &testutil.MockMsg{} - err = parsed.FromMessage(received) - require.NoError(t, err) - messages = append(messages, parsed) - received.Ack() - } - return messages - } - - // Receive all messages from both queues - messages1 := receiveAllMessages(sub1, 100*time.Millisecond) - messages2 := receiveAllMessages(sub2, 100*time.Millisecond) - - // Check queue1 - if assert.Len(t, messages1, 1, "queue1 should have exactly 1 message") { - assert.Equal(t, msg1.ID, messages1[0].ID, "queue1 should have msg1") - } - - // Check queue2 - if assert.Len(t, messages2, 1, "queue2 should have exactly 1 message") { - assert.Equal(t, msg2.ID, messages2[0].ID, "queue2 should have msg2") - } - }) -} From f46a59542a2144b78c66ac86b12de3d9fa29442a Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 9 Jun 2025 20:12:30 +0700 Subject: [PATCH 07/27] refactor: Differentiate mq infra config vs data-plane config --- cmd/e2e/configs/basic.go | 4 +- go.mod | 10 ++ go.sum | 40 ++++++ internal/app/app.go | 4 +- internal/config/config.go | 4 +- internal/config/mq.go | 166 +++++------------------- internal/config/mqconfig_aws.go | 63 +++++++++ internal/config/mqconfig_gcp.go | 69 ++++++++++ internal/config/mqconfig_rabbitmq.go | 54 ++++++++ internal/config/validation.go | 15 ++- internal/config/validation_test.go | 2 +- internal/infra/infra.go | 5 +- internal/mqinfra/awssqs.go | 16 ++- internal/mqinfra/gcppubsub.go | 3 +- internal/mqinfra/mqinfra.go | 62 +++++++-- internal/mqinfra/mqinfra_test.go | 170 ++++++++++++++++++------- internal/mqinfra/rabbitmq.go | 3 +- internal/services/api/api.go | 6 +- internal/services/delivery/delivery.go | 26 ++-- internal/services/log/log.go | 7 +- 20 files changed, 512 insertions(+), 217 deletions(-) create mode 100644 internal/config/mqconfig_aws.go create mode 100644 internal/config/mqconfig_gcp.go create mode 100644 internal/config/mqconfig_rabbitmq.go diff --git a/cmd/e2e/configs/basic.go b/cmd/e2e/configs/basic.go index 27191c5b..966735e3 100644 --- a/cmd/e2e/configs/basic.go +++ b/cmd/e2e/configs/basic.go @@ -75,8 +75,8 @@ func Basic(t *testing.T, opts BasicOpts) config.Config { // Setup cleanup t.Cleanup(func() { if err := infra.Teardown(context.Background(), infra.Config{ - DeliveryMQ: c.MQs.GetDeliveryQueueConfig(), - LogMQ: c.MQs.GetLogQueueConfig(), + DeliveryMQ: c.MQs.ToInfraConfig("deliverymq"), + LogMQ: c.MQs.ToInfraConfig("logmq"), }); err != nil { log.Println("Teardown failed:", err) } diff --git a/go.mod b/go.mod index 359c3714..88fe48f0 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,9 @@ go 1.23.0 require ( cloud.google.com/go/pubsub v1.41.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 + github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.7.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus v1.2.0 github.com/ClickHouse/clickhouse-go/v2 v2.29.0 github.com/Masterminds/sprig/v3 v3.3.0 github.com/alicebob/miniredis/v2 v2.33.0 @@ -77,7 +80,12 @@ require ( cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.13 // indirect dario.cat/mergo v1.0.1 // indirect + github.com/Azure/azure-amqp-common-go/v3 v3.2.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/go-amqp v1.0.5 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/ClickHouse/ch-go v0.61.5 // indirect github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -142,6 +150,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/lufia/plan9stats v0.0.0-20240819163618-b1d8f4d146e7 // indirect @@ -167,6 +176,7 @@ require ( github.com/paulmach/orb v0.11.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/redis/go-redis/extra/rediscmd/v9 v9.5.3 // indirect diff --git a/go.sum b/go.sum index 3e36ef2f..c739b5b9 100644 --- a/go.sum +++ b/go.sum @@ -629,8 +629,36 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/azure-amqp-common-go/v3 v3.2.3 h1:uDF62mbd9bypXWi19V1bN5NZEO84JqgmI5G73ibAmrk= +github.com/Azure/azure-amqp-common-go/v3 v3.2.3/go.mod h1:7rPmbSfszeovxGfc5fSAXE4ehlXQZHpMja2OtxC2Tas= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.7.1 h1:o/Ws6bEqMeKZUfj1RRm3mQ51O8JGU5w+Qdg2AhHib6A= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.7.1/go.mod h1:6QAMYBAbQeeKX+REFJMZ1nFWu9XLw/PPcjYpuc9RDFs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus v1.2.0 h1:jngSeKBnzC7qIk3rvbWHsLI7eeasEucORHWr2CHX0Yg= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus v1.2.0/go.mod h1:1YXAxWw6baox+KafeQU2scy21/4IHvqXoIJuCpcvpMQ= +github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= +github.com/Azure/go-amqp v1.0.5 h1:po5+ljlcNSU8xtapHTe8gIc8yHxCzC03E8afH2g1ftU= +github.com/Azure/go-amqp v1.0.5/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= @@ -765,6 +793,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dhui/dktest v0.4.4 h1:+I4s6JRE1yGuqflzwqG+aIaMdgXIorCf5P98JnaAWa8= @@ -806,6 +835,9 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -889,6 +921,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -1059,6 +1092,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -1139,6 +1174,8 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1344,6 +1381,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -2082,6 +2120,8 @@ modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= +nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= +nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/internal/app/app.go b/internal/app/app.go index 793af6a5..4c54ccab 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -54,8 +54,8 @@ func run(mainContext context.Context, cfg *config.Config) error { } if err := infra.Declare(mainContext, infra.Config{ - DeliveryMQ: cfg.MQs.GetDeliveryQueueConfig(), - LogMQ: cfg.MQs.GetLogQueueConfig(), + DeliveryMQ: cfg.MQs.ToInfraConfig("deliverymq"), + LogMQ: cfg.MQs.ToInfraConfig("logmq"), }); err != nil { return err } diff --git a/internal/config/config.go b/internal/config/config.go index 26d3b029..12adbfc1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -62,7 +62,7 @@ type Config struct { Redis RedisConfig `yaml:"redis"` ClickHouse ClickHouseConfig `yaml:"clickhouse"` PostgresURL string `yaml:"postgres" env:"POSTGRES_URL" desc:"Connection URL for PostgreSQL, used as an alternative log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'. Required if ClickHouse is not configured and log storage is needed." required:"C"` - MQs MQsConfig `yaml:"mqs"` + MQs *MQsConfig `yaml:"mqs"` // PublishMQ PublishMQ PublishMQConfig `yaml:"publishmq"` @@ -122,7 +122,7 @@ func (c *Config) InitDefaults() { c.ClickHouse = ClickHouseConfig{ Database: "outpost", } - c.MQs = MQsConfig{ + c.MQs = &MQsConfig{ RabbitMQ: RabbitMQConfig{ Exchange: "outpost", DeliveryQueue: "outpost-delivery", diff --git a/internal/config/mq.go b/internal/config/mq.go index ed46a989..660e9d37 100644 --- a/internal/config/mq.go +++ b/internal/config/mq.go @@ -1,157 +1,61 @@ package config import ( - "fmt" + "context" + "github.com/hookdeck/outpost/internal/mqinfra" "github.com/hookdeck/outpost/internal/mqs" ) -// MQ Infrastructure configs -type AWSSQSConfig struct { - AccessKeyID string `yaml:"access_key_id" env:"AWS_SQS_ACCESS_KEY_ID" desc:"AWS Access Key ID for SQS. Required if AWS SQS is the chosen MQ provider." required:"C"` - SecretAccessKey string `yaml:"secret_access_key" env:"AWS_SQS_SECRET_ACCESS_KEY" desc:"AWS Secret Access Key for SQS. Required if AWS SQS is the chosen MQ provider." required:"C"` - Region string `yaml:"region" env:"AWS_SQS_REGION" desc:"AWS Region for SQS. Required if AWS SQS is the chosen MQ provider." required:"C"` - Endpoint string `yaml:"endpoint" env:"AWS_SQS_ENDPOINT" desc:"Custom AWS SQS endpoint URL. Optional, typically used for local testing (e.g., LocalStack)." required:"N"` - DeliveryQueue string `yaml:"delivery_queue" env:"AWS_SQS_DELIVERY_QUEUE" desc:"Name of the SQS queue for delivery events." required:"N"` - LogQueue string `yaml:"log_queue" env:"AWS_SQS_LOG_QUEUE" desc:"Name of the SQS queue for log events." required:"N"` -} - -type GCPPubSubConfig struct { - Project string `yaml:"project" env:"GCP_PUBSUB_PROJECT" desc:"GCP Project ID for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider." required:"C"` - ServiceAccountCredentials string `yaml:"service_account_credentials" env:"GCP_PUBSUB_SERVICE_ACCOUNT_CREDENTIALS" desc:"JSON string or path to a file containing GCP service account credentials for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider and not running in an environment with implicit credentials (e.g., GCE, GKE)." required:"C"` - DeliveryTopic string `yaml:"delivery_topic" env:"GCP_PUBSUB_DELIVERY_TOPIC" desc:"Name of the GCP Pub/Sub topic for delivery events." required:"N"` - DeliverySubscription string `yaml:"delivery_subscription" env:"GCP_PUBSUB_DELIVERY_SUBSCRIPTION" desc:"Name of the GCP Pub/Sub subscription for delivery events." required:"N"` - LogTopic string `yaml:"log_topic" env:"GCP_PUBSUB_LOG_TOPIC" desc:"Name of the GCP Pub/Sub topic for log events." required:"N"` - LogSubscription string `yaml:"log_subscription" env:"GCP_PUBSUB_LOG_SUBSCRIPTION" desc:"Name of the GCP Pub/Sub subscription for log events." required:"N"` -} - -type RabbitMQConfig struct { - ServerURL string `yaml:"server_url" env:"RABBITMQ_SERVER_URL" desc:"RabbitMQ server connection URL (e.g., 'amqp://user:pass@host:port/vhost'). Required if RabbitMQ is the chosen MQ provider." required:"C"` - Exchange string `yaml:"exchange" env:"RABBITMQ_EXCHANGE" desc:"Name of the RabbitMQ exchange to use." required:"N"` - DeliveryQueue string `yaml:"delivery_queue" env:"RABBITMQ_DELIVERY_QUEUE" desc:"Name of the RabbitMQ queue for delivery events." required:"N"` - LogQueue string `yaml:"log_queue" env:"RABBITMQ_LOG_QUEUE" desc:"Name of the RabbitMQ queue for log events." required:"N"` +type MQConfigAdapter interface { + ToInfraConfig(queueType string) *mqinfra.MQInfraConfig + ToQueueConfig(ctx context.Context, queueType string) (*mqs.QueueConfig, error) + GetProviderType() string + IsConfigured() bool } type MQsConfig struct { - AWSSQS AWSSQSConfig `yaml:"aws_sqs" desc:"Configuration for using AWS SQS as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured." required:"N"` - GCPPubSub GCPPubSubConfig `yaml:"gcp_pubsub" desc:"Configuration for using GCP Pub/Sub as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured." required:"N"` - RabbitMQ RabbitMQConfig `yaml:"rabbitmq" desc:"Configuration for using RabbitMQ as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured." required:"N"` -} + AWSSQS AWSSQSConfig `yaml:"aws_sqs" desc:"Configuration for using AWS SQS as the message queue. Only one MQ provider should be configured." required:"N"` + GCPPubSub GCPPubSubConfig `yaml:"gcp_pubsub" desc:"Configuration for using GCP Pub/Sub as the message queue. Only one MQ provider should be configured." required:"N"` + RabbitMQ RabbitMQConfig `yaml:"rabbitmq" desc:"Configuration for using RabbitMQ as the message queue. Only one MQ provider should be configured." required:"N"` -func (c MQsConfig) GetInfraType() string { - if hasAWSSQSConfig(c.AWSSQS) { - return "awssqs" - } - if hasGCPPubSubConfig(c.GCPPubSub) { - return "gcppubsub" - } - if hasRabbitMQConfig(c.RabbitMQ) { - return "rabbitmq" - } - return "" + adapter MQConfigAdapter } -// getQueueConfig returns a queue config for the given queue type -// queueType can be "deliverymq" or "logmq" -func (c *MQsConfig) getQueueConfig(queueType string) *mqs.QueueConfig { - if c == nil { - return nil +func (c *MQsConfig) init() { + if c.adapter != nil { + return } - infraType := c.GetInfraType() - switch infraType { - case "awssqs": - queue := "" - if queueType == "deliverymq" { - queue = c.AWSSQS.DeliveryQueue - } else if queueType == "logmq" { - queue = c.AWSSQS.LogQueue - } - - creds := fmt.Sprintf("%s:%s:", c.AWSSQS.AccessKeyID, c.AWSSQS.SecretAccessKey) - return &mqs.QueueConfig{ - AWSSQS: &mqs.AWSSQSConfig{ - Endpoint: c.AWSSQS.Endpoint, - Region: c.AWSSQS.Region, - ServiceAccountCredentials: creds, - Topic: queue, - }, - } - case "gcppubsub": - topic := "" - subscription := "" - if queueType == "deliverymq" { - topic = c.GCPPubSub.DeliveryTopic - subscription = c.GCPPubSub.DeliverySubscription - } else if queueType == "logmq" { - topic = c.GCPPubSub.LogTopic - subscription = c.GCPPubSub.LogSubscription - } - return &mqs.QueueConfig{ - GCPPubSub: &mqs.GCPPubSubConfig{ - ProjectID: c.GCPPubSub.Project, - ServiceAccountCredentials: c.GCPPubSub.ServiceAccountCredentials, - TopicID: topic, - SubscriptionID: subscription, - }, - } - case "rabbitmq": - queue := "" - if queueType == "deliverymq" { - queue = c.RabbitMQ.DeliveryQueue - } else if queueType == "logmq" { - queue = c.RabbitMQ.LogQueue - } - return &mqs.QueueConfig{ - RabbitMQ: &mqs.RabbitMQConfig{ - ServerURL: c.RabbitMQ.ServerURL, - Exchange: c.RabbitMQ.Exchange, - Queue: queue, - }, - } - default: - return nil + if c.AWSSQS.IsConfigured() { + c.adapter = &c.AWSSQS + } else if c.GCPPubSub.IsConfigured() { + c.adapter = &c.GCPPubSub + } else if c.RabbitMQ.IsConfigured() { + c.adapter = &c.RabbitMQ } } -func (c MQsConfig) GetDeliveryQueueConfig() *mqs.QueueConfig { - infraType := c.GetInfraType() - switch infraType { - case "awssqs": - return c.getQueueConfig("deliverymq") - case "gcppubsub": - return c.getQueueConfig("deliverymq") - case "rabbitmq": - return c.getQueueConfig("deliverymq") - default: - return nil +func (c *MQsConfig) GetInfraType() string { + c.init() + if c.adapter == nil { + return "" } + return c.adapter.GetProviderType() } -func (c MQsConfig) GetLogQueueConfig() *mqs.QueueConfig { - infraType := c.GetInfraType() - switch infraType { - case "awssqs": - return c.getQueueConfig("logmq") - case "gcppubsub": - return c.getQueueConfig("logmq") - case "rabbitmq": - return c.getQueueConfig("logmq") - default: +func (c *MQsConfig) ToInfraConfig(queueType string) *mqinfra.MQInfraConfig { + c.init() + if c.adapter == nil { return nil } + return c.adapter.ToInfraConfig(queueType) } -// Helper functions to check for required fields -func hasAWSSQSConfig(config AWSSQSConfig) bool { - return config.AccessKeyID != "" && - config.SecretAccessKey != "" && config.Region != "" -} - -func hasGCPPubSubConfig(config GCPPubSubConfig) bool { - return config.Project != "" -} - -func hasRabbitMQConfig(config RabbitMQConfig) bool { - return config.ServerURL != "" +func (c *MQsConfig) ToQueueConfig(ctx context.Context, queueType string) (*mqs.QueueConfig, error) { + c.init() + if c.adapter == nil { + return nil, nil + } + return c.adapter.ToQueueConfig(ctx, queueType) } diff --git a/internal/config/mqconfig_aws.go b/internal/config/mqconfig_aws.go new file mode 100644 index 00000000..10541069 --- /dev/null +++ b/internal/config/mqconfig_aws.go @@ -0,0 +1,63 @@ +package config + +import ( + "context" + "fmt" + + "github.com/hookdeck/outpost/internal/mqinfra" + "github.com/hookdeck/outpost/internal/mqs" +) + +type AWSSQSConfig struct { + AccessKeyID string `yaml:"access_key_id" env:"AWS_SQS_ACCESS_KEY_ID" desc:"AWS Access Key ID for SQS. Required if AWS SQS is the chosen MQ provider." required:"C"` + SecretAccessKey string `yaml:"secret_access_key" env:"AWS_SQS_SECRET_ACCESS_KEY" desc:"AWS Secret Access Key for SQS. Required if AWS SQS is the chosen MQ provider." required:"C"` + Region string `yaml:"region" env:"AWS_SQS_REGION" desc:"AWS Region for SQS. Required if AWS SQS is the chosen MQ provider." required:"C"` + Endpoint string `yaml:"endpoint" env:"AWS_SQS_ENDPOINT" desc:"Custom AWS SQS endpoint URL. Optional, typically used for local testing (e.g., LocalStack)." required:"N"` + DeliveryQueue string `yaml:"delivery_queue" env:"AWS_SQS_DELIVERY_QUEUE" desc:"Name of the SQS queue for delivery events." required:"N"` + LogQueue string `yaml:"log_queue" env:"AWS_SQS_LOG_QUEUE" desc:"Name of the SQS queue for log events." required:"N"` +} + +func (c *AWSSQSConfig) getQueueName(queueType string) string { + switch queueType { + case "deliverymq": + return c.DeliveryQueue + case "logmq": + return c.LogQueue + default: + return "" + } +} + +func (c *AWSSQSConfig) getCredentials() string { + return fmt.Sprintf("%s:%s:", c.AccessKeyID, c.SecretAccessKey) +} + +func (c *AWSSQSConfig) ToInfraConfig(queueType string) *mqinfra.MQInfraConfig { + return &mqinfra.MQInfraConfig{ + AWSSQS: &mqinfra.AWSSQSInfraConfig{ + Endpoint: c.Endpoint, + Region: c.Region, + ServiceAccountCredentials: c.getCredentials(), + Topic: c.getQueueName(queueType), + }, + } +} + +func (c *AWSSQSConfig) ToQueueConfig(ctx context.Context, queueType string) (*mqs.QueueConfig, error) { + return &mqs.QueueConfig{ + AWSSQS: &mqs.AWSSQSConfig{ + Endpoint: c.Endpoint, + Region: c.Region, + ServiceAccountCredentials: c.getCredentials(), + Topic: c.getQueueName(queueType), + }, + }, nil +} + +func (c *AWSSQSConfig) GetProviderType() string { + return "awssqs" +} + +func (c *AWSSQSConfig) IsConfigured() bool { + return c.AccessKeyID != "" && c.SecretAccessKey != "" && c.Region != "" +} diff --git a/internal/config/mqconfig_gcp.go b/internal/config/mqconfig_gcp.go new file mode 100644 index 00000000..f277f4ed --- /dev/null +++ b/internal/config/mqconfig_gcp.go @@ -0,0 +1,69 @@ +package config + +import ( + "context" + + "github.com/hookdeck/outpost/internal/mqinfra" + "github.com/hookdeck/outpost/internal/mqs" +) + +type GCPPubSubConfig struct { + Project string `yaml:"project" env:"GCP_PUBSUB_PROJECT" desc:"GCP Project ID for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider." required:"C"` + ServiceAccountCredentials string `yaml:"service_account_credentials" env:"GCP_PUBSUB_SERVICE_ACCOUNT_CREDENTIALS" desc:"JSON string or path to a file containing GCP service account credentials for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider and not running in an environment with implicit credentials (e.g., GCE, GKE)." required:"C"` + DeliveryTopic string `yaml:"delivery_topic" env:"GCP_PUBSUB_DELIVERY_TOPIC" desc:"Name of the GCP Pub/Sub topic for delivery events." required:"N"` + DeliverySubscription string `yaml:"delivery_subscription" env:"GCP_PUBSUB_DELIVERY_SUBSCRIPTION" desc:"Name of the GCP Pub/Sub subscription for delivery events." required:"N"` + LogTopic string `yaml:"log_topic" env:"GCP_PUBSUB_LOG_TOPIC" desc:"Name of the GCP Pub/Sub topic for log events." required:"N"` + LogSubscription string `yaml:"log_subscription" env:"GCP_PUBSUB_LOG_SUBSCRIPTION" desc:"Name of the GCP Pub/Sub subscription for log events." required:"N"` +} + +func (c *GCPPubSubConfig) getTopicByQueueType(queueType string) string { + switch queueType { + case "deliverymq": + return c.DeliveryTopic + case "logmq": + return c.LogTopic + default: + return "" + } +} + +func (c *GCPPubSubConfig) getSubscriptionByQueueType(queueType string) string { + switch queueType { + case "deliverymq": + return c.DeliverySubscription + case "logmq": + return c.LogSubscription + default: + return "" + } +} + +func (c *GCPPubSubConfig) ToInfraConfig(queueType string) *mqinfra.MQInfraConfig { + return &mqinfra.MQInfraConfig{ + GCPPubSub: &mqinfra.GCPPubSubInfraConfig{ + ProjectID: c.Project, + ServiceAccountCredentials: c.ServiceAccountCredentials, + TopicID: c.getTopicByQueueType(queueType), + SubscriptionID: c.getSubscriptionByQueueType(queueType), + }, + } +} + +func (c *GCPPubSubConfig) ToQueueConfig(ctx context.Context, queueType string) (*mqs.QueueConfig, error) { + return &mqs.QueueConfig{ + GCPPubSub: &mqs.GCPPubSubConfig{ + ProjectID: c.Project, + ServiceAccountCredentials: c.ServiceAccountCredentials, + TopicID: c.getTopicByQueueType(queueType), + SubscriptionID: c.getSubscriptionByQueueType(queueType), + }, + }, nil +} + +func (c *GCPPubSubConfig) GetProviderType() string { + return "gcppubsub" +} + +func (c *GCPPubSubConfig) IsConfigured() bool { + return c.Project != "" +} diff --git a/internal/config/mqconfig_rabbitmq.go b/internal/config/mqconfig_rabbitmq.go new file mode 100644 index 00000000..b11df46c --- /dev/null +++ b/internal/config/mqconfig_rabbitmq.go @@ -0,0 +1,54 @@ +package config + +import ( + "context" + + "github.com/hookdeck/outpost/internal/mqinfra" + "github.com/hookdeck/outpost/internal/mqs" +) + +type RabbitMQConfig struct { + ServerURL string `yaml:"server_url" env:"RABBITMQ_SERVER_URL" desc:"RabbitMQ server connection URL (e.g., 'amqp://user:pass@host:port/vhost'). Required if RabbitMQ is the chosen MQ provider." required:"C"` + Exchange string `yaml:"exchange" env:"RABBITMQ_EXCHANGE" desc:"Name of the RabbitMQ exchange to use." required:"N"` + DeliveryQueue string `yaml:"delivery_queue" env:"RABBITMQ_DELIVERY_QUEUE" desc:"Name of the RabbitMQ queue for delivery events." required:"N"` + LogQueue string `yaml:"log_queue" env:"RABBITMQ_LOG_QUEUE" desc:"Name of the RabbitMQ queue for log events." required:"N"` +} + +func (c *RabbitMQConfig) getQueueName(queueType string) string { + switch queueType { + case "deliverymq": + return c.DeliveryQueue + case "logmq": + return c.LogQueue + default: + return "" + } +} + +func (c *RabbitMQConfig) ToInfraConfig(queueType string) *mqinfra.MQInfraConfig { + return &mqinfra.MQInfraConfig{ + RabbitMQ: &mqinfra.RabbitMQInfraConfig{ + ServerURL: c.ServerURL, + Exchange: c.Exchange, + Queue: c.getQueueName(queueType), + }, + } +} + +func (c *RabbitMQConfig) ToQueueConfig(ctx context.Context, queueType string) (*mqs.QueueConfig, error) { + return &mqs.QueueConfig{ + RabbitMQ: &mqs.RabbitMQConfig{ + ServerURL: c.ServerURL, + Exchange: c.Exchange, + Queue: c.getQueueName(queueType), + }, + }, nil +} + +func (c *RabbitMQConfig) GetProviderType() string { + return "rabbitmq" +} + +func (c *RabbitMQConfig) IsConfigured() bool { + return c.ServerURL != "" +} diff --git a/internal/config/validation.go b/internal/config/validation.go index 467d1581..8b33235c 100644 --- a/internal/config/validation.go +++ b/internal/config/validation.go @@ -1,6 +1,7 @@ package config import ( + "context" "fmt" "net/url" ) @@ -92,13 +93,23 @@ func (c *Config) validateLogStorage() error { // validateMQs validates the MQs configuration func (c *Config) validateMQs() error { + ctx := context.Background() + // Check delivery queue - if c.MQs.GetDeliveryQueueConfig() == nil { + deliveryConfig, err := c.MQs.ToQueueConfig(ctx, "deliverymq") + if err != nil { + return fmt.Errorf("failed to validate delivery queue config: %w", err) + } + if deliveryConfig == nil { return ErrMissingMQs } // Check log queue - if c.MQs.GetLogQueueConfig() == nil { + logConfig, err := c.MQs.ToQueueConfig(ctx, "logmq") + if err != nil { + return fmt.Errorf("failed to validate log queue config: %w", err) + } + if logConfig == nil { return ErrMissingMQs } diff --git a/internal/config/validation_test.go b/internal/config/validation_test.go index cb368f32..95a4b59c 100644 --- a/internal/config/validation_test.go +++ b/internal/config/validation_test.go @@ -199,7 +199,7 @@ func TestMQs(t *testing.T) { name: "missing mqs config", config: func() *config.Config { c := validConfig() - c.MQs = config.MQsConfig{} + c.MQs = &config.MQsConfig{} return c }(), wantErr: config.ErrMissingMQs, diff --git a/internal/infra/infra.go b/internal/infra/infra.go index d2a83776..eec8558d 100644 --- a/internal/infra/infra.go +++ b/internal/infra/infra.go @@ -4,12 +4,11 @@ import ( "context" "github.com/hookdeck/outpost/internal/mqinfra" - "github.com/hookdeck/outpost/internal/mqs" ) type Config struct { - DeliveryMQ *mqs.QueueConfig - LogMQ *mqs.QueueConfig + DeliveryMQ *mqinfra.MQInfraConfig + LogMQ *mqinfra.MQInfraConfig } func (cfg *Config) SetSensiblePolicyDefaults() { diff --git a/internal/mqinfra/awssqs.go b/internal/mqinfra/awssqs.go index 31c4e923..33300e76 100644 --- a/internal/mqinfra/awssqs.go +++ b/internal/mqinfra/awssqs.go @@ -10,7 +10,7 @@ import ( ) type infraAWSSQS struct { - cfg *mqs.QueueConfig + cfg *MQInfraConfig } func (infra *infraAWSSQS) Declare(ctx context.Context) error { @@ -18,7 +18,12 @@ func (infra *infraAWSSQS) Declare(ctx context.Context) error { return errors.New("failed assertion: cfg.AWSSQS != nil") // IMPOSSIBLE } - sqsClient, err := awsutil.SQSClientFromConfig(ctx, infra.cfg.AWSSQS) + sqsClient, err := awsutil.SQSClientFromConfig(ctx, &mqs.AWSSQSConfig{ + Endpoint: infra.cfg.AWSSQS.Endpoint, + Region: infra.cfg.AWSSQS.Region, + ServiceAccountCredentials: infra.cfg.AWSSQS.ServiceAccountCredentials, + Topic: infra.cfg.AWSSQS.Topic, + }) if err != nil { return err } @@ -54,7 +59,12 @@ func (infra *infraAWSSQS) TearDown(ctx context.Context) error { return errors.New("failed assertion: cfg.AWSSQS != nil") // IMPOSSIBLE } - sqsClient, err := awsutil.SQSClientFromConfig(ctx, infra.cfg.AWSSQS) + sqsClient, err := awsutil.SQSClientFromConfig(ctx, &mqs.AWSSQSConfig{ + Endpoint: infra.cfg.AWSSQS.Endpoint, + Region: infra.cfg.AWSSQS.Region, + ServiceAccountCredentials: infra.cfg.AWSSQS.ServiceAccountCredentials, + Topic: infra.cfg.AWSSQS.Topic, + }) if err != nil { return err } diff --git a/internal/mqinfra/gcppubsub.go b/internal/mqinfra/gcppubsub.go index 2c5cecbb..bfab4f5b 100644 --- a/internal/mqinfra/gcppubsub.go +++ b/internal/mqinfra/gcppubsub.go @@ -7,14 +7,13 @@ import ( "time" "cloud.google.com/go/pubsub" - "github.com/hookdeck/outpost/internal/mqs" "google.golang.org/api/option" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) type infraGCPPubSub struct { - cfg *mqs.QueueConfig + cfg *MQInfraConfig } func (infra *infraGCPPubSub) Declare(ctx context.Context) error { diff --git a/internal/mqinfra/mqinfra.go b/internal/mqinfra/mqinfra.go index b0490258..c7616079 100644 --- a/internal/mqinfra/mqinfra.go +++ b/internal/mqinfra/mqinfra.go @@ -3,8 +3,6 @@ package mqinfra import ( "context" "fmt" - - "github.com/hookdeck/outpost/internal/mqs" ) type MQInfra interface { @@ -12,12 +10,57 @@ type MQInfra interface { TearDown(ctx context.Context) error } -func New(cfg *mqs.QueueConfig) MQInfra { +type MQInfraConfig struct { + AWSSQS *AWSSQSInfraConfig + AzureServiceBus *AzureServiceBusInfraConfig + GCPPubSub *GCPPubSubInfraConfig + RabbitMQ *RabbitMQInfraConfig + + Policy Policy +} + +type Policy struct { + VisibilityTimeout int + RetryLimit int +} + +type AWSSQSInfraConfig struct { + Endpoint string + Region string + ServiceAccountCredentials string + Topic string +} + +type AzureServiceBusInfraConfig struct { + TenantID string + ClientID string + ClientSecret string + SubscriptionID string + ResourceGroup string + Namespace string + Topic string + Subscription string +} + +type GCPPubSubInfraConfig struct { + ProjectID string + TopicID string + SubscriptionID string + ServiceAccountCredentials string +} + +type RabbitMQInfraConfig struct { + ServerURL string + Exchange string + Queue string +} + +func New(cfg *MQInfraConfig) MQInfra { if cfg.AWSSQS != nil { return &infraAWSSQS{cfg: cfg} } if cfg.AzureServiceBus != nil { - return &infraUnimplemented{} + return &infraAzureServiceBus{cfg: cfg} } if cfg.GCPPubSub != nil { return &infraGCPPubSub{cfg: cfg} @@ -40,15 +83,16 @@ func (infra *infraInvalid) TearDown(ctx context.Context) error { return ErrInvalidConfig } -type infraUnimplemented struct { +type infraAzureServiceBus struct { + cfg *MQInfraConfig } -func (infra *infraUnimplemented) Declare(ctx context.Context) error { - return ErrInvalidConfig +func (infra *infraAzureServiceBus) Declare(ctx context.Context) error { + return ErrInfraUnimplemented } -func (infra *infraUnimplemented) TearDown(ctx context.Context) error { - return ErrInvalidConfig +func (infra *infraAzureServiceBus) TearDown(ctx context.Context) error { + return ErrInfraUnimplemented } var ( diff --git a/internal/mqinfra/mqinfra_test.go b/internal/mqinfra/mqinfra_test.go index 5e90ae0d..41365cfc 100644 --- a/internal/mqinfra/mqinfra_test.go +++ b/internal/mqinfra/mqinfra_test.go @@ -17,20 +17,25 @@ import ( const retryLimit = 5 -func testMQInfra(t *testing.T, mqConfig mqs.QueueConfig, dlqConfig mqs.QueueConfig) { +type Config struct { + infra mqinfra.MQInfraConfig + mq mqs.QueueConfig +} + +func testMQInfra(t *testing.T, mqConfig *Config, dlqConfig *Config) { t.Parallel() t.Cleanup(testinfra.Start(t)) t.Run("should create queue", func(t *testing.T) { ctx := context.Background() - infra := mqinfra.New(&mqConfig) + infra := mqinfra.New(&mqConfig.infra) require.NoError(t, infra.Declare(ctx)) t.Cleanup(func() { require.NoError(t, infra.TearDown(ctx)) }) - mq := mqs.NewQueue(&mqConfig) + mq := mqs.NewQueue(&mqConfig.mq) cleanup, err := mq.Init(ctx) require.NoError(t, err) t.Cleanup(cleanup) @@ -76,14 +81,14 @@ func testMQInfra(t *testing.T, mqConfig mqs.QueueConfig, dlqConfig mqs.QueueConf // - Afterwards, reading the DLQ should return the message. t.Run("should create dlq queue", func(t *testing.T) { ctx := context.Background() - infra := mqinfra.New(&mqConfig) + infra := mqinfra.New(&mqConfig.infra) require.NoError(t, infra.Declare(ctx)) t.Cleanup(func() { require.NoError(t, infra.TearDown(ctx)) }) - mq := mqs.NewQueue(&mqConfig) + mq := mqs.NewQueue(&mqConfig.mq) cleanup, err := mq.Init(ctx) require.NoError(t, err) t.Cleanup(cleanup) @@ -126,7 +131,7 @@ func testMQInfra(t *testing.T, mqConfig mqs.QueueConfig, dlqConfig mqs.QueueConf } require.Equal(t, retryLimit+1, msgCount) - dlmq := mqs.NewQueue(&dlqConfig) + dlmq := mqs.NewQueue(&dlqConfig.mq) dlsubscription, err := dlmq.Subscribe(ctx) require.NoError(t, err) dlmsgchan := make(chan *testutil.MockMsg) @@ -168,21 +173,42 @@ func TestIntegrationMQInfra_RabbitMQ(t *testing.T) { queue := uuid.New().String() testMQInfra(t, - mqs.QueueConfig{ - RabbitMQ: &mqs.RabbitMQConfig{ - ServerURL: testinfra.EnsureRabbitMQ(), - Exchange: exchange, - Queue: queue, + &Config{ + infra: mqinfra.MQInfraConfig{ + RabbitMQ: &mqinfra.RabbitMQInfraConfig{ + ServerURL: testinfra.EnsureRabbitMQ(), + Exchange: exchange, + Queue: queue, + }, + Policy: mqinfra.Policy{ + RetryLimit: retryLimit, + }, }, - Policy: mqs.Policy{ - RetryLimit: retryLimit, + mq: mqs.QueueConfig{ + RabbitMQ: &mqs.RabbitMQConfig{ + ServerURL: testinfra.EnsureRabbitMQ(), + Exchange: exchange, + Queue: queue, + }, + Policy: mqs.Policy{ + RetryLimit: retryLimit, + }, }, }, - mqs.QueueConfig{ - RabbitMQ: &mqs.RabbitMQConfig{ - ServerURL: testinfra.EnsureRabbitMQ(), - Exchange: exchange, - Queue: queue + ".dlq", + &Config{ + infra: mqinfra.MQInfraConfig{ + RabbitMQ: &mqinfra.RabbitMQInfraConfig{ + ServerURL: testinfra.EnsureRabbitMQ(), + Exchange: exchange, + Queue: queue + ".dlq", + }, + }, + mq: mqs.QueueConfig{ + RabbitMQ: &mqs.RabbitMQConfig{ + ServerURL: testinfra.EnsureRabbitMQ(), + Exchange: exchange, + Queue: queue + ".dlq", + }, }, }, ) @@ -192,24 +218,48 @@ func TestIntegrationMQInfra_AWSSQS(t *testing.T) { q := uuid.New().String() testMQInfra(t, - mqs.QueueConfig{ - AWSSQS: &mqs.AWSSQSConfig{ - Endpoint: testinfra.EnsureLocalStack(), - ServiceAccountCredentials: "test:test:", - Region: "us-east-1", - Topic: q, + &Config{ + infra: mqinfra.MQInfraConfig{ + AWSSQS: &mqinfra.AWSSQSInfraConfig{ + Endpoint: testinfra.EnsureLocalStack(), + ServiceAccountCredentials: "test:test:", + Region: "us-east-1", + Topic: q, + }, + Policy: mqinfra.Policy{ + RetryLimit: retryLimit, + VisibilityTimeout: 1, + }, }, - Policy: mqs.Policy{ - RetryLimit: retryLimit, - VisibilityTimeout: 1, + mq: mqs.QueueConfig{ + AWSSQS: &mqs.AWSSQSConfig{ + Endpoint: testinfra.EnsureLocalStack(), + ServiceAccountCredentials: "test:test:", + Region: "us-east-1", + Topic: q, + }, + Policy: mqs.Policy{ + RetryLimit: retryLimit, + VisibilityTimeout: 1, + }, }, }, - mqs.QueueConfig{ - AWSSQS: &mqs.AWSSQSConfig{ - Endpoint: testinfra.EnsureLocalStack(), - ServiceAccountCredentials: "test:test:", - Region: "us-east-1", - Topic: q + "-dlq", + &Config{ + infra: mqinfra.MQInfraConfig{ + AWSSQS: &mqinfra.AWSSQSInfraConfig{ + Endpoint: testinfra.EnsureLocalStack(), + ServiceAccountCredentials: "test:test:", + Region: "us-east-1", + Topic: q + "-dlq", + }, + }, + mq: mqs.QueueConfig{ + AWSSQS: &mqs.AWSSQSConfig{ + Endpoint: testinfra.EnsureLocalStack(), + ServiceAccountCredentials: "test:test:", + Region: "us-east-1", + Topic: q + "-dlq", + }, }, }, ) @@ -223,24 +273,48 @@ func TestIntegrationMQInfra_GCPPubSub(t *testing.T) { subscriptionID := topicID + "-subscription" testMQInfra(t, - mqs.QueueConfig{ - GCPPubSub: &mqs.GCPPubSubConfig{ - ProjectID: "test-project", - TopicID: topicID, - SubscriptionID: subscriptionID, - ServiceAccountCredentials: "", + &Config{ + infra: mqinfra.MQInfraConfig{ + GCPPubSub: &mqinfra.GCPPubSubInfraConfig{ + ProjectID: "test-project", + TopicID: topicID, + SubscriptionID: subscriptionID, + ServiceAccountCredentials: "", + }, + Policy: mqinfra.Policy{ + RetryLimit: retryLimit, + VisibilityTimeout: 10, + }, }, - Policy: mqs.Policy{ - RetryLimit: retryLimit, - VisibilityTimeout: 10, + mq: mqs.QueueConfig{ + GCPPubSub: &mqs.GCPPubSubConfig{ + ProjectID: "test-project", + TopicID: topicID, + SubscriptionID: subscriptionID, + ServiceAccountCredentials: "", + }, + Policy: mqs.Policy{ + RetryLimit: retryLimit, + VisibilityTimeout: 10, + }, }, }, - mqs.QueueConfig{ - GCPPubSub: &mqs.GCPPubSubConfig{ - ProjectID: "test-project", - TopicID: topicID + "-dlq", - SubscriptionID: topicID + "-dlq-sub", - ServiceAccountCredentials: "", + &Config{ + infra: mqinfra.MQInfraConfig{ + GCPPubSub: &mqinfra.GCPPubSubInfraConfig{ + ProjectID: "test-project", + TopicID: topicID + "-dlq", + SubscriptionID: topicID + "-dlq-sub", + ServiceAccountCredentials: "", + }, + }, + mq: mqs.QueueConfig{ + GCPPubSub: &mqs.GCPPubSubConfig{ + ProjectID: "test-project", + TopicID: topicID + "-dlq", + SubscriptionID: topicID + "-dlq-sub", + ServiceAccountCredentials: "", + }, }, }, ) diff --git a/internal/mqinfra/rabbitmq.go b/internal/mqinfra/rabbitmq.go index 62d46e87..f0067952 100644 --- a/internal/mqinfra/rabbitmq.go +++ b/internal/mqinfra/rabbitmq.go @@ -4,12 +4,11 @@ import ( "context" "errors" - "github.com/hookdeck/outpost/internal/mqs" "github.com/rabbitmq/amqp091-go" ) type infraRabbitMQ struct { - cfg *mqs.QueueConfig + cfg *MQInfraConfig } func (infra *infraRabbitMQ) Declare(ctx context.Context) error { diff --git a/internal/services/api/api.go b/internal/services/api/api.go index 2cb288cb..f4ed5576 100644 --- a/internal/services/api/api.go +++ b/internal/services/api/api.go @@ -56,7 +56,11 @@ func NewService(ctx context.Context, wg *sync.WaitGroup, cfg *config.Config, log return nil, err } - deliveryMQ := deliverymq.New(deliverymq.WithQueue(cfg.MQs.GetDeliveryQueueConfig())) + deliveryQueueConfig, err := cfg.MQs.ToQueueConfig(ctx, "deliverymq") + if err != nil { + return nil, err + } + deliveryMQ := deliverymq.New(deliverymq.WithQueue(deliveryQueueConfig)) cleanupDeliveryMQ, err := deliveryMQ.Init(ctx) if err != nil { return nil, err diff --git a/internal/services/delivery/delivery.go b/internal/services/delivery/delivery.go index a65c251a..6bd9bb09 100644 --- a/internal/services/delivery/delivery.go +++ b/internal/services/delivery/delivery.go @@ -52,13 +52,29 @@ func NewService(ctx context.Context, return nil, err } - logMQ := logmq.New(logmq.WithQueue(cfg.MQs.GetLogQueueConfig())) + logmqConfig, err := cfg.MQs.ToQueueConfig(ctx, "logmq") + if err != nil { + return nil, err + } + deliverymqConfig, err := cfg.MQs.ToQueueConfig(ctx, "deliverymq") + if err != nil { + return nil, err + } + + logMQ := logmq.New(logmq.WithQueue(logmqConfig)) cleanupLogMQ, err := logMQ.Init(ctx) if err != nil { return nil, err } cleanupFuncs = append(cleanupFuncs, cleanupLogMQ) + deliveryMQ := deliverymq.New(deliverymq.WithQueue(deliverymqConfig)) + cleanupDeliveryMQ, err := deliveryMQ.Init(ctx) + if err != nil { + return nil, err + } + cleanupFuncs = append(cleanupFuncs, cleanupDeliveryMQ) + if handler == nil { registry := destregistry.NewRegistry(&destregistry.Config{ DestinationMetadataPath: cfg.Destinations.MetadataPath, @@ -94,12 +110,6 @@ func NewService(ctx context.Context, return nil, err } - deliveryMQ := deliverymq.New(deliverymq.WithQueue(cfg.MQs.GetDeliveryQueueConfig())) - cleanupDeliveryMQ, err := deliveryMQ.Init(ctx) - if err != nil { - return nil, err - } - cleanupFuncs = append(cleanupFuncs, cleanupDeliveryMQ) retryScheduler := deliverymq.NewRetryScheduler(deliveryMQ, cfg.Redis.ToConfig()) if err := retryScheduler.Init(ctx); err != nil { return nil, err @@ -146,7 +156,7 @@ func NewService(ctx context.Context, Logger: logger, RedisClient: redisClient, Handler: handler, - DeliveryMQ: deliverymq.New(deliverymq.WithQueue(cfg.MQs.GetDeliveryQueueConfig())), + DeliveryMQ: deliveryMQ, consumerOptions: &consumerOptions{ concurreny: cfg.DeliveryMaxConcurrency, }, diff --git a/internal/services/log/log.go b/internal/services/log/log.go index cd3088b1..f8f72a1e 100644 --- a/internal/services/log/log.go +++ b/internal/services/log/log.go @@ -84,10 +84,15 @@ func NewService(ctx context.Context, } }) + logQueueConfig, err := cfg.MQs.ToQueueConfig(ctx, "logmq") + if err != nil { + return nil, err + } + service := &LogService{} service.logger = logger service.redisClient = redisClient - service.logMQ = logmq.New(logmq.WithQueue(cfg.MQs.GetLogQueueConfig())) + service.logMQ = logmq.New(logmq.WithQueue(logQueueConfig)) service.consumerOptions = &consumerOptions{ concurreny: cfg.DeliveryMaxConcurrency, } From e9a6fc161aa14237e80eb0d58735e3009ab19c1c Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 9 Jun 2025 20:19:12 +0700 Subject: [PATCH 08/27] chore: gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 331706db..707a2c9c 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ terraform.tfstate.* .DS_Store *.log cmd/dev +.ai/ .bruno/ .cursor/ -.ai/ +.claude/ +*.local.* From dfd6717001cc62091436be14e02a543910212720 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 9 Jun 2025 20:47:20 +0700 Subject: [PATCH 09/27] chore: Remove policy from data-plan mq config --- internal/mqinfra/mqinfra_test.go | 11 ----------- internal/mqs/config.go | 7 ------- 2 files changed, 18 deletions(-) diff --git a/internal/mqinfra/mqinfra_test.go b/internal/mqinfra/mqinfra_test.go index 41365cfc..d3402cbf 100644 --- a/internal/mqinfra/mqinfra_test.go +++ b/internal/mqinfra/mqinfra_test.go @@ -190,9 +190,6 @@ func TestIntegrationMQInfra_RabbitMQ(t *testing.T) { Exchange: exchange, Queue: queue, }, - Policy: mqs.Policy{ - RetryLimit: retryLimit, - }, }, }, &Config{ @@ -238,10 +235,6 @@ func TestIntegrationMQInfra_AWSSQS(t *testing.T) { Region: "us-east-1", Topic: q, }, - Policy: mqs.Policy{ - RetryLimit: retryLimit, - VisibilityTimeout: 1, - }, }, }, &Config{ @@ -293,10 +286,6 @@ func TestIntegrationMQInfra_GCPPubSub(t *testing.T) { SubscriptionID: subscriptionID, ServiceAccountCredentials: "", }, - Policy: mqs.Policy{ - RetryLimit: retryLimit, - VisibilityTimeout: 10, - }, }, }, &Config{ diff --git a/internal/mqs/config.go b/internal/mqs/config.go index 0854be1c..5208c7cd 100644 --- a/internal/mqs/config.go +++ b/internal/mqs/config.go @@ -6,13 +6,6 @@ type QueueConfig struct { GCPPubSub *GCPPubSubConfig RabbitMQ *RabbitMQConfig InMemory *InMemoryConfig // mainly for testing purposes - - Policy Policy -} - -type Policy struct { - VisibilityTimeout int // seconds - RetryLimit int } type AzureServiceBusConfig struct { From 5296fd50c6df8d46bb6e04216e58eab4cba40033 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 9 Jun 2025 23:25:05 +0700 Subject: [PATCH 10/27] build: pin air version in dev dockerfile --- build/dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/dev/Dockerfile b/build/dev/Dockerfile index f55ab15c..aae625c4 100644 --- a/build/dev/Dockerfile +++ b/build/dev/Dockerfile @@ -1,5 +1,5 @@ FROM golang:1.23-alpine AS fetch -RUN go install github.com/air-verse/air@latest +RUN go install github.com/air-verse/air@v1.61.1 WORKDIR /app COPY . . CMD ["air"] From b902b5e06208ac2b414008adb04703d6d3728ba3 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 10 Jun 2025 00:41:34 +0700 Subject: [PATCH 11/27] feat: azure mq & mqinfra --- internal/config/config.go | 6 + internal/config/mq.go | 9 +- internal/config/mqconfig_azure.go | 145 +++++++++++++++ internal/config/validation.go | 2 +- internal/mqinfra/azureservicebus.go | 258 ++++++++++++++++++++++++++ internal/mqinfra/mqinfra.go | 12 -- internal/mqinfra/mqinfra_test.go | 74 ++++++++ internal/mqs/config.go | 23 --- internal/mqs/queue.go | 14 +- internal/mqs/queue_azureservicebus.go | 149 +++++++++++++++ internal/mqs/queue_gcppubsub.go | 7 + internal/mqs/queue_test.go | 33 +++- internal/util/testinfra/testinfra.go | 9 +- 13 files changed, 697 insertions(+), 44 deletions(-) create mode 100644 internal/config/mqconfig_azure.go create mode 100644 internal/mqinfra/azureservicebus.go delete mode 100644 internal/mqs/config.go create mode 100644 internal/mqs/queue_azureservicebus.go diff --git a/internal/config/config.go b/internal/config/config.go index 12adbfc1..6fbff182 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -138,6 +138,12 @@ func (c *Config) InitDefaults() { LogTopic: "outpost-log", LogSubscription: "outpost-log-sub", }, + AzureServiceBus: AzureServiceBusConfig{ + DeliveryTopic: "outpost-delivery", + DeliverySubscription: "outpost-delivery-sub", + LogTopic: "outpost-log", + LogSubscription: "outpost-log-sub", + }, } c.PublishMaxConcurrency = 1 c.DeliveryMaxConcurrency = 1 diff --git a/internal/config/mq.go b/internal/config/mq.go index 660e9d37..a2312a99 100644 --- a/internal/config/mq.go +++ b/internal/config/mq.go @@ -15,9 +15,10 @@ type MQConfigAdapter interface { } type MQsConfig struct { - AWSSQS AWSSQSConfig `yaml:"aws_sqs" desc:"Configuration for using AWS SQS as the message queue. Only one MQ provider should be configured." required:"N"` - GCPPubSub GCPPubSubConfig `yaml:"gcp_pubsub" desc:"Configuration for using GCP Pub/Sub as the message queue. Only one MQ provider should be configured." required:"N"` - RabbitMQ RabbitMQConfig `yaml:"rabbitmq" desc:"Configuration for using RabbitMQ as the message queue. Only one MQ provider should be configured." required:"N"` + AWSSQS AWSSQSConfig `yaml:"aws_sqs" desc:"Configuration for using AWS SQS as the message queue. Only one MQ provider should be configured." required:"N"` + AzureServiceBus AzureServiceBusConfig `yaml:"azure_servicebus" desc:"Configuration for using Azure Service Bus as the message queue. Only one MQ provider should be configured." required:"N"` + GCPPubSub GCPPubSubConfig `yaml:"gcp_pubsub" desc:"Configuration for using GCP Pub/Sub as the message queue. Only one MQ provider should be configured." required:"N"` + RabbitMQ RabbitMQConfig `yaml:"rabbitmq" desc:"Configuration for using RabbitMQ as the message queue. Only one MQ provider should be configured." required:"N"` adapter MQConfigAdapter } @@ -29,6 +30,8 @@ func (c *MQsConfig) init() { if c.AWSSQS.IsConfigured() { c.adapter = &c.AWSSQS + } else if c.AzureServiceBus.IsConfigured() { + c.adapter = &c.AzureServiceBus } else if c.GCPPubSub.IsConfigured() { c.adapter = &c.GCPPubSub } else if c.RabbitMQ.IsConfigured() { diff --git a/internal/config/mqconfig_azure.go b/internal/config/mqconfig_azure.go new file mode 100644 index 00000000..a8144bf7 --- /dev/null +++ b/internal/config/mqconfig_azure.go @@ -0,0 +1,145 @@ +package config + +import ( + "context" + "fmt" + "sync" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus" + "github.com/hookdeck/outpost/internal/mqinfra" + "github.com/hookdeck/outpost/internal/mqs" +) + +type AzureServiceBusConfig struct { + TenantID string `yaml:"tenant_id" desc:"Azure Active Directory tenant ID" required:"Y"` + ClientID string `yaml:"client_id" desc:"Service principal client ID" required:"Y"` + ClientSecret string `yaml:"client_secret" desc:"Service principal client secret" required:"Y"` + SubscriptionID string `yaml:"subscription_id" desc:"Azure subscription ID" required:"Y"` + ResourceGroup string `yaml:"resource_group" desc:"Azure resource group name" required:"Y"` + Namespace string `yaml:"namespace" desc:"Azure Service Bus namespace" required:"Y"` + + DeliveryTopic string `yaml:"delivery_topic" desc:"Topic name for delivery queue" required:"N" default:"outpost-delivery"` + DeliverySubscription string `yaml:"delivery_subscription" desc:"Subscription name for delivery queue" required:"N" default:"outpost-delivery-subscription"` + LogTopic string `yaml:"log_topic" desc:"Topic name for log queue" required:"N" default:"outpost-log"` + LogSubscription string `yaml:"log_subscription" desc:"Subscription name for log queue" required:"N" default:"outpost-log-subscription"` + + connectionStringOnce sync.Once + connectionString string + connectionStringError error +} + +func (c *AzureServiceBusConfig) IsConfigured() bool { + return c.TenantID != "" && c.ClientID != "" && c.ClientSecret != "" && c.SubscriptionID != "" && c.ResourceGroup != "" && c.Namespace != "" +} + +func (c *AzureServiceBusConfig) GetProviderType() string { + return "azure_service_bus" +} + +func (c *AzureServiceBusConfig) getTopicByQueueType(queueType string) string { + switch queueType { + case "deliverymq": + return c.DeliveryTopic + case "logmq": + return c.LogTopic + default: + return "" + } +} + +func (c *AzureServiceBusConfig) getSubscriptionByQueueType(queueType string) string { + switch queueType { + case "deliverymq": + return c.DeliverySubscription + case "logmq": + return c.LogSubscription + default: + return "" + } +} + +func (c *AzureServiceBusConfig) ToInfraConfig(queueType string) *mqinfra.MQInfraConfig { + if !c.IsConfigured() { + return nil + } + + topic := c.getTopicByQueueType(queueType) + subscription := c.getSubscriptionByQueueType(queueType) + + return &mqinfra.MQInfraConfig{ + AzureServiceBus: &mqinfra.AzureServiceBusInfraConfig{ + TenantID: c.TenantID, + ClientID: c.ClientID, + ClientSecret: c.ClientSecret, + SubscriptionID: c.SubscriptionID, + ResourceGroup: c.ResourceGroup, + Namespace: c.Namespace, + Topic: topic, + Subscription: subscription, + }, + } +} + +func (c *AzureServiceBusConfig) ToQueueConfig(ctx context.Context, queueType string) (*mqs.QueueConfig, error) { + if !c.IsConfigured() { + return nil, nil + } + + // connectionString, err := c.getConnectionString(ctx) + // if err != nil { + // return nil, err + // } + + topic := c.getTopicByQueueType(queueType) + subscription := c.getSubscriptionByQueueType(queueType) + + return &mqs.QueueConfig{ + AzureServiceBus: &mqs.AzureServiceBusConfig{ + Topic: topic, + Subscription: subscription, + TenantID: c.TenantID, + ClientID: c.ClientID, + ClientSecret: c.ClientSecret, + SubscriptionID: c.SubscriptionID, + ResourceGroup: c.ResourceGroup, + Namespace: c.Namespace, + }, + }, nil +} + +func (c *AzureServiceBusConfig) getConnectionString(ctx context.Context) (string, error) { + c.connectionStringOnce.Do(func() { + cred, err := azidentity.NewClientSecretCredential( + c.TenantID, + c.ClientID, + c.ClientSecret, + nil, + ) + if err != nil { + c.connectionStringError = fmt.Errorf("failed to create credential: %w", err) + return + } + + sbClient, err := armservicebus.NewNamespacesClient(c.SubscriptionID, cred, nil) + if err != nil { + c.connectionStringError = fmt.Errorf("failed to create servicebus client: %w", err) + return + } + + keysResp, err := sbClient.ListKeys(ctx, c.ResourceGroup, c.Namespace, "RootManageSharedAccessKey", nil) + if err != nil { + c.connectionStringError = fmt.Errorf("failed to get keys: %w", err) + return + } + + if keysResp.PrimaryConnectionString == nil { + c.connectionStringError = fmt.Errorf("no connection string found") + return + } + + c.connectionString = *keysResp.PrimaryConnectionString + }) + + return c.connectionString, c.connectionStringError +} diff --git a/internal/config/validation.go b/internal/config/validation.go index 8b33235c..d474a636 100644 --- a/internal/config/validation.go +++ b/internal/config/validation.go @@ -94,7 +94,7 @@ func (c *Config) validateLogStorage() error { // validateMQs validates the MQs configuration func (c *Config) validateMQs() error { ctx := context.Background() - + // Check delivery queue deliveryConfig, err := c.MQs.ToQueueConfig(ctx, "deliverymq") if err != nil { diff --git a/internal/mqinfra/azureservicebus.go b/internal/mqinfra/azureservicebus.go new file mode 100644 index 00000000..01ce76ee --- /dev/null +++ b/internal/mqinfra/azureservicebus.go @@ -0,0 +1,258 @@ +package mqinfra + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus" +) + +type infraAzureServiceBus struct { + cfg *MQInfraConfig +} + +func (infra *infraAzureServiceBus) Declare(ctx context.Context) error { + if infra.cfg == nil || infra.cfg.AzureServiceBus == nil { + return fmt.Errorf("failed assertion: cfg.AzureServiceBus != nil") + } + + cfg := infra.cfg.AzureServiceBus + + // Create credential for authentication + cred, err := azidentity.NewClientSecretCredential( + cfg.TenantID, + cfg.ClientID, + cfg.ClientSecret, + nil, + ) + if err != nil { + return fmt.Errorf("failed to create credential: %w", err) + } + + // Create clients for topic and subscription management + topicClient, err := armservicebus.NewTopicsClient(cfg.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create topic client: %w", err) + } + + subClient, err := armservicebus.NewSubscriptionsClient(cfg.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create subscription client: %w", err) + } + + // Declare main topic (upsert) + topicName := cfg.Topic + err = infra.declareTopic(ctx, topicClient, cfg.ResourceGroup, cfg.Namespace, topicName) + if err != nil { + return fmt.Errorf("failed to declare topic: %w", err) + } + + // Configure retry policy settings + lockDuration := "PT1M" // 1 minute default + if infra.cfg.Policy.VisibilityTimeout > 0 { + lockDuration = fmt.Sprintf("PT%dS", infra.cfg.Policy.VisibilityTimeout) + } + + // Set maximum delivery count (Azure default is 10, but we apply our retry policy) + maxDeliveryCount := int32(10) // Azure default + if infra.cfg.Policy.RetryLimit > 0 { + // Adding 1 because Azure counts the initial delivery as attempt #1 + maxDeliveryCount = int32(infra.cfg.Policy.RetryLimit + 1) + if maxDeliveryCount < 1 { + maxDeliveryCount = 1 + } + } + + // Create main subscription with DLQ configuration + subName := cfg.Subscription + subConfig := &armservicebus.SBSubscription{ + Properties: &armservicebus.SBSubscriptionProperties{ + LockDuration: to.Ptr(lockDuration), + DefaultMessageTimeToLive: to.Ptr("P14D"), // 14 days + DeadLetteringOnMessageExpiration: to.Ptr(true), + MaxDeliveryCount: to.Ptr(maxDeliveryCount), + EnableBatchedOperations: to.Ptr(true), + RequiresSession: to.Ptr(false), + // Azure Service Bus has built-in dead letter queue functionality + // Dead lettered messages automatically go to the subscription's DLQ + }, + } + + err = infra.declareSubscription(ctx, subClient, cfg.ResourceGroup, cfg.Namespace, topicName, subName, subConfig) + if err != nil { + return fmt.Errorf("failed to declare subscription: %w", err) + } + + return nil +} + +func (infra *infraAzureServiceBus) TearDown(ctx context.Context) error { + if infra.cfg == nil || infra.cfg.AzureServiceBus == nil { + return fmt.Errorf("failed assertion: cfg.AzureServiceBus != nil") + } + + cfg := infra.cfg.AzureServiceBus + + // Create credential for authentication + cred, err := azidentity.NewClientSecretCredential( + cfg.TenantID, + cfg.ClientID, + cfg.ClientSecret, + nil, + ) + if err != nil { + return fmt.Errorf("failed to create credential: %w", err) + } + + // Create clients for topic and subscription management + topicClient, err := armservicebus.NewTopicsClient(cfg.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create topic client: %w", err) + } + + subClient, err := armservicebus.NewSubscriptionsClient(cfg.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create subscription client: %w", err) + } + + topicName := cfg.Topic + + // Delete main subscription + err = infra.deleteSubscription(ctx, subClient, cfg.ResourceGroup, cfg.Namespace, topicName, cfg.Subscription) + if err != nil { + return fmt.Errorf("failed to delete subscription: %w", err) + } + + // Delete main topic + err = infra.deleteTopic(ctx, topicClient, cfg.ResourceGroup, cfg.Namespace, topicName) + if err != nil { + return fmt.Errorf("failed to delete topic: %w", err) + } + + return nil +} + +// Helper methods for resource management + +func (infra *infraAzureServiceBus) declareTopic(ctx context.Context, client *armservicebus.TopicsClient, resourceGroup, namespace, topicName string) error { + // First, try to get the existing topic + _, err := client.Get(ctx, resourceGroup, namespace, topicName, nil) + if err == nil { + // Topic already exists, no need to create + return nil + } + + // If it's not a "not found" error, return the error + if !isNotFoundError(err) { + return fmt.Errorf("failed to check topic %s existence: %w", topicName, err) + } + + // Topic doesn't exist, create it + _, err = client.CreateOrUpdate( + ctx, + resourceGroup, + namespace, + topicName, + armservicebus.SBTopic{ + Properties: &armservicebus.SBTopicProperties{ + MaxSizeInMegabytes: to.Ptr[int32](1024), + DefaultMessageTimeToLive: to.Ptr("P14D"), // 14 days + EnablePartitioning: to.Ptr(false), + RequiresDuplicateDetection: to.Ptr(false), + DuplicateDetectionHistoryTimeWindow: to.Ptr("PT10M"), // 10 minutes + SupportOrdering: to.Ptr(true), + }, + }, + nil, + ) + if err != nil { + return fmt.Errorf("failed to create topic %s: %w", topicName, err) + } + + return nil +} + +func (infra *infraAzureServiceBus) declareSubscription(ctx context.Context, client *armservicebus.SubscriptionsClient, resourceGroup, namespace, topicName, subName string, config *armservicebus.SBSubscription) error { + // First, try to get the existing subscription + _, err := client.Get(ctx, resourceGroup, namespace, topicName, subName, nil) + if err == nil { + // Subscription already exists, no need to create + return nil + } + + // If it's not a "not found" error, return the error + if !isNotFoundError(err) { + return fmt.Errorf("failed to check subscription %s existence: %w", subName, err) + } + + // Subscription doesn't exist, create it + _, err = client.CreateOrUpdate( + ctx, + resourceGroup, + namespace, + topicName, + subName, + *config, + nil, + ) + if err != nil { + return fmt.Errorf("failed to create subscription %s: %w", subName, err) + } + + return nil +} + +func (infra *infraAzureServiceBus) deleteTopic(ctx context.Context, client *armservicebus.TopicsClient, resourceGroup, namespace, topicName string) error { + // Delete topic directly - no existence check needed + _, err := client.Delete(ctx, resourceGroup, namespace, topicName, nil) + if err != nil { + // If topic doesn't exist, that's fine - deletion is idempotent + if isNotFoundError(err) { + return nil + } + return fmt.Errorf("failed to delete topic %s: %w", topicName, err) + } + + return nil +} + +func (infra *infraAzureServiceBus) deleteSubscription(ctx context.Context, client *armservicebus.SubscriptionsClient, resourceGroup, namespace, topicName, subName string) error { + // Delete subscription directly - no existence check needed + _, err := client.Delete(ctx, resourceGroup, namespace, topicName, subName, nil) + if err != nil { + // If subscription doesn't exist, that's fine - deletion is idempotent + if isNotFoundError(err) { + return nil + } + return fmt.Errorf("failed to delete subscription %s: %w", subName, err) + } + + return nil +} + +// Helper function to check if error is a "not found" error +func isNotFoundError(err error) bool { + if err == nil { + return false + } + + // Check for common Azure "not found" error patterns + errStr := err.Error() + + // HTTP 404 Not Found + if strings.Contains(errStr, "404") || strings.Contains(errStr, "Not Found") { + return true + } + + // Azure-specific error codes + if strings.Contains(errStr, "ResourceNotFound") || + strings.Contains(errStr, "EntityNotFound") || + strings.Contains(errStr, "MessagingEntityNotFound") { + return true + } + + return false +} diff --git a/internal/mqinfra/mqinfra.go b/internal/mqinfra/mqinfra.go index c7616079..8840b8d8 100644 --- a/internal/mqinfra/mqinfra.go +++ b/internal/mqinfra/mqinfra.go @@ -83,18 +83,6 @@ func (infra *infraInvalid) TearDown(ctx context.Context) error { return ErrInvalidConfig } -type infraAzureServiceBus struct { - cfg *MQInfraConfig -} - -func (infra *infraAzureServiceBus) Declare(ctx context.Context) error { - return ErrInfraUnimplemented -} - -func (infra *infraAzureServiceBus) TearDown(ctx context.Context) error { - return ErrInfraUnimplemented -} - var ( ErrInvalidConfig = fmt.Errorf("invalid config") ErrInfraUnimplemented = fmt.Errorf("unimplemented infra") diff --git a/internal/mqinfra/mqinfra_test.go b/internal/mqinfra/mqinfra_test.go index d3402cbf..5471aeda 100644 --- a/internal/mqinfra/mqinfra_test.go +++ b/internal/mqinfra/mqinfra_test.go @@ -308,3 +308,77 @@ func TestIntegrationMQInfra_GCPPubSub(t *testing.T) { }, ) } + +func TestIntegrationMQInfra_AzureServiceBus(t *testing.T) { + t.Skip("skip AzureServiceBus integration test for now since there's no emulator yet") + + topic := uuid.New().String() + subscription := topic + "-subscription" + + const ( + tenantID = "" + clientID = "" + clientSecret = "" + subscriptionID = "" + resourceGroup = "" + namespace = "" + ) + + testMQInfra(t, + &Config{ + infra: mqinfra.MQInfraConfig{ + AzureServiceBus: &mqinfra.AzureServiceBusInfraConfig{ + TenantID: tenantID, + ClientID: clientID, + ClientSecret: clientSecret, + SubscriptionID: subscriptionID, + ResourceGroup: resourceGroup, + Namespace: namespace, + Topic: topic, + Subscription: subscription, + }, + Policy: mqinfra.Policy{ + RetryLimit: retryLimit, + VisibilityTimeout: 10, + }, + }, + mq: mqs.QueueConfig{ + AzureServiceBus: &mqs.AzureServiceBusConfig{ + TenantID: tenantID, + ClientID: clientID, + ClientSecret: clientSecret, + SubscriptionID: subscriptionID, + ResourceGroup: resourceGroup, + Namespace: namespace, + Topic: topic, + Subscription: subscription, + }, + }, + }, + &Config{ + infra: mqinfra.MQInfraConfig{ + AzureServiceBus: &mqinfra.AzureServiceBusInfraConfig{ + TenantID: tenantID, + ClientID: clientID, + ClientSecret: clientSecret, + SubscriptionID: subscriptionID, + ResourceGroup: resourceGroup, + Namespace: namespace, + Topic: topic, // Same topic as main queue + Subscription: subscription, // Same subscription as main queue + }}, + mq: mqs.QueueConfig{ + AzureServiceBus: &mqs.AzureServiceBusConfig{ + TenantID: tenantID, + ClientID: clientID, + ClientSecret: clientSecret, + SubscriptionID: subscriptionID, + ResourceGroup: resourceGroup, + Namespace: namespace, + Topic: topic, // Same topic as main queue + Subscription: subscription, // Same subscription as main queue + DLQ: true, // Enable DLQ mode + }}, + }, + ) +} diff --git a/internal/mqs/config.go b/internal/mqs/config.go deleted file mode 100644 index 5208c7cd..00000000 --- a/internal/mqs/config.go +++ /dev/null @@ -1,23 +0,0 @@ -package mqs - -type QueueConfig struct { - AWSSQS *AWSSQSConfig - AzureServiceBus *AzureServiceBusConfig - GCPPubSub *GCPPubSubConfig - RabbitMQ *RabbitMQConfig - InMemory *InMemoryConfig // mainly for testing purposes -} - -type AzureServiceBusConfig struct { -} - -type GCPPubSubConfig struct { - ProjectID string - TopicID string - SubscriptionID string - ServiceAccountCredentials string // JSON key file content -} - -type InMemoryConfig struct { - Name string -} diff --git a/internal/mqs/queue.go b/internal/mqs/queue.go index 2374f722..fc0dd19d 100644 --- a/internal/mqs/queue.go +++ b/internal/mqs/queue.go @@ -11,6 +11,18 @@ import ( _ "gocloud.dev/pubsub/mempubsub" ) +type QueueConfig struct { + AWSSQS *AWSSQSConfig + AzureServiceBus *AzureServiceBusConfig + GCPPubSub *GCPPubSubConfig + RabbitMQ *RabbitMQConfig + InMemory *InMemoryConfig // mainly for testing purposes +} + +type InMemoryConfig struct { + Name string +} + type Queue interface { Init(ctx context.Context) (func(), error) Publish(ctx context.Context, msg IncomingMessage) error @@ -45,7 +57,7 @@ func NewQueue(config *QueueConfig) Queue { if config.AWSSQS != nil { return NewAWSQueue(config.AWSSQS) } else if config.AzureServiceBus != nil { - return &UnimplementedQueue{} + return NewAzureServiceBusQueue(config.AzureServiceBus) } else if config.GCPPubSub != nil { return NewGCPPubSubQueue(config.GCPPubSub) } else if config.RabbitMQ != nil { diff --git a/internal/mqs/queue_azureservicebus.go b/internal/mqs/queue_azureservicebus.go new file mode 100644 index 00000000..691eb1ec --- /dev/null +++ b/internal/mqs/queue_azureservicebus.go @@ -0,0 +1,149 @@ +package mqs + +import ( + "context" + "fmt" + "sync" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus" + "gocloud.dev/pubsub" + "gocloud.dev/pubsub/azuresb" +) + +type AzureServiceBusConfig struct { + Topic string + Subscription string + DLQ bool // Set to true to subscribe to the dead letter queue + + // Credentials + ConnectionString string + // or + TenantID string + ClientID string + ClientSecret string + SubscriptionID string + ResourceGroup string + Namespace string +} + +type AzureServiceBusQueue struct { + base *wrappedBaseQueue + once *sync.Once + client *azservicebus.Client + config *AzureServiceBusConfig + topic *pubsub.Topic +} + +var _ Queue = &AzureServiceBusQueue{} + +func (q *AzureServiceBusQueue) Init(ctx context.Context) (func(), error) { + var err error + q.once.Do(func() { + err = q.InitClient(ctx) + }) + if err != nil { + return nil, err + } + + sender, err := q.client.NewSender(q.config.Topic, nil) + if err != nil { + return nil, err + } + + q.topic, err = azuresb.OpenTopic(ctx, sender, nil) + if err != nil { + return nil, err + } + + return func() { + if q.client != nil { + q.client.Close(ctx) + } + if q.topic != nil { + q.topic.Shutdown(ctx) + } + }, nil +} + +func (q *AzureServiceBusQueue) Publish(ctx context.Context, incomingMessage IncomingMessage) error { + return q.base.Publish(ctx, q.topic, incomingMessage, nil) +} + +func (q *AzureServiceBusQueue) Subscribe(ctx context.Context) (Subscription, error) { + var err error + q.once.Do(func() { + err = q.InitClient(ctx) + }) + if err != nil { + return nil, err + } + + // Configure receiver options based on DLQ setting + var receiverOptions *azservicebus.ReceiverOptions + if q.config.DLQ { + // Subscribe to the dead letter queue + receiverOptions = &azservicebus.ReceiverOptions{ + SubQueue: azservicebus.SubQueueDeadLetter, + } + } + + receiver, err := q.client.NewReceiverForSubscription(q.config.Topic, q.config.Subscription, receiverOptions) + if err != nil { + return nil, err + } + + subscription, err := azuresb.OpenSubscription(ctx, q.client, receiver, nil) + if err != nil { + return nil, err + } + + return q.base.Subscribe(ctx, subscription) +} + +func (q *AzureServiceBusQueue) InitClient(ctx context.Context) error { + // Case 1: Use connection string if provided + if q.config.ConnectionString != "" { + client, err := azservicebus.NewClientFromConnectionString(q.config.ConnectionString, nil) + if err != nil { + return fmt.Errorf("failed to create client from connection string: %w", err) + } + q.client = client + return nil + } + + // Case 2: Use service principal credentials directly + if q.config.TenantID != "" && q.config.ClientID != "" && q.config.ClientSecret != "" && q.config.Namespace != "" { + // Create credential + cred, err := azidentity.NewClientSecretCredential( + q.config.TenantID, + q.config.ClientID, + q.config.ClientSecret, + nil, + ) + if err != nil { + return fmt.Errorf("failed to create credential: %w", err) + } + + // Create client using credential and namespace FQDN + namespaceEndpoint := q.config.Namespace + ".servicebus.windows.net" + client, err := azservicebus.NewClient(namespaceEndpoint, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client from credentials: %w", err) + } + q.client = client + return nil + } + + // Case 3: Neither connection string nor credentials provided + return fmt.Errorf("azure service bus configuration incomplete: must provide either connection_string or (tenant_id, client_id, client_secret, namespace)") +} + +func NewAzureServiceBusQueue(config *AzureServiceBusConfig) *AzureServiceBusQueue { + var once sync.Once + return &AzureServiceBusQueue{ + config: config, + once: &once, + base: newWrappedBaseQueue(), + } +} diff --git a/internal/mqs/queue_gcppubsub.go b/internal/mqs/queue_gcppubsub.go index 61771d3a..b3b1c65e 100644 --- a/internal/mqs/queue_gcppubsub.go +++ b/internal/mqs/queue_gcppubsub.go @@ -12,6 +12,13 @@ import ( "google.golang.org/grpc" ) +type GCPPubSubConfig struct { + ProjectID string + TopicID string + SubscriptionID string + ServiceAccountCredentials string // JSON key file content +} + type GCPPubSubQueue struct { once *sync.Once base *wrappedBaseQueue diff --git a/internal/mqs/queue_test.go b/internal/mqs/queue_test.go index 8d38495c..e8d497b7 100644 --- a/internal/mqs/queue_test.go +++ b/internal/mqs/queue_test.go @@ -34,20 +34,49 @@ func TestIntegrationMQ_RabbitMQ(t *testing.T) { testMQ(t, func() mqs.QueueConfig { return config }) } -func TestIntegrationMQ_AWS(t *testing.T) { +func TestIntegrationMQ_AWSSQS(t *testing.T) { t.Parallel() t.Cleanup(testinfra.Start(t)) config := testinfra.NewMQAWSConfig(t, nil) testMQ(t, func() mqs.QueueConfig { return config }) } -func TestIntegrationMQ_GCP(t *testing.T) { +func TestIntegrationMQ_GCPPubSub(t *testing.T) { t.Parallel() t.Cleanup(testinfra.Start(t)) config := testinfra.NewMQGCPConfig(t, nil) testMQ(t, func() mqs.QueueConfig { return config }) } +func TestIntegrationMQ_AzureServiceBus(t *testing.T) { + t.Skip("skip AzureServiceBus integration test for now since there's no emulator yet") + + t.Parallel() + t.Cleanup(testinfra.Start(t)) + // config := testinfra.NewMQAzureServiceBusConfig(t, nil) + + // config := mqs.QueueConfig{ + // AzureServiceBus: &mqs.AzureServiceBusConfig{ + // ConnectionString: "", + // Topic: "test-topic", + // Subscription: "test-subscription", + // }, + // } + config := mqs.QueueConfig{ + AzureServiceBus: &mqs.AzureServiceBusConfig{ + TenantID: "", + ClientID: "", + ClientSecret: "", + SubscriptionID: "", + ResourceGroup: "", + Namespace: "", + Topic: "test-topic", + Subscription: "test-subscription", + }, + } + testMQ(t, func() mqs.QueueConfig { return config }) +} + type Msg struct { ID string Data map[string]string diff --git a/internal/util/testinfra/testinfra.go b/internal/util/testinfra/testinfra.go index 0da35fde..2745976d 100644 --- a/internal/util/testinfra/testinfra.go +++ b/internal/util/testinfra/testinfra.go @@ -88,8 +88,13 @@ func Start(t *testing.T) func() { return func() { suiteCounter -= 1 if suiteCounter == 0 { - for _, fn := range cfg.cleanupFns { - fn() + // Ensure cfg is initialized and not nil before accessing cleanupFns + if cfg != nil && cfg.cleanupFns != nil { + for _, fn := range cfg.cleanupFns { + if fn != nil { + fn() + } + } } } } From 47b1cccfcf3017c928f6d6e414b388a53e3ae054 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 10 Jun 2025 02:44:34 +0700 Subject: [PATCH 12/27] chore: go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 88fe48f0..577a0ecf 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.23.0 require ( cloud.google.com/go/pubsub v1.41.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.7.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus v1.2.0 @@ -81,7 +82,6 @@ require ( cloud.google.com/go/iam v1.1.13 // indirect dario.cat/mergo v1.0.1 // indirect github.com/Azure/azure-amqp-common-go/v3 v3.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/go-amqp v1.0.5 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect From 23cb811d61af81c4d3bf9b77bee6ac0e2a08716f Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 10 Jun 2025 02:48:19 +0700 Subject: [PATCH 13/27] chore: azure env --- .env.example | 11 +++++++++++ internal/config/mqconfig_azure.go | 31 +++++++++++++++---------------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/.env.example b/.env.example index 175dd3f5..c0b442f2 100644 --- a/.env.example +++ b/.env.example @@ -54,6 +54,17 @@ RABBITMQ_LOG_QUEUE="outpost-log" # GCP_PUBSUB_LOG_TOPIC="outpost-log" # GCP_PUBSUB_LOG_SUBSCRIPTION="outpost-log-sub" +## Azure ServiceBus +AZURE_SERVICEBUS_TENANT_ID="" +AZURE_SERVICEBUS_CLIENT_ID="" +AZURE_SERVICEBUS_CLIENT_SECRET="" +AZURE_SERVICEBUS_SUBSCRIPTION_ID="" +AZURE_SERVICEBUS_RESOURCE_GROUP="" +AZURE_SERVICEBUS_NAMESPACE="" +AZURE_SERVICEBUS_DELIVERY_TOPIC="outpost-delivery" +AZURE_SERVICEBUS_DELIVERY_SUBSCRIPTION="outpost-delivery-sub" +AZURE_SERVICEBUS_LOG_TOPIC="outpost-log" +AZURE_SERVICEBUS_LOG_SUBSCRIPTION="outpost-log-sub" # ============================== PublishMQ ============================== diff --git a/internal/config/mqconfig_azure.go b/internal/config/mqconfig_azure.go index a8144bf7..0aebe2a5 100644 --- a/internal/config/mqconfig_azure.go +++ b/internal/config/mqconfig_azure.go @@ -3,7 +3,6 @@ package config import ( "context" "fmt" - "sync" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus" @@ -12,21 +11,21 @@ import ( ) type AzureServiceBusConfig struct { - TenantID string `yaml:"tenant_id" desc:"Azure Active Directory tenant ID" required:"Y"` - ClientID string `yaml:"client_id" desc:"Service principal client ID" required:"Y"` - ClientSecret string `yaml:"client_secret" desc:"Service principal client secret" required:"Y"` - SubscriptionID string `yaml:"subscription_id" desc:"Azure subscription ID" required:"Y"` - ResourceGroup string `yaml:"resource_group" desc:"Azure resource group name" required:"Y"` - Namespace string `yaml:"namespace" desc:"Azure Service Bus namespace" required:"Y"` - - DeliveryTopic string `yaml:"delivery_topic" desc:"Topic name for delivery queue" required:"N" default:"outpost-delivery"` - DeliverySubscription string `yaml:"delivery_subscription" desc:"Subscription name for delivery queue" required:"N" default:"outpost-delivery-subscription"` - LogTopic string `yaml:"log_topic" desc:"Topic name for log queue" required:"N" default:"outpost-log"` - LogSubscription string `yaml:"log_subscription" desc:"Subscription name for log queue" required:"N" default:"outpost-log-subscription"` - - connectionStringOnce sync.Once - connectionString string - connectionStringError error + TenantID string `yaml:"tenant_id" env:"AZURE_SERVICEBUS_TENANT_ID" desc:"Azure Active Directory tenant ID" required:"Y"` + ClientID string `yaml:"client_id" env:"AZURE_SERVICEBUS_CLIENT_ID" desc:"Service principal client ID" required:"Y"` + ClientSecret string `yaml:"client_secret" env:"AZURE_SERVICEBUS_CLIENT_SECRET" desc:"Service principal client secret" required:"Y"` + SubscriptionID string `yaml:"subscription_id" env:"AZURE_SERVICEBUS_SUBSCRIPTION_ID" desc:"Azure subscription ID" required:"Y"` + ResourceGroup string `yaml:"resource_group" env:"AZURE_SERVICEBUS_RESOURCE_GROUP" desc:"Azure resource group name" required:"Y"` + Namespace string `yaml:"namespace" env:"AZURE_SERVICEBUS_NAMESPACE" desc:"Azure Service Bus namespace" required:"Y"` + + DeliveryTopic string `yaml:"delivery_topic" env:"AZURE_SERVICEBUS_DELIVERY_TOPIC" desc:"Topic name for delivery queue" required:"N" default:"outpost-delivery"` + DeliverySubscription string `yaml:"delivery_subscription" env:"AZURE_SERVICEBUS_DELIVERY_SUBSCRIPTION" desc:"Subscription name for delivery queue" required:"N" default:"outpost-delivery-subscription"` + LogTopic string `yaml:"log_topic" env:"AZURE_SERVICEBUS_LOG_TOPIC" desc:"Topic name for log queue" required:"N" default:"outpost-log"` + LogSubscription string `yaml:"log_subscription" env:"AZURE_SERVICEBUS_LOG_SUBSCRIPTION" desc:"Subscription name for log queue" required:"N" default:"outpost-log-subscription"` + + // connectionStringOnce sync.Once + // connectionString string + // connectionStringError error } func (c *AzureServiceBusConfig) IsConfigured() bool { From 51f0894c41f97322c990ac75a14cb4f23117dd71 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 10 Jun 2025 02:55:26 +0700 Subject: [PATCH 14/27] fix: revert --- internal/config/mqconfig_azure.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/config/mqconfig_azure.go b/internal/config/mqconfig_azure.go index 0aebe2a5..716ff426 100644 --- a/internal/config/mqconfig_azure.go +++ b/internal/config/mqconfig_azure.go @@ -3,6 +3,7 @@ package config import ( "context" "fmt" + "sync" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus" @@ -23,9 +24,9 @@ type AzureServiceBusConfig struct { LogTopic string `yaml:"log_topic" env:"AZURE_SERVICEBUS_LOG_TOPIC" desc:"Topic name for log queue" required:"N" default:"outpost-log"` LogSubscription string `yaml:"log_subscription" env:"AZURE_SERVICEBUS_LOG_SUBSCRIPTION" desc:"Subscription name for log queue" required:"N" default:"outpost-log-subscription"` - // connectionStringOnce sync.Once - // connectionString string - // connectionStringError error + connectionStringOnce sync.Once + connectionString string + connectionStringError error } func (c *AzureServiceBusConfig) IsConfigured() bool { From 9492f140ac6737a1eb999fa1c95be4ca03e5b555 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 10 Jun 2025 03:04:24 +0700 Subject: [PATCH 15/27] chore: azureservicebus subscription config --- internal/mqs/queue_azureservicebus.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/mqs/queue_azureservicebus.go b/internal/mqs/queue_azureservicebus.go index 691eb1ec..e40c9971 100644 --- a/internal/mqs/queue_azureservicebus.go +++ b/internal/mqs/queue_azureservicebus.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus" @@ -93,7 +94,9 @@ func (q *AzureServiceBusQueue) Subscribe(ctx context.Context) (Subscription, err return nil, err } - subscription, err := azuresb.OpenSubscription(ctx, q.client, receiver, nil) + subscription, err := azuresb.OpenSubscription(ctx, q.client, receiver, &azuresb.SubscriptionOptions{ + ListenerTimeout: 10 * time.Second, // Increased for cross-region scenarios + }) if err != nil { return nil, err } From 559f6c6ffe7651bb4d6591f570f61ee0ccb5b8fa Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 9 Jun 2025 21:29:39 +0100 Subject: [PATCH 16/27] chore: update docs gen to only use reflection --- cmd/configdocsgen/main.go | 918 ++++++++++++++++++-------------------- 1 file changed, 438 insertions(+), 480 deletions(-) diff --git a/cmd/configdocsgen/main.go b/cmd/configdocsgen/main.go index c7a70cd2..cf5c8bf6 100644 --- a/cmd/configdocsgen/main.go +++ b/cmd/configdocsgen/main.go @@ -1,5 +1,4 @@ // Known Limitations/Further Improvements: -// Embedded Structs (AST): Fields from anonymously embedded structs are not fully resolved during AST parsing for documentation as part of the parent struct. // Complex Slice/Map YAML Formatting: Default value formatting for slices is basic. Maps are not explicitly formatted for YAML beyond their default string representation. package main @@ -7,74 +6,78 @@ package main import ( "flag" "fmt" - "go/ast" - "go/parser" - "go/token" "log" "os" - "path/filepath" "reflect" "sort" "strconv" "strings" + "time" "github.com/hookdeck/outpost/internal/config" // Import your project's config package ) var ( + // inputDir is no longer used for AST parsing but kept for potential future use or to avoid breaking existing scripts if any. inputDir string outputFile string ) +// ReflectionFieldInfo is an intermediate struct to hold data extracted via reflection. +type ReflectionFieldInfo struct { + Path string // Full dot-notation path from root config. e.g. "Redis.Host" + FieldName string // Original Go field name. e.g. "Host" + FieldTypeStr string // String representation of field type. e.g. "string", "int", "config.RedisConfig" + StructName string // Name of the immediate struct this field belongs to. e.g. "Config", "RedisConfig" + StructPkgPath string // Package path of the struct this field belongs to. + YAMLName string + EnvName string + EnvSeparator string + Description string + Required string // "true", "false", or "Y", "N", "C" from tag + DefaultValue interface{} // Actual default value + IsEmbeddedField bool // True if this field comes from an embedded struct and should be inlined in YAML +} + +// ConfigField represents a field in a configuration struct (used by generateDocs) +type ConfigField struct { + Name string + Type string + YAMLName string + EnvName string + EnvSeparator string + Description string + Required string // Y, N, C + DefaultValue string // String representation +} + +// ParsedConfig represents a parsed configuration struct (used by generateDocs) +type ParsedConfig struct { + FileName string // Can be set to "reflection" or similar + Name string // Go struct name (e.g., "Config", "RedisConfig") + Fields []ConfigField + IsTopLevel bool // Flag to identify the root config.Config struct for pathing + GoStructName string // Same as Name +} + func main() { - // Default paths are relative to the project root, where `go run cmd/configdocsgen/main.go` is expected to be executed. - defaultInputDir := "internal/config" + defaultInputDir := "internal/config" // Retained for consistency, though not directly used for parsing config.Config defaultOutputFile := "docs/pages/references/configuration.mdx" - flag.StringVar(&inputDir, "input-dir", defaultInputDir, "Directory containing the Go configuration source files.") + flag.StringVar(&inputDir, "input-dir", defaultInputDir, "Directory containing the Go configuration source files (usage changed with reflection).") flag.StringVar(&outputFile, "output-file", defaultOutputFile, "Path to the output Markdown file.") flag.Parse() - fmt.Println("Configuration Documentation Generator") - log.Printf("Input directory: %s", inputDir) + fmt.Println("Configuration Documentation Generator (Reflection-based)") log.Printf("Output file: %s", outputFile) - // TODO: - // 2. Implement parsing of Go files in inputDir - // - Use go/parser and go/ast - // 3. Identify config structs - // 4. Extract field information (name, yaml, env, desc, required tags, data type) - // 5. Instantiate config.Config and call InitDefaults() - // - Use reflection to get default values - // 6. Handle "one-of" types (e.g., MQsConfig) - // 7. Generate Markdown: - // - Environment Variables Table - // - YAML Configuration Section - // 8. Write to output file - - parsedConfigs, err := parseConfigFiles(inputDir) + allFieldInfos, err := parseConfigWithReflection() if err != nil { - log.Fatalf("Error parsing config files: %v", err) + log.Fatalf("Error parsing config with reflection: %v", err) } - // For now, just print what was found (later this will be processed) - for _, cfg := range parsedConfigs { - log.Printf("Found config struct: %s in file %s", cfg.Name, cfg.FileName) - } + parsedConfigs := transformToParsedConfigs(allFieldInfos) - // Attempt to get default values and integrate them BEFORE generating docs - log.Println("Attempting to load and reflect on config.Config for default values...") - defaults, err := getConfigDefaults() - if err != nil { - log.Printf("Warning: Could not get config defaults: %v", err) - log.Println("Default values will be missing or incorrect in the generated documentation.") - } else { - log.Printf("Successfully reflected on config.Config. Total default keys found: %d", len(defaults)) - // Integrate defaults into parsedConfigs - integrateDefaults(parsedConfigs, defaults) // This modifies parsedConfigs in place - } - - // Now generate docs with populated defaults err = generateDocs(parsedConfigs, outputFile) if err != nil { log.Fatalf("Error generating docs: %v", err) @@ -83,355 +86,260 @@ func main() { fmt.Printf("Successfully generated documentation to %s\n", outputFile) } -// ConfigField represents a field in a configuration struct -type ConfigField struct { - Name string - Type string - YAMLName string - EnvName string - EnvSeparator string - Description string - Required string // Y, N, C - DefaultValue string -} +func parseConfigWithReflection() ([]ReflectionFieldInfo, error) { + var infos []ReflectionFieldInfo + cfg := config.Config{} + cfg.InitDefaults() -// ParsedConfig represents a parsed configuration struct -type ParsedConfig struct { - FileName string - Name string // Go struct name - Fields []ConfigField - IsTopLevel bool // Flag to identify the root config.Config struct for pathing - GoStructName string // To help build paths for defaults map - // We might need to store the original ast.StructType for default value resolution later + cfgType := reflect.TypeOf(cfg) + cfgValue := reflect.ValueOf(cfg) + targetPkgPath := cfgType.PkgPath() + + err := extractFieldsRecursive(cfgValue, cfgType, "", cfgType.Name(), cfgType.PkgPath(), targetPkgPath, &infos, false) + if err != nil { + return nil, err + } + return infos, nil } -func parseConfigFiles(dirPath string) ([]ParsedConfig, error) { - var configs []ParsedConfig - fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, dirPath, func(fi os.FileInfo) bool { - return !fi.IsDir() && strings.HasSuffix(fi.Name(), ".go") && !strings.HasSuffix(fi.Name(), "_test.go") - }, parser.ParseComments) +func extractFieldsRecursive(currentVal reflect.Value, currentType reflect.Type, pathPrefix string, structName string, structPkgPath string, targetPkgPath string, infos *[]ReflectionFieldInfo, isEmbedded bool) error { + if currentType.Kind() == reflect.Ptr { + if currentVal.IsNil() { + // If the pointer is nil, create a zero value of the element type to inspect its structure + // This ensures fields of this struct are documented, albeit with zero/nil defaults. + currentType = currentType.Elem() + currentVal = reflect.Zero(currentType) + } else { + currentVal = currentVal.Elem() + currentType = currentType.Elem() + } + } - if err != nil { - return nil, fmt.Errorf("failed to parse directory %s: %w", dirPath, err) + if currentType.Kind() != reflect.Struct { + return fmt.Errorf("expected a struct or pointer to a struct, got %s", currentType.Kind()) } - for _, pkg := range pkgs { - for fileName, file := range pkg.Files { - log.Printf("Processing file: %s", fileName) - ast.Inspect(file, func(n ast.Node) bool { - ts, ok := n.(*ast.TypeSpec) - if !ok || ts.Type == nil { - return true // Continue traversal - } + for i := 0; i < currentType.NumField(); i++ { + fieldSpec := currentType.Field(i) + fieldVal := currentVal.Field(i) - s, ok := ts.Type.(*ast.StructType) - if !ok { - return true // Continue traversal - } + // Skip unexported fields + if !fieldVal.CanInterface() { + continue + } - // Found a struct - structName := ts.Name.Name - log.Printf("Found struct: %s in file %s", structName, fileName) - isTopLevel := (structName == "Config") // Simple check for the main Config - - var fields []ConfigField - for _, field := range s.Fields.List { - if len(field.Names) > 0 { // Ensure it's a named field - fieldName := field.Names[0].Name - var fieldTypeStr string - switch typeExpr := field.Type.(type) { - case *ast.Ident: - fieldTypeStr = typeExpr.Name - case *ast.SelectorExpr: // For types like pkg.Type - if pkgIdent, ok := typeExpr.X.(*ast.Ident); ok { - fieldTypeStr = pkgIdent.Name + "." + typeExpr.Sel.Name - } else { - fieldTypeStr = fmt.Sprintf("%s", field.Type) // Fallback - } - case *ast.ArrayType: - // Further inspect Elt for element type if needed - if eltIdent, ok := typeExpr.Elt.(*ast.Ident); ok { - fieldTypeStr = "[]" + eltIdent.Name - } else { - fieldTypeStr = fmt.Sprintf("%s", field.Type) // Fallback for complex array/slice types - } - case *ast.MapType: - // Further inspect Key and Value for their types - fieldTypeStr = fmt.Sprintf("%s", field.Type) // Fallback for complex map types - default: - fieldTypeStr = fmt.Sprintf("%s", field.Type) // Fallback - } - - var yamlName, envName, envSeparator, description, requiredValue string - - if field.Tag != nil { - tagValue := field.Tag.Value // Raw string like `yaml:"name" env:"VAR"` - // Remove backticks - tagValue = strings.Trim(tagValue, "`") - - // The line below was unused and caused a compiler error. - // tags := strings.Fields(tagValue) - // A more robust way is to use reflect.StructTag, but that requires an instance. - // For AST parsing, manual parsing or a dedicated tag parser is common. - // Let's do a simplified manual parse for now. - - parsedTag := parseStructTag(tagValue) - yamlName = parsedTag["yaml"] - envName = parsedTag["env"] - envSeparator = parsedTag["envSeparator"] - description = parsedTag["desc"] - requiredValue = parsedTag["required"] - } - - // TODO: Handle embedded structs (field.Names would be empty) - - log.Printf(" Field: %s, Type: %s, YAML: '%s', Env: '%s', Desc: '%s', Required: '%s'", - fieldName, fieldTypeStr, yamlName, envName, description, requiredValue) - - fields = append(fields, ConfigField{ - Name: fieldName, - Type: fieldTypeStr, - YAMLName: yamlName, - EnvName: envName, - EnvSeparator: envSeparator, - Description: description, - Required: requiredValue, - }) - } else { - // This could be an embedded struct - // field.Type would give its type. - // Example: `MyEmbeddedStruct` or `pkg.MyEmbeddedStruct` - // We need to decide how to represent these. - // For now, we are only processing fields with names. - // If an embedded struct's fields should be "flattened" into the parent, - // this logic needs to recursively call a part of parseConfigFiles or similar. - // The current reflection for defaults *will* see flattened fields. - // The AST parsing needs to align if we want to document them as part of the parent. - if typeIdent, ok := field.Type.(*ast.Ident); ok { - log.Printf(" Found embedded-like field (no name): Type: %s", typeIdent.Name) - // Here, you might look up `typeIdent.Name` in `allConfigs` (if populated first pass) - // and then merge its fields. This gets complex with ordering and paths. - // For now, we'll skip documenting fields of embedded structs directly here, - // assuming they are separate `ParsedConfig` entries if they are config structs themselves. - } - } - } - configs = append(configs, ParsedConfig{ - FileName: filepath.Base(fileName), - Name: structName, // This is the Go struct name - GoStructName: structName, - Fields: fields, - IsTopLevel: isTopLevel, - }) - return false // Stop traversal for this struct, already processed - }) + fieldName := fieldSpec.Name + fullPath := fieldName + if pathPrefix != "" { + fullPath = pathPrefix + "." + fieldName } - } - return configs, nil -} -// parseStructTag parses a struct tag string and returns a map of key-value pairs. -// It handles quoted values that may contain spaces. -func parseStructTag(tagStr string) map[string]string { - tags := make(map[string]string) - for tagStr != "" { - // Skip leading spaces. - i := 0 - for i < len(tagStr) && tagStr[i] == ' ' { - i++ - } - tagStr = tagStr[i:] - if tagStr == "" { - break - } - - // Find the key. - i = 0 - for i < len(tagStr) && tagStr[i] != ' ' && tagStr[i] != ':' { - i++ - } - if i == 0 || i+1 >= len(tagStr) || tagStr[i] != ':' || tagStr[i+1] != '"' { - // Malformed tag or no value, skip to next potential tag part - // This might happen if a tag is just `key` without `:"value"` - // Or if it's not a quoted value. For simplicity, we assume all values are quoted. - nextSpace := strings.Index(tagStr, " ") - if nextSpace == -1 { - break // No more tags + // Handle anonymous/embedded structs + if fieldSpec.Anonymous { + // For embedded structs, recurse with the same pathPrefix (or adjusted if needed) + // and mark fields as embedded so they can be inlined in YAML. + err := extractFieldsRecursive(fieldVal, fieldSpec.Type, pathPrefix, structName, structPkgPath, targetPkgPath, infos, true) + if err != nil { + return fmt.Errorf("error recursing into embedded struct %s: %w", fieldName, err) } - tagStr = tagStr[nextSpace:] - continue + continue // Skip adding the embedded struct itself as a field } - key := tagStr[:i] - tagStr = tagStr[i+1:] // Skip key and ':' - // Find the value (quoted). - if tagStr[0] != '"' { - // Malformed tag: value not quoted - nextSpace := strings.Index(tagStr, " ") - if nextSpace == -1 { - break - } - tagStr = tagStr[nextSpace:] + yamlTag := fieldSpec.Tag.Get("yaml") + yamlName := strings.Split(yamlTag, ",")[0] // Get name part, ignore omitempty etc. + if yamlName == "-" { // Skip fields explicitly ignored by YAML continue } - i = 1 // Skip leading quote - for i < len(tagStr) && tagStr[i] != '"' { - if tagStr[i] == '\\' { // Handle escaped quotes - i++ - } - i++ + if yamlName == "" { // Default to field name if yaml tag is missing or empty + // This behavior might need adjustment based on how YAML typically marshals + // For documentation, often we want to show the Go field name if no YAML name. + // However, for config, usually explicit YAML names are preferred. + // Let's assume if yaml tag is missing, it's not a primary config field for YAML docs. + // Or, for full documentation, one might want to include it. + // For now, if no yamlName, we might skip or use Go name. + // Let's use Go field name if yamlName is empty after split (e.g. tag was just ",omitempty") + // but if tag was entirely missing, it's less clear. + // The current AST parser implies fields without YAML tags are still processed. + // Let's default to Go field name if yamlName is empty. + yamlName = fieldName } - if i >= len(tagStr) { - // Malformed tag: unclosed quote - break + + fieldTypeStr := formatFieldType(fieldSpec.Type) + + // Create info for the current field itself + // This applies whether it's a basic type, a struct from another package, or a struct we'll recurse into. + currentFieldInfo := ReflectionFieldInfo{ + Path: fullPath, + FieldName: fieldName, + FieldTypeStr: fieldTypeStr, + StructName: structName, // The struct this field belongs to (e.g., "Config") + StructPkgPath: structPkgPath, + YAMLName: yamlName, + EnvName: fieldSpec.Tag.Get("env"), + EnvSeparator: fieldSpec.Tag.Get("envSeparator"), + Description: fieldSpec.Tag.Get("desc"), + Required: fieldSpec.Tag.Get("required"), + DefaultValue: fieldVal.Interface(), // Capture the default value of this field + IsEmbeddedField: isEmbedded, // This is about whether *this field* is part of an embedding context + } + // If fieldSpec.Anonymous was true, we already 'continue'd earlier. + // So, this currentFieldInfo is for a named field. + *infos = append(*infos, currentFieldInfo) + + // If the field's type is a struct from the target package (and not an interface), recurse into its members. + // These members will have their StructName as fieldSpec.Type.Name() (e.g., "RedisConfig") + // and their Path will be prefixed with fullPath (e.g., "Redis.Host"). + actualFieldType := fieldSpec.Type + if actualFieldType.Kind() == reflect.Ptr { + actualFieldType = actualFieldType.Elem() } - value := tagStr[1:i] - tags[key] = value - // Move to next tag. - tagStr = tagStr[i+1:] + if fieldSpec.Type.Kind() != reflect.Interface && actualFieldType.Kind() == reflect.Struct && actualFieldType.PkgPath() == targetPkgPath && actualFieldType.Name() != "Time" { + // Use actualFieldType.Name() for the StructName of the members + err := extractFieldsRecursive(fieldVal, fieldSpec.Type, fullPath, actualFieldType.Name(), actualFieldType.PkgPath(), targetPkgPath, infos, false) // isEmbedded is false for members of a regularly named struct field + if err != nil { + return fmt.Errorf("error recursing into members of struct field %s (type %s): %w", fullPath, actualFieldType.Name(), err) + } + } } - return tags + return nil } -// getConfigDefaults attempts to instantiate config.Config, run InitDefaults, -// and extract default values using reflection. -func getConfigDefaults() (map[string]interface{}, error) { - cfg := config.Config{} // Create an instance of the actual Config struct - cfg.InitDefaults() // Initialize it with default values - - defaults := make(map[string]interface{}) - extractDefaultsRecursive(reflect.ValueOf(cfg), defaults, "") - - // TODO: Map these defaults back to the ParsedConfig fields, - // potentially using a path like "Redis.Host" or by matching struct and field names. - - return defaults, nil +func formatFieldType(t reflect.Type) string { + // This produces types like "string", "int", "[]string", "map[string]int", "config.RedisConfig", "*config.MQsConfig" + // which is generally good for documentation. + return t.String() } -func extractDefaultsRecursive(val reflect.Value, defaultsMap map[string]interface{}, prefix string) { - // Dereference pointers if any - if val.Kind() == reflect.Ptr { - val = val.Elem() +func isTargetPkgStruct(t reflect.Type, targetPkgPath string) bool { + if t.Kind() == reflect.Ptr { + t = t.Elem() } + return t.Kind() == reflect.Struct && t.PkgPath() == targetPkgPath && t.Name() != "Time" // Exclude time.Time +} - // Ensure we are dealing with a struct - if val.Kind() != reflect.Struct { - return +func formatDefaultValueToString(value interface{}, goType string) string { + if value == nil { + return "" // generateDocs will convert this to `nil` } - typ := val.Type() - for i := 0; i < val.NumField(); i++ { - field := typ.Field(i) - fieldVal := val.Field(i) + val := reflect.ValueOf(value) + if val.Kind() == reflect.Ptr && val.IsNil() { + return "" + } - // Skip unexported fields - if !fieldVal.CanInterface() { - continue + // Handle time.Time specifically if needed, otherwise %v is usually okay. + if t, ok := value.(time.Time); ok { + if t.IsZero() { + return "" // Represent zero time as empty for cleaner defaults } + return t.Format(time.RFC3339) // Or another suitable format + } - currentPath := field.Name - if prefix != "" { - currentPath = prefix + "." + field.Name + // Handle slices + if val.Kind() == reflect.Slice { + if val.IsNil() { // Explicitly nil slice + return "" } - - if fieldVal.Kind() == reflect.Struct { - // Check if it's a time.Time struct or other common struct we don't want to recurse into deeply for this purpose - // For example, time.Time has unexported fields that would cause issues or are not relevant as "defaults" - if fieldVal.Type().PkgPath() == "time" && fieldVal.Type().Name() == "Time" { - defaultsMap[currentPath] = fieldVal.Interface() - } else { - // Recurse for nested structs - extractDefaultsRecursive(fieldVal, defaultsMap, currentPath) - } - } else if fieldVal.Kind() == reflect.Slice || fieldVal.Kind() == reflect.Array { - // Handle slices/arrays - store them directly - // For complex slice elements (structs), further handling might be needed if defaults per element are relevant - defaultsMap[currentPath] = fieldVal.Interface() - } else { - defaultsMap[currentPath] = fieldVal.Interface() + if val.Len() == 0 { // Empty slice [] + return "[]" + } + // For non-empty slices, fmt.Sprintf("%v", value) gives "[elem1 elem2 ...]" + // We might want commas: "[elem1, elem2]" + var parts []string + for i := 0; i < val.Len(); i++ { + parts = append(parts, fmt.Sprintf("%v", val.Index(i).Interface())) } + return "[" + strings.Join(parts, ", ") + "]" } -} - -func integrateDefaults(parsedConfigs []ParsedConfig, defaults map[string]interface{}) { - // Find the top-level "Config" struct first to establish base for paths - var topLevelConfig *ParsedConfig - otherConfigs := make(map[string]*ParsedConfig) // Map by GoStructName - for i := range parsedConfigs { - pc := &parsedConfigs[i] // Get a pointer to modify in place - if pc.GoStructName == "Config" { // Assuming "Config" is the main one - topLevelConfig = pc - } - otherConfigs[pc.GoStructName] = pc + // Handle booleans to ensure "true" or "false" + if val.Kind() == reflect.Bool { + return strconv.FormatBool(val.Bool()) } - if topLevelConfig == nil { - log.Println("Warning: Top-level 'Config' struct not found in parsed files. Cannot accurately map all defaults.") - return + // Default string representation + strVal := fmt.Sprintf("%v", value) + + // Avoid Go's default "" string for non-interface nils (e.g. nil map/slice that wasn't caught above) + if strVal == "" { + return "" } - // Recursive function to build paths and assign defaults - var assignDefaults func(currentFields *[]ConfigField, currentPathPrefix string) - assignDefaults = func(currentFields *[]ConfigField, currentPathPrefix string) { - for i := range *currentFields { - field := &(*currentFields)[i] // Pointer to modify the field + return strVal +} - defaultPathKey := field.Name // Default path is just the field name for top-level - if currentPathPrefix != "" { - defaultPathKey = currentPathPrefix + "." + field.Name - } +func transformToParsedConfigs(infos []ReflectionFieldInfo) []ParsedConfig { + groupedByStruct := make(map[string][]ReflectionFieldInfo) + structOrder := []string{} // To maintain an order, e.g., "Config" first - if defaultValue, ok := defaults[defaultPathKey]; ok { - field.DefaultValue = fmt.Sprintf("%v", defaultValue) - log.Printf("Assigned default for %s: %s", defaultPathKey, field.DefaultValue) + for _, info := range infos { + if _, exists := groupedByStruct[info.StructName]; !exists { + groupedByStruct[info.StructName] = []ReflectionFieldInfo{} + if info.StructName == "Config" { // Ensure "Config" is first if present + structOrder = append([]string{info.StructName}, structOrder...) } else { - // If not found directly, it might be a nested struct defined elsewhere. - // The current `extractDefaultsRecursive` stores nested struct fields as "Parent.Field". - // The AST parsing gives us field types like "RedisConfig". We need to bridge this. - // For now, this simple pathing works for direct fields and first-level nesting if paths align. - // More complex mapping might be needed if AST field type (e.g. "RedisConfig") needs to be part of path. - // The current `extractDefaultsRecursive` builds paths like "Redis.Host", "Redis.Port". - // Our AST parsing gives us `Config.Redis` (type `RedisConfig`), then `RedisConfig.Host`. - // We need to ensure the path construction aligns. - - // If field.Type is a known struct type from our parsed configs, recurse. - // This part needs careful alignment of how paths are constructed in `extractDefaultsRecursive` - // and how we look them up here. - // The current `extractDefaultsRecursive` uses Go field names for paths. - // So, if `Config` has a field `Redis RedisConfig`, the path for redis host is `Redis.Host`. - - // Let's assume field.Type is the Go type name of the struct (e.g., "RedisConfig") - // And that our `extractDefaultsRecursive` has built paths like "Redis.Host" - // where "Redis" is the field name in the parent struct (`Config`) - // and "Host" is the field name in the child struct (`RedisConfig`). - - // If the field itself is a struct type we parsed, we need to recurse into its fields. - // The `defaultPathKey` for fields of this nested struct would be `ParentField.NestedField`. - // Example: Config.Redis (field.Name = "Redis"), its type is "RedisConfig". - // We need to find the ParsedConfig for "RedisConfig" and then iterate its fields. - // The path prefix for fields of RedisConfig would be "Redis". - - if nestedStructInfo, isStruct := otherConfigs[field.Type]; isStruct { - log.Printf("Recursing for nested struct field %s (type %s) with path prefix %s", field.Name, field.Type, defaultPathKey) - // The prefix for the children of this field is the full path to this field. - assignDefaults(&nestedStructInfo.Fields, defaultPathKey) - } else { - log.Printf("No direct default found for %s (path key: %s)", field.Name, defaultPathKey) - } + structOrder = append(structOrder, info.StructName) } } + groupedByStruct[info.StructName] = append(groupedByStruct[info.StructName], info) } - // Start with the top-level config, no prefix for its direct fields - assignDefaults(&topLevelConfig.Fields, "") + // Sort structOrder to have "Config" first, then alphabetically for others + sort.SliceStable(structOrder, func(i, j int) bool { + if structOrder[i] == "Config" { + return true + } + if structOrder[j] == "Config" { + return false + } + return structOrder[i] < structOrder[j] + }) - // After processing top-level, some nested structs might have had their defaults assigned - // by the recursion above if their parent field was processed. - // We might need a more robust way to ensure all parsed structs are processed if they can be standalone. - // For now, this focuses on defaults reachable from `config.Config`. + var parsedConfigs []ParsedConfig + for _, structName := range structOrder { + fieldsInfo := groupedByStruct[structName] + var configFields []ConfigField + var goStructPkgPath string // Take from the first field, should be consistent + + // Sort fields by their original Go field name for consistent order + // This is important if ReflectionFieldInfo Path was used for sorting, + // but here we sort by FieldName within each struct. + sort.Slice(fieldsInfo, func(i, j int) bool { + // A simple path sort might be better if fields from embedded structs are mixed. + // For now, FieldName within the current struct context. + return fieldsInfo[i].FieldName < fieldsInfo[j].FieldName + }) + + for _, info := range fieldsInfo { + if goStructPkgPath == "" { + goStructPkgPath = info.StructPkgPath + } + configFields = append(configFields, ConfigField{ + Name: info.FieldName, // Use the Go field name + Type: info.FieldTypeStr, + YAMLName: info.YAMLName, + EnvName: info.EnvName, + EnvSeparator: info.EnvSeparator, + Description: info.Description, + Required: info.Required, + DefaultValue: formatDefaultValueToString(info.DefaultValue, info.FieldTypeStr), + }) + } + + // Determine if this struct is the top-level "Config" + // The main config.Config struct from internal/config + isTopLevel := (structName == "Config" && goStructPkgPath == "github.com/hookdeck/outpost/internal/config") + + parsedConfigs = append(parsedConfigs, ParsedConfig{ + FileName: "reflection-generated", + Name: structName, + GoStructName: structName, + Fields: configFields, + IsTopLevel: isTopLevel, + }) + } + return parsedConfigs } const ( @@ -458,8 +366,6 @@ func generateDocs(parsedConfigs []ParsedConfig, outputPath string) error { } else { // Escape special characters for Markdown table cells defaultValueText = strings.ReplaceAll(defaultValueText, "|", "\\|") - // defaultValueText = strings.ReplaceAll(defaultValueText, "{", "\\{") - // defaultValueText = strings.ReplaceAll(defaultValueText, "}", "\\}") // Enclose in backticks if not already `nil` defaultValueText = fmt.Sprintf("`%s`", defaultValueText) } @@ -478,66 +384,95 @@ func generateDocs(parsedConfigs []ParsedConfig, outputPath string) error { )) } envVarsContent := strings.TrimRight(envVarsBuilder.String(), "\n") + // --- Generate YAML Content (including fences) --- var yamlBuilder strings.Builder - // The "## YAML" header should be manually placed in the MDX file. yamlBuilder.WriteString("```yaml\n") yamlBuilder.WriteString("# Outpost Configuration Example (Generated)\n") yamlBuilder.WriteString("# This example shows all available keys with their default values where applicable.\n\n") + var mainConfigInfo *ParsedConfig configInfoMap := make(map[string]*ParsedConfig) for i := range parsedConfigs { - pc := &parsedConfigs[i] + pc := &parsedConfigs[i] // operate on a copy to avoid modifying the slice configInfoMap[pc.GoStructName] = pc - if pc.GoStructName == "Config" { + if pc.IsTopLevel { // Use IsTopLevel flag mainConfigInfo = pc } } + if mainConfigInfo != nil { generateYAMLPart(&yamlBuilder, mainConfigInfo, configInfoMap, 0, true) } else { - yamlBuilder.WriteString("# ERROR: Main 'Config' struct not found. Cannot generate YAML structure.\n") + // Fallback if IsTopLevel wasn't set correctly, try finding "Config" by name + if cfgByName, ok := configInfoMap["Config"]; ok { + log.Println("Warning: Main 'Config' struct not found by IsTopLevel flag, falling back to name 'Config'.") + generateYAMLPart(&yamlBuilder, cfgByName, configInfoMap, 0, true) + } else { + yamlBuilder.WriteString("# ERROR: Main 'Config' struct not found. Cannot generate YAML structure.\n") + log.Println("Error: Main 'Config' struct not found by IsTopLevel flag or by name 'Config'.") + } } - yamlBuilder.WriteString("```\n") // Add closing fence - yamlContent := strings.TrimRight(yamlBuilder.String(), "\n") + yamlBuilder.WriteString("```\n") + yamlContent := yamlBuilder.String() - // --- Read existing file and replace placeholder sections --- - existingContentBytes, err := os.ReadFile(outputPath) + // --- Read existing MDX file --- + mdxBytes, err := os.ReadFile(outputPath) if err != nil { - log.Printf("Warning: Could not read existing output file %s. Will create a new one. Error: %v", outputPath, err) - // If file doesn't exist, create it with placeholders and content - var newFileBuilder strings.Builder - newFileBuilder.WriteString("---\ntitle: \"Outpost Configuration\"\n---\n\n") // Basic frontmatter - newFileBuilder.WriteString("\n\n") - newFileBuilder.WriteString(envVarsStartPlaceholder + "\n") - newFileBuilder.WriteString(envVarsContent) - newFileBuilder.WriteString(envVarsEndPlaceholder + "\n\n") - newFileBuilder.WriteString("\n\n") - newFileBuilder.WriteString(yamlStartPlaceholder + "\n") - newFileBuilder.WriteString(yamlContent) - newFileBuilder.WriteString(yamlEndPlaceholder + "\n\n") - newFileBuilder.WriteString("\n") - return os.WriteFile(outputPath, []byte(newFileBuilder.String()), 0644) - } + if os.IsNotExist(err) { + log.Printf("Warning: Output file %s does not exist. A new file will be created with placeholders.", outputPath) + // Create a template with placeholders if file doesn't exist + templateContent := fmt.Sprintf(`--- +title: Configuration Reference +description: Detailed configuration options for Outpost. +--- + +This document outlines all the configuration options available for Outpost, settable via environment variables or a YAML configuration file. + +## Environment Variables + +%s + +%s + +%s + +## YAML Configuration + +Below is an example YAML configuration file showing all available options and their default values. - existingContent := string(existingContentBytes) +%s - finalContent, envReplaced := replacePlaceholder(existingContent, envVarsStartPlaceholder, envVarsEndPlaceholder, envVarsContent) - if !envReplaced { - log.Printf("Warning: ENV VARS placeholders not found in %s. ENV VARS section not updated.", outputPath) +%s + +%s +`, envVarsStartPlaceholder, envVarsContent, envVarsEndPlaceholder, yamlStartPlaceholder, yamlContent, yamlEndPlaceholder) + mdxBytes = []byte(templateContent) + } else { + return fmt.Errorf("failed to read output file %s: %w", outputPath, err) + } } + mdxContent := string(mdxBytes) - finalContent, yamlReplaced := replacePlaceholder(finalContent, yamlStartPlaceholder, yamlEndPlaceholder, yamlContent) - if !yamlReplaced { - log.Printf("Warning: YAML placeholders not found in %s. YAML section not updated.", outputPath) + // --- Replace placeholders --- + var changed bool + mdxContent, changed = replacePlaceholder(mdxContent, envVarsStartPlaceholder, envVarsEndPlaceholder, "\n"+envVarsContent+"\n") + if !changed { + log.Printf("Warning: ENV vars placeholder '%s' not found or content unchanged.", envVarsStartPlaceholder) } - if !envReplaced && !yamlReplaced { - log.Printf("Neither ENV VARS nor YAML placeholders were found. File %s was not modified.", outputPath) - return nil // Or return an error if placeholders are mandatory + mdxContent, changed = replacePlaceholder(mdxContent, yamlStartPlaceholder, yamlEndPlaceholder, "\n"+yamlContent+"\n") + if !changed { + log.Printf("Warning: YAML placeholder '%s' not found or content unchanged.", yamlStartPlaceholder) } - return os.WriteFile(outputPath, []byte(finalContent), 0644) + // --- Write updated content back to MDX file --- + err = os.WriteFile(outputPath, []byte(mdxContent), 0644) + if err != nil { + return fmt.Errorf("failed to write updated content to %s: %w", outputPath, err) + } + + return nil } func replacePlaceholder(content, startPlaceholder, endPlaceholder, newBlockContent string) (string, bool) { @@ -545,141 +480,161 @@ func replacePlaceholder(content, startPlaceholder, endPlaceholder, newBlockConte endIndex := strings.Index(content, endPlaceholder) if startIndex != -1 && endIndex != -1 && endIndex > startIndex { - // Preserve the placeholders themselves, replace content between them - newContent := content[:startIndex+len(startPlaceholder)] + "\n" + newBlockContent + "\n" + content[endIndex:] - return newContent, true + // Include the start placeholder, replace content until end placeholder, then add end placeholder + newContent := content[:startIndex+len(startPlaceholder)] + + newBlockContent + + content[endIndex:] + return newContent, newContent != content } - return content, false // Placeholders not found or in wrong order + return content, false // Placeholder not found or content is the same } func generateYAMLPart(builder *strings.Builder, configInfo *ParsedConfig, allConfigs map[string]*ParsedConfig, indentLevel int, isRoot bool) { indent := strings.Repeat(" ", indentLevel) - // Special handling for MQsConfig and PublishMQConfig to show "one of" - if configInfo.GoStructName == "MQsConfig" || configInfo.GoStructName == "PublishMQConfig" { - builder.WriteString(fmt.Sprintf("%s# Choose one of the following MQ providers:\n", indent)) - } + // Sort fields by YAMLName for consistent YAML output + // This is important because map iteration order is not guaranteed in Go for allConfigs + // and field order from reflection is Go struct order, not necessarily desired YAML order. + // However, for struct fields, we use the order from ParsedConfig.Fields which is + // now sorted by Go FieldName in transformToParsedConfigs. + // For truly aesthetic YAML, one might want a custom sort order. + // For now, using the order from ParsedConfig.Fields. - // Sort fields by YAML name for consistent output - sortedFields := make([]ConfigField, len(configInfo.Fields)) - copy(sortedFields, configInfo.Fields) - sort.Slice(sortedFields, func(i, j int) bool { - if sortedFields[i].YAMLName == "" { - return false - } - if sortedFields[j].YAMLName == "" { - return true - } - return sortedFields[i].YAMLName < sortedFields[j].YAMLName - }) - - for _, field := range sortedFields { - if field.YAMLName == "" { + for _, field := range configInfo.Fields { + if field.YAMLName == "" || field.YAMLName == "-" { // Skip if no YAML name or explicitly ignored continue } - + // Field description as a comment if field.Description != "" { + // Ensure multi-line descriptions are commented correctly descLines := strings.Split(field.Description, "\n") for _, line := range descLines { - builder.WriteString(fmt.Sprintf("%s# %s\n", indent, line)) + builder.WriteString(fmt.Sprintf("%s# %s\n", indent, strings.TrimSpace(line))) } } - if nestedConfigInfo, ok := allConfigs[field.Type]; ok { - builder.WriteString(fmt.Sprintf("%s%s:\n", indent, field.YAMLName)) - generateYAMLPart(builder, nestedConfigInfo, allConfigs, indentLevel+1, false) - } else if strings.HasPrefix(field.Type, "[]") { // Handle slices - builder.WriteString(fmt.Sprintf("%s%s:\n", indent, field.YAMLName)) - // Attempt to use the actual reflected default value for slices - // This part is tricky because `field.DefaultValue` is just a string. - // We need a way to get the actual `reflect.Value` of the default. - // For now, we'll rely on a placeholder or a very simple interpretation of field.DefaultValue - // A proper solution would involve passing the `defaults` map (from getConfigDefaults) - // down to here and looking up the field's default by its path. - - // Let's assume field.DefaultValue is a string like "[]" or "[item1 item2]" - // This is a simplification. - if field.DefaultValue == "[]" || field.DefaultValue == "" || field.DefaultValue == "" { - builder.WriteString(fmt.Sprintf("%s [] # Empty list\n", indent)) + // Default value as a comment, if available and not a struct type that will be expanded + // isStructTypeField := false + // Check if the field's type (after dereferencing if it's a pointer) corresponds to a known struct configuration + // actualFieldTypeForCheck := strings.TrimPrefix(field.Type, "*") + // if _, isKnownStruct := allConfigs[actualFieldTypeForCheck]; isKnownStruct { + // isStructTypeField = true + // } + + // Removed default value comments as per feedback. The value itself is shown. + // if field.DefaultValue != "" && !isStructTypeField { + // builder.WriteString(fmt.Sprintf("%s# Default: %s\n", indent, field.DefaultValue)) + // } else if field.DefaultValue == "" && !isStructTypeField && field.Required != "Y" && field.Required != "true" { + // // Indicate if no default and not required (optional) + // builder.WriteString(fmt.Sprintf("%s# Default: (none)\n", indent)) + // } + + // Required status as a comment + if field.Required != "" && field.Required != "N" && field.Required != "false" { + builder.WriteString(fmt.Sprintf("%s# Required: %s\n", indent, field.Required)) + } + + // Field line + builder.WriteString(fmt.Sprintf("%s%s:", indent, field.YAMLName)) + + var nestedStructShortName string + fieldTypeForLookup := field.Type // This is string like "config.RedisConfig" or "*config.MQsConfig" + if strings.HasPrefix(fieldTypeForLookup, "*") { + fieldTypeForLookup = fieldTypeForLookup[1:] // remove "*" + } + // Further strip package path if present, e.g. "config.RedisConfig" -> "RedisConfig" + // or "time.Time" -> "Time" + parts := strings.Split(fieldTypeForLookup, ".") + if len(parts) > 0 { + nestedStructShortName = parts[len(parts)-1] // Get the last part + } + + if nestedStructShortName != "" { + if nestedConfig, ok := allConfigs[nestedStructShortName]; ok && field.Type != "string" && nestedStructShortName != "Time" { // also ensure we don't try to expand time.Time as a custom struct + // It's a nested struct we should expand + builder.WriteString("\n") + generateYAMLPart(builder, nestedConfig, allConfigs, indentLevel+1, false) } else { - // Attempt to parse a simple string representation of a slice. - // This is very basic and will likely need improvement. - trimmed := strings.Trim(field.DefaultValue, "[]") - if trimmed != "" { - elements := strings.Fields(trimmed) // Splits by space, assumes simple elements - for _, elem := range elements { - // Try to format element based on assumed type (e.g., string if it's not number/bool) - elemType := strings.TrimPrefix(field.Type, "[]") // e.g. "string" from "[]string" - builder.WriteString(fmt.Sprintf("%s - %s\n", indent, formatSimpleValueToYAML(elem, elemType))) - } + // Scalar, slice, map, or a struct from a different package (or time.Time) + valueStr := field.DefaultValue + if valueStr == "" || (valueStr == "[]" && strings.HasPrefix(field.Type, "[]")) { + valueStr = getYAMLPlaceholderForType(field.Type) } else { - builder.WriteString(fmt.Sprintf("%s [] # Empty list from default value: %s\n", indent, field.DefaultValue)) + valueStr = formatSimpleValueToYAML(valueStr, field.Type) } + builder.WriteString(fmt.Sprintf(" %s\n", valueStr)) } - } else { // Simple field - defaultValueText := field.DefaultValue - if defaultValueText == "" || defaultValueText == "" { - defaultValueText = getYAMLPlaceholderForType(field.Type) + } else { + // Fallback if short name extraction failed (should not happen for valid types) + valueStr := field.DefaultValue + if valueStr == "" || (valueStr == "[]" && strings.HasPrefix(field.Type, "[]")) { + valueStr = getYAMLPlaceholderForType(field.Type) } else { - defaultValueText = formatSimpleValueToYAML(defaultValueText, field.Type) + valueStr = formatSimpleValueToYAML(valueStr, field.Type) } - builder.WriteString(fmt.Sprintf("%s%s: %s\n", indent, field.YAMLName, defaultValueText)) - } - if indentLevel == 0 && isRoot { - builder.WriteString("\n") + builder.WriteString(fmt.Sprintf(" %s\n", valueStr)) } + builder.WriteString("\n") // Add a blank line after each top-level entry in a struct for readability } } func getYAMLPlaceholderForType(goType string) string { - if strings.HasPrefix(goType, "[]") { // Slice - return "[] # Empty list or no default" - } - switch goType { - case "string": - return `""` - case "int", "int64", "int32", "uint", "uint64", "uint32": + switch { + case strings.HasPrefix(goType, "[]string"): + return "[item1, item2]" + case strings.HasPrefix(goType, "[]"): // Generic slice + return "[]" + case strings.HasPrefix(goType, "map["): + return "{key: value}" + case goType == "string": + return "\"\"" + case goType == "int", goType == "int64", goType == "float64": return "0" - case "bool": + case goType == "bool": return "false" - case "map[string]string", "map[string]interface{}": // Basic map types - return "{} # Empty map or no default" default: - // For unknown or complex struct types not handled as nested. - // Check if it's a known config struct type (should have been handled by nestedConfigInfo) - // If not, it's likely a type we don't have specific YAML for. - return "null # Check type and provide appropriate default" + return "# <" + goType + ">" } } func formatSimpleValueToYAML(value, goType string) string { - if value == "" { // Handle explicit nil from reflection - return "null" - } + // If it's a string, ensure it's quoted if it contains special chars or is empty if goType == "string" { - // Ensure strings are quoted, unless they are already clearly a YAML string that doesn't need quotes - // (e.g. simple alphanumeric, or already quoted). This is a simplification. - if _, err := strconv.ParseFloat(value, 64); err != nil && value != "true" && value != "false" && value != "null" { - if !(strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`)) && - !(strings.HasPrefix(value, `'`) && strings.HasSuffix(value, `'`)) { - return fmt.Sprintf(`"%s"`, strings.ReplaceAll(value, `"`, `\"`)) - } + if value == "" { + return "\"\"" // Explicitly empty string } - return value // Return as is if it looks like a number, bool, null, or already quoted + // Always quote non-empty strings to handle all special characters correctly for YAML. + return strconv.Quote(value) + } + if value == "[]" && strings.HasPrefix(goType, "[]") { // Empty slice from default + return "[]" } - // For non-string simple types, assume DefaultValue is already a good representation + // For non-string types (numbers, booleans), the default fmt.Sprintf("%v") representation is usually fine for YAML. return value } -// collectEnvVarFields flattens all fields that have an environment variable name. -// This helps in case fields are defined across multiple ParsedConfig structs but should be in one table. +// collectEnvVarFields gathers all fields that have an EnvName, from all ParsedConfig structs. func collectEnvVarFields(parsedConfigs []ParsedConfig) []ConfigField { var allFields []ConfigField - seenEnvVars := make(map[string]bool) // To avoid duplicates if somehow an env var is on multiple fields + seenEnvVars := make(map[string]bool) // To avoid duplicates if structs are processed multiple times or nested weirdly - // It's better to iterate based on the main config structure to get a somewhat logical order - // For now, a simple iteration. Order might need refinement. - for _, pc := range parsedConfigs { + // Process top-level "Config" first if available, then others. + // This helps in establishing a somewhat predictable order if paths were involved. + // With the current flat list from reflection, order of parsedConfigs matters less here. + + sortedParsedConfigs := make([]ParsedConfig, len(parsedConfigs)) + copy(sortedParsedConfigs, parsedConfigs) + sort.Slice(sortedParsedConfigs, func(i, j int) bool { + if sortedParsedConfigs[i].IsTopLevel { + return true + } + if sortedParsedConfigs[j].IsTopLevel { + return false + } + return sortedParsedConfigs[i].Name < sortedParsedConfigs[j].Name + }) + + for _, pc := range sortedParsedConfigs { for _, field := range pc.Fields { if field.EnvName != "" && !seenEnvVars[field.EnvName] { allFields = append(allFields, field) @@ -687,7 +642,7 @@ func collectEnvVarFields(parsedConfigs []ParsedConfig) []ConfigField { } } } - // Sort fields alphabetically by EnvName for consistent output + // Sort by EnvName for consistent table output sort.Slice(allFields, func(i, j int) bool { return allFields[i].EnvName < allFields[j].EnvName }) @@ -695,14 +650,17 @@ func collectEnvVarFields(parsedConfigs []ParsedConfig) []ConfigField { } func formatRequiredText(reqStatus string) string { - switch reqStatus { - case "Y": + switch strings.ToUpper(reqStatus) { + case "Y", "TRUE": return "Yes" - case "N": + case "N", "FALSE": return "No" case "C": - return "Conditional (see description)" + return "Conditional" default: - return reqStatus // Or "Unknown" + if reqStatus != "" { + return reqStatus // Show as is if not recognized + } + return "No" // Default to No if empty } } From d9ffee3ac5299851749f258b631274a6b1d9c75f Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 9 Jun 2025 21:29:58 +0100 Subject: [PATCH 17/27] chore(docs): update config via docsgen --- docs/pages/references/configuration.mdx | 244 ++++++++++++++++++++---- 1 file changed, 202 insertions(+), 42 deletions(-) diff --git a/docs/pages/references/configuration.mdx b/docs/pages/references/configuration.mdx index ac60ddc1..71b43c64 100644 --- a/docs/pages/references/configuration.mdx +++ b/docs/pages/references/configuration.mdx @@ -24,13 +24,13 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `API_KEY` | API key for authenticating requests to the Outpost API. | `nil` | Yes | | `API_PORT` | Port number for the API server to listen on. | `3333` | No | | `AUDIT_LOG` | Enables or disables audit logging for significant events. | `true` | No | -| `AWS_SQS_ACCESS_KEY_ID` | AWS Access Key ID for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional (see description) | +| `AWS_SQS_ACCESS_KEY_ID` | AWS Access Key ID for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional | | `AWS_SQS_DELIVERY_QUEUE` | Name of the SQS queue for delivery events. | `outpost-delivery` | No | | `AWS_SQS_ENDPOINT` | Custom AWS SQS endpoint URL. Optional, typically used for local testing (e.g., LocalStack). | `nil` | No | | `AWS_SQS_LOG_QUEUE` | Name of the SQS queue for log events. | `outpost-log` | No | -| `AWS_SQS_REGION` | AWS Region for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional (see description) | -| `AWS_SQS_SECRET_ACCESS_KEY` | AWS Secret Access Key for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional (see description) | -| `CLICKHOUSE_ADDR` | Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage. | `nil` | Conditional (see description) | +| `AWS_SQS_REGION` | AWS Region for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional | +| `AWS_SQS_SECRET_ACCESS_KEY` | AWS Secret Access Key for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional | +| `CLICKHOUSE_ADDR` | Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage. | `nil` | Conditional | | `CLICKHOUSE_DATABASE` | Database name in ClickHouse to use. | `outpost` | No | | `CLICKHOUSE_PASSWORD` | Password for ClickHouse authentication. | `nil` | No | | `CLICKHOUSE_USERNAME` | Username for ClickHouse authentication. | `nil` | No | @@ -53,8 +53,8 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `GCP_PUBSUB_DELIVERY_TOPIC` | Name of the GCP Pub/Sub topic for delivery events. | `outpost-delivery` | No | | `GCP_PUBSUB_LOG_SUBSCRIPTION` | Name of the GCP Pub/Sub subscription for log events. | `outpost-log-sub` | No | | `GCP_PUBSUB_LOG_TOPIC` | Name of the GCP Pub/Sub topic for log events. | `outpost-log` | No | -| `GCP_PUBSUB_PROJECT` | GCP Project ID for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider. | `nil` | Conditional (see description) | -| `GCP_PUBSUB_SERVICE_ACCOUNT_CREDENTIALS` | JSON string or path to a file containing GCP service account credentials for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider and not running in an environment with implicit credentials (e.g., GCE, GKE). | `nil` | Conditional (see description) | +| `GCP_PUBSUB_PROJECT` | GCP Project ID for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider. | `nil` | Conditional | +| `GCP_PUBSUB_SERVICE_ACCOUNT_CREDENTIALS` | JSON string or path to a file containing GCP service account credentials for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider and not running in an environment with implicit credentials (e.g., GCE, GKE). | `nil` | Conditional | | `GIN_MODE` | Sets the Gin framework mode (e.g., 'debug', 'release', 'test'). See Gin documentation for details. | `release` | No | | `HTTP_USER_AGENT` | Custom HTTP User-Agent string for outgoing webhook deliveries. If unset, a default (OrganizationName/Version) is used. | `nil` | No | | `LOG_BATCH_SIZE` | Maximum number of log entries to batch together before writing to storage. | `1000` | No | @@ -64,8 +64,8 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `MAX_DESTINATIONS_PER_TENANT` | Maximum number of destinations allowed per tenant/organization. | `20` | No | | `MAX_RETRY_LIMIT` | Maximum number of retry attempts for a single event delivery before giving up. | `10` | No | | `ORGANIZATION_NAME` | Name of the organization, used for display purposes and potentially in user agent strings. | `nil` | No | -| `OTEL_EXPORTER` | Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. | `nil` | Conditional (see description) | -| `OTEL_PROTOCOL` | Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. | `nil` | Conditional (see description) | +| `OTEL_EXPORTER` | Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. | `nil` | Conditional | +| `OTEL_PROTOCOL` | Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. | `nil` | Conditional | | `OTEL_SERVICE_NAME` | The service name reported to OpenTelemetry. If set, OpenTelemetry will be enabled. | `nil` | No | | `PORTAL_BRAND_COLOR` | Primary brand color (hex code) for theming the Outpost Portal (e.g., '#6122E7'). Also referred to as Accent Color in some contexts. | `nil` | No | | `PORTAL_DISABLE_OUTPOST_BRANDING` | If true, disables Outpost branding in the portal. | `false` | No | @@ -75,25 +75,25 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `PORTAL_LOGO_DARK` | URL for the dark-mode logo to be displayed in the Outpost Portal. | `nil` | No | | `PORTAL_ORGANIZATION_NAME` | Organization name displayed in the Outpost Portal. | `nil` | No | | `PORTAL_PROXY_URL` | URL to proxy the Outpost Portal through. If set, Outpost serves the portal assets, and this URL is used as the base. Must be a valid URL. | `nil` | No | -| `PORTAL_REFERER_URL` | The expected Referer URL for accessing the portal. This is a security measure. Required if the Outpost Portal is enabled/used. Example: 'https://admin.example.com'. | `nil` | Conditional (see description) | -| `POSTGRES_URL` | Connection URL for PostgreSQL, used as an alternative log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'. Required if ClickHouse is not configured and log storage is needed. | `nil` | Conditional (see description) | -| `PUBLISH_AWS_SQS_ACCESS_KEY_ID` | AWS Access Key ID for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional (see description) | +| `PORTAL_REFERER_URL` | The expected Referer URL for accessing the portal. This is a security measure. Required if the Outpost Portal is enabled/used. Example: 'https://admin.example.com'. | `nil` | Conditional | +| `POSTGRES_URL` | Connection URL for PostgreSQL, used as an alternative log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'. Required if ClickHouse is not configured and log storage is needed. | `nil` | Conditional | +| `PUBLISH_AWS_SQS_ACCESS_KEY_ID` | AWS Access Key ID for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional | | `PUBLISH_AWS_SQS_ENDPOINT` | Custom AWS SQS endpoint URL for the publish queue. Optional. | `nil` | No | -| `PUBLISH_AWS_SQS_QUEUE` | Name of the SQS queue for publishing events. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional (see description) | -| `PUBLISH_AWS_SQS_REGION` | AWS Region for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional (see description) | -| `PUBLISH_AWS_SQS_SECRET_ACCESS_KEY` | AWS Secret Access Key for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional (see description) | -| `PUBLISH_GCP_PUBSUB_PROJECT` | GCP Project ID for the Pub/Sub publish topic. Required if GCP Pub/Sub is the chosen publish MQ provider. | `nil` | Conditional (see description) | -| `PUBLISH_GCP_PUBSUB_SERVICE_ACCOUNT_CREDENTIALS` | JSON string or path to a file containing GCP service account credentials for the Pub/Sub publish topic. Required if GCP Pub/Sub is chosen and not using implicit credentials. | `nil` | Conditional (see description) | -| `PUBLISH_GCP_PUBSUB_SUBSCRIPTION` | Name of the GCP Pub/Sub subscription to read published events from. Required if GCP Pub/Sub is the chosen publish MQ provider. | `nil` | Conditional (see description) | -| `PUBLISH_GCP_PUBSUB_TOPIC` | Name of the GCP Pub/Sub topic for publishing events. Required if GCP Pub/Sub is the chosen publish MQ provider. | `nil` | Conditional (see description) | +| `PUBLISH_AWS_SQS_QUEUE` | Name of the SQS queue for publishing events. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional | +| `PUBLISH_AWS_SQS_REGION` | AWS Region for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional | +| `PUBLISH_AWS_SQS_SECRET_ACCESS_KEY` | AWS Secret Access Key for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional | +| `PUBLISH_GCP_PUBSUB_PROJECT` | GCP Project ID for the Pub/Sub publish topic. Required if GCP Pub/Sub is the chosen publish MQ provider. | `nil` | Conditional | +| `PUBLISH_GCP_PUBSUB_SERVICE_ACCOUNT_CREDENTIALS` | JSON string or path to a file containing GCP service account credentials for the Pub/Sub publish topic. Required if GCP Pub/Sub is chosen and not using implicit credentials. | `nil` | Conditional | +| `PUBLISH_GCP_PUBSUB_SUBSCRIPTION` | Name of the GCP Pub/Sub subscription to read published events from. Required if GCP Pub/Sub is the chosen publish MQ provider. | `nil` | Conditional | +| `PUBLISH_GCP_PUBSUB_TOPIC` | Name of the GCP Pub/Sub topic for publishing events. Required if GCP Pub/Sub is the chosen publish MQ provider. | `nil` | Conditional | | `PUBLISH_MAX_CONCURRENCY` | Maximum number of messages to process concurrently from the publish queue. | `1` | No | | `PUBLISH_RABBITMQ_EXCHANGE` | Name of the RabbitMQ exchange for the publish queue. | `nil` | No | -| `PUBLISH_RABBITMQ_QUEUE` | Name of the RabbitMQ queue for publishing events. Required if RabbitMQ is the chosen publish MQ provider. | `nil` | Conditional (see description) | -| `PUBLISH_RABBITMQ_SERVER_URL` | RabbitMQ server connection URL for the publish queue. Required if RabbitMQ is the chosen publish MQ provider. | `nil` | Conditional (see description) | +| `PUBLISH_RABBITMQ_QUEUE` | Name of the RabbitMQ queue for publishing events. Required if RabbitMQ is the chosen publish MQ provider. | `nil` | Conditional | +| `PUBLISH_RABBITMQ_SERVER_URL` | RabbitMQ server connection URL for the publish queue. Required if RabbitMQ is the chosen publish MQ provider. | `nil` | Conditional | | `RABBITMQ_DELIVERY_QUEUE` | Name of the RabbitMQ queue for delivery events. | `outpost-delivery` | No | | `RABBITMQ_EXCHANGE` | Name of the RabbitMQ exchange to use. | `outpost` | No | | `RABBITMQ_LOG_QUEUE` | Name of the RabbitMQ queue for log events. | `outpost-log` | No | -| `RABBITMQ_SERVER_URL` | RabbitMQ server connection URL (e.g., 'amqp://user:pass@host:port/vhost'). Required if RabbitMQ is the chosen MQ provider. | `nil` | Conditional (see description) | +| `RABBITMQ_SERVER_URL` | RabbitMQ server connection URL (e.g., 'amqp://user:pass@host:port/vhost'). Required if RabbitMQ is the chosen MQ provider. | `nil` | Conditional | | `REDIS_DATABASE` | Redis database number to select after connecting. | `0` | Yes | | `REDIS_HOST` | Hostname or IP address of the Redis server. | `127.0.0.1` | Yes | | `REDIS_PASSWORD` | Password for Redis authentication, if required by the server. | `nil` | Yes | @@ -104,7 +104,7 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `TELEMETRY_BATCH_SIZE` | Maximum number of telemetry events to batch before sending. | `100` | No | | `TELEMETRY_HOOKDECK_SOURCE_URL` | The Hookdeck Source URL to send anonymous usage telemetry data to. Set to empty to disable sending to Hookdeck. | `https://hkdk.events/yhk665ljz3rn6l` | No | | `TELEMETRY_SENTRY_DSN` | Sentry DSN for error reporting. If provided and telemetry is not disabled, Sentry integration will be enabled. | `https://examplePublicKey@o0.ingest.sentry.io/0` | No | -| `TOPICS` | Comma-separated list of topics that this Outpost instance should subscribe to for event processing. | `[]` | No | +| `TOPICS` | Comma-separated list of topics that this Outpost instance should subscribe to for event processing. | `nil` | No | {/* END AUTOGENERATED CONFIG ENV VARS */} ## YAML @@ -115,38 +115,49 @@ Global configurations are provided through env variables or a YAML file. ConfigM # This example shows all available keys with their default values where applicable. # A 16, 24, or 32 byte secret key used for AES encryption of sensitive data at rest. +# Required: Y aes_encryption_secret: "" -alert: - # If true, automatically disables a destination after 'consecutive_failure_count' is reached. - auto_disable_destination: true - # URL to which Outpost will send a POST request when an alert is triggered (e.g., for destination failures). - callback_url: "" - # Number of consecutive delivery failures for a destination before triggering an alert and potentially disabling it. - consecutive_failure_count: 20 - # Secret key for signing and verifying JWTs if JWT authentication is used for the API. +# Required: Y api_jwt_secret: "" # API key for authenticating requests to the Outpost API. +# Required: Y api_key: "" # Port number for the API server to listen on. api_port: 3333 +alert: + # If true, automatically disables a destination after 'consecutive_failure_count' is reached. + auto_disable_destination: true + + # URL to which Outpost will send a POST request when an alert is triggered (e.g., for destination failures). + callback_url: "" + + # Number of consecutive delivery failures for a destination before triggering an alert and potentially disabling it. + consecutive_failure_count: 20 + + # Enables or disables audit logging for significant events. audit_log: true clickhouse: # Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage. + # Required: C addr: "" + # Database name in ClickHouse to use. database: "outpost" + # Password for ClickHouse authentication. password: "" + # Username for ClickHouse authentication. username: "" + # Maximum number of delivery attempts to process concurrently. delivery_max_concurrency: 1 @@ -161,29 +172,42 @@ destinations: aws_kinesis: # If true, includes Outpost metadata (event ID, topic, etc.) within the Kinesis record payload. metadata_in_payload: true + + # Path to the directory containing custom destination type definitions. This can be overridden by the root-level 'destination_metadata_path' if also set. metadata_path: "config/outpost/destinations" + # Configuration specific to webhook destinations. webhook: # If true, disables adding the default 'X-Outpost-Event-Id' header to webhook requests. disable_default_event_id_header: false + # If true, disables adding the default 'X-Outpost-Signature' header to webhook requests. disable_default_signature_header: false + # If true, disables adding the default 'X-Outpost-Timestamp' header to webhook requests. disable_default_timestamp_header: false + # If true, disables adding the default 'X-Outpost-Topic' header to webhook requests. disable_default_topic_header: false + # Prefix for custom headers added to webhook requests (e.g., 'X-MyOrg-'). header_prefix: "x-outpost-" + # Algorithm used for signing webhook requests (e.g., 'hmac-sha256'). signature_algorithm: "hmac-sha256" + # Go template for constructing the content to be signed for webhook requests. signature_content_template: "{{.Timestamp.Unix}}.{{.Body}}" + # Encoding for the signature (e.g., 'hex', 'base64'). signature_encoding: "hex" + # Go template for the value of the signature header. signature_header_template: "t={{.Timestamp.Unix}},v0={{.Signatures | join \",\"}}" + + # Global flag to disable all telemetry (anonymous usage statistics to Hookdeck and error reporting to Sentry). If true, overrides 'telemetry.disabled'. disable_telemetry: false @@ -205,144 +229,275 @@ log_level: "info" # Maximum number of log writing operations to process concurrently. log_max_concurrency: 1 -# Maximum number of destinations allowed per tenant/organization. -max_destinations_per_tenant: 20 - mqs: - # Choose one of the following MQ providers: # Configuration for using AWS SQS as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured. aws_sqs: # AWS Access Key ID for SQS. Required if AWS SQS is the chosen MQ provider. + # Required: C access_key_id: "" + # Name of the SQS queue for delivery events. delivery_queue: "outpost-delivery" + # Custom AWS SQS endpoint URL. Optional, typically used for local testing (e.g., LocalStack). endpoint: "" + # Name of the SQS queue for log events. log_queue: "outpost-log" + # AWS Region for SQS. Required if AWS SQS is the chosen MQ provider. + # Required: C region: "" + # AWS Secret Access Key for SQS. Required if AWS SQS is the chosen MQ provider. + # Required: C secret_access_key: "" + + # Configuration for using GCP Pub/Sub as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured. gcp_pubsub: # Name of the GCP Pub/Sub subscription for delivery events. delivery_subscription: "outpost-delivery-sub" + # Name of the GCP Pub/Sub topic for delivery events. delivery_topic: "outpost-delivery" + # Name of the GCP Pub/Sub subscription for log events. log_subscription: "outpost-log-sub" + # Name of the GCP Pub/Sub topic for log events. log_topic: "outpost-log" + # GCP Project ID for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider. + # Required: C project: "" + # JSON string or path to a file containing GCP service account credentials for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider and not running in an environment with implicit credentials (e.g., GCE, GKE). + # Required: C service_account_credentials: "" + + # Configuration for using RabbitMQ as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured. rabbitmq: # Name of the RabbitMQ queue for delivery events. delivery_queue: "outpost-delivery" + # Name of the RabbitMQ exchange to use. exchange: "outpost" + # Name of the RabbitMQ queue for log events. log_queue: "outpost-log" + # RabbitMQ server connection URL (e.g., 'amqp://user:pass@host:port/vhost'). Required if RabbitMQ is the chosen MQ provider. + # Required: C server_url: "" -# Name of the organization, used for display purposes and potentially in user agent strings. -organization_name: "" + + +# Maximum number of destinations allowed per tenant/organization. +max_destinations_per_tenant: 20 otel: # OpenTelemetry configuration specific to logs. logs: # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C exporter: "" + + # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C + exporter: "" + + # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C + exporter: "" + + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C + protocol: "" + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C protocol: "" + + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C + protocol: "" + + # OpenTelemetry configuration specific to metrics. metrics: # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C + exporter: "" + + # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C + exporter: "" + + # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C exporter: "" + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C protocol: "" + + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C + protocol: "" + + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C + protocol: "" + + # The service name reported to OpenTelemetry. If set, OpenTelemetry will be enabled. service_name: "" + # OpenTelemetry configuration specific to traces. traces: # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C + exporter: "" + + # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C exporter: "" + + # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. + # Required: C + exporter: "" + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C protocol: "" + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C + protocol: "" + + # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. + # Required: C + protocol: "" + + + +# Name of the organization, used for display purposes and potentially in user agent strings. +organization_name: "" + portal: # Primary brand color (hex code) for theming the Outpost Portal (e.g., '#6122E7'). Also referred to as Accent Color in some contexts. brand_color: "" + # If true, disables Outpost branding in the portal. disable_outpost_branding: false + # URL for the favicon to be used in the Outpost Portal. favicon_url: "" + # Force a specific theme for the Outpost Portal (e.g., 'light', 'dark'). force_theme: "" + # URL for the light-mode logo to be displayed in the Outpost Portal. logo: "" + # URL for the dark-mode logo to be displayed in the Outpost Portal. logo_dark: "" + # Organization name displayed in the Outpost Portal. org_name: "" + # URL to proxy the Outpost Portal through. If set, Outpost serves the portal assets, and this URL is used as the base. Must be a valid URL. proxy_url: "" + # The expected Referer URL for accessing the portal. This is a security measure. Required if the Outpost Portal is enabled/used. Example: 'https://admin.example.com'. + # Required: C referer_url: "" + # Connection URL for PostgreSQL, used as an alternative log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'. Required if ClickHouse is not configured and log storage is needed. +# Required: C postgres: "" -# Maximum number of messages to process concurrently from the publish queue. -publish_max_concurrency: 1 - publishmq: - # Choose one of the following MQ providers: # Configuration for using AWS SQS as the publish message queue. Only one publish MQ provider should be configured. aws_sqs: # AWS Access Key ID for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. + # Required: C access_key_id: "" + # Custom AWS SQS endpoint URL for the publish queue. Optional. endpoint: "" + # Name of the SQS queue for publishing events. Required if AWS SQS is the chosen publish MQ provider. + # Required: C queue: "" + # AWS Region for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. + # Required: C region: "" + # AWS Secret Access Key for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. + # Required: C secret_access_key: "" + + # Configuration for using GCP Pub/Sub as the publish message queue. Only one publish MQ provider should be configured. gcp_pubsub: # GCP Project ID for the Pub/Sub publish topic. Required if GCP Pub/Sub is the chosen publish MQ provider. + # Required: C project: "" + # JSON string or path to a file containing GCP service account credentials for the Pub/Sub publish topic. Required if GCP Pub/Sub is chosen and not using implicit credentials. + # Required: C service_account_credentials: "" + # Name of the GCP Pub/Sub subscription to read published events from. Required if GCP Pub/Sub is the chosen publish MQ provider. + # Required: C subscription: "" + # Name of the GCP Pub/Sub topic for publishing events. Required if GCP Pub/Sub is the chosen publish MQ provider. + # Required: C topic: "" + + # Configuration for using RabbitMQ as the publish message queue. Only one publish MQ provider should be configured. rabbitmq: # Name of the RabbitMQ exchange for the publish queue. exchange: "" + # Name of the RabbitMQ queue for publishing events. Required if RabbitMQ is the chosen publish MQ provider. + # Required: C queue: "" + # RabbitMQ server connection URL for the publish queue. Required if RabbitMQ is the chosen publish MQ provider. + # Required: C server_url: "" + + +# Maximum number of messages to process concurrently from the publish queue. +publish_max_concurrency: 1 + redis: # Redis database number to select after connecting. + # Required: Y database: 0 + # Hostname or IP address of the Redis server. + # Required: Y host: "127.0.0.1" + # Password for Redis authentication, if required by the server. + # Required: Y password: "" + # Port number for the Redis server. + # Required: Y port: 6379 + # Interval in seconds between delivery retry attempts for failed webhooks. retry_interval_seconds: 30 @@ -355,18 +510,23 @@ service: "" telemetry: # Maximum time in seconds to wait before sending a batch of telemetry events if batch size is not reached. batch_interval: 5 + # Maximum number of telemetry events to batch before sending. batch_size: 100 + # Disables telemetry within the 'telemetry' block (Hookdeck usage stats and Sentry). Can be overridden by the global 'disable_telemetry' flag at the root of the configuration. disabled: false + # The Hookdeck Source URL to send anonymous usage telemetry data to. Set to empty to disable sending to Hookdeck. hookdeck_source_url: "https://hkdk.events/yhk665ljz3rn6l" + # Sentry DSN for error reporting. If provided and telemetry is not disabled, Sentry integration will be enabled. sentry_dsn: "https://examplePublicKey@o0.ingest.sentry.io/0" + # Comma-separated list of topics that this Outpost instance should subscribe to for event processing. -topics: - [] # Empty list +topics: [item1, item2] ``` + {/* END AUTOGENERATED CONFIG YAML */} From 1780e88d5d73ca9f961f905a917f275f37b58e1c Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 9 Jun 2025 21:44:35 +0100 Subject: [PATCH 18/27] chore: enhance placeholder handling in MDX content generation with improved logging --- cmd/configdocsgen/main.go | 43 +++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/cmd/configdocsgen/main.go b/cmd/configdocsgen/main.go index cf5c8bf6..5c2dcac0 100644 --- a/cmd/configdocsgen/main.go +++ b/cmd/configdocsgen/main.go @@ -455,15 +455,36 @@ Below is an example YAML configuration file showing all available options and th mdxContent := string(mdxBytes) // --- Replace placeholders --- - var changed bool - mdxContent, changed = replacePlaceholder(mdxContent, envVarsStartPlaceholder, envVarsEndPlaceholder, "\n"+envVarsContent+"\n") - if !changed { - log.Printf("Warning: ENV vars placeholder '%s' not found or content unchanged.", envVarsStartPlaceholder) + envStartIndex := strings.Index(mdxContent, envVarsStartPlaceholder) + envEndIndex := strings.Index(mdxContent, envVarsEndPlaceholder) + + if envStartIndex != -1 && envEndIndex != -1 && envEndIndex > envStartIndex { + newMdxContent, changed := replacePlaceholder(mdxContent, envVarsStartPlaceholder, envVarsEndPlaceholder, "\n"+envVarsContent+"\n") + if !changed { + // Placeholder found, but content was already up-to-date. + log.Printf("Info: The content for the ENV vars placeholder '%s' was already up-to-date. No changes made to this block.", envVarsStartPlaceholder) + } else { + mdxContent = newMdxContent // Content was updated. + } + } else { + // Placeholder not found or in wrong order. + log.Printf("Warning: The ENV vars placeholder '%s' (and/or its corresponding end tag '%s') was not found or is in the wrong order in the output file. The ENV vars block will not be updated.", envVarsStartPlaceholder, envVarsEndPlaceholder) } - mdxContent, changed = replacePlaceholder(mdxContent, yamlStartPlaceholder, yamlEndPlaceholder, "\n"+yamlContent+"\n") - if !changed { - log.Printf("Warning: YAML placeholder '%s' not found or content unchanged.", yamlStartPlaceholder) + yamlStartIndex := strings.Index(mdxContent, yamlStartPlaceholder) + yamlEndIndex := strings.Index(mdxContent, yamlEndPlaceholder) + + if yamlStartIndex != -1 && yamlEndIndex != -1 && yamlEndIndex > yamlStartIndex { + newMdxContent, changed := replacePlaceholder(mdxContent, yamlStartPlaceholder, yamlEndPlaceholder, "\n"+yamlContent+"\n") + if !changed { + // Placeholder found, but content was already up-to-date. + log.Printf("Info: The content for the YAML placeholder '%s' was already up-to-date. No changes made to this block.", yamlStartPlaceholder) + } else { + mdxContent = newMdxContent // Content was updated. + } + } else { + // Placeholder not found or in wrong order. + log.Printf("Warning: The YAML placeholder '%s' (and/or its corresponding end tag '%s') was not found or is in the wrong order in the output file. The YAML block will not be updated.", yamlStartPlaceholder, yamlEndPlaceholder) } // --- Write updated content back to MDX file --- @@ -513,14 +534,6 @@ func generateYAMLPart(builder *strings.Builder, configInfo *ParsedConfig, allCon } } - // Default value as a comment, if available and not a struct type that will be expanded - // isStructTypeField := false - // Check if the field's type (after dereferencing if it's a pointer) corresponds to a known struct configuration - // actualFieldTypeForCheck := strings.TrimPrefix(field.Type, "*") - // if _, isKnownStruct := allConfigs[actualFieldTypeForCheck]; isKnownStruct { - // isStructTypeField = true - // } - // Removed default value comments as per feedback. The value itself is shown. // if field.DefaultValue != "" && !isStructTypeField { // builder.WriteString(fmt.Sprintf("%s# Default: %s\n", indent, field.DefaultValue)) From e9df6ca1b32fbd01b65baaabd9fa79fe8092af40 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 9 Jun 2025 21:44:39 +0100 Subject: [PATCH 19/27] chore: add command to generate documentation configuration --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index ebe42f88..1141cc5a 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,9 @@ test/coverage: test/coverage/html: go tool cover -html=coverage.out +docs/generate/config: + go run cmd/configdocsgen/main.go + network: docker network create outpost From 9a655e9ce203acd88ca77690ea97b811a9eac8d2 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 9 Jun 2025 22:37:23 +0100 Subject: [PATCH 20/27] chore: refactor ReflectionFieldInfo and extractFieldsRecursive for improved clarity and structure --- cmd/configdocsgen/main.go | 305 ++++++++++++++++++++++---------------- 1 file changed, 174 insertions(+), 131 deletions(-) diff --git a/cmd/configdocsgen/main.go b/cmd/configdocsgen/main.go index 5c2dcac0..5319b5fd 100644 --- a/cmd/configdocsgen/main.go +++ b/cmd/configdocsgen/main.go @@ -25,18 +25,19 @@ var ( // ReflectionFieldInfo is an intermediate struct to hold data extracted via reflection. type ReflectionFieldInfo struct { - Path string // Full dot-notation path from root config. e.g. "Redis.Host" - FieldName string // Original Go field name. e.g. "Host" - FieldTypeStr string // String representation of field type. e.g. "string", "int", "config.RedisConfig" - StructName string // Name of the immediate struct this field belongs to. e.g. "Config", "RedisConfig" - StructPkgPath string // Package path of the struct this field belongs to. - YAMLName string - EnvName string - EnvSeparator string - Description string - Required string // "true", "false", or "Y", "N", "C" from tag - DefaultValue interface{} // Actual default value - IsEmbeddedField bool // True if this field comes from an embedded struct and should be inlined in YAML + FieldPath string // Full dot-notation path from root config. e.g. "OTEL.Traces.Exporter" + FieldName string // Original Go field name. e.g. "Exporter" + FieldTypeStr string // String representation of field type. e.g. "string", "int", "config.OTELSignalExporterConfig" + ParentGoStructName string // Go type name of the struct this field directly belongs to. e.g. "OTELSignalConfig" + ParentGoStructPkgPath string // Package path of the Go struct this field directly belongs to. + ParentStructInstancePath string // Unique path to the instance of the parent struct. e.g. "Config.OTEL.Traces" + YAMLName string + EnvName string + EnvSeparator string + Description string + Required string // "true", "false", or "Y", "N", "C" from tag + DefaultValue interface{} // Actual default value + IsEmbeddedField bool // True if this field comes from an embedded struct and should be inlined in YAML } // ConfigField represents a field in a configuration struct (used by generateDocs) @@ -95,18 +96,16 @@ func parseConfigWithReflection() ([]ReflectionFieldInfo, error) { cfgValue := reflect.ValueOf(cfg) targetPkgPath := cfgType.PkgPath() - err := extractFieldsRecursive(cfgValue, cfgType, "", cfgType.Name(), cfgType.PkgPath(), targetPkgPath, &infos, false) + err := extractFieldsRecursive(cfgValue, cfgType, cfgType.Name(), cfgType.Name(), cfgType.PkgPath(), targetPkgPath, &infos, false) // Initial pathPrefix is cfgType.Name() if err != nil { return nil, err } return infos, nil } -func extractFieldsRecursive(currentVal reflect.Value, currentType reflect.Type, pathPrefix string, structName string, structPkgPath string, targetPkgPath string, infos *[]ReflectionFieldInfo, isEmbedded bool) error { +func extractFieldsRecursive(currentVal reflect.Value, currentType reflect.Type, parentStructInstancePath string, parentGoStructName string, parentGoStructPkgPath string, targetPkgPath string, infos *[]ReflectionFieldInfo, isEmbedded bool) error { if currentType.Kind() == reflect.Ptr { if currentVal.IsNil() { - // If the pointer is nil, create a zero value of the element type to inspect its structure - // This ensures fields of this struct are documented, albeit with zero/nil defaults. currentType = currentType.Elem() currentVal = reflect.Zero(currentType) } else { @@ -123,82 +122,69 @@ func extractFieldsRecursive(currentVal reflect.Value, currentType reflect.Type, fieldSpec := currentType.Field(i) fieldVal := currentVal.Field(i) - // Skip unexported fields if !fieldVal.CanInterface() { continue } fieldName := fieldSpec.Name - fullPath := fieldName - if pathPrefix != "" { - fullPath = pathPrefix + "." + fieldName + fieldPath := fieldName + if parentStructInstancePath != "" { + fieldPath = parentStructInstancePath + "." + fieldName + } else { + // This case should ideally only happen for the root struct's direct fields if parentStructInstancePath for root is empty. + // However, we initialize parentStructInstancePath to cfgType.Name() for the root. } - // Handle anonymous/embedded structs if fieldSpec.Anonymous { - // For embedded structs, recurse with the same pathPrefix (or adjusted if needed) - // and mark fields as embedded so they can be inlined in YAML. - err := extractFieldsRecursive(fieldVal, fieldSpec.Type, pathPrefix, structName, structPkgPath, targetPkgPath, infos, true) + // For embedded structs, recurse with the same parentStructInstancePath + // The parentGoStructName and parentGoStructPkgPath also remain the same for the fields of the embedded struct. + err := extractFieldsRecursive(fieldVal, fieldSpec.Type, parentStructInstancePath, parentGoStructName, parentGoStructPkgPath, targetPkgPath, infos, true) if err != nil { return fmt.Errorf("error recursing into embedded struct %s: %w", fieldName, err) } - continue // Skip adding the embedded struct itself as a field + continue } yamlTag := fieldSpec.Tag.Get("yaml") - yamlName := strings.Split(yamlTag, ",")[0] // Get name part, ignore omitempty etc. - if yamlName == "-" { // Skip fields explicitly ignored by YAML + yamlName := strings.Split(yamlTag, ",")[0] + if yamlName == "-" { continue } - if yamlName == "" { // Default to field name if yaml tag is missing or empty - // This behavior might need adjustment based on how YAML typically marshals - // For documentation, often we want to show the Go field name if no YAML name. - // However, for config, usually explicit YAML names are preferred. - // Let's assume if yaml tag is missing, it's not a primary config field for YAML docs. - // Or, for full documentation, one might want to include it. - // For now, if no yamlName, we might skip or use Go name. - // Let's use Go field name if yamlName is empty after split (e.g. tag was just ",omitempty") - // but if tag was entirely missing, it's less clear. - // The current AST parser implies fields without YAML tags are still processed. - // Let's default to Go field name if yamlName is empty. + if yamlName == "" { yamlName = fieldName } fieldTypeStr := formatFieldType(fieldSpec.Type) - // Create info for the current field itself - // This applies whether it's a basic type, a struct from another package, or a struct we'll recurse into. currentFieldInfo := ReflectionFieldInfo{ - Path: fullPath, - FieldName: fieldName, - FieldTypeStr: fieldTypeStr, - StructName: structName, // The struct this field belongs to (e.g., "Config") - StructPkgPath: structPkgPath, - YAMLName: yamlName, - EnvName: fieldSpec.Tag.Get("env"), - EnvSeparator: fieldSpec.Tag.Get("envSeparator"), - Description: fieldSpec.Tag.Get("desc"), - Required: fieldSpec.Tag.Get("required"), - DefaultValue: fieldVal.Interface(), // Capture the default value of this field - IsEmbeddedField: isEmbedded, // This is about whether *this field* is part of an embedding context - } - // If fieldSpec.Anonymous was true, we already 'continue'd earlier. - // So, this currentFieldInfo is for a named field. + FieldPath: fieldPath, + FieldName: fieldName, + FieldTypeStr: fieldTypeStr, + ParentGoStructName: parentGoStructName, // Go type name of the struct this field *directly* belongs to + ParentGoStructPkgPath: parentGoStructPkgPath, + ParentStructInstancePath: parentStructInstancePath, // Path to the instance of the parent struct + YAMLName: yamlName, + EnvName: fieldSpec.Tag.Get("env"), + EnvSeparator: fieldSpec.Tag.Get("envSeparator"), + Description: fieldSpec.Tag.Get("desc"), + Required: fieldSpec.Tag.Get("required"), + DefaultValue: fieldVal.Interface(), + IsEmbeddedField: isEmbedded, + } *infos = append(*infos, currentFieldInfo) - // If the field's type is a struct from the target package (and not an interface), recurse into its members. - // These members will have their StructName as fieldSpec.Type.Name() (e.g., "RedisConfig") - // and their Path will be prefixed with fullPath (e.g., "Redis.Host"). actualFieldType := fieldSpec.Type if actualFieldType.Kind() == reflect.Ptr { actualFieldType = actualFieldType.Elem() } + // Recurse if it's a struct from the target package (and not time.Time) if fieldSpec.Type.Kind() != reflect.Interface && actualFieldType.Kind() == reflect.Struct && actualFieldType.PkgPath() == targetPkgPath && actualFieldType.Name() != "Time" { - // Use actualFieldType.Name() for the StructName of the members - err := extractFieldsRecursive(fieldVal, fieldSpec.Type, fullPath, actualFieldType.Name(), actualFieldType.PkgPath(), targetPkgPath, infos, false) // isEmbedded is false for members of a regularly named struct field + // The new parentStructInstancePath for the recursive call is the FieldPath of the current struct field. + // The new parentGoStructName for the recursive call is the Go type name of this nested struct field. + err := extractFieldsRecursive(fieldVal, fieldSpec.Type, fieldPath, actualFieldType.Name(), actualFieldType.PkgPath(), targetPkgPath, infos, false) if err != nil { - return fmt.Errorf("error recursing into members of struct field %s (type %s): %w", fullPath, actualFieldType.Name(), err) + return fmt.Errorf("error recursing into members of struct field %s (type %s): %w", fieldPath, actualFieldType.Name(), err) } } } @@ -270,53 +256,86 @@ func formatDefaultValueToString(value interface{}, goType string) string { } func transformToParsedConfigs(infos []ReflectionFieldInfo) []ParsedConfig { - groupedByStruct := make(map[string][]ReflectionFieldInfo) - structOrder := []string{} // To maintain an order, e.g., "Config" first + groupedByInstancePath := make(map[string][]ReflectionFieldInfo) + structInstancePathOrder := []string{} // To maintain an order + + rootConfigInstancePath := "" // Will be determined by the first info, assuming root is processed first or by checking PkgPath for _, info := range infos { - if _, exists := groupedByStruct[info.StructName]; !exists { - groupedByStruct[info.StructName] = []ReflectionFieldInfo{} - if info.StructName == "Config" { // Ensure "Config" is first if present - structOrder = append([]string{info.StructName}, structOrder...) - } else { - structOrder = append(structOrder, info.StructName) + // Attempt to identify the root config instance path (e.g., "Config") + if rootConfigInstancePath == "" && info.ParentGoStructName == "Config" && info.ParentGoStructPkgPath == "github.com/hookdeck/outpost/internal/config" { + // The ParentStructInstancePath for fields directly under Config will be "Config" + // This logic assumes the initial call to extractFieldsRecursive for config.Config uses "Config" as pathPrefix. + if strings.Count(info.ParentStructInstancePath, ".") == 0 { // e.g. "Config", not "Config.Foo" + rootConfigInstancePath = info.ParentStructInstancePath + } + } + + if _, exists := groupedByInstancePath[info.ParentStructInstancePath]; !exists { + groupedByInstancePath[info.ParentStructInstancePath] = []ReflectionFieldInfo{} + structInstancePathOrder = append(structInstancePathOrder, info.ParentStructInstancePath) + } + groupedByInstancePath[info.ParentStructInstancePath] = append(groupedByInstancePath[info.ParentStructInstancePath], info) + } + if rootConfigInstancePath == "" && len(structInstancePathOrder) > 0 { + // Fallback: assume the shortest path is the root, or the one named "Config" if available + for _, p := range structInstancePathOrder { + if p == "Config" { // Default root struct name + rootConfigInstancePath = p + break + } + } + if rootConfigInstancePath == "" { + // As a last resort, pick the first one if only one, or sort and pick shortest. + // This part might need refinement if the root config isn't named "Config". + sort.Strings(structInstancePathOrder) // Sort alphabetically to have a deterministic order + if len(structInstancePathOrder) > 0 { + rootConfigInstancePath = structInstancePathOrder[0] // Default to first after sort + log.Printf("Warning: Could not definitively determine root config instance path. Defaulting to '%s'. Ensure root config is named 'Config' or initial pathPrefix is set correctly.", rootConfigInstancePath) } } - groupedByStruct[info.StructName] = append(groupedByStruct[info.StructName], info) } - // Sort structOrder to have "Config" first, then alphabetically for others - sort.SliceStable(structOrder, func(i, j int) bool { - if structOrder[i] == "Config" { + // Sort structInstancePathOrder: root path first, then alphabetically. + sort.SliceStable(structInstancePathOrder, func(i, j int) bool { + pathI := structInstancePathOrder[i] + pathJ := structInstancePathOrder[j] + if pathI == rootConfigInstancePath { return true } - if structOrder[j] == "Config" { + if pathJ == rootConfigInstancePath { return false } - return structOrder[i] < structOrder[j] + // Sort by depth first (fewer dots), then alphabetically + depthI := strings.Count(pathI, ".") + depthJ := strings.Count(pathJ, ".") + if depthI != depthJ { + return depthI < depthJ + } + return pathI < pathJ }) var parsedConfigs []ParsedConfig - for _, structName := range structOrder { - fieldsInfo := groupedByStruct[structName] + for _, instancePath := range structInstancePathOrder { + fieldsInfo := groupedByInstancePath[instancePath] + if len(fieldsInfo) == 0 { // Should not happen if instancePath came from the map keys + continue + } var configFields []ConfigField - var goStructPkgPath string // Take from the first field, should be consistent + // All fields in fieldsInfo share the same ParentGoStructName and ParentStructInstancePath. + // The ParentGoStructName is the Go type of the struct instance represented by 'instancePath'. + parentGoStructName := fieldsInfo[0].ParentGoStructName // Safe due to check above - // Sort fields by their original Go field name for consistent order - // This is important if ReflectionFieldInfo Path was used for sorting, - // but here we sort by FieldName within each struct. + // Sort fields by their original Go field name for consistent order within this instance sort.Slice(fieldsInfo, func(i, j int) bool { - // A simple path sort might be better if fields from embedded structs are mixed. - // For now, FieldName within the current struct context. return fieldsInfo[i].FieldName < fieldsInfo[j].FieldName }) for _, info := range fieldsInfo { - if goStructPkgPath == "" { - goStructPkgPath = info.StructPkgPath - } + // We only add fields that directly belong to this ParentStructInstancePath. + // Embedded fields are handled by IsEmbeddedField flag if needed later, but here we list them. configFields = append(configFields, ConfigField{ - Name: info.FieldName, // Use the Go field name + Name: info.FieldName, Type: info.FieldTypeStr, YAMLName: info.YAMLName, EnvName: info.EnvName, @@ -327,14 +346,12 @@ func transformToParsedConfigs(infos []ReflectionFieldInfo) []ParsedConfig { }) } - // Determine if this struct is the top-level "Config" - // The main config.Config struct from internal/config - isTopLevel := (structName == "Config" && goStructPkgPath == "github.com/hookdeck/outpost/internal/config") + isTopLevel := (instancePath == rootConfigInstancePath) parsedConfigs = append(parsedConfigs, ParsedConfig{ FileName: "reflection-generated", - Name: structName, - GoStructName: structName, + Name: instancePath, // Unique instance path, e.g., "Config.OTEL.Traces" + GoStructName: parentGoStructName, // Go type name, e.g., "OTELSignalConfig" Fields: configFields, IsTopLevel: isTopLevel, }) @@ -392,25 +409,65 @@ func generateDocs(parsedConfigs []ParsedConfig, outputPath string) error { yamlBuilder.WriteString("# This example shows all available keys with their default values where applicable.\n\n") var mainConfigInfo *ParsedConfig - configInfoMap := make(map[string]*ParsedConfig) + configInfoMap := make(map[string]*ParsedConfig) // Keyed by ParsedConfig.Name (ParentStructInstancePath) + rootConfigInstancePath := "" + for i := range parsedConfigs { - pc := &parsedConfigs[i] // operate on a copy to avoid modifying the slice - configInfoMap[pc.GoStructName] = pc - if pc.IsTopLevel { // Use IsTopLevel flag + pc := &parsedConfigs[i] + configInfoMap[pc.Name] = pc // pc.Name is now the ParentStructInstancePath + if pc.IsTopLevel { mainConfigInfo = pc + rootConfigInstancePath = pc.Name // Store the root instance path } } if mainConfigInfo != nil { generateYAMLPart(&yamlBuilder, mainConfigInfo, configInfoMap, 0, true) } else { - // Fallback if IsTopLevel wasn't set correctly, try finding "Config" by name - if cfgByName, ok := configInfoMap["Config"]; ok { - log.Println("Warning: Main 'Config' struct not found by IsTopLevel flag, falling back to name 'Config'.") - generateYAMLPart(&yamlBuilder, cfgByName, configInfoMap, 0, true) - } else { - yamlBuilder.WriteString("# ERROR: Main 'Config' struct not found. Cannot generate YAML structure.\n") - log.Println("Error: Main 'Config' struct not found by IsTopLevel flag or by name 'Config'.") + foundRoot := false + // Attempt to find the root config using rootConfigInstancePath if IsTopLevel set it + if rootConfigInstancePath != "" { + if cfgByRootPath, ok := configInfoMap[rootConfigInstancePath]; ok { + log.Printf("Info: Main config not directly found by IsTopLevel flag, but using identified root instance path '%s'.", rootConfigInstancePath) + generateYAMLPart(&yamlBuilder, cfgByRootPath, configInfoMap, 0, true) + foundRoot = true + } else { + log.Printf("Warning: rootConfigInstancePath '%s' was set (likely by IsTopLevel processing) but its corresponding entry was not found in configInfoMap. Proceeding with other fallbacks.", rootConfigInstancePath) + } + } + + // If not found via rootConfigInstancePath, try falling back to "Config" by name + if !foundRoot { + if cfgByName, ok := configInfoMap["Config"]; ok { + log.Println("Warning: Main config not found by IsTopLevel flag or specific root path. Falling back to instance path 'Config'.") + generateYAMLPart(&yamlBuilder, cfgByName, configInfoMap, 0, true) + foundRoot = true + } + } + + // If still not found, and parsedConfigs exist, try the shortest path as a heuristic + if !foundRoot && len(parsedConfigs) > 0 { + // Create a copy for sorting, as parsedConfigs itself might be used elsewhere or iterating over it. + sortedConfigs := make([]ParsedConfig, len(parsedConfigs)) + copy(sortedConfigs, parsedConfigs) + sort.SliceStable(sortedConfigs, func(i, j int) bool { + lenI := len(sortedConfigs[i].Name) + lenJ := len(sortedConfigs[j].Name) + if lenI != lenJ { + return lenI < lenJ // Shorter paths first + } + return sortedConfigs[i].Name < sortedConfigs[j].Name // Then alphabetically + }) + potentialRoot := &sortedConfigs[0] + log.Printf("Warning: Main config not found by IsTopLevel, specific root path, or 'Config' name. Falling back to first parsed config by path length: '%s'.", potentialRoot.Name) + generateYAMLPart(&yamlBuilder, potentialRoot, configInfoMap, 0, true) + foundRoot = true + } + + // If no root could be determined by any means + if !foundRoot { + yamlBuilder.WriteString("# ERROR: Main configuration struct not found. Cannot generate YAML structure.\n") + log.Println("Error: Main configuration struct not found. Cannot determine entry point for YAML generation.") } } yamlBuilder.WriteString("```\n") @@ -550,37 +607,23 @@ func generateYAMLPart(builder *strings.Builder, configInfo *ParsedConfig, allCon // Field line builder.WriteString(fmt.Sprintf("%s%s:", indent, field.YAMLName)) - var nestedStructShortName string - fieldTypeForLookup := field.Type // This is string like "config.RedisConfig" or "*config.MQsConfig" - if strings.HasPrefix(fieldTypeForLookup, "*") { - fieldTypeForLookup = fieldTypeForLookup[1:] // remove "*" - } - // Further strip package path if present, e.g. "config.RedisConfig" -> "RedisConfig" - // or "time.Time" -> "Time" - parts := strings.Split(fieldTypeForLookup, ".") - if len(parts) > 0 { - nestedStructShortName = parts[len(parts)-1] // Get the last part - } - - if nestedStructShortName != "" { - if nestedConfig, ok := allConfigs[nestedStructShortName]; ok && field.Type != "string" && nestedStructShortName != "Time" { // also ensure we don't try to expand time.Time as a custom struct - // It's a nested struct we should expand - builder.WriteString("\n") - generateYAMLPart(builder, nestedConfig, allConfigs, indentLevel+1, false) - } else { - // Scalar, slice, map, or a struct from a different package (or time.Time) - valueStr := field.DefaultValue - if valueStr == "" || (valueStr == "[]" && strings.HasPrefix(field.Type, "[]")) { - valueStr = getYAMLPlaceholderForType(field.Type) - } else { - valueStr = formatSimpleValueToYAML(valueStr, field.Type) - } - builder.WriteString(fmt.Sprintf(" %s\n", valueStr)) - } + // Construct the potential instance path for the nested struct. + // configInfo.Name is the instance path of the current struct (e.g., "Config.OTEL"). + // field.Name is the Go field name of the current field (e.g., "Traces"). + nestedStructInstancePath := configInfo.Name + "." + field.Name + + // Check if this instance path exists in our map of parsed configs. + // allConfigs is keyed by ParentStructInstancePath (which is ParsedConfig.Name). + if nestedConfig, ok := allConfigs[nestedStructInstancePath]; ok && nestedConfig.GoStructName != "Time" { + // It's a nested struct instance we should expand, and it's not time.Time. + // The ParsedConfig for this instance (nestedConfig) contains its fields. + builder.WriteString("\n") + generateYAMLPart(builder, nestedConfig, allConfigs, indentLevel+1, false) } else { - // Fallback if short name extraction failed (should not happen for valid types) + // It's a scalar, slice, map, time.Time, or a struct from a different package/type not further detailed by a ParsedConfig entry. valueStr := field.DefaultValue if valueStr == "" || (valueStr == "[]" && strings.HasPrefix(field.Type, "[]")) { + // If default is empty or an empty slice representation for a slice type, use placeholder. valueStr = getYAMLPlaceholderForType(field.Type) } else { valueStr = formatSimpleValueToYAML(valueStr, field.Type) From e8fed564560047d991b6c7dcd7f0aea5fd27c1ec Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 9 Jun 2025 22:37:29 +0100 Subject: [PATCH 21/27] chore: clean up redundant OTLP exporter and protocol entries in configuration documentation --- docs/pages/references/configuration.mdx | 48 ------------------------- 1 file changed, 48 deletions(-) diff --git a/docs/pages/references/configuration.mdx b/docs/pages/references/configuration.mdx index 71b43c64..3ec98f81 100644 --- a/docs/pages/references/configuration.mdx +++ b/docs/pages/references/configuration.mdx @@ -304,22 +304,6 @@ otel: # Required: C exporter: "" - # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. - # Required: C - exporter: "" - - # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. - # Required: C - exporter: "" - - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. - # Required: C - protocol: "" - - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. - # Required: C - protocol: "" - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. # Required: C protocol: "" @@ -331,22 +315,6 @@ otel: # Required: C exporter: "" - # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. - # Required: C - exporter: "" - - # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. - # Required: C - exporter: "" - - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. - # Required: C - protocol: "" - - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. - # Required: C - protocol: "" - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. # Required: C protocol: "" @@ -361,22 +329,6 @@ otel: # Required: C exporter: "" - # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. - # Required: C - exporter: "" - - # Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT. - # Required: C - exporter: "" - - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. - # Required: C - protocol: "" - - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. - # Required: C - protocol: "" - # Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL. # Required: C protocol: "" From 6718b3533f325e00aeefa35ecf37871c2ece79d2 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 10 Jun 2025 15:51:06 +0700 Subject: [PATCH 22/27] chore: Comment unused method --- internal/config/mqconfig_azure.go | 91 +++++++++++++++---------------- 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/internal/config/mqconfig_azure.go b/internal/config/mqconfig_azure.go index 716ff426..f74f22fa 100644 --- a/internal/config/mqconfig_azure.go +++ b/internal/config/mqconfig_azure.go @@ -2,11 +2,7 @@ package config import ( "context" - "fmt" - "sync" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/servicebus/armservicebus" "github.com/hookdeck/outpost/internal/mqinfra" "github.com/hookdeck/outpost/internal/mqs" ) @@ -24,9 +20,9 @@ type AzureServiceBusConfig struct { LogTopic string `yaml:"log_topic" env:"AZURE_SERVICEBUS_LOG_TOPIC" desc:"Topic name for log queue" required:"N" default:"outpost-log"` LogSubscription string `yaml:"log_subscription" env:"AZURE_SERVICEBUS_LOG_SUBSCRIPTION" desc:"Subscription name for log queue" required:"N" default:"outpost-log-subscription"` - connectionStringOnce sync.Once - connectionString string - connectionStringError error + // connectionStringOnce sync.Once + // connectionString string + // connectionStringError error } func (c *AzureServiceBusConfig) IsConfigured() bool { @@ -86,11 +82,6 @@ func (c *AzureServiceBusConfig) ToQueueConfig(ctx context.Context, queueType str return nil, nil } - // connectionString, err := c.getConnectionString(ctx) - // if err != nil { - // return nil, err - // } - topic := c.getTopicByQueueType(queueType) subscription := c.getSubscriptionByQueueType(queueType) @@ -108,38 +99,44 @@ func (c *AzureServiceBusConfig) ToQueueConfig(ctx context.Context, queueType str }, nil } -func (c *AzureServiceBusConfig) getConnectionString(ctx context.Context) (string, error) { - c.connectionStringOnce.Do(func() { - cred, err := azidentity.NewClientSecretCredential( - c.TenantID, - c.ClientID, - c.ClientSecret, - nil, - ) - if err != nil { - c.connectionStringError = fmt.Errorf("failed to create credential: %w", err) - return - } - - sbClient, err := armservicebus.NewNamespacesClient(c.SubscriptionID, cred, nil) - if err != nil { - c.connectionStringError = fmt.Errorf("failed to create servicebus client: %w", err) - return - } - - keysResp, err := sbClient.ListKeys(ctx, c.ResourceGroup, c.Namespace, "RootManageSharedAccessKey", nil) - if err != nil { - c.connectionStringError = fmt.Errorf("failed to get keys: %w", err) - return - } - - if keysResp.PrimaryConnectionString == nil { - c.connectionStringError = fmt.Errorf("no connection string found") - return - } - - c.connectionString = *keysResp.PrimaryConnectionString - }) - - return c.connectionString, c.connectionStringError -} +// getConnectionString fetches the namespace's primary connection string using ARM API. +// This method is not currently used because we've adopted direct Service Principal authentication. +// +// Permission requirements: +// - Connection string: Requires management plane access (e.g., "Contributor" role) to list namespace keys +// - Direct auth: Requires only data plane access ("Azure Service Bus Data Owner" role) +// func (c *AzureServiceBusConfig) getConnectionString(ctx context.Context) (string, error) { +// c.connectionStringOnce.Do(func() { +// cred, err := azidentity.NewClientSecretCredential( +// c.TenantID, +// c.ClientID, +// c.ClientSecret, +// nil, +// ) +// if err != nil { +// c.connectionStringError = fmt.Errorf("failed to create credential: %w", err) +// return +// } + +// sbClient, err := armservicebus.NewNamespacesClient(c.SubscriptionID, cred, nil) +// if err != nil { +// c.connectionStringError = fmt.Errorf("failed to create servicebus client: %w", err) +// return +// } + +// keysResp, err := sbClient.ListKeys(ctx, c.ResourceGroup, c.Namespace, "RootManageSharedAccessKey", nil) +// if err != nil { +// c.connectionStringError = fmt.Errorf("failed to get keys: %w", err) +// return +// } + +// if keysResp.PrimaryConnectionString == nil { +// c.connectionStringError = fmt.Errorf("no connection string found") +// return +// } + +// c.connectionString = *keysResp.PrimaryConnectionString +// }) + +// return c.connectionString, c.connectionStringError +// } From 2292671f106dc95cac773888dba44bdcf8223dc4 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Tue, 10 Jun 2025 10:19:43 +0100 Subject: [PATCH 23/27] feat(docs): Guide to configure Azure Service Bus as internal MQ --- docs/pages/guides.mdx | 2 + docs/pages/guides/service-bus-internal-mq.mdx | 169 ++++++++++++++++++ docs/pages/references/configuration.mdx | 55 +++++- docs/zudoku.config.ts | 9 +- 4 files changed, 230 insertions(+), 5 deletions(-) create mode 100644 docs/pages/guides/service-bus-internal-mq.mdx diff --git a/docs/pages/guides.mdx b/docs/pages/guides.mdx index 3e0ef9e6..aae2316d 100644 --- a/docs/pages/guides.mdx +++ b/docs/pages/guides.mdx @@ -12,6 +12,8 @@ Welcome to the Outpost guides section. These guides will help you get the most o - [Publish from RabbitMQ](/guides/publish-from-rabbitmq) - [Publish from SQS](/guides/publish-from-sqs) +- [Publish from GCP Pub/Sub](/guides/publish-from-gcp-pubsub) +- [Use Azure Service Bus as an Internal MQ](/guides/service-bus-internal-mq) ## Migration Guides diff --git a/docs/pages/guides/service-bus-internal-mq.mdx b/docs/pages/guides/service-bus-internal-mq.mdx new file mode 100644 index 00000000..d277fd9d --- /dev/null +++ b/docs/pages/guides/service-bus-internal-mq.mdx @@ -0,0 +1,169 @@ +--- +title: "Configure Azure Service Bus as the Outpost Internal Message Queue" +--- + +## Azure Resource Concepts + +Azure's resource hierarchy: + +- **Tenant**: The top-level organizational boundary representing a company or project +- **Subscription**: A billing account that contains and manages resources within a tenant +- **Service Principal**: An identity used for programmatic access, similar to an IAM role in other clouds +- **Resource Group**: A logical container that groups related Azure resources for easier management +- **Service Bus Namespace**: A messaging container within Azure Service Bus where you create topics and subscriptions for event routing + +This implementation leverages Azure Service Bus topics and subscriptions within a namespace to provide reliable message delivery with Outpost's multi-tenant architecture. + +## User Setup Requirements + +Outpost requires specific Azure permissions to manage Service Bus resources on your behalf: + +**Required Role**: + +- Azure Service Bus Data Owner on the Service Bus namespace + +**Required Permissions**: + +- Create and delete topics and subscriptions +- Publish messages to topics +- Consume messages from subscriptions + +**Recommended Setup**: + +- Optionally create a dedicated resource group to contain Outpost-related resources +- Create a dedicated Service Bus namespace for Outpost +- Assign the Service Principal used by Outpost the Azure Service Bus Data Owner role scoped to the namespace + +This scoped permission model ensures Outpost can fully manage message routing within its designated namespace while maintaining security boundaries with your other Azure resources. + +## Azure Service Bus Setup for Outpost + +### Prerequisites + +Install [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli). + +The following steps also make use of [`jq`](https://jqlang.org/) for JSON parsing, so ensure it is installed on your system. + +Login to Azure: + +```bash +az login +``` + +### 1. Create Resource Group + +Set variables: + +```bash +RESOURCE_GROUP="outpost-rg" +LOCATION="eastus" +``` + +Create resource group: + +```bash +az group create --name $RESOURCE_GROUP --location $LOCATION +``` + +### 2. Create Service Bus Namespace + +Generate unique namespace name (must be globally unique) + +```bash +RANDOM_SUFFIX=$(openssl rand -hex 4) +NAMESPACE_NAME="outpost-servicebus-$RANDOM_SUFFIX" +``` + +Create Service Bus namespace: + +```bash +az servicebus namespace create \ + --resource-group $RESOURCE_GROUP \ + --name $NAMESPACE_NAME \ + --location $LOCATION \ + --sku Standard +``` + +### 3. Create Service Principal + +```bash +APP_NAME="outpost-service-principal" +``` + +Create service principal and capture output: + +```bash +SP_OUTPUT=$(az ad sp create-for-rbac --name $APP_NAME) +CLIENT_ID=$(echo $SP_OUTPUT | jq -r '.appId') +CLIENT_SECRET=$(echo $SP_OUTPUT | jq -r '.password') +TENANT_ID=$(echo $SP_OUTPUT | jq -r '.tenant') +``` + +### 4. Assign Permissions + +Get the namespace resource ID: + +```bash +NAMESPACE_ID=$(az servicebus namespace show \ + --resource-group $RESOURCE_GROUP \ + --name $NAMESPACE_NAME \ + --query id -o tsv) +``` + +Assign Azure Service Bus Data Owner role: + +```bash +az role assignment create \ + --assignee $CLIENT_ID \ + --role "Azure Service Bus Data Owner" \ + --scope $NAMESPACE_ID +``` + +### 5. Configuration Values for Outpost + +Get subscription ID: + +```bash +SUBSCRIPTION_ID=$(az account show --query id -o tsv) +``` + +Echo all required values: + +```bash +echo "tenant_id: $TENANT_ID" +echo "client_id: $CLIENT_ID" +echo "client_secret: $CLIENT_SECRET" +echo "subscription_id: $SUBSCRIPTION_ID" +echo "resource_group: $RESOURCE_GROUP" +echo "namespace: $NAMESPACE_NAME" +``` + +### 6. Configure Outpost + +In the Outpost configuration, use the values obtained above. + +#### Example YAML + +```yaml +mqs: + azure_servicebus: + client_id: "" + client_secret: "" + namespace: "" + resource_group: "" + subscription_id: "" + tenant_id: "" +``` + +#### Example Environment Variables + +``` +AZURE_SERVICEBUS_CLIENT_ID="" +AZURE_SERVICEBUS_CLIENT_SECRET="" +AZURE_SERVICEBUS_NAMESPACE="" +AZURE_SERVICEBUS_RESOURCE_GROUP="" +AZURE_SERVICEBUS_SUBSCRIPTION_ID="" +AZURE_SERVICEBUS_TENANT_ID="" +``` + +This configuration allows Outpost to connect to your Azure Service Bus namespace and manage topics and subscriptions for internal message queuing. diff --git a/docs/pages/references/configuration.mdx b/docs/pages/references/configuration.mdx index 3ec98f81..a1be891e 100644 --- a/docs/pages/references/configuration.mdx +++ b/docs/pages/references/configuration.mdx @@ -30,6 +30,16 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `AWS_SQS_LOG_QUEUE` | Name of the SQS queue for log events. | `outpost-log` | No | | `AWS_SQS_REGION` | AWS Region for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional | | `AWS_SQS_SECRET_ACCESS_KEY` | AWS Secret Access Key for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional | +| `AZURE_SERVICEBUS_CLIENT_ID` | Service principal client ID | `nil` | Yes | +| `AZURE_SERVICEBUS_CLIENT_SECRET` | Service principal client secret | `nil` | Yes | +| `AZURE_SERVICEBUS_DELIVERY_SUBSCRIPTION` | Subscription name for delivery queue | `outpost-delivery-sub` | No | +| `AZURE_SERVICEBUS_DELIVERY_TOPIC` | Topic name for delivery queue | `outpost-delivery` | No | +| `AZURE_SERVICEBUS_LOG_SUBSCRIPTION` | Subscription name for log queue | `outpost-log-sub` | No | +| `AZURE_SERVICEBUS_LOG_TOPIC` | Topic name for log queue | `outpost-log` | No | +| `AZURE_SERVICEBUS_NAMESPACE` | Azure Service Bus namespace | `nil` | Yes | +| `AZURE_SERVICEBUS_RESOURCE_GROUP` | Azure resource group name | `nil` | Yes | +| `AZURE_SERVICEBUS_SUBSCRIPTION_ID` | Azure subscription ID | `nil` | Yes | +| `AZURE_SERVICEBUS_TENANT_ID` | Azure Active Directory tenant ID | `nil` | Yes | | `CLICKHOUSE_ADDR` | Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage. | `nil` | Conditional | | `CLICKHOUSE_DATABASE` | Database name in ClickHouse to use. | `outpost` | No | | `CLICKHOUSE_PASSWORD` | Password for ClickHouse authentication. | `nil` | No | @@ -230,7 +240,7 @@ log_level: "info" log_max_concurrency: 1 mqs: - # Configuration for using AWS SQS as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured. + # Configuration for using AWS SQS as the message queue. Only one MQ provider should be configured. aws_sqs: # AWS Access Key ID for SQS. Required if AWS SQS is the chosen MQ provider. # Required: C @@ -254,7 +264,46 @@ mqs: secret_access_key: "" - # Configuration for using GCP Pub/Sub as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured. + # Configuration for using Azure Service Bus as the message queue. Only one MQ provider should be configured. + azure_servicebus: + # Service principal client ID + # Required: Y + client_id: "" + + # Service principal client secret + # Required: Y + client_secret: "" + + # Subscription name for delivery queue + delivery_subscription: "outpost-delivery-sub" + + # Topic name for delivery queue + delivery_topic: "outpost-delivery" + + # Subscription name for log queue + log_subscription: "outpost-log-sub" + + # Topic name for log queue + log_topic: "outpost-log" + + # Azure Service Bus namespace + # Required: Y + namespace: "" + + # Azure resource group name + # Required: Y + resource_group: "" + + # Azure subscription ID + # Required: Y + subscription_id: "" + + # Azure Active Directory tenant ID + # Required: Y + tenant_id: "" + + + # Configuration for using GCP Pub/Sub as the message queue. Only one MQ provider should be configured. gcp_pubsub: # Name of the GCP Pub/Sub subscription for delivery events. delivery_subscription: "outpost-delivery-sub" @@ -277,7 +326,7 @@ mqs: service_account_credentials: "" - # Configuration for using RabbitMQ as the message queue. Only one MQ provider (AWSSQS, GCPPubSub, RabbitMQ) should be configured. + # Configuration for using RabbitMQ as the message queue. Only one MQ provider should be configured. rabbitmq: # Name of the RabbitMQ queue for delivery events. delivery_queue: "outpost-delivery" diff --git a/docs/zudoku.config.ts b/docs/zudoku.config.ts index df1caf99..0a2f7790 100644 --- a/docs/zudoku.config.ts +++ b/docs/zudoku.config.ts @@ -126,6 +126,11 @@ const config: ZudokuConfig = { collapsible: false, link: "guides", items: [ + { + type: "doc", + label: "Deployment", + id: "guides/deployment", + }, { type: "doc", label: "Migrate to Outpost", @@ -148,8 +153,8 @@ const config: ZudokuConfig = { }, { type: "doc", - label: "Deployment", - id: "guides/deployment", + label: "Using Azure Service Bus as an Internal MQ", + id: "guides/service-bus-internal-mq", }, { type: "doc", From b9c06e45b0691c857f828a50e1e0f2b1ceb27f7d Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Tue, 10 Jun 2025 10:19:58 +0100 Subject: [PATCH 24/27] chore(docs): Update documentation to include Azure Service Bus in relevant sections --- docs/pages/concepts.mdx | 4 +-- docs/pages/guides/deployment.mdx | 2 +- docs/pages/references/roadmap.mdx | 53 +++++++++++++++++-------------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/docs/pages/concepts.mdx b/docs/pages/concepts.mdx index 0e070bc8..9772e694 100644 --- a/docs/pages/concepts.mdx +++ b/docs/pages/concepts.mdx @@ -32,7 +32,7 @@ Outpost consists of 3 services that can either be run as a single deployment or - AWS SQS - RabbitMQ - GCP Pub/Sub - - [Azure ServiceBus (planned)](https://github.com/hookdeck/outpost/issues/139) + - Azure Service Bus ### Log Service @@ -50,7 +50,7 @@ Event destination types belonging to Outpost tenants where events are delivered. - **AWS SQS** - **Hookdeck Event Gateway** - **[Amazon EventBridge (planned)](https://github.com/hookdeck/outpost/issues/201)** -- **[Azure ServiceBus (planned)](https://github.com/hookdeck/outpost/issues/241)** +- **[Azure Service Bus (planned)](https://github.com/hookdeck/outpost/issues/241)** - **[GCP Pub/Sub (planned)](https://github.com/hookdeck/outpost/issues/140)** - **[Kafka (planned)](https://github.com/hookdeck/outpost/issues/141)** diff --git a/docs/pages/guides/deployment.mdx b/docs/pages/guides/deployment.mdx index c940d0f0..2f4f1d1a 100644 --- a/docs/pages/guides/deployment.mdx +++ b/docs/pages/guides/deployment.mdx @@ -42,7 +42,7 @@ Outpost requires the following dependencies: - Redis - PostgreSQL -- One of the following message queue systems (for the queues listed below): RabbitMQ, AWS SQS, or GCP Pub/Sub +- One of the following message queue systems (for the queues listed below): RabbitMQ, AWS SQS, Azure Service Bus, or GCP Pub/Sub - Log queue - Delivery queue - Publish queue (optional) diff --git a/docs/pages/references/roadmap.mdx b/docs/pages/references/roadmap.mdx index b03cf79a..06a9d488 100644 --- a/docs/pages/references/roadmap.mdx +++ b/docs/pages/references/roadmap.mdx @@ -2,47 +2,54 @@ title: "Outpost Roadmap" --- -## Upcoming Releases +## Upcoming Features -### v0.1.0 +### Core Functionality -#### Internal Message Queues +- [Postgres partition management](https://github.com/hookdeck/outpost/issues/249) -- ✅ [GCP Pub/Sub](https://github.com/hookdeck/outpost/issues/138) +### Publish Message Queues -#### Publish Message Queues +- [Azure Service Bus](https://github.com/hookdeck/outpost/issues/139) -- ✅ [GCP Pub/Sub](https://github.com/hookdeck/outpost/issues/325) +### Destination Types -#### Destination Types +- [GCP Pub/Sub](https://github.com/hookdeck/outpost/issues/140) +- [Azure Service Bus](https://github.com/hookdeck/outpost/issues/241) +- [Amazon EventBridge](https://github.com/hookdeck/outpost/issues/201) +- [Kafka](https://github.com/hookdeck/outpost/issues/141) +- [S3](https://github.com/orgs/hookdeck/projects/21/views/1?filterQuery=s3&pane=issue&itemId=113373337&issue=hookdeck%7Coutpost%7C418) -- ✅ [Hookdeck Event Gateway](https://github.com/hookdeck/outpost/issues/136) -- ✅ [Amazon Kinesis](https://github.com/hookdeck/outpost/issues/305) +## Later -### v0.2.0 +### Core Functionality -#### Core Functionality +- [ClickHouse log storage](https://github.com/hookdeck/outpost/issues/52) -- ❌ [Postgres partition management](https://github.com/hookdeck/outpost/issues/249) +## Previous Notable Milestones -#### Publish Message Queues +### v0.3.0 -- ❌ [Azure ServiceBus](https://github.com/hookdeck/outpost/issues/139) +#### Internal Message Queues -#### Destination Types +- ✅ [Azure Service Bus](https://github.com/orgs/hookdeck/projects/21?pane=issue&itemId=101269468&issue=hookdeck%7Coutpost%7C139) -- ❌ [GCP Pub/Sub](https://github.com/hookdeck/outpost/issues/140) -- ❌ [Azure ServiceBus](https://github.com/hookdeck/outpost/issues/241) -- ❌ [Amazon EventBridge](https://github.com/hookdeck/outpost/issues/201) -- ❌ [Kafka](https://github.com/hookdeck/outpost/issues/141) +### v0.1.0 -### Later +See [v0.1.0 release notes](https://github.com/hookdeck/outpost/releases/tag/v0.1.0). -#### Core Functionality +#### Internal Message Queues -- 🚧 [ClickHouse log storage](https://github.com/hookdeck/outpost/issues/52) +- ✅ [GCP Pub/Sub](https://github.com/hookdeck/outpost/issues/138) -## Previous Notable Milestones +#### Publish Message Queues + +- ✅ [GCP Pub/Sub](https://github.com/hookdeck/outpost/issues/325) + +#### Destination Types + +- ✅ [Hookdeck Event Gateway](https://github.com/hookdeck/outpost/issues/136) +- ✅ [Amazon Kinesis](https://github.com/hookdeck/outpost/issues/305) ### ALPHA From b7c41f35644b5923880a8f1c5def3886f0a3765a Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Tue, 10 Jun 2025 10:22:27 +0100 Subject: [PATCH 25/27] chore(docs): Add troubleshooting section for Azure Service Bus topic creation errors --- docs/pages/guides/service-bus-internal-mq.mdx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/pages/guides/service-bus-internal-mq.mdx b/docs/pages/guides/service-bus-internal-mq.mdx index d277fd9d..137ab262 100644 --- a/docs/pages/guides/service-bus-internal-mq.mdx +++ b/docs/pages/guides/service-bus-internal-mq.mdx @@ -167,3 +167,12 @@ AZURE_SERVICEBUS_TENANT_ID="" ``` This configuration allows Outpost to connect to your Azure Service Bus namespace and manage topics and subscriptions for internal message queuing. + +## Troubleshooting + +### failed to declare topic: failed to create topic outpost-delivery + +This error can happen due to a race condition of multiple Outpost instances trying to create the same topic concurrently. To resolve this you can either: + +1. Run the services one at a time to ensure only one instance is creating the topic. +2. Running the services again often resolves the issue as the topic may have been created by another instance. From 44608df58b26cd83cf5102efdcf95ca025a05699 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Tue, 10 Jun 2025 10:22:48 +0100 Subject: [PATCH 26/27] chore(docs): Correct grammar in troubleshooting instructions for topic creation errors --- docs/pages/guides/service-bus-internal-mq.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/guides/service-bus-internal-mq.mdx b/docs/pages/guides/service-bus-internal-mq.mdx index 137ab262..4aae971c 100644 --- a/docs/pages/guides/service-bus-internal-mq.mdx +++ b/docs/pages/guides/service-bus-internal-mq.mdx @@ -175,4 +175,4 @@ This configuration allows Outpost to connect to your Azure Service Bus namespace This error can happen due to a race condition of multiple Outpost instances trying to create the same topic concurrently. To resolve this you can either: 1. Run the services one at a time to ensure only one instance is creating the topic. -2. Running the services again often resolves the issue as the topic may have been created by another instance. +2. Run the services again often resolves the issue as the topic may have been created by another instance. From 80b4d76d496d894d0a9dc09e01703532daacc7c9 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Tue, 10 Jun 2025 10:23:54 +0100 Subject: [PATCH 27/27] chore(docs): Add reference to avoid concurrent infra provisioning requests in Azure Service Bus guide --- docs/pages/guides/service-bus-internal-mq.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pages/guides/service-bus-internal-mq.mdx b/docs/pages/guides/service-bus-internal-mq.mdx index 4aae971c..9193bf6a 100644 --- a/docs/pages/guides/service-bus-internal-mq.mdx +++ b/docs/pages/guides/service-bus-internal-mq.mdx @@ -176,3 +176,5 @@ This error can happen due to a race condition of multiple Outpost instances tryi 1. Run the services one at a time to ensure only one instance is creating the topic. 2. Run the services again often resolves the issue as the topic may have been created by another instance. + +Also see [Avoid concurrent infra provisioning requests](https://github.com/hookdeck/outpost/issues/427).