FEAT: Discovery Video Service Subscription

2026-08-29 16:50:58 +03:00
parent 98950aba20
commit 6f9e04ee97
34 changed files with 1754 additions and 65 deletions
@@ -854,6 +854,158 @@ func main() {
 			return err
 		}

+		// The catalogue topic: where cms announces a video once its transcode
+		// has finished. A topic rather than a queue so the announcement fans
+		// out — the read side subscribes the queue below, and a second reader
+		// (search indexing, notifications) can subscribe its own without cms
+		// knowing it exists or this wiring changing.
+		catalogueTopicArgs := &sns.TopicArgs{}
+		catalogueQueueArgs := &sqs.QueueArgs{}
+		catalogueDeadLetterArgs := &sqs.QueueArgs{}
+		if localstack {
+			// docker-compose.yml and the tests/ suite refer to these literally.
+			catalogueTopicArgs.Name = pulumi.String("catalogue-events")
+			catalogueQueueArgs.Name = pulumi.String("discovery-catalogue-events")
+			catalogueDeadLetterArgs.Name = pulumi.String("discovery-catalogue-events-dlq")
+		}
+
+		catalogueTopic, err := sns.NewTopic(ctx, "catalogue-events", catalogueTopicArgs)
+		if err != nil {
+			return err
+		}
+
+		catalogueDeadLetter, err := sqs.NewQueue(ctx, "discovery-catalogue-events-dlq", catalogueDeadLetterArgs)
+		if err != nil {
+			return err
+		}
+
+		catalogueQueueArgs.VisibilityTimeoutSeconds = pulumi.Int(60)
+		catalogueQueueArgs.RedrivePolicy = catalogueDeadLetter.Arn.ApplyT(func(arn string) (string, error) {
+			b, err := json.Marshal(map[string]any{
+				"deadLetterTargetArn": arn,
+				"maxReceiveCount":     5,
+			})
+			return string(b), err
+		}).(pulumi.StringOutput)
+
+		// The read side's own queue on the catalogue topic. It exists before
+		// discovery has code to drain it so the announcements pile up rather
+		// than being dropped on the floor — a topic delivers only to the
+		// subscriptions that exist when it publishes.
+		catalogueQueue, err := sqs.NewQueue(ctx, "discovery-catalogue-events", catalogueQueueArgs)
+		if err != nil {
+			return err
+		}
+
+		// SNS is not an IAM principal the queue trusts by default; without this
+		// the subscription is created and silently delivers nothing.
+		catalogueQueuePolicy := pulumi.All(catalogueQueue.Arn, catalogueTopic.Arn).ApplyT(
+			func(args []any) (string, error) {
+				queueArn := args[0].(string)
+				topicArn := args[1].(string)
+
+				doc := map[string]any{
+					"Version": "2012-10-17",
+					"Statement": []map[string]any{
+						{
+							"Sid":       "AllowCatalogueTopicToSend",
+							"Effect":    "Allow",
+							"Principal": map[string]string{"Service": "sns.amazonaws.com"},
+							"Action":    "sqs:SendMessage",
+							"Resource":  queueArn,
+							"Condition": map[string]any{
+								"ArnEquals": map[string]string{"aws:SourceArn": topicArn},
+							},
+						},
+					},
+				}
+				b, err := json.Marshal(doc)
+				return string(b), err
+			},
+		).(pulumi.StringOutput)
+
+		_, err = sqs.NewQueuePolicy(ctx, "discovery-catalogue-events-queue-policy", &sqs.QueuePolicyArgs{
+			QueueUrl: catalogueQueue.ID(),
+			Policy:   catalogueQueuePolicy,
+		})
+		if err != nil {
+			return err
+		}
+
+		_, err = sns.NewTopicSubscription(ctx, "discovery-catalogue-events-subscription", &sns.TopicSubscriptionArgs{
+			Topic:    catalogueTopic.Arn,
+			Protocol: pulumi.String("sqs"),
+			Endpoint: catalogueQueue.Arn,
+			// The queue receives the announcement itself rather than an SNS
+			// envelope carrying it as a JSON string, so a consumer parses one
+			// document instead of unwrapping two.
+			RawMessageDelivery: pulumi.Bool(true),
+		})
+		if err != nil {
+			return err
+		}
+
+		// A second subscriber, for the tests/ suite only. The BDD scenarios
+		// assert on the announcement itself — the field names cms publishes,
+		// which no assertion through discovery's API can pin — and a consumer
+		// destroys what it reads. Sharing discovery's queue would mean the two
+		// racing for every message and each seeing about half, so the suite
+		// gets its own subscription. This is what the topic is for.
+		//
+		// LocalStack only: in AWS this queue would fill up unread, since
+		// nothing runs the suite there.
+		if localstack {
+			testsCatalogueQueue, err := sqs.NewQueue(ctx, "tests-catalogue-events", &sqs.QueueArgs{
+				Name: pulumi.String("tests-catalogue-events"),
+			})
+			if err != nil {
+				return err
+			}
+
+			testsCatalogueQueuePolicy := pulumi.All(testsCatalogueQueue.Arn, catalogueTopic.Arn).ApplyT(
+				func(args []any) (string, error) {
+					queueArn := args[0].(string)
+					topicArn := args[1].(string)
+
+					doc := map[string]any{
+						"Version": "2012-10-17",
+						"Statement": []map[string]any{
+							{
+								"Sid":       "AllowCatalogueTopicToSend",
+								"Effect":    "Allow",
+								"Principal": map[string]string{"Service": "sns.amazonaws.com"},
+								"Action":    "sqs:SendMessage",
+								"Resource":  queueArn,
+								"Condition": map[string]any{
+									"ArnEquals": map[string]string{"aws:SourceArn": topicArn},
+								},
+							},
+						},
+					}
+					b, err := json.Marshal(doc)
+					return string(b), err
+				},
+			).(pulumi.StringOutput)
+
+			_, err = sqs.NewQueuePolicy(ctx, "tests-catalogue-events-queue-policy", &sqs.QueuePolicyArgs{
+				QueueUrl: testsCatalogueQueue.ID(),
+				Policy:   testsCatalogueQueuePolicy,
+			})
+			if err != nil {
+				return err
+			}
+
+			_, err = sns.NewTopicSubscription(ctx, "tests-catalogue-events-subscription", &sns.TopicSubscriptionArgs{
+				Topic:              catalogueTopic.Arn,
+				Protocol:           pulumi.String("sqs"),
+				Endpoint:           testsCatalogueQueue.Arn,
+				RawMessageDelivery: pulumi.Bool(true),
+			})
+			if err != nil {
+				return err
+			}
+		}
+
 		// SNS is not an IAM principal the queue trusts by default; without this
 		// the subscription is created and silently delivers nothing.
 		jobEventsQueuePolicy := pulumi.All(jobEventsQueue.Arn, jobEventsTopic.Arn).ApplyT(
@@ -969,11 +1121,12 @@ func main() {
 			return err
 		}

-		cmsTaskPolicy := pulumi.All(rawUploadsBucket.Arn, mediaConvertRole.Arn, jobEventsQueue.Arn).ApplyT(
+		cmsTaskPolicy := pulumi.All(rawUploadsBucket.Arn, mediaConvertRole.Arn, jobEventsQueue.Arn, catalogueTopic.Arn).ApplyT(
 			func(args []any) (string, error) {
 				rawUploadsArn := args[0].(string)
 				mediaConvertRoleArn := args[1].(string)
 				jobEventsQueueArn := args[2].(string)
+				catalogueTopicArn := args[3].(string)

 				doc := map[string]any{
 					"Version": "2012-10-17",
@@ -1009,6 +1162,15 @@ func main() {
 							},
 							"Resource": jobEventsQueueArn,
 						},
+						{
+							"Sid":    "AnnounceReadyVideos",
+							"Effect": "Allow",
+							"Action": []string{
+								"sns:Publish",
+								"sns:GetTopicAttributes",
+							},
+							"Resource": catalogueTopicArn,
+						},
 						{
 							"Sid":      "PassMediaConvertRole",
 							"Effect":   "Allow",
@@ -1035,6 +1197,45 @@ func main() {
 			return err
 		}

+		// discovery's own ECS task role. It used to have none — it touched no
+		// AWS API at all — and it still gets nothing but the one queue it
+		// drains: no S3, no MediaConvert, and no ability to publish back onto
+		// the catalogue topic. It is a subscriber, not a participant.
+		discoveryTaskRole, err := iam.NewRole(ctx, "discovery-task-role", &iam.RoleArgs{
+			AssumeRolePolicy: pulumi.String(execRoleAssumePolicy),
+		})
+		if err != nil {
+			return err
+		}
+
+		discoveryTaskPolicy := catalogueQueue.Arn.ApplyT(func(catalogueQueueArn string) (string, error) {
+			doc := map[string]any{
+				"Version": "2012-10-17",
+				"Statement": []map[string]any{
+					{
+						"Sid":    "ConsumeCatalogueAnnouncements",
+						"Effect": "Allow",
+						"Action": []string{
+							"sqs:ReceiveMessage",
+							"sqs:DeleteMessage",
+							"sqs:GetQueueAttributes",
+						},
+						"Resource": catalogueQueueArn,
+					},
+				},
+			}
+			b, err := json.Marshal(doc)
+			return string(b), err
+		}).(pulumi.StringOutput)
+
+		_, err = iam.NewRolePolicy(ctx, "discovery-task-role-policy", &iam.RolePolicyArgs{
+			Role:   discoveryTaskRole.ID(),
+			Policy: discoveryTaskPolicy,
+		})
+		if err != nil {
+			return err
+		}
+
 		// The two ECS services and the CI user that deploys them. Under
 		// LocalStack the equivalent of all this is the cms service in
 		// docker-compose.yml, which reads the same env vars from .env.
@@ -1046,6 +1247,7 @@ func main() {
 				{Name: "MEDIACONVERT_OUTPUT_BUCKET", Value: bucket.ID().ToStringOutput()},
 				{Name: "MEDIACONVERT_ROLE_ARN", Value: mediaConvertRole.Arn},
 				{Name: "MEDIACONVERT_EVENTS_QUEUE_URL", Value: jobEventsQueue.Url},
+				{Name: "CATALOGUE_EVENTS_TOPIC_ARN", Value: catalogueTopic.Arn},
 				// The CDN in front of the encoded bucket, which is the only way
 				// that bucket is readable — it blocks public access and grants
 				// only CloudFront's OAC principal. cms rewrites the s3:// paths
@@ -1093,8 +1295,13 @@ func main() {
 			// discovery embeds its own golang-migrate migrations and applies
 			// them through `./discovery migrate`, so ECS runs that container to
 			// completion before the service accepts traffic.
+			discoveryExtraEnv := []envVar{
+				{Name: "AWS_REGION", Value: pulumi.String(awsRegion).ToStringOutput()},
+				{Name: "CATALOGUE_EVENTS_QUEUE_URL", Value: catalogueQueue.Url},
+			}
+
 			discoveryRepo, discoveryService, discoveryAlb, err = deployFargateService(ctx, "discovery", 8080,
-				cluster, execRole, nil, nil, vpcID, subnetIDs, albSecurityGroup, serviceSecurityGroup,
+				cluster, execRole, discoveryTaskRole, discoveryExtraEnv, vpcID, subnetIDs, albSecurityGroup, serviceSecurityGroup,
 				db.Address, db.Port, discoveryPassword, true)
 			if err != nil {
 				return err