FEAT: Update Status Of Video Row On Media Convert Update

2026-08-27 20:04:38 +03:00
parent 5d35d4029e
commit 3f5301afb6
25 changed files with 1444 additions and 14 deletions
@@ -12,7 +12,11 @@ config:
  aws:accessKey: test
  aws:secretKey: test
  aws:skipCredentialsValidation: "true"
-  aws:skipRequestingAccountId: "true"
+  # SNS builds topic ARNs from the caller's account id, so unlike the other
+  # services it cannot work without one — "true" here makes the provider hand
+  # SNS an empty account and every topic call fails. sts is pointed at
+  # LocalStack just below, so asking is cheap and answers 000000000000.
+  aws:skipRequestingAccountId: "false"
  aws:skipMetadataApiCheck: "true"
  # Virtual-host addressing would resolve <bucket>.localhost.localstack.cloud,
  # which points at 127.0.0.1 — the wrong container from inside compose.
@@ -21,6 +25,8 @@ config:
    - iam: http://localhost.localstack.cloud:4566
      rds: http://localhost.localstack.cloud:4566
      s3: http://localhost.localstack.cloud:4566
+      sns: http://localhost.localstack.cloud:4566
+      sqs: http://localhost.localstack.cloud:4566
      sts: http://localhost.localstack.cloud:4566
  thamanyah:localstack: "true"
 encryptionsalt: v1:QKoqcSNBvDc=:v1:bTGZko5jHuGNRVoq:2oQ4pFb8MYwspEbxipYeO+rVqVnLBg==
@@ -18,6 +18,8 @@ import (
 	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/rds"
 	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/s3"
 	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/secretsmanager"
+	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/sns"
+	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/sqs"
 	"github.com/pulumi/pulumi-postgresql/sdk/v3/go/postgresql"
 	"github.com/pulumi/pulumi-random/sdk/v4/go/random"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
@@ -808,6 +810,154 @@ func main() {
 			return err
 		}

+		// MediaConvert job events: MediaConvert -> EventBridge -> SNS -> SQS ->
+		// cms. cms *consumes a queue* rather than being called back on an HTTPS
+		// endpoint, so nothing new is exposed on the ALB, authenticity comes
+		// from IAM instead of SNS message signatures, and a failed handler
+		// leaves the message on the queue to be retried instead of dropping the
+		// update. The topic sits between the rule and the queue so a second
+		// consumer — discovery, when it has code — can subscribe its own queue
+		// without touching this wiring or MediaConvert.
+		jobEventsTopicArgs := &sns.TopicArgs{}
+		jobEventsQueueArgs := &sqs.QueueArgs{}
+		jobEventsDeadLetterArgs := &sqs.QueueArgs{}
+		if localstack {
+			// docker-compose.yml and the tests/ suite refer to these literally.
+			jobEventsTopicArgs.Name = pulumi.String("mediaconvert-job-events")
+			jobEventsQueueArgs.Name = pulumi.String("cms-mediaconvert-events")
+			jobEventsDeadLetterArgs.Name = pulumi.String("cms-mediaconvert-events-dlq")
+		}
+
+		jobEventsTopic, err := sns.NewTopic(ctx, "mediaconvert-job-events", jobEventsTopicArgs)
+		if err != nil {
+			return err
+		}
+
+		// Where a message lands after maxReceiveCount failed handlings, so one
+		// event cms cannot process never blocks the ones behind it.
+		jobEventsDeadLetter, err := sqs.NewQueue(ctx, "cms-mediaconvert-events-dlq", jobEventsDeadLetterArgs)
+		if err != nil {
+			return err
+		}
+
+		jobEventsQueueArgs.VisibilityTimeoutSeconds = pulumi.Int(60)
+		jobEventsQueueArgs.RedrivePolicy = jobEventsDeadLetter.Arn.ApplyT(func(arn string) (string, error) {
+			b, err := json.Marshal(map[string]any{
+				"deadLetterTargetArn": arn,
+				"maxReceiveCount":     5,
+			})
+			return string(b), err
+		}).(pulumi.StringOutput)
+
+		jobEventsQueue, err := sqs.NewQueue(ctx, "cms-mediaconvert-events", jobEventsQueueArgs)
+		if err != nil {
+			return err
+		}
+
+		// SNS is not an IAM principal the queue trusts by default; without this
+		// the subscription is created and silently delivers nothing.
+		jobEventsQueuePolicy := pulumi.All(jobEventsQueue.Arn, jobEventsTopic.Arn).ApplyT(
+			func(args []any) (string, error) {
+				queueArn := args[0].(string)
+				topicArn := args[1].(string)
+
+				doc := map[string]any{
+					"Version": "2012-10-17",
+					"Statement": []map[string]any{
+						{
+							"Sid":       "AllowJobEventsTopicToSend",
+							"Effect":    "Allow",
+							"Principal": map[string]string{"Service": "sns.amazonaws.com"},
+							"Action":    "sqs:SendMessage",
+							"Resource":  queueArn,
+							"Condition": map[string]any{
+								"ArnEquals": map[string]string{"aws:SourceArn": topicArn},
+							},
+						},
+					},
+				}
+				b, err := json.Marshal(doc)
+				return string(b), err
+			},
+		).(pulumi.StringOutput)
+
+		_, err = sqs.NewQueuePolicy(ctx, "cms-mediaconvert-events-queue-policy", &sqs.QueuePolicyArgs{
+			QueueUrl: jobEventsQueue.ID(),
+			Policy:   jobEventsQueuePolicy,
+		})
+		if err != nil {
+			return err
+		}
+
+		_, err = sns.NewTopicSubscription(ctx, "cms-mediaconvert-events-subscription", &sns.TopicSubscriptionArgs{
+			Topic:    jobEventsTopic.Arn,
+			Protocol: pulumi.String("sqs"),
+			Endpoint: jobEventsQueue.Arn,
+			// The queue receives the EventBridge event itself rather than an SNS
+			// envelope carrying it as a JSON string, so the consumer parses one
+			// document instead of unwrapping two.
+			RawMessageDelivery: pulumi.Bool(true),
+		})
+		if err != nil {
+			return err
+		}
+
+		jobEventsTopicPolicy := jobEventsTopic.Arn.ApplyT(func(topicArn string) (string, error) {
+			doc := map[string]any{
+				"Version": "2012-10-17",
+				"Statement": []map[string]any{
+					{
+						"Sid":       "AllowEventBridgeToPublish",
+						"Effect":    "Allow",
+						"Principal": map[string]string{"Service": "events.amazonaws.com"},
+						"Action":    "sns:Publish",
+						"Resource":  topicArn,
+					},
+				},
+			}
+			b, err := json.Marshal(doc)
+			return string(b), err
+		}).(pulumi.StringOutput)
+
+		_, err = sns.NewTopicPolicy(ctx, "mediaconvert-job-events-topic-policy", &sns.TopicPolicyArgs{
+			Arn:    jobEventsTopic.Arn,
+			Policy: jobEventsTopicPolicy,
+		})
+		if err != nil {
+			return err
+		}
+
+		// MediaConvert does not publish to SNS itself: it reports state changes
+		// to EventBridge, and this rule is what forwards them to the topic.
+		// Skipped under LocalStack, whose MediaConvert emulation does not emit
+		// these — the suite in tests/ publishes to the topic directly instead,
+		// which is the same seam from cms's side.
+		if !localstack {
+			jobStateChangePattern, err := json.Marshal(map[string]any{
+				"source":      []string{"aws.mediaconvert"},
+				"detail-type": []string{"MediaConvert Job State Change"},
+			})
+			if err != nil {
+				return err
+			}
+
+			jobEventsRule, err := cloudwatch.NewEventRule(ctx, "mediaconvert-job-state-change", &cloudwatch.EventRuleArgs{
+				Description:  pulumi.String("MediaConvert job state changes, forwarded to the cms job events topic"),
+				EventPattern: pulumi.String(string(jobStateChangePattern)),
+			})
+			if err != nil {
+				return err
+			}
+
+			_, err = cloudwatch.NewEventTarget(ctx, "mediaconvert-job-state-change-to-topic", &cloudwatch.EventTargetArgs{
+				Rule: jobEventsRule.Name,
+				Arn:  jobEventsTopic.Arn,
+			})
+			if err != nil {
+				return err
+			}
+		}
+
 		// cms's own ECS task role (distinct from the shared execRole, which
 		// is only for the ECS agent's pull/logs/secrets access): lets the
 		// running container call S3 and MediaConvert directly. discovery
@@ -819,10 +969,11 @@ func main() {
 			return err
 		}

-		cmsTaskPolicy := pulumi.All(rawUploadsBucket.Arn, mediaConvertRole.Arn).ApplyT(
+		cmsTaskPolicy := pulumi.All(rawUploadsBucket.Arn, mediaConvertRole.Arn, jobEventsQueue.Arn).ApplyT(
 			func(args []any) (string, error) {
 				rawUploadsArn := args[0].(string)
 				mediaConvertRoleArn := args[1].(string)
+				jobEventsQueueArn := args[2].(string)

 				doc := map[string]any{
 					"Version": "2012-10-17",
@@ -848,6 +999,16 @@ func main() {
 							"Action":   "mediaconvert:CreateJob",
 							"Resource": "*",
 						},
+						{
+							"Sid":    "ConsumeJobEvents",
+							"Effect": "Allow",
+							"Action": []string{
+								"sqs:ReceiveMessage",
+								"sqs:DeleteMessage",
+								"sqs:GetQueueAttributes",
+							},
+							"Resource": jobEventsQueueArn,
+						},
 						{
 							"Sid":      "PassMediaConvertRole",
 							"Effect":   "Allow",
@@ -884,6 +1045,7 @@ func main() {
 				{Name: "MEDIACONVERT_INPUT_BUCKET", Value: rawUploadsBucket.ID().ToStringOutput()},
 				{Name: "MEDIACONVERT_OUTPUT_BUCKET", Value: bucket.ID().ToStringOutput()},
 				{Name: "MEDIACONVERT_ROLE_ARN", Value: mediaConvertRole.Arn},
+				{Name: "MEDIACONVERT_EVENTS_QUEUE_URL", Value: jobEventsQueue.Url},
 			}

 			cmsRepo, cmsService, cmsAlb, err = deployFargateService(ctx, "cms", 8081,