diff --git a/infrastructure/Pulumi.local.yaml b/infrastructure/Pulumi.local.yaml index c0b71b5..11bbd1b 100644 --- a/infrastructure/Pulumi.local.yaml +++ b/infrastructure/Pulumi.local.yaml @@ -25,6 +25,10 @@ config: - iam: http://localhost.localstack.cloud:4566 rds: http://localhost.localstack.cloud:4566 s3: http://localhost.localstack.cloud:4566 + # aws@7 reads a bucket's tags back through S3 Control's + # ListTagsForResource, so without this override the create call + # goes to real AWS and 403s — even though no tags are set here. + s3control: http://localhost.localstack.cloud:4566 sns: http://localhost.localstack.cloud:4566 sqs: http://localhost.localstack.cloud:4566 sts: http://localhost.localstack.cloud:4566 diff --git a/infrastructure/main.go b/infrastructure/main.go index e320ad5..04fffda 100644 --- a/infrastructure/main.go +++ b/infrastructure/main.go @@ -1046,6 +1046,11 @@ func main() { {Name: "MEDIACONVERT_OUTPUT_BUCKET", Value: bucket.ID().ToStringOutput()}, {Name: "MEDIACONVERT_ROLE_ARN", Value: mediaConvertRole.Arn}, {Name: "MEDIACONVERT_EVENTS_QUEUE_URL", Value: jobEventsQueue.Url}, + // The CDN in front of the encoded bucket, which is the only way + // that bucket is readable — it blocks public access and grants + // only CloudFront's OAC principal. cms rewrites the s3:// paths + // MediaConvert reports onto this host. + {Name: "PLAYBACK_BASE_URL", Value: pulumi.Sprintf("https://%s", distribution.DomainName).ToStringOutput()}, } cmsRepo, cmsService, cmsAlb, err = deployFargateService(ctx, "cms", 8081, diff --git a/tests/client_test.go b/tests/client_test.go index f0f427c..3d56f09 100644 --- a/tests/client_test.go +++ b/tests/client_test.go @@ -84,6 +84,7 @@ type videoBody struct { StorageKey string `json:"storageKey"` MediaConvertJobID string `json:"mediaConvertJobId"` Status string `json:"status"` + PlaybackURL string `json:"playbackUrl"` SizeBytes int64 `json:"sizeBytes"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` diff --git a/tests/events_test.go b/tests/events_test.go index 65355a1..6110501 100644 --- a/tests/events_test.go +++ b/tests/events_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path" "strings" "time" @@ -24,11 +25,35 @@ import ( // client does still goes over plain HTTP. const ( - defaultEndpointURL = "http://localhost.localstack.cloud:4566" - defaultTopicARN = "arn:aws:sns:us-east-1:000000000000:mediaconvert-job-events" - defaultRegion = "us-east-1" + defaultEndpointURL = "http://localhost.localstack.cloud:4566" + defaultTopicARN = "arn:aws:sns:us-east-1:000000000000:mediaconvert-job-events" + defaultRegion = "us-east-1" + defaultOutputBucket = "encoded-bucket" + defaultPlaybackBaseURL = "http://localhost.localstack.cloud:4566/encoded-bucket" ) +// playbackBaseURL is the public host the encoded output is served from — the +// CloudFront distribution in AWS, LocalStack's own S3 endpoint here. It has to +// match what cms is configured with, since the whole point of the rewrite is +// that the two agree. +func playbackBaseURL() string { + return strings.TrimRight(envOr("PLAYBACK_BASE_URL", defaultPlaybackBaseURL), "/") +} + +// playlistPathFor is where the HLS output group writes the master playlist for +// a given source key: one folder per video, named after the source object. +// QueueEncodingJob derives the job's destination the same way. +func playlistPathFor(storageKey string) string { + base := strings.TrimSuffix(storageKey, path.Ext(storageKey)) + return fmt.Sprintf("s3://%s/%s/index.m3u8", envOr("MEDIACONVERT_OUTPUT_BUCKET", defaultOutputBucket), base) +} + +// playbackURLFor is what cms should end up storing for that same key. +func playbackURLFor(storageKey string) string { + base := strings.TrimSuffix(storageKey, path.Ext(storageKey)) + return fmt.Sprintf("%s/%s/index.m3u8", playbackBaseURL(), base) +} + func envOr(name, fallback string) string { if v := strings.TrimSpace(os.Getenv(name)); v != "" { return v @@ -51,11 +76,19 @@ type jobStateChange struct { } type jobStateDetail struct { - Timestamp int64 `json:"timestamp"` - AccountID string `json:"accountId"` - Queue string `json:"queue"` - JobID string `json:"jobId"` - Status string `json:"status"` + Timestamp int64 `json:"timestamp"` + AccountID string `json:"accountId"` + Queue string `json:"queue"` + JobID string `json:"jobId"` + Status string `json:"status"` + OutputGroupDetails []outputGroup `json:"outputGroupDetails,omitempty"` +} + +// outputGroup is one output group's result. An HLS group reports the manifests +// it wrote under playlistFilePaths; the segments themselves are not listed. +type outputGroup struct { + Type string `json:"type"` + PlaylistFilePaths []string `json:"playlistFilePaths,omitempty"` } // eventPublisher puts job state changes on the topic cms's queue subscribes to. @@ -85,7 +118,10 @@ func newEventPublisher(ctx context.Context) (*eventPublisher, error) { }, nil } -func (p *eventPublisher) publishJobState(ctx context.Context, jobID, state string) error { +// publishJobState sends the event MediaConvert would send. playlistPath is the +// master playlist an HLS output group reports on COMPLETE; pass "" to send a +// COMPLETE that names no playlist, which is what a job with no HLS group does. +func (p *eventPublisher) publishJobState(ctx context.Context, jobID, state, playlistPath string) error { event := jobStateChange{ Version: "0", ID: fmt.Sprintf("test-%d", time.Now().UnixNano()), @@ -103,6 +139,13 @@ func (p *eventPublisher) publishJobState(ctx context.Context, jobID, state strin }, } + if playlistPath != "" { + event.Detail.OutputGroupDetails = []outputGroup{{ + Type: "HLS_GROUP", + PlaylistFilePaths: []string{playlistPath}, + }} + } + body, err := json.Marshal(event) if err != nil { return err diff --git a/tests/features/transcoding_status.feature b/tests/features/transcoding_status.feature index d795a46..942f939 100644 --- a/tests/features/transcoding_status.feature +++ b/tests/features/transcoding_status.feature @@ -13,6 +13,27 @@ Feature: Transcoding status When MediaConvert reports that the job reached "COMPLETE" Then the video eventually has status "ready" + # The COMPLETE event carries the paths MediaConvert actually wrote. The + # playlist among them is an s3:// URI into a private bucket, so what the + # catalogue stores is that path rewritten onto the public delivery host. + Scenario: A finished job gives the video an HLS playback URL + Given I have registered a video that is being transcoded + When MediaConvert reports that the job reached "COMPLETE" + Then the video eventually has status "ready" + And the video has a playback URL for its HLS playlist + + Scenario: A finished job that reported no playlist still marks the video ready + Given I have registered a video that is being transcoded + When MediaConvert reports that the job reached "COMPLETE" without a playlist + Then the video eventually has status "ready" + And the video has no playback URL + + Scenario: A failed job leaves the video with no playback URL + Given I have registered a video that is being transcoded + When MediaConvert reports that the job reached "ERROR" + Then the video eventually has status "failed" + And the video has no playback URL + Scenario Outline: A job that did not finish cleanly marks the video failed Given I have registered a video that is being transcoded When MediaConvert reports that the job reached "" diff --git a/tests/steps_test.go b/tests/steps_test.go index 2a07ab1..5d8087c 100644 --- a/tests/steps_test.go +++ b/tests/steps_test.go @@ -429,15 +429,32 @@ func mediaConvertReportsState(t gobdd.StepTest, ctx gobdd.Context, state string) t.Fatalf("no transcoding job has been started, so there is no job to report on") return } - publishJobState(t, w.video.MediaConvertJobID, state) + + // A real COMPLETE from an HLS output group always names the manifests it + // wrote; the other states carry no output at all. + playlist := "" + if state == "COMPLETE" { + playlist = playlistPathFor(w.video.StorageKey) + } + publishJobState(t, w.video.MediaConvertJobID, state, playlist) +} + +func mediaConvertReportsStateWithoutPlaylist(t gobdd.StepTest, ctx gobdd.Context, state string) { + w := worldOf(t, ctx) + + if strings.TrimSpace(w.video.MediaConvertJobID) == "" { + t.Fatalf("no transcoding job has been started, so there is no job to report on") + return + } + publishJobState(t, w.video.MediaConvertJobID, state, "") } func mediaConvertReportsStateForJob(t gobdd.StepTest, ctx gobdd.Context, jobID, state string) { worldOf(t, ctx) // keeps the step honest about needing a scenario world - publishJobState(t, jobID, state) + publishJobState(t, jobID, state, "") } -func publishJobState(t gobdd.StepTest, jobID, state string) { +func publishJobState(t gobdd.StepTest, jobID, state, playlistPath string) { background := context.Background() publisher, err := newEventPublisher(background) @@ -445,7 +462,7 @@ func publishJobState(t gobdd.StepTest, jobID, state string) { t.Fatalf("could not reach the job events topic: %s", err) return } - if err := publisher.publishJobState(background, jobID, state); err != nil { + if err := publisher.publishJobState(background, jobID, state, playlistPath); err != nil { t.Fatalf("could not publish the job state change: %s", err) } } @@ -481,25 +498,57 @@ func theVideoKeepsStatus(t gobdd.StepTest, ctx gobdd.Context, want string) { } } -// currentStatus re-reads the video over the API. It deliberately does not -// touch w.last: these steps assert on the record, not on an exchange. -func currentStatus(t gobdd.StepTest, w *world) string { +// currentVideo re-reads the video over the API. It deliberately does not touch +// w.last: these steps assert on the record, not on an exchange. +func currentVideo(t gobdd.StepTest, w *world) videoBody { result, err := w.client.get("/api/videos/" + w.video.ID) if err != nil { t.Fatalf("could not re-read the video: %s", err) - return "" + return videoBody{} } if result.status != 200 { t.Fatalf("could not re-read the video: %s", result.summary()) - return "" + return videoBody{} } var body videoBody if err := result.json(&body); err != nil { t.Fatalf("could not decode the video: %s", err) - return "" + return videoBody{} + } + return body +} + +func currentStatus(t gobdd.StepTest, w *world) string { + return currentVideo(t, w).Status +} + +func theVideoHasAPlaybackURL(t gobdd.StepTest, ctx gobdd.Context) { + w := worldOf(t, ctx) + + video := currentVideo(t, w) + want := playbackURLFor(video.StorageKey) + + if video.PlaybackURL != want { + t.Errorf("expected the playback URL %q, got %q", want, video.PlaybackURL) + return + } + // The point of the rewrite: what is stored has to be something a player + // can fetch, not the s3:// path MediaConvert reported. + if !strings.HasSuffix(video.PlaybackURL, ".m3u8") { + t.Errorf("the playback URL %q is not an HLS playlist", video.PlaybackURL) + } + if strings.HasPrefix(video.PlaybackURL, "s3://") { + t.Errorf("the playback URL %q is still an S3 URI, which no player can fetch", video.PlaybackURL) + } +} + +func theVideoHasNoPlaybackURL(t gobdd.StepTest, ctx gobdd.Context) { + w := worldOf(t, ctx) + + if got := currentVideo(t, w).PlaybackURL; got != "" { + t.Errorf("expected no playback URL, got %q", got) } - return body.Status } // askForThatVideoWrittenAs re-reads the video by a different, still-valid diff --git a/tests/suite_test.go b/tests/suite_test.go index 046379a..33e7c62 100644 --- a/tests/suite_test.go +++ b/tests/suite_test.go @@ -145,9 +145,12 @@ func TestVideoUpload(t *testing.T) { suite.AddStep(`^I have registered a video$`, registerVideoBeingTranscoded) suite.AddStep(`^I ask the CMS for that video with its id written (.*)$`, askForThatVideoWrittenAs) suite.AddStep(`^MediaConvert reports that the job reached "(.*)"$`, mediaConvertReportsState) + suite.AddStep(`^MediaConvert reports that the job reached "(.*)" without a playlist$`, mediaConvertReportsStateWithoutPlaylist) suite.AddStep(`^MediaConvert reports that the job "(.*)" reached "(.*)"$`, mediaConvertReportsStateForJob) suite.AddStep(`^the video eventually has status "(.*)"$`, theVideoEventuallyHasStatus) suite.AddStep(`^the video keeps status "(.*)"$`, theVideoKeepsStatus) + suite.AddStep(`^the video has a playback URL for its HLS playlist$`, theVideoHasAPlaybackURL) + suite.AddStep(`^the video has no playback URL$`, theVideoHasNoPlaybackURL) suite.AddStep(`^I register the uploaded video titled "(.*)" under categories "(.*)"$`, registerUploadedVideo) suite.AddStep(`^I register the uploaded video titled "(.*)" under category id (\d+)$`, registerUnderCategoryID) suite.AddStep(`^I register a video titled "(.*)" with the key "(.*)" under categories "(.*)"$`, registerWithKey)