Compare commits

..

55 Commits

Author SHA1 Message Date
fahd c05b7cf196 TEST 2026-09-04 22:55:27 +03:00
FahdShalhoub db50e015bf FEAT: Swicthed QUERY Method To GET
Build, Push and Deploy Discovery / build-push-deploy (push) Successful in 5m2s
2026-08-30 10:56:51 +03:00
FahdShalhoub 35d4d6756e FEAT: Added Redis Caching
Deploy Infrastructure / pulumi-up (push) Successful in 3s
Build, Push and Deploy Discovery / build-push-deploy (push) Failing after 34m37s
2026-08-29 23:12:11 +03:00
FahdShalhoub f66f3b555d FIX: Missing CORS
Deploy Infrastructure / pulumi-up (push) Successful in 2s
2026-08-29 18:51:52 +03:00
FahdShalhoub c3465bedb1 FEAT: Video Search In Discovery
Build, Push and Deploy Discovery / build-push-deploy (push) Successful in 1m57s
2026-08-29 17:56:11 +03:00
FahdShalhoub 6f9e04ee97 FEAT: Discovery Video Service Subscription
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m35s
Build, Push and Deploy Discovery / build-push-deploy (push) Successful in 1m55s
Deploy Infrastructure / pulumi-up (push) Successful in 2s
2026-08-29 16:50:58 +03:00
FahdShalhoub 98950aba20 FEAT: Init Discovery Service
Build, Push and Deploy Discovery / build-push-deploy (push) Successful in 1m45s
Deploy Infrastructure / pulumi-up (push) Successful in 2s
2026-08-27 22:03:41 +03:00
FahdShalhoub 0cfd89f200 FEAT: Return Playback URL in Get Videos
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m28s
Deploy Infrastructure / pulumi-up (push) Successful in 2s
2026-08-27 21:31:13 +03:00
FahdShalhoub 8c75d7500f FEAT: Return Playback URL in Get Videos 2026-08-27 21:30:56 +03:00
FahdShalhoub 3f5301afb6 FEAT: Update Status Of Video Row On Media Convert Update 2026-08-27 20:04:38 +03:00
FahdShalhoub 5d35d4029e CHANGE: Removed Default Set Variables
Deploy Infrastructure / pulumi-up (push) Successful in 2s
2026-08-27 19:04:36 +03:00
FahdShalhoub 33aa92a29c FIX: Corrected URL Of pulumi deployment 2026-08-27 19:03:50 +03:00
FahdShalhoub 653edcbc51 INFRA: Corrected Deploy Job
Deploy Infrastructure / pulumi-up (push) Successful in 2s
2026-08-27 18:58:51 +03:00
FahdShalhoub 88c64adf38 FIX: Gap Of Uploading The Same File Key Twice
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m19s
2026-08-27 18:56:08 +03:00
FahdShalhoub b68086eea4 FIX: Uploading File With Unkown Catagory Id
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m22s
2026-08-27 18:43:04 +03:00
FahdShalhoub ae129c8f61 FEAT: Init BDD Tests 2026-08-27 18:42:35 +03:00
FahdShalhoub 334d9c1afc INFRA: Changed the INfra Pipeline from Running the Deployment Locally in The Gitea Runner To Triggeirng it in Pulumi
Deploy Infrastructure / pulumi-up (push) Successful in 2s
2026-08-27 12:19:01 +03:00
FahdShalhoub 92e3b3283f FEAT: Installed LocalStack For Development
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m30s
Deploy Infrastructure / pulumi-up (push) Failing after 19s
2026-08-27 12:06:23 +03:00
FahdShalhoub 4303f005bd TEST: Init Test Harness Using Localstack
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m40s
Deploy Infrastructure / pulumi-up (push) Failing after 35s
2026-08-26 20:03:02 +03:00
FahdShalhoub 9d7e1825f1 REFACTOR: Deleted Redundant Repo Interfaces
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m46s
2026-08-26 19:02:40 +03:00
FahdShalhoub 858b277c9d FIX: Checked If File Exists Before Submitting Queue Job 2026-08-26 18:45:32 +03:00
FahdShalhoub cb3ab1f442 AI: Updated Claude Code 2026-08-26 18:36:22 +03:00
FahdShalhoub 4132c6071e REFACTOR: Seperated Catagories And Video DB Repositories
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m45s
2026-08-26 08:51:40 +03:00
FahdShalhoub b77afbfed6 REFACTOR: Optimized Catagory IDs INput Validation
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m47s
Deploy Infrastructure / pulumi-up (push) Failing after 26s
2026-08-17 01:17:08 +03:00
FahdShalhoub 56a2992206 REFACTOR: Optimized Catagory IDs
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m19s
2026-08-17 00:51:11 +03:00
FahdShalhoub aa32bb6ed8 FEAT: Added MOre Video Statuses
Build, Push and Deploy CMS / build-push-deploy (push) Failing after 2m2s
2026-08-17 00:13:22 +03:00
FahdShalhoub 4b719f712d FIX: Changed Content Type From Static mp4 To Input
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m18s
2026-08-16 23:56:21 +03:00
FahdShalhoub f26ea98b52 FEAT: Added Swagger Docs
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m20s
2026-08-16 23:42:00 +03:00
FahdShalhoub cdc5315401 CHANGE: Corrected The Use Of Title In The ProblemDetail Response
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m0s
2026-08-16 23:28:46 +03:00
FahdShalhoub 0dae4abb74 CHANGE: Change Error To ProblemDetails
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 1m52s
2026-08-16 23:21:09 +03:00
FahdShalhoub acae765948 CHANGE: Switched The Interface From HTML to JSON API
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 1m53s
2026-08-16 23:15:53 +03:00
FahdShalhoub 8bf0eae425 REFACTOR: Create Single Connection For App Lifetime
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m6s
2026-08-16 23:05:15 +03:00
FahdShalhoub 785154cec6 REFACTOR: Changed Video ID Creation To DB
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m11s
2026-08-16 22:49:50 +03:00
FahdShalhoub 27fd28d426 FEAT: Load Catagories from backend
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m11s
2026-08-16 22:42:17 +03:00
FahdShalhoub 724351c271 FEAT: Change Catagories to a list
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m17s
2026-08-16 22:29:35 +03:00
FahdShalhoub 96cabc5716 FEAT: Added Database To CMS Serbove
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m17s
2026-08-16 22:19:03 +03:00
FahdShalhoub c0d4401f4a FIX: Added Missing MaxBitrate
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 1m58s
2026-08-16 20:45:08 +03:00
FahdShalhoub a558a0fe88 LOG: Added Logs
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m0s
2026-08-16 20:26:29 +03:00
FahdShalhoub 29fefcaf2b FIX: Added Correct CORS rules For Raw Uploaded Videos Bucket 2026-08-16 20:17:41 +03:00
FahdShalhoub a07113d061 FIX: Permissions To s3 And MediaConvert]
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 2m2s
2026-08-16 20:04:04 +03:00
FahdShalhoub 1a44783b1a FIX: Added Step To install AWS Cli
Build, Push and Deploy CMS / build-push-deploy (push) Successful in 1m57s
2026-08-16 19:44:23 +03:00
FahdShalhoub be5a78a923 FIX: Added Missing Env Variables
Build, Push and Deploy CMS / build-push-deploy (push) Failing after 1m47s
2026-08-16 19:32:36 +03:00
FahdShalhoub 4ae93d4e39 FIX: Added Missing Permission To Gitea User 2026-08-16 19:31:15 +03:00
FahdShalhoub 32506730c7 TEST: Pipeline
Build, Push and Deploy CMS / build-push-deploy (push) Failing after 4m15s
2026-08-16 19:26:51 +03:00
fahd 1d4ddd7c47 CHANGE: Deleted Gitea Access Keys 2026-08-16 19:19:10 +03:00
fahd 9c95e0539a INFRA: Dockerfile For CMS
Build, Push and Deploy CMS / build-push-deploy (push) Failing after 49s
2026-08-16 18:58:10 +03:00
fahd 3cba235723 FEAT: Media Convert + s3 Integration 2026-08-16 18:57:40 +03:00
fahd 7ca76b62ec FEAT: Deploy ECS For CMS and Discovery 2026-08-12 21:23:11 +03:00
fahd 0ade647bd4 FIX: Corrected postgres-sg Description 2026-08-12 19:13:38 +03:00
fahd cb0dc37a8e GITIGNORE 2026-08-12 19:03:23 +03:00
fahd dc2d1c6e33 FEAT: Initial Upload Page 2026-08-12 19:03:02 +03:00
fahd 7754051e9c CHANGE: Deleted Media Convert Queue 2026-08-12 15:21:49 +03:00
fahd 7e2aea0c56 FIX: Media Convert Plan Name 2026-08-12 14:20:04 +03:00
fahd 6bd8f3d2a3 FEAT: Created Databases And Roles For Each App 2026-08-12 14:13:12 +03:00
fahd 979f541a6c DEP: Updated Dependcies 2026-08-12 14:01:37 +03:00
91 changed files with 10757 additions and 127 deletions
+121
View File
@@ -0,0 +1,121 @@
---
description: The ATDD loop this repo develops features with — write the Gherkin scenario first, watch it fail, write just enough code in cms/ to pass it, re-run, move to the next scenario. Use when adding or changing an endpoint, a validation rule, an error response, or any user-visible behaviour in cms/.
---
## The loop
One scenario at a time. Never write two scenarios' worth of code in one pass.
1. **Write the scenario** in `tests/features/*.feature`, in the language of the
feature rather than of HTTP where you can. Name it for the behaviour
(`Refusing to register a video whose file was never uploaded`), not the
mechanism.
2. **Run it and watch it fail.** `cd tests && go test -count=1 -v ./...`
Confirm it fails for the reason you expect. A scenario that passes before
you have written any code is testing nothing — go back to step 1.
3. **Write just enough code in `cms/`** to turn that one scenario green.
Not the next scenario's code. Not a generalisation you have no test for.
4. **Re-run the suite** — the whole file, not just the new scenario. Green
means done; a scenario that used to pass and now does not means the new
code changed published behaviour, which is a decision, not an accident.
5. **Next scenario.** Repeat until the feature is described.
Only then tidy: extract helpers, rename, split files. The suite is what makes
that safe, so do it with the tests green and re-run after.
## Where things go
Tests are **black box**. `tests/` is its own Go module (`thamanyah/tests`) that
imports nothing from `cms/` and speaks only HTTP. Do not add `_test.go` files
inside `cms/` — if a behaviour cannot be reached through the API, that is a
finding about the design, not a reason to reach past it.
| file | what goes in it |
|---|---|
| `tests/features/*.feature` | the scenarios |
| `tests/suite_test.go` | step registration, the per-scenario `world` |
| `tests/steps_test.go` | what each step does |
| `tests/client_test.go` | HTTP plumbing and wire types |
Production code keeps to the seams in `CLAUDE.md`: wire structs in
`internal/api`, domain types in `internal/models`, HTTP in `internal/handlers`,
SQL in `internal/db/repositories`, AWS in `internal/services`. "Just enough
code" still lands on the right seam — a handler that runs its own SQL is not
minimal, it is misplaced.
## Running it
```bash
docker compose up -d # localstack + infra + cms (needs LOCALSTACK_AUTH_TOKEN)
docker exec cms ./cms migrate # see Migrations below
cd tests && go test -count=1 -v ./...
```
`-count=1` defeats Go's test caching, which otherwise hides a re-run against a
changed service. `-v` prints the Gherkin: gobdd nests a subtest per feature,
scenario and step, so the step text appears in the output.
`CMS_BASE_URL` (default `http://localhost:8081`) points the suite elsewhere. If
nothing is serving, the suite skips rather than fails.
**Migrations.** Compose starts `cms` in server mode only — the migrate
container exists in the ECS task definition, not in `docker-compose.yml`. A new
migration is **not** applied by `docker compose watch` rebuilding the image.
Run `docker exec cms ./cms migrate` after adding one, or every scenario fails
on `relation "…" does not exist`.
**The spec.** Changing a handler, its annotations, or an `internal/api` struct
means regenerating the OpenAPI spec, or the build drifts from the published
contract:
```bash
$(go env GOPATH)/bin/swag init --generalInfo main.go --dir ./ --parseInternal --output ./docs
```
## Reading a red run
gobdd does **not** stop a scenario at the first failing step. A step that fails
leaves the later steps running against stale state, so one real failure
produces a cascade of noisy ones. **Read the first failure in a scenario and
ignore the rest** until it is fixed.
`cannot find step definition for step: …` is a legitimate red: the scenario is
written and the step is not. Add the function to `steps_test.go`, register it
in `TestVideoUpload`, and run again.
## Writing steps
- **Anchor every pattern** with `^…$`. gobdd matches unanchored and the
first registered pattern that matches wins, so an unanchored pattern can
silently shadow another step.
- State moves between steps through the `*world` in the gobdd context, never
package variables. It is held by **pointer** because gobdd clones the context
between the `Background` and the scenario steps.
- A `Scenario Outline` runs every example row in **one** world. Steps must
overwrite state, never accumulate it.
- Reuse a step before writing a new one. Two steps that differ only in wording
are two ways for the feature file to drift.
## Asserting errors
Errors are RFC 9457 Problem Details on `application/problem+json`. `title` is
held identical across every occurrence of a given problem so clients can branch
on it — so **assert the exact title**, not just the status. A new error
condition means a new stable title, and a new `@Failure` annotation on the
handler.
When a status alone would be ambiguous, distinguish the cases in code rather
than in the assertion: a client's mistake and a server fault reaching the same
`writeProblem` call is the bug, and a sentinel error in
`internal/handlers/errors.go` plus `errors.Is` at the call site is how this repo
separates them.
## `@known-gap`
Tag a scenario `@known-gap` when it pins what the service does **today** and
that differs from what it is documented to do. It still passes — it is a record,
not a failure — and it carries a comment saying what the contract promises and
what changing it would take. Fixing the code means changing the scenario in the
same commit and dropping the tag.
Do not use the tag to park a scenario you could not make pass.
+64
View File
@@ -0,0 +1,64 @@
name: Build, Push and Deploy CMS
on:
push:
branches: [main]
paths:
- "cms/**"
- ".gitea/workflows/cms-deploy.yml"
env:
AWS_REGION: us-east-1
ECR_REPOSITORY: cms
ECS_CLUSTER_ARN: arn:aws:ecs:us-east-1:887083795404:cluster/app-cluster-384acc4
CMS_ECS_SERVICE_ARN: arn:aws:ecs:us-east-1:887083795404:service/app-cluster-384acc4/cms-service-350b51b
jobs:
build-push-deploy:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
- name: Log in to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: ./cms
file: ./cms/Dockerfile
push: true
tags: |
${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:latest
${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ gitea.sha }}
- name: Install AWS CLI
run: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip -q awscliv2.zip
sudo ./aws/install
rm -rf awscliv2.zip aws
# cluster/service ARNs come from `pulumi stack output ecsClusterArn` /
# `cmsServiceArn` in the infrastructure project — set once as repo
# variables (Settings > Actions > Variables), not secrets, since ARNs
# aren't sensitive.
- name: Deploy to ECS
run: |
aws ecs update-service \
--cluster "${{ env.ECS_CLUSTER_ARN }}" \
--service "${{ env.CMS_ECS_SERVICE_ARN }}" \
--force-new-deployment \
--region "${{ env.AWS_REGION }}"
+60
View File
@@ -0,0 +1,60 @@
name: Build, Push and Deploy Discovery
on:
push:
branches: [main]
paths:
- "discovery/**"
- ".gitea/workflows/discovery-deploy.yml"
env:
AWS_REGION: us-east-1
ECR_REPOSITORY: discovery
ECS_CLUSTER_ARN: arn:aws:ecs:us-east-1:887083795404:cluster/app-cluster-384acc4
DISCOVERY_ECS_SERVICE_ARN: arn:aws:ecs:us-east-1:887083795404:service/app-cluster-384acc4/discovery-service-52aa4f6
jobs:
build-push-deploy:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
- name: Log in to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: ./discovery
file: ./discovery/Dockerfile
push: true
tags: |
${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:latest
${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ gitea.sha }}
- name: Install AWS CLI
run: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip -q awscliv2.zip
sudo ./aws/install
rm -rf awscliv2.zip aws
- name: Deploy to ECS
run: |
aws ecs update-service \
--cluster "${{ env.ECS_CLUSTER_ARN }}" \
--service "${{ env.DISCOVERY_ECS_SERVICE_ARN }}" \
--force-new-deployment \
--region "${{ env.AWS_REGION }}"
@@ -0,0 +1,30 @@
name: Deploy Infrastructure
on:
push:
branches: [main]
paths:
- "infrastructure/**"
- ".gitea/workflows/infrastructure-deploy.yml"
# Pulumi state isn't safe to update concurrently; serialize runs and let a
# newer push supersede one still queued (not one already applying).
concurrency:
group: pulumi-thamanyah-main
cancel-in-progress: false
env:
AWS_REGION: us-east-1
jobs:
pulumi-up:
runs-on: ubuntu-latest
steps:
- name: Send Deployment Request
run: |
curl -X POST https://api.pulumi.com/api/stacks/fahdshalhoub123-gmail-com/thamanyah/main/deployments \
-H "Authorization: token ${{ secrets.PULUMI_ACCESS_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{
"operation": "update"
}'
+1
View File
@@ -0,0 +1 @@
/volume
+500
View File
@@ -0,0 +1,500 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Repo layout
```
.
├── cms/ Go service: video ingestion/CMS (implemented)
├── discovery/ Go service: catalogue discovery/read side (skeleton)
├── infrastructure/ Pulumi (Go) program provisioning all AWS resources
├── tests/ Go module: Gherkin/gobdd scenarios run against a live cms
└── .gitea/workflows/ Gitea Actions CI/CD pipelines
```
`discovery` is the read side. It owns no ingestion: it learns what exists by
draining `discovery-catalogue-events`, the queue subscribed to the catalogue
topic `cms` announces ready videos on, and keeps its own copy in its own
database. Neither service reads the other's tables and nothing polls `cms`.
It serves `GET /health`, `GET /api/videos/{id}`, `GET /api/videos` (the
catalogue search — see below) and the Swagger UI. Migration `0001_init` still
creates nothing (it predates the domain and exists only because
`internal/db/migrate.go` embeds `migrations/*.sql`, which will not compile
against an empty directory); `0002_create_videos_table` is where the schema
actually starts, and `0003`–`0005` add what the search needs. It now has an `internal/services` (SQS and Redis) and an
`internal/consumers`, and a task role scoped to that one queue — receive,
delete, get-attributes, and nothing else. It cannot publish back onto the
topic: it is a subscriber, not a participant. The Redis client needs nothing
from that role: ElastiCache is reached over the Redis protocol on the private
network, so the security group is what grants access, not IAM.
## Commands
### cms (Go 1.25, module `thamanyah/cms/v2`)
```bash
cd cms
go run . # serves on :8081 (requires DB_*/S3_*/MEDIACONVERT_* env vars — see below)
go run . migrate # applies pending DB migrations, then exits (no HTTP server)
go build ./...
go vet ./...
```
### discovery (Go 1.25, module `thamanyah/discovery`)
Same commands as `cms`, and the same two entry paths (`runServer`,
`runMigrate`) in `discovery/main.go` — only the port and the dependencies
differ. Note the module path carries no `/v2`: unlike `cms`, this module has
never had a v1.
```bash
cd discovery
go run . # serves on :8080 (requires DB_* plus AWS_REGION/CATALOGUE_EVENTS_QUEUE_URL/REDIS_ADDR — no S3/MediaConvert)
go run . migrate # applies pending DB migrations, then exits (no HTTP server)
go build ./...
go vet ./...
```
Its OpenAPI spec is generated into `discovery/docs` by the same `swag init`
invocation as `cms`, run from `discovery/`. In `docker-compose.yml` it is a
service of its own on `127.0.0.1:8080`, wired to the `discovery` database and
role LocalStack provisions, plus the AWS_* vars its consumer needs and
`REDIS_ADDR` pointing at the `redis` container — and, like
`cms`, started in server mode only, so a freshly created local database needs
`docker exec discovery ./discovery migrate` once. Until that runs, the consumer
logs `relation "videos" does not exist` per announcement and leaves them on the
queue; they are picked up once the table exists.
`discovery`'s data model is one table, `videos`: the catalogue's copy of an
announced video, keyed by **the id `cms` issued** (no `DEFAULT` — the id
arrives in the announcement and is reused verbatim, so both services name the
same video the same way). `categories` is a denormalized `TEXT[]` of names:
`cms` owns the vocabulary and announces names, so there is nothing to join to.
`repositories.SaveVideo` is an **upsert** — the topic delivers at-least-once,
so the same announcement can arrive twice, and a plain `INSERT` would fail on
the second, wedging every message queued behind it. A later announcement wins.
Three more columns/indexes exist purely for the search (`0003`–`0005`):
`search_vector` is a **generated** `tsvector` column (`to_tsvector('arabic',
title)`, `STORED`) with a GIN index; `categories` has a GIN index for the `&&`
overlap; and `videos_recent_idx` is a btree on `(created_at DESC, id DESC)`.
The text search configuration is `'arabic'`, not `'english'` or `'simple'`: it
stems Arabic while leaving Latin-script tokens as written, which suits a mixed
catalogue. The cost is that English is **not** stemmed, so "documentaries" does
not find "documentary". It is named explicitly in the column *and* in every
query — a query built under a different configuration stems its terms
differently and silently matches nothing.
### Catalogue search (`GET /api/videos`)
Served on **GET**, not POST: a search is safe and idempotent, which POST is not,
and its parameters fit a query string. It shares the path with
`GET /api/videos/{id}`; `ServeMux` keeps them apart, the more specific pattern
winning. Two things to know before touching it:
- This endpoint was on **QUERY** until the method was changed to GET. The
parameters are now a query string, not a JSON body, so there is no
`api.SearchRequest` any more — `handlers.parseSearchQuery` reads
`r.URL.Query()` into an unexported `searchQuery`. Anything still sending a
JSON body gets the whole catalogue, since every parameter is optional and it
supplied none of them.
- It **does** appear in the OpenAPI spec now, as `@Router /api/videos [get]`
with one `@Param … query` per parameter (`categories` carries
`collectionFormat(multi)`). That is the reason the change is worth anything
beyond method purity: QUERY had no `PathItem` slot in Swagger 2.0 or
OpenAPI 3.x, so `swag` rejected the annotation with `invalid method: QUERY`
and the endpoint had to be described in prose instead.
`categories` is **repeated**, not comma-separated
(`?categories=news&categories=documentary`), so a category name containing a
comma stays one name; an empty value is dropped rather than treated as a
category named `""`. `limit` is the only parameter that can be malformed — a
non-numeric one is a 400 `Malformed Search`; zero or negative reads as "no
preference" and takes the default, as omitting it does.
Response `{videos, nextCursor}`.
`title` is matched lexically, ANDing the words with the **last one as a prefix**
(`desert & fal:*`) so a part-typed word still matches; `categories` narrows to
videos filed under **any** of the names (`&&`, not `@>`); `limit` defaults to 20
and is **capped at 100** rather than refused. Paging is **keyset**, not offset:
the opaque cursor carries the last row's `(rank, created_at, id)`, so pages stay
stable while the catalogue is being written to. The query asks for `limit+1`
rows and hands back a cursor only when the extra row appears, which is how the
last page reports itself as last.
`tsQueryFor` builds the tsquery by splitting input on everything that is not a
letter or digit. That drops every character tsquery gives a meaning to, which is
what makes it safe to interpolate — `websearch_to_tsquery` parses user syntax
but cannot express a prefix match.
A measured caveat: for a **ranked** search, deep paging is not a seek. `ts_rank`
is computed per row, so the keyset comparison lands as a filter rather than an
index condition and every matching row is still scanned. Keyset paging still
buys stability and avoids OFFSET's growing discard cost, but only the unranked
browse path (no title term) is a true index seek — that is what
`videos_recent_idx` is for, and it takes a 200k-row browse from ~15 ms to
~0.03 ms.
#### Search cache (Redis / ElastiCache)
Because of that caveat, the search reads through a Redis cache before it
touches Postgres — `handlers.cachedSearch`/`cacheSearch` around the repository
call, storing the `api.SearchResults` a page serialises to. `REDIS_ADDR` points
at the ElastiCache node `infrastructure/main.go` provisions (a `redis`
container under compose). Locally it takes a repeated ranked search from
~6.8 ms to ~0.5 ms.
- **The key is `discovery:search:v1:` + a canonical JSON encoding of
`{t: title, c: categories, l: limit, p: cursor}`.** The cursor *is* the page
identity — keyset paging has no page number to key on. `limit` is in the key
because the same title/categories/cursor at a different limit is a different
set of rows. JSON rather than pasted-together separators so a category name
containing `,` or `|` cannot be mistaken for a boundary. The title is
lowercased (`tsQueryFor` folds case anyway) but the **categories are not**:
`categories && $2` compares them verbatim, so `News` and `news` really are
different searches. Categories are sorted and deduped — `&&` means "any of
these", so order never changed the answer — and never nil, so `"categories":
[]` and an omitted member share an entry.
- **TTL is 60 s and nothing invalidates on write.** The consumer writes rows
continuously and would have to know which cached pages a new title belongs
on — for a ranked search, every page it outranks. Expiry is cheaper and
bounds staleness to roughly the announcement's own delivery lag.
- **Only 200s are cached**, so a malformed cursor still reaches the repository
and is still a 400.
- **Every cache failure is a miss.** A Get/Set error is logged and the search
is answered from Postgres; the service also *starts* with an unreachable
cache, warning rather than panicking as the DB and SQS assertions do — a slow
read side beats no read side. Each call is bounded by
`services.cacheOperationTimeout` (100 ms, retries off), because an unbounded
cache miss on a dead node costs the dial timeout *plus* the query it was
avoiding. A total cache outage adds ~200 ms per search (a failed read and a
failed write), not seconds.
`cms`, `discovery` and `infrastructure` hold no test files of their own. The
only tests in the repo are the black-box BDD scenarios in `tests/` — see below.
`cms` is a JSON API only — it serves no HTML and has no static assets. The
OpenAPI spec is generated from swaggo annotations on the handlers into
`cms/docs`, a committed, compiled-in Go package. If you add or change a
handler, its annotation comments, or a request/response struct, regenerate it:
```bash
go install github.com/swaggo/swag/cmd/swag@v1.16.6 # match go.mod; not installed by default
swag init --generalInfo main.go --dir ./ --parseInternal --output ./docs
```
`--parseInternal` is required — the handlers live under `internal/`, which
swag skips without it. Keep the `swaggo/swag` version in `go.mod` and the CLI
in lockstep: `http-swagger/v2` transitively pulls a much older `swag` whose
`swag.Spec` struct lacks the `LeftDelim`/`RightDelim` fields newer generators
emit, and the build breaks outright if the two drift. (The CLI's `--version`
misreports itself as v1.16.4; `go version -m $(go env GOPATH)/bin/swag` gives
the real one.)
### tests (Go, module `thamanyah/tests`)
```bash
cd tests
go test ./... # runs features/*.feature against CMS_BASE_URL (default http://localhost:8081)
go test -v ./... # -v prints the Gherkin: gobdd nests a subtest per feature/scenario/step
CMS_BASE_URL=… go test ./...
```
Black-box BDD covering the video upload feature: its own Go module importing
nothing from `cms/`, talking to a running service over HTTP only, so the same
scenarios run against compose and against a deployed environment. Skips (does
not fail) when nothing is serving. Uploading is a plain `PUT` to the presigned
URL, no AWS SDK. Scenarios can be tagged `@known-gap` to pin current behaviour
that differs from the documented contract — see `tests/README.md`.
Note compose starts `cms` in server mode only: the migrate container exists in
the ECS task definition, not in `docker-compose.yml`, so a freshly created
local database needs `docker exec cms ./cms migrate` once or every scenario
fails on `relation "categories" does not exist`.
### infrastructure (Go, Pulumi, module `thamanyah`)
```bash
cd infrastructure
pulumi preview # plan changes against stack "main"
pulumi up # apply — this touches real AWS resources, confirm with the user first
pulumi stack output # e.g. ecsClusterArn, cmsServiceArn
```
Deploys run in CI (`.gitea/workflows/infrastructure-deploy.yml`) on push to
`main` touching `infrastructure/**`. Treat local `pulumi up` as something to
confirm with the user, not a routine dev command — it mutates shared cloud
state and Pulumi state isn't safe to update concurrently with CI.
## Architecture
### cms package layout
Four `internal` packages, split by the kind of thing they hold — keep new code
on the same seams:
| package | holds |
|---|---|
| `internal/api` | the **wire contract**: request/response structs with their JSON + swaggo tags. No logic. |
| `internal/models` | the **domain types** the rest of the code passes around: `Video`, `Category`, `VideoStatus`. No JSON tags, no SQL. |
| `internal/handlers` | HTTP: decode `api.*`, validate, call repositories/services, encode `api.*`. |
| `internal/db` | the `*sql.DB` connection lifecycle and migrations; `internal/db/repositories` holds all SQL. |
| `internal/services` | **AWS only** — S3, MediaConvert and SQS. Nothing database-related lives here. |
`api` and `models` are deliberately separate types even where the fields look
alike: `api.CompleteResponse` is the published schema, `models.Video` is the
row. Handlers translate between them field by field, so renaming a `models`
field doesn't move the published API and vice versa.
### cms service
Plain `net/http` (Go 1.22+ pattern-based `ServeMux`), no framework. Entry
point `cms/main.go` builds the dependencies and assigns them to package-level
interface vars that the handlers call through:
- `services.S3Client` (`services.S3`) ← `*services.S3Concrete`
- `services.MediaConvertClient` (`services.MediaConvert`) ← `*services.MediaConvertConcrete`
- `repositories.VideoRepo` (`repositories.VideoRepository`) ← `*repositories.ConcreteVideoRepository`
- `repositories.CatagoriesRepo` (`repositories.CatagoriesRepository`) ← `*repositories.ConcreteCatagoriesRepository`
Each interface has exactly one implementation; the indirection is what makes
the handlers substitutable in tests, even though no tests exist yet. Note the
spelling: the categories repository is `Catagories`/`CatagoriesRepo` in
`repositories/catagories.go` (and `handlers.validateCatagoryIDS`) — the
misspelling is load-bearing for compilation, so match it rather than "fixing"
it piecemeal.
On boot, `S3Concrete.AssertSuccessfulConnection` proactively exercises
head-bucket/put/get/presign against the bucket (writing a throwaway
`.s3-connectivity-check` object), and `db.AssertSuccessfulConnection` pings
Postgres — both panic on failure rather than letting the service come up in a
broken state.
`cms/main.go` has two entry paths, dispatched on `os.Args[1]`: the default
path (`runServer`) boots the HTTP server; `./cms migrate` (`runMigrate`)
only opens the DB connection, applies pending migrations via
`cms/internal/db.Migrate` (golang-migrate, `iofs` source, SQL files embedded
from `cms/internal/db/migrations/*.sql`), and exits — it does not touch
S3/MediaConvert or start the server. This is run as its own ECS container
before the main container starts (see infrastructure below), so `runServer`
never runs migrations itself, only `AssertSuccessfulConnection`.
The DB connection is opened once for the process lifetime by
`db.CreateDBConnection(connString)` and closed by `db.CloseConnection` — both
plain functions over `*sql.DB` in `internal/db/client.go`, not methods on a
wrapper type. Repositories take that `*sql.DB` as their `SQLDB` field.
Routes (`cms/main.go`): `GET /health`, `GET /api/categories`,
`POST /api/videos/presign`, `POST /api/videos`, plus Swagger UI at
`GET /swagger/` (`/swagger/doc.json` serves the spec). The UI assets are
embedded in the binary by `swaggo/files`, so nothing is read from disk and
nothing is fetched from a CDN at runtime.
Handlers live in `cms/internal/handlers` — `handlers.go` holds `Health` and
the shared response writers, `videos.go` the categories and upload endpoints.
The wire structs they serve live in `cms/internal/api` (`api.go` for
`HealthResponse` and `ProblemDetails`, `videos.go` for `Category`,
`CategoriesResponse`, `PresignRequest`/`PresignResponse` and
`CompleteRequest`/`CompleteResponse`), referenced by the handlers' swaggo
annotations as `api.CompleteResponse` and so on — so the generated spec's
definition names track that package, and renaming a type there changes the
published schema names.
There is no view layer: the `internal/views` templ package, the `static/`
directory, and the htmx frontend were all removed when the service became a
JSON API, along with the `templ` dependency.
Error responses are RFC 9457 Problem Details objects
(`application/problem+json`), written by
`writeProblem(w, status, title, detail)`. `type` is always `"about:blank"`;
`title` is a short summary held identical across every occurrence of a given
problem, so clients can branch on it; `detail` is the only member that varies
with request data. Success responses go through `writeJSON`
(`application/json`). Both share `writeJSONContent`. Note this deviates
slightly from RFC 9457, which pairs an `about:blank` type with a title that is
just the HTTP status phrase — meaningful titles like these are supposed to
carry a real `type` URI. Adding per-problem type URIs is the conforming fix if
it ever matters.
### Data model (Postgres, `cms/internal/db/migrations/`, two migrations: `0001`, `0002`)
- `videos` — one row per uploaded video: `title`, `description`, `tags`
(free-text, comma-separated — not normalized), `file_name`, `storage_key`
(the S3 key, unique), `mediaconvert_job_id`, `status` (written once as
`"processing"` on insert — see Known gaps), `size_bytes`, timestamps.
`id` is a `UUID` filled by the column's own `DEFAULT gen_random_uuid()`
(v4) and read back through `RETURNING id` — the application does not
generate it. (An earlier revision generated UUIDv7 application-side; that
was reverted in `785154c`, so primary keys are random, not time-ordered.)
`status` is a plain `TEXT NOT NULL DEFAULT 'processing'` column with **no
CHECK constraint** — the closed set exists only in Go, as the
`models.VideoStatus` string type in `cms/internal/models/video.go`
(`processing`, `ready`, `failed`). Extending it means adding a constant
there and extending the `enums` annotation on `api.CompleteResponse.Status`
before regenerating the spec. (The doc comment on `VideoStatus` claims a
`videos_status_check` constraint enforces it in the database — that
constraint does not exist; nothing has ever created it.)
- `categories` — a small fixed lookup table (`documentary`, `news`,
`entertainment`, `podcast`, `other`), seeded by migration `0001`. Its
`SMALLSERIAL` ids are part of the public API: `GET /api/categories` returns
`{id, name}` pairs (ordered by name, not id) and `POST /api/videos` takes
`categoryIds`, so the seed order in migration `0001` is what fixes which id
means which name — never renumber it.
- `video_categories` — join table (`video_id`, `category_id`, composite PK,
`ON DELETE CASCADE`) added in migration `0002`: a video can belong to
*multiple* categories, not just one. `ConcreteVideoRepository.CreateVideo`
inserts the `videos` row and its `video_categories` links inside a single
transaction, taking the category ids as given — `handlers.validateCatagoryIDS`
is what checks them against `CatagoriesRepo.ListCategoriesIDs` before the
transcode job is queued, leaving the foreign key and the composite PK as
backstops (see Known gaps for how those failures surface).
### Video upload → transcode pipeline
1. Client calls `POST /api/videos/presign` → cms returns a presigned S3 `PUT`
URL for `raw-uploads-bucket`, key `videos/<random-hex>.<ext>`. The
submitted `contentType` (only `video/mp4` or `video/quicktime`) is signed
into the URL, so the client's `PUT` must send the identical header.
2. Client `PUT`s the file directly to S3 (from a browser this requires the
bucket's CORS rule, set up in `infrastructure/main.go`,
its allowed origin is now stale). The file never passes through cms.
3. Client calls `POST /api/videos` with the metadata (categories given as
`categoryIds` from `GET /api/categories`) + key → cms validates the
category ids, calls `MediaConvertClient.QueueEncodingJob(key)`, submitting
a MediaConvert job `s3://raw-uploads-bucket/<key>` →
`s3://encoded-bucket/<key>` (H.264/AAC → MP4, QVBR rate control — QVBR
requires `MaxBitrate` to be set explicitly), then
`repositories.VideoRepo.CreateVideo` persists the `videos` row (status
`"processing"`) and its `video_categories` links.
4. Finished output lands in `encoded-bucket`, served via CloudFront.
5. MediaConvert reports the job's state changes to an SNS topic that fans out
to `cms-mediaconvert-events`, which `internal/consumers` reads for the
lifetime of the process: it records the outcome (`ready`/`failed`) and the
playback URL, rewriting the `s3://` playlist path onto `PLAYBACK_BASE_URL`.
A job that came out **ready** is then announced on the `catalogue-events`
SNS topic via `services.CatalogueClient` — `{videoId, title, playbackUrl,
categories}`, categories by **name** so a subscriber needs nothing from
cms to interpret them. Failed jobs are recorded but never announced. A
topic, not a queue, so the announcement fans out: `discovery-catalogue-events`
subscribes today (raw delivery, so there is no SNS envelope to unwrap), and
a second reader can subscribe its own queue without cms changing. Note a
topic only delivers to subscriptions that exist when it publishes, so a new
subscriber's queue has to be in place before the announcement, not after.
Delivery is at-least-once: a failed publish leaves the *job* event on the
consumer's queue, and the outcome update is idempotent, so a redelivery
re-announces and a subscriber can see a video twice.
Config wiring: cms reads `S3_BUCKET`, `MEDIACONVERT_INPUT_BUCKET`,
`MEDIACONVERT_OUTPUT_BUCKET`, `MEDIACONVERT_ROLE_ARN`, `AWS_REGION`,
`MEDIACONVERT_EVENTS_QUEUE_URL`, `CATALOGUE_EVENTS_TOPIC_ARN`,
`PLAYBACK_BASE_URL`,
`DB_HOST`, `DB_PORT`, `DB_NAME`, `DB_USER`, `DB_PASSWORD` from env vars
injected by the ECS task definition (`extraEnv` and the unconditional DB_*
vars in `deployFargateService`, `infrastructure/main.go`) — `main.go` panics
on boot if any required var is empty.
### infrastructure (`infrastructure/main.go`, single Pulumi Go program, region `us-east-1`, stack `main`)
- **Postgres**: one shared RDS instance (`db.t3.micro`, single-AZ, no
backups — intentionally minimal). Each app (`cms`, `discovery`) gets its
own login role and same-named database via the `postgresql` provider
(`newServiceDatabase`), so services never share DB credentials.
- **ElastiCache**: one `cache.t4g.micro` Redis node (`search-cache`, engine
7.1, single-AZ, no replica, no snapshots) that `discovery` answers repeated
catalogue searches from. Its contents are derivable from Postgres by
definition, so there is nothing to back up. Its security group admits 6379
from `ecs-service-sg` **only** — narrower than the database's, which also
admits the deployer's IP for the `postgresql` provider; there is nothing to
administer here from a laptop. The endpoint is the single node's address
(`CacheNodes[0]`, not `ConfigurationEndpoint` — that is a Memcached thing),
exported as `searchCacheAddress` and injected as `REDIS_ADDR`. Skipped under
LocalStack, where docker-compose runs a plain `redis:7-alpine` container
instead: ElastiCache picks its own endpoint, and compose needs a literal
`REDIS_ADDR` before anything is provisioned.
- **ECS Fargate**: one cluster (`app-cluster`), one ALB *per service* (each
gets its own DNS name rather than sharing a load balancer on different
ports). `deployFargateService(...)` is the shared helper building a
service's ECR repo, CloudWatch log group, task definition, ECS service,
and ALB. `taskRole` is optional (nil = no AWS identity beyond the shared
execution role); `extraEnv` appends container env vars beyond the DB_* set;
`runMigrations` (true for both services) adds a second,
non-essential `<name>-migrate` container to the task — same image,
`command: ["migrate"]` — with the main container's `dependsOn` set to
`condition: "COMPLETE"` on it. This is ECS's container-dependency
mechanism, the Fargate equivalent of a Kubernetes init container: ECS runs
the migrate container to completion (exit 0) before starting the main
container, so schema migrations always finish before the service accepts
traffic. No separate CI/Docker migration step exists — `Dockerfile`'s
`ENTRYPOINT ["./cms"]` plus the container's `command` override composes to
`./cms migrate`.
- **S3 + CloudFront**: `encoded-bucket` holds finished transcoded output,
served publicly via CloudFront using Origin Access Control (OAC) — the
bucket itself blocks all public access; only CloudFront's OAC principal
can read it. The distribution carries a **response headers policy** adding
permissive CORS headers (`*`, `GET`/`HEAD`/`OPTIONS`), and `OPTIONS` is in
the behaviour's allowed/cached methods so CloudFront answers preflights
itself: HLS is fetched by JavaScript, so a playlist or segment served
without `Access-Control-Allow-Origin` is discarded by the browser. The
bucket also carries its own equivalent CORS rule, which is what the
LocalStack stack relies on — there is no CloudFront there and
`PLAYBACK_BASE_URL` points the player straight at S3.
- **S3 raw uploads**: `raw-uploads-bucket` is a separate, private bucket for
pre-transcode uploads — deliberately kept apart from `encoded-bucket` so
raw source video is never reachable through the public CDN. CORS is
scoped to `PUT` only, from the `cms` ALB's own origin — correct back when
cms served the upload page itself, but stale now that it serves no UI (see
Known gaps).
- **IAM roles** — five distinct roles/users, each scoped narrowly, don't
conflate them:
- `ecs-task-execution-role` — shared by both services' ECS *agent* (image
pull, log write, Secrets Manager read for DB password). Not usable by
application code inside the container.
- `cms-task-role` — the `cms` container's own AWS identity: S3
`ListBucket`/`PutObject`/`GetObject` on `raw-uploads-bucket` only,
`mediaconvert:CreateJob`, `iam:PassRole` scoped to the MediaConvert
service role (`iam:PassedToService` condition), receive/delete on
`cms-mediaconvert-events`, and `sns:Publish` on the `catalogue-events`
topic — the only thing it writes to. It has no access to
`discovery-catalogue-events`, the queue subscribed to that topic: cms
publishes, it does not reach into a subscriber.
- `discovery-task-role` — the `discovery` container's own AWS identity, and
its only one: receive/delete/get-attributes on
`discovery-catalogue-events`. No S3, no MediaConvert, and no `sns:Publish`
— it consumes the catalogue, it does not add to it. The search cache is
absent from it on purpose: ElastiCache is reached over the Redis protocol
inside the VPC, so `search-cache-sg` is the grant, not IAM.
- `mediaconvert-service-role` — trusted by `mediaconvert.amazonaws.com`,
not by ECS; the role MediaConvert itself assumes (passed as
`CreateJobInput.Role`) to read `raw-uploads-bucket` and write
`encoded-bucket`. Distinct from `cms-task-role` by design: one is "cms
calling AWS", the other is "AWS calling AWS on cms's behalf".
- `gitea-ci-user` — an IAM **user** (static access keys, not OIDC — the
Gitea Actions runner doesn't support instance-profile auth) scoped to
just ECR push (`cms`/`discovery` repos) and
`ecs:UpdateService`/`ecs:DescribeServices` on the two ECS services.
Never broaden this to `ecr:*`/`ecs:*`.
### CI/CD (`.gitea/workflows/`)
- **`cms-deploy.yml`**: push to `main` touching `cms/**`. Builds/pushes the
Docker image to ECR using `gitea-ci-user`, installs the AWS CLI (not
preinstalled on the runner image — via AWS's official install script, not
a third-party action), then `aws ecs update-service --force-new-deployment`.
Cluster/service ARNs come from `pulumi stack output` and are set as repo
*variables* (not secrets — ARNs aren't sensitive).
- **`discovery-deploy.yml`**: the same pipeline for `discovery`, on pushes
touching `discovery/**`. One difference: the ECS service ARN is read from
the `DISCOVERY_ECS_SERVICE_ARN` repo variable rather than pinned in the
workflow, so it has to be set (from `pulumi stack output
discoveryServiceArn`) before the first deploy can succeed.
- **`infrastructure-deploy.yml`**: push to `main` touching
`infrastructure/**`. Runs `pulumi up` using a separate, broader AWS
credential (`PULUMI_AWS_ACCESS_KEY_ID`/`SECRET`) than `gitea-ci-user`,
since provisioning IAM/RDS/ECS/CloudFront needs wider permissions than
pushing images and forcing deployments. Guarded with a concurrency group
since Pulumi state isn't safe to update concurrently.
- The runner's `ubuntu-latest` label maps to a docker image configured on
the runner host (outside this repo) — currently minimal, lacking the AWS
CLI, hence the manual install step in `cms-deploy.yml`.
+1 -1
View File
@@ -1 +1 @@
Hello Hello There
+35
View File
@@ -0,0 +1,35 @@
/uploads/
# Binaries
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool
*.out
coverage.txt
# Dependency directories
vendor/
# Go workspace file
go.work
go.work.sum
# Env files
.env
.env.local
# Build output
/bin/
/dist/
# IDE
.idea/
.vscode/
*.swp
+19
View File
@@ -0,0 +1,19 @@
# syntax=docker/dockerfile:1
FROM golang:1.25-alpine AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o /out/cms .
FROM alpine:3.20
RUN apk add --no-cache ca-certificates
WORKDIR /app
COPY --from=builder /out/cms ./cms
EXPOSE 8081
ENTRYPOINT ["./cms"]
+490
View File
@@ -0,0 +1,490 @@
// Package docs Code generated by swaggo/swag. DO NOT EDIT
package docs
import "github.com/swaggo/swag"
const docTemplate = `{
"schemes": {{ marshal .Schemes }},
"swagger": "2.0",
"info": {
"description": "{{escape .Description}}",
"title": "{{.Title}}",
"contact": {},
"version": "{{.Version}}"
},
"host": "{{.Host}}",
"basePath": "{{.BasePath}}",
"paths": {
"/api/categories": {
"get": {
"description": "Returns the fixed lookup set of categories a video can belong to, each with the id used to reference it. The ` + "`" + `id` + "`" + ` values from this list are the only ones accepted in the ` + "`" + `categoryIds` + "`" + ` field of POST /api/videos.",
"produces": [
"application/json"
],
"tags": [
"categories"
],
"summary": "List video categories",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.CategoriesResponse"
}
},
"500": {
"description": "Categories could not be read from the database",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/api/videos": {
"post": {
"description": "Step 2 of the upload flow. Takes the storage key returned by POST /api/videos/presign — after the file has been PUT to the presigned URL — submits a MediaConvert transcoding job, and persists the video with its category links. The returned record has status \"processing\" — one of the states \"processing\", \"ready\" or \"failed\" — though nothing currently moves it on once transcoding finishes.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Register an uploaded video",
"parameters": [
{
"description": "Video metadata plus the storage key from the presign step. title, key and categoryIds are required; categoryIds must hold at least one id from GET /api/categories.",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.CompleteRequest"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"$ref": "#/definitions/api.CompleteResponse"
}
},
"400": {
"description": "Request body was not valid JSON",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"404": {
"description": "categoryIds referenced a category that does not exist",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"409": {
"description": "key has already been registered by an earlier request",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"422": {
"description": "title was empty, key was missing or not a key issued by this API, or categoryIds was empty",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "Categories could not be read, the key could not be checked, the transcoding job could not be queued, or the video record could not be saved",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/api/videos/presign": {
"post": {
"description": "Step 1 of the upload flow. Returns a short-lived presigned S3 URL that the client PUTs the video file to directly, plus the storage key identifying it. The submitted contentType is signed into that URL, so the PUT must carry an identical Content-Type header or S3 rejects it as a signature mismatch. Once the PUT succeeds, pass the key to POST /api/videos to register the video and start transcoding. The file itself never passes through this API.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Create a presigned upload URL",
"parameters": [
{
"description": "Name and media type of the file to be uploaded. contentType must be exactly 'video/mp4' or 'video/quicktime'.",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.PresignRequest"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.PresignResponse"
}
},
"400": {
"description": "Request body was not valid JSON",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"422": {
"description": "contentType was not 'video/mp4' or 'video/quicktime'",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "Upload URL could not be issued",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/api/videos/{id}": {
"get": {
"description": "Returns the catalogue record for one video, including the status its transcoding job has reached. This is how a client follows an upload after POST /api/videos: the record starts \"processing\" and moves to \"ready\" or \"failed\" once MediaConvert reports the job finished.",
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Read a video",
"parameters": [
{
"type": "string",
"description": "The video's id, as returned by POST /api/videos",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.VideoResponse"
}
},
"404": {
"description": "No video has that id",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "The video could not be read from the database",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/health": {
"get": {
"description": "Reports that the service is up and serving. Used as the ALB target group health check; it does not verify the database or S3 connections.",
"produces": [
"application/json"
],
"tags": [
"system"
],
"summary": "Health check",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.HealthResponse"
}
}
}
}
}
},
"definitions": {
"api.CategoriesResponse": {
"type": "object",
"properties": {
"categories": {
"type": "array",
"items": {
"$ref": "#/definitions/api.Category"
}
}
}
},
"api.Category": {
"type": "object",
"properties": {
"id": {
"type": "integer",
"example": 1
},
"name": {
"type": "string",
"example": "documentary"
}
}
},
"api.CompleteRequest": {
"type": "object",
"properties": {
"categoryIds": {
"type": "array",
"items": {
"type": "integer"
},
"example": [
1,
2
]
},
"description": {
"type": "string",
"example": "A behind-the-scenes look at the evening bulletin."
},
"fileName": {
"type": "string",
"example": "interview-cut.mov"
},
"key": {
"type": "string",
"example": "videos/a1b2c3d4e5f6.mov"
},
"tags": {
"type": "string",
"example": "media, press, riyadh"
},
"title": {
"type": "string",
"example": "Inside the Newsroom"
}
}
},
"api.CompleteResponse": {
"type": "object",
"properties": {
"categoryIds": {
"type": "array",
"items": {
"type": "integer"
},
"example": [
1,
2
]
},
"createdAt": {
"type": "string"
},
"description": {
"type": "string",
"example": "A behind-the-scenes look at the evening bulletin."
},
"fileName": {
"type": "string",
"example": "interview-cut.mov"
},
"id": {
"type": "string",
"example": "0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f"
},
"mediaConvertJobId": {
"type": "string",
"example": "1755300000000-abcdef"
},
"sizeBytes": {
"type": "integer",
"example": 60
},
"status": {
"type": "string",
"enum": [
"processing",
"ready",
"failed"
],
"example": "processing"
},
"storageKey": {
"type": "string",
"example": "videos/a1b2c3d4e5f6.mov"
},
"tags": {
"type": "string",
"example": "media, press, riyadh"
},
"title": {
"type": "string",
"example": "Inside the Newsroom"
},
"updatedAt": {
"type": "string"
}
}
},
"api.HealthResponse": {
"type": "object",
"properties": {
"status": {
"type": "string",
"example": "ok"
}
}
},
"api.PresignRequest": {
"type": "object",
"properties": {
"contentType": {
"type": "string",
"enum": [
"video/mp4",
"video/quicktime"
],
"example": "video/quicktime"
},
"fileName": {
"type": "string",
"example": "interview-cut.mov"
}
}
},
"api.PresignResponse": {
"type": "object",
"properties": {
"key": {
"type": "string",
"example": "videos/a1b2c3d4e5f6.mov"
},
"uploadUrl": {
"type": "string",
"example": "https://raw-uploads-bucket.s3.amazonaws.com/videos/a1b2....mov?X-Amz-Signature=..."
}
}
},
"api.ProblemDetails": {
"type": "object",
"properties": {
"detail": {
"type": "string",
"example": "The 'title' field was absent or contained only whitespace. Every video needs a non-empty title."
},
"status": {
"type": "integer",
"example": 422
},
"title": {
"type": "string",
"example": "Title is required."
},
"type": {
"type": "string",
"example": "about:blank"
}
}
},
"api.VideoResponse": {
"type": "object",
"properties": {
"categoryIds": {
"type": "array",
"items": {
"type": "integer"
},
"example": [
1,
2
]
},
"createdAt": {
"type": "string"
},
"description": {
"type": "string",
"example": "A behind-the-scenes look at the evening bulletin."
},
"fileName": {
"type": "string",
"example": "interview-cut.mov"
},
"id": {
"type": "string",
"example": "0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f"
},
"mediaConvertJobId": {
"type": "string",
"example": "1755300000000-abcdef"
},
"playbackUrl": {
"description": "PlaybackURL is the HLS master playlist, ready for an m3u8-capable\nplayer. Empty until the transcoding job reports success, and for a job\nthat failed.",
"type": "string",
"example": "https://d111111abcdef8.cloudfront.net/videos/a1b2c3d4e5f6/index.m3u8"
},
"sizeBytes": {
"type": "integer",
"example": 60
},
"status": {
"type": "string",
"enum": [
"processing",
"ready",
"failed"
],
"example": "processing"
},
"storageKey": {
"type": "string",
"example": "videos/a1b2c3d4e5f6.mov"
},
"tags": {
"type": "string",
"example": "media, press, riyadh"
},
"title": {
"type": "string",
"example": "Inside the Newsroom"
},
"updatedAt": {
"type": "string"
}
}
}
}
}`
// SwaggerInfo holds exported Swagger Info so clients can modify it
var SwaggerInfo = &swag.Spec{
Version: "1.0",
Host: "",
BasePath: "/",
Schemes: []string{},
Title: "Thamanyah CMS API",
Description: "JSON API for ingesting videos into the Thamanyah catalogue. Uploads are two-step: presign, PUT the file straight to S3, then register the video to queue transcoding.",
InfoInstanceName: "swagger",
SwaggerTemplate: docTemplate,
LeftDelim: "{{",
RightDelim: "}}",
}
func init() {
swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
}
+465
View File
@@ -0,0 +1,465 @@
{
"swagger": "2.0",
"info": {
"description": "JSON API for ingesting videos into the Thamanyah catalogue. Uploads are two-step: presign, PUT the file straight to S3, then register the video to queue transcoding.",
"title": "Thamanyah CMS API",
"contact": {},
"version": "1.0"
},
"basePath": "/",
"paths": {
"/api/categories": {
"get": {
"description": "Returns the fixed lookup set of categories a video can belong to, each with the id used to reference it. The `id` values from this list are the only ones accepted in the `categoryIds` field of POST /api/videos.",
"produces": [
"application/json"
],
"tags": [
"categories"
],
"summary": "List video categories",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.CategoriesResponse"
}
},
"500": {
"description": "Categories could not be read from the database",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/api/videos": {
"post": {
"description": "Step 2 of the upload flow. Takes the storage key returned by POST /api/videos/presign — after the file has been PUT to the presigned URL — submits a MediaConvert transcoding job, and persists the video with its category links. The returned record has status \"processing\" — one of the states \"processing\", \"ready\" or \"failed\" — though nothing currently moves it on once transcoding finishes.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Register an uploaded video",
"parameters": [
{
"description": "Video metadata plus the storage key from the presign step. title, key and categoryIds are required; categoryIds must hold at least one id from GET /api/categories.",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.CompleteRequest"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"$ref": "#/definitions/api.CompleteResponse"
}
},
"400": {
"description": "Request body was not valid JSON",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"404": {
"description": "categoryIds referenced a category that does not exist",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"409": {
"description": "key has already been registered by an earlier request",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"422": {
"description": "title was empty, key was missing or not a key issued by this API, or categoryIds was empty",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "Categories could not be read, the key could not be checked, the transcoding job could not be queued, or the video record could not be saved",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/api/videos/presign": {
"post": {
"description": "Step 1 of the upload flow. Returns a short-lived presigned S3 URL that the client PUTs the video file to directly, plus the storage key identifying it. The submitted contentType is signed into that URL, so the PUT must carry an identical Content-Type header or S3 rejects it as a signature mismatch. Once the PUT succeeds, pass the key to POST /api/videos to register the video and start transcoding. The file itself never passes through this API.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Create a presigned upload URL",
"parameters": [
{
"description": "Name and media type of the file to be uploaded. contentType must be exactly 'video/mp4' or 'video/quicktime'.",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.PresignRequest"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.PresignResponse"
}
},
"400": {
"description": "Request body was not valid JSON",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"422": {
"description": "contentType was not 'video/mp4' or 'video/quicktime'",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "Upload URL could not be issued",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/api/videos/{id}": {
"get": {
"description": "Returns the catalogue record for one video, including the status its transcoding job has reached. This is how a client follows an upload after POST /api/videos: the record starts \"processing\" and moves to \"ready\" or \"failed\" once MediaConvert reports the job finished.",
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Read a video",
"parameters": [
{
"type": "string",
"description": "The video's id, as returned by POST /api/videos",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.VideoResponse"
}
},
"404": {
"description": "No video has that id",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "The video could not be read from the database",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/health": {
"get": {
"description": "Reports that the service is up and serving. Used as the ALB target group health check; it does not verify the database or S3 connections.",
"produces": [
"application/json"
],
"tags": [
"system"
],
"summary": "Health check",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.HealthResponse"
}
}
}
}
}
},
"definitions": {
"api.CategoriesResponse": {
"type": "object",
"properties": {
"categories": {
"type": "array",
"items": {
"$ref": "#/definitions/api.Category"
}
}
}
},
"api.Category": {
"type": "object",
"properties": {
"id": {
"type": "integer",
"example": 1
},
"name": {
"type": "string",
"example": "documentary"
}
}
},
"api.CompleteRequest": {
"type": "object",
"properties": {
"categoryIds": {
"type": "array",
"items": {
"type": "integer"
},
"example": [
1,
2
]
},
"description": {
"type": "string",
"example": "A behind-the-scenes look at the evening bulletin."
},
"fileName": {
"type": "string",
"example": "interview-cut.mov"
},
"key": {
"type": "string",
"example": "videos/a1b2c3d4e5f6.mov"
},
"tags": {
"type": "string",
"example": "media, press, riyadh"
},
"title": {
"type": "string",
"example": "Inside the Newsroom"
}
}
},
"api.CompleteResponse": {
"type": "object",
"properties": {
"categoryIds": {
"type": "array",
"items": {
"type": "integer"
},
"example": [
1,
2
]
},
"createdAt": {
"type": "string"
},
"description": {
"type": "string",
"example": "A behind-the-scenes look at the evening bulletin."
},
"fileName": {
"type": "string",
"example": "interview-cut.mov"
},
"id": {
"type": "string",
"example": "0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f"
},
"mediaConvertJobId": {
"type": "string",
"example": "1755300000000-abcdef"
},
"sizeBytes": {
"type": "integer",
"example": 60
},
"status": {
"type": "string",
"enum": [
"processing",
"ready",
"failed"
],
"example": "processing"
},
"storageKey": {
"type": "string",
"example": "videos/a1b2c3d4e5f6.mov"
},
"tags": {
"type": "string",
"example": "media, press, riyadh"
},
"title": {
"type": "string",
"example": "Inside the Newsroom"
},
"updatedAt": {
"type": "string"
}
}
},
"api.HealthResponse": {
"type": "object",
"properties": {
"status": {
"type": "string",
"example": "ok"
}
}
},
"api.PresignRequest": {
"type": "object",
"properties": {
"contentType": {
"type": "string",
"enum": [
"video/mp4",
"video/quicktime"
],
"example": "video/quicktime"
},
"fileName": {
"type": "string",
"example": "interview-cut.mov"
}
}
},
"api.PresignResponse": {
"type": "object",
"properties": {
"key": {
"type": "string",
"example": "videos/a1b2c3d4e5f6.mov"
},
"uploadUrl": {
"type": "string",
"example": "https://raw-uploads-bucket.s3.amazonaws.com/videos/a1b2....mov?X-Amz-Signature=..."
}
}
},
"api.ProblemDetails": {
"type": "object",
"properties": {
"detail": {
"type": "string",
"example": "The 'title' field was absent or contained only whitespace. Every video needs a non-empty title."
},
"status": {
"type": "integer",
"example": 422
},
"title": {
"type": "string",
"example": "Title is required."
},
"type": {
"type": "string",
"example": "about:blank"
}
}
},
"api.VideoResponse": {
"type": "object",
"properties": {
"categoryIds": {
"type": "array",
"items": {
"type": "integer"
},
"example": [
1,
2
]
},
"createdAt": {
"type": "string"
},
"description": {
"type": "string",
"example": "A behind-the-scenes look at the evening bulletin."
},
"fileName": {
"type": "string",
"example": "interview-cut.mov"
},
"id": {
"type": "string",
"example": "0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f"
},
"mediaConvertJobId": {
"type": "string",
"example": "1755300000000-abcdef"
},
"playbackUrl": {
"description": "PlaybackURL is the HLS master playlist, ready for an m3u8-capable\nplayer. Empty until the transcoding job reports success, and for a job\nthat failed.",
"type": "string",
"example": "https://d111111abcdef8.cloudfront.net/videos/a1b2c3d4e5f6/index.m3u8"
},
"sizeBytes": {
"type": "integer",
"example": 60
},
"status": {
"type": "string",
"enum": [
"processing",
"ready",
"failed"
],
"example": "processing"
},
"storageKey": {
"type": "string",
"example": "videos/a1b2c3d4e5f6.mov"
},
"tags": {
"type": "string",
"example": "media, press, riyadh"
},
"title": {
"type": "string",
"example": "Inside the Newsroom"
},
"updatedAt": {
"type": "string"
}
}
}
}
}
+347
View File
@@ -0,0 +1,347 @@
basePath: /
definitions:
api.CategoriesResponse:
properties:
categories:
items:
$ref: '#/definitions/api.Category'
type: array
type: object
api.Category:
properties:
id:
example: 1
type: integer
name:
example: documentary
type: string
type: object
api.CompleteRequest:
properties:
categoryIds:
example:
- 1
- 2
items:
type: integer
type: array
description:
example: A behind-the-scenes look at the evening bulletin.
type: string
fileName:
example: interview-cut.mov
type: string
key:
example: videos/a1b2c3d4e5f6.mov
type: string
tags:
example: media, press, riyadh
type: string
title:
example: Inside the Newsroom
type: string
type: object
api.CompleteResponse:
properties:
categoryIds:
example:
- 1
- 2
items:
type: integer
type: array
createdAt:
type: string
description:
example: A behind-the-scenes look at the evening bulletin.
type: string
fileName:
example: interview-cut.mov
type: string
id:
example: 0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f
type: string
mediaConvertJobId:
example: 1755300000000-abcdef
type: string
sizeBytes:
example: 60
type: integer
status:
enum:
- processing
- ready
- failed
example: processing
type: string
storageKey:
example: videos/a1b2c3d4e5f6.mov
type: string
tags:
example: media, press, riyadh
type: string
title:
example: Inside the Newsroom
type: string
updatedAt:
type: string
type: object
api.HealthResponse:
properties:
status:
example: ok
type: string
type: object
api.PresignRequest:
properties:
contentType:
enum:
- video/mp4
- video/quicktime
example: video/quicktime
type: string
fileName:
example: interview-cut.mov
type: string
type: object
api.PresignResponse:
properties:
key:
example: videos/a1b2c3d4e5f6.mov
type: string
uploadUrl:
example: https://raw-uploads-bucket.s3.amazonaws.com/videos/a1b2....mov?X-Amz-Signature=...
type: string
type: object
api.ProblemDetails:
properties:
detail:
example: The 'title' field was absent or contained only whitespace. Every
video needs a non-empty title.
type: string
status:
example: 422
type: integer
title:
example: Title is required.
type: string
type:
example: about:blank
type: string
type: object
api.VideoResponse:
properties:
categoryIds:
example:
- 1
- 2
items:
type: integer
type: array
createdAt:
type: string
description:
example: A behind-the-scenes look at the evening bulletin.
type: string
fileName:
example: interview-cut.mov
type: string
id:
example: 0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f
type: string
mediaConvertJobId:
example: 1755300000000-abcdef
type: string
playbackUrl:
description: |-
PlaybackURL is the HLS master playlist, ready for an m3u8-capable
player. Empty until the transcoding job reports success, and for a job
that failed.
example: https://d111111abcdef8.cloudfront.net/videos/a1b2c3d4e5f6/index.m3u8
type: string
sizeBytes:
example: 60
type: integer
status:
enum:
- processing
- ready
- failed
example: processing
type: string
storageKey:
example: videos/a1b2c3d4e5f6.mov
type: string
tags:
example: media, press, riyadh
type: string
title:
example: Inside the Newsroom
type: string
updatedAt:
type: string
type: object
info:
contact: {}
description: 'JSON API for ingesting videos into the Thamanyah catalogue. Uploads
are two-step: presign, PUT the file straight to S3, then register the video to
queue transcoding.'
title: Thamanyah CMS API
version: "1.0"
paths:
/api/categories:
get:
description: Returns the fixed lookup set of categories a video can belong to,
each with the id used to reference it. The `id` values from this list are
the only ones accepted in the `categoryIds` field of POST /api/videos.
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/api.CategoriesResponse'
"500":
description: Categories could not be read from the database
schema:
$ref: '#/definitions/api.ProblemDetails'
summary: List video categories
tags:
- categories
/api/videos:
post:
consumes:
- application/json
description: Step 2 of the upload flow. Takes the storage key returned by POST
/api/videos/presign — after the file has been PUT to the presigned URL — submits
a MediaConvert transcoding job, and persists the video with its category links.
The returned record has status "processing" — one of the states "processing",
"ready" or "failed" — though nothing currently moves it on once transcoding
finishes.
parameters:
- description: Video metadata plus the storage key from the presign step. title,
key and categoryIds are required; categoryIds must hold at least one id
from GET /api/categories.
in: body
name: request
required: true
schema:
$ref: '#/definitions/api.CompleteRequest'
produces:
- application/json
responses:
"201":
description: Created
schema:
$ref: '#/definitions/api.CompleteResponse'
"400":
description: Request body was not valid JSON
schema:
$ref: '#/definitions/api.ProblemDetails'
"404":
description: categoryIds referenced a category that does not exist
schema:
$ref: '#/definitions/api.ProblemDetails'
"409":
description: key has already been registered by an earlier request
schema:
$ref: '#/definitions/api.ProblemDetails'
"422":
description: title was empty, key was missing or not a key issued by this
API, or categoryIds was empty
schema:
$ref: '#/definitions/api.ProblemDetails'
"500":
description: Categories could not be read, the key could not be checked,
the transcoding job could not be queued, or the video record could not
be saved
schema:
$ref: '#/definitions/api.ProblemDetails'
summary: Register an uploaded video
tags:
- videos
/api/videos/{id}:
get:
description: 'Returns the catalogue record for one video, including the status
its transcoding job has reached. This is how a client follows an upload after
POST /api/videos: the record starts "processing" and moves to "ready" or "failed"
once MediaConvert reports the job finished.'
parameters:
- description: The video's id, as returned by POST /api/videos
in: path
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/api.VideoResponse'
"404":
description: No video has that id
schema:
$ref: '#/definitions/api.ProblemDetails'
"500":
description: The video could not be read from the database
schema:
$ref: '#/definitions/api.ProblemDetails'
summary: Read a video
tags:
- videos
/api/videos/presign:
post:
consumes:
- application/json
description: Step 1 of the upload flow. Returns a short-lived presigned S3 URL
that the client PUTs the video file to directly, plus the storage key identifying
it. The submitted contentType is signed into that URL, so the PUT must carry
an identical Content-Type header or S3 rejects it as a signature mismatch.
Once the PUT succeeds, pass the key to POST /api/videos to register the video
and start transcoding. The file itself never passes through this API.
parameters:
- description: Name and media type of the file to be uploaded. contentType must
be exactly 'video/mp4' or 'video/quicktime'.
in: body
name: request
required: true
schema:
$ref: '#/definitions/api.PresignRequest'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/api.PresignResponse'
"400":
description: Request body was not valid JSON
schema:
$ref: '#/definitions/api.ProblemDetails'
"422":
description: contentType was not 'video/mp4' or 'video/quicktime'
schema:
$ref: '#/definitions/api.ProblemDetails'
"500":
description: Upload URL could not be issued
schema:
$ref: '#/definitions/api.ProblemDetails'
summary: Create a presigned upload URL
tags:
- videos
/health:
get:
description: Reports that the service is up and serving. Used as the ALB target
group health check; it does not verify the database or S3 connections.
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/api.HealthResponse'
summary: Health check
tags:
- system
swagger: "2.0"
+48
View File
@@ -0,0 +1,48 @@
module thamanyah/cms/v2
go 1.25.12
require (
github.com/aws/aws-sdk-go-v2 v1.45.1
github.com/aws/aws-sdk-go-v2/config v1.32.36
github.com/aws/aws-sdk-go-v2/service/mediaconvert v1.97.2
github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1
github.com/aws/aws-sdk-go-v2/service/sqs v1.47.0
github.com/golang-migrate/migrate/v4 v4.19.1
github.com/google/uuid v1.6.0
github.com/lib/pq v1.12.3
github.com/swaggo/http-swagger/v2 v2.0.2
github.com/swaggo/swag v1.16.6
)
require (
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.35 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.5.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.44.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.33.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.45.5 // indirect
github.com/aws/smithy-go v1.28.1 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/spec v0.20.6 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/swaggo/files/v2 v2.0.0 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/tools v0.38.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
+167
View File
@@ -0,0 +1,167 @@
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/aws/aws-sdk-go-v2 v1.44.0 h1:4IbaHhtzy+4h37z4JQyO9a2QsiCml3CNYHtq5hIHigo=
github.com/aws/aws-sdk-go-v2 v1.44.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
github.com/aws/aws-sdk-go-v2 v1.45.1 h1:iIoG3NaLhV6UZpPXyPXlDj2I9oS8tV/nMcMnITCC6Ks=
github.com/aws/aws-sdk-go-v2 v1.45.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17 h1:mn+Vxb9zgz/FE/yDTcFim3DZ1qpcrxR+qBQkBrl6bzA=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17/go.mod h1:eDfmEFxu+BSVsUGLbzJhWjpOurv1mqczClS97yI8wdk=
github.com/aws/aws-sdk-go-v2/config v1.32.36 h1:mX6ietU7UlB4w/2IUaexJdsyUDvhTd+jYPjVePiyi6s=
github.com/aws/aws-sdk-go-v2/config v1.32.36/go.mod h1:rMpV4xk7ZK59edraSaHP0jsWrztWTT5tbCwWY495hug=
github.com/aws/aws-sdk-go-v2/credentials v1.19.35 h1:Cxua2RVdRwL0sfjHM/SnQoOnQ7xKng9m5EQBO8BnZlg=
github.com/aws/aws-sdk-go-v2/credentials v1.19.35/go.mod h1:9XQ+RSIGPkycr+oCJYnB1uTv5kMVVR+rd2vYK0Hxj2w=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36 h1:gucL1KH/PAYbpTpBg09CiVpBdTu4qkCl8C7xOTBixUg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36/go.mod h1:usTB+PHhNMhrx2dxUeHcM7OrT5pySvmjYI++IsefPN0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 h1:UIXlbijuB2XK1Kr57fo8iIxCuaSHJzwZ1uo+2tbEYIk=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40/go.mod h1:wcEsL6jscjZjVUinb0Q5qD/GXOG1yT3GNfmT9HuDwzU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 h1:pc138gM1CW+XPc60rEwUlwwuwWFQK16CI1T7v1F9Oec=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1/go.mod h1:1+koxpPIbfBdfzP6vojm5/zTpTQ/micYwlxIiNB3TxI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 h1:xLQVRDs2NddDmK9BEyh5KSlJ1Gpy5/GIJXrV6WcVGAE=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40/go.mod h1:XRXnpFVFGLaEVK+olDdFIM1vNa04ETW452oFGEPUxAo=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 h1:K0JsbZQj+1h208Ro1zHeA4l7bMp0NvRffHQ91q8Ol1s=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1/go.mod h1:W3/vL6EtCIatICGy9ab29QhMuae+cOKPWcMxv02CO+Q=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37 h1:oyd3ke4V9AhKcRR7rRgxk1VyI+DjK2CBQtbxh3OkdaA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37/go.mod h1:aA9D7SqfG9IC1b7FLD7Iyc8Q4JN0a8gHhNjN4zPlIaI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16 h1:iE4NGbvqUZnHDqddQAauZzCILYtFjOHwRM5MOOKLB5A=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16/go.mod h1:VsjEgrP+ibcou8TlWA4tYaB+0OojuhirsmCe+U60hTA=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29 h1:E65Hj648dOV6FuUfI0mYXXhQRHbsi7n+B9h6fZPJO/E=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29/go.mod h1:xLrF9yNTCs92VZSpdEd68EJbgcdw3SMR74RO6QDzWHE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36 h1:fx2ujmozWn+C/GtfXfz5k6Ckzza40ElOpIW7d92fLWQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36/go.mod h1:QT2ufGVJ+xTRxtXPHTQ1kHkAdWIKPCmD+BqYAXWv8/4=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37 h1:KGHa9iZCrgtkOsFfXb0S4ywsjostA/hau7WE9aSb43E=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37/go.mod h1:FV79f0DSnZIEGsQjWenENGtUycrasyAaJZO+zRanLHA=
github.com/aws/aws-sdk-go-v2/service/mediaconvert v1.97.2 h1:0LYlFn1lcMqwiBUJHqe6CRzpZlNKllF97mwJxvU84V0=
github.com/aws/aws-sdk-go-v2/service/mediaconvert v1.97.2/go.mod h1:GUTvcSEX0M8Y8F9Om+IFp9oTrMRuJNN9XGKZP/OWqOU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1 h1:VUTtUJMuRNMkb/7NIKmd8NQaeQLPGCMoTJxkYKre4qM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1/go.mod h1:WvUaO0lP5GNMs1R6cs6qvB3mqo16GLta8yfOuf55Rpc=
github.com/aws/aws-sdk-go-v2/service/signin v1.5.5 h1:0VTFBfOgPJrUSpGMgzoi8qLcXF5dbmiBuxpo14eBWUw=
github.com/aws/aws-sdk-go-v2/service/signin v1.5.5/go.mod h1:sNZYlBxoohYMBYl47BO/bFtAM6I8HSsPa1qwwPPRGoQ=
github.com/aws/aws-sdk-go-v2/service/sns v1.44.1 h1:dMIcbUQ8fPJPbX9jZV19JtL2lCAgEh1LUNlpe6sdgqE=
github.com/aws/aws-sdk-go-v2/service/sns v1.44.1/go.mod h1:ucBUMGW8avqGmbyQoXyoC6Cgt+WsNBrhL9DA4Bb+jN4=
github.com/aws/aws-sdk-go-v2/service/sqs v1.47.0 h1:vNsYthHgT4sUo0KVqpkZlz+8ZDqy/MdlqdvZdP6IoAc=
github.com/aws/aws-sdk-go-v2/service/sqs v1.47.0/go.mod h1:FSB4mnod1TCBhs3vp2tWVVGHbqxluzA0Fo6LBOXZByw=
github.com/aws/aws-sdk-go-v2/service/sso v1.33.5 h1:jDQARFp1mJ2PEnllQf01nfFXGfWMJ59e0/HCHUTTZCk=
github.com/aws/aws-sdk-go-v2/service/sso v1.33.5/go.mod h1:OcT2AhgTuxGAwZk5hgxaNLGpS33W8s8dUQadGVDVY9I=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5 h1:8xo1q9ttkYqMJ6vOXX67FPSpVEI7BWKVTKh77g82w+8=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5/go.mod h1:hbBeEUrZg6VddXYZpbKPyF0tl4XEnM+Dbx92RW3vmZI=
github.com/aws/aws-sdk-go-v2/service/sts v1.45.5 h1:eQ5BtXDrPg2wK0AjtVPzeBhUpYPeqHE/ptiH7xJRGek=
github.com/aws/aws-sdk-go-v2/service/sts v1.45.5/go.mod h1:f9ImhnOISY7BuTZLM8qHepCYnglHBVLk5wVzatmP++w=
github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ=
github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
github.com/swaggo/http-swagger/v2 v2.0.2 h1:FKCdLsl+sFCx60KFsyM0rDarwiUSZ8DqbfSyIKC9OBg=
github.com/swaggo/http-swagger/v2 v2.0.2/go.mod h1:r7/GBkAWIfK6E/OLnE8fXnviHiDeAHmgIyooa4xm3AQ=
github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+23
View File
@@ -0,0 +1,23 @@
// Package api holds the request and response bodies of the HTTP API — the
// wire contract, kept apart from the handlers that serve it. The swaggo
// annotations on the handlers reference these types by name (api.CompleteResponse
// and so on), so renaming one changes the generated spec.
package api
// HealthResponse is the body of GET /health.
type HealthResponse struct {
Status string `json:"status" example:"ok"`
}
// ProblemDetails is an error body in the RFC 9457 "Problem Details for HTTP
// APIs" format. Type stays "about:blank" — the value RFC 9457 defines for
// problems with no dedicated documentation URI. Title is a short summary that
// stays identical for every occurrence of the same problem, so clients can
// branch on it; Detail explains this particular occurrence and is the only
// member that varies with request data.
type ProblemDetails struct {
Type string `json:"type" example:"about:blank"`
Title string `json:"title" example:"Title is required."`
Status int `json:"status" example:"422"`
Detail string `json:"detail,omitempty" example:"The 'title' field was absent or contained only whitespace. Every video needs a non-empty title."`
}
+80
View File
@@ -0,0 +1,80 @@
package api
import "time"
// Category is one entry of the fixed category lookup table. Its ID is what
// clients send back in CompleteRequest.CategoryIDs.
type Category struct {
ID int16 `json:"id" example:"1"`
Name string `json:"name" example:"documentary"`
}
// CategoriesResponse is the body of GET /api/categories.
type CategoriesResponse struct {
Categories []Category `json:"categories"`
}
// PresignRequest is the body of POST /api/videos/presign.
type PresignRequest struct {
FileName string `json:"fileName" example:"interview-cut.mov"`
ContentType string `json:"contentType" enums:"video/mp4,video/quicktime" example:"video/quicktime"`
}
// PresignResponse is the body returned by POST /api/videos/presign.
type PresignResponse struct {
UploadURL string `json:"uploadUrl" example:"https://raw-uploads-bucket.s3.amazonaws.com/videos/a1b2....mov?X-Amz-Signature=..."`
Key string `json:"key" example:"videos/a1b2c3d4e5f6.mov"`
}
// CompleteRequest is the body of POST /api/videos. CategoryIDs holds ids from
// GET /api/categories, not category names, and must carry at least one — a
// video with no category is rejected.
type CompleteRequest struct {
Title string `json:"title" example:"Inside the Newsroom"`
Description string `json:"description" example:"A behind-the-scenes look at the evening bulletin."`
CategoryIDs []int16 `json:"categoryIds" example:"1,2"`
Tags string `json:"tags" example:"media, press, riyadh"`
FileName string `json:"fileName" example:"interview-cut.mov"`
Key string `json:"key" example:"videos/a1b2c3d4e5f6.mov"`
}
// CompleteResponse is a persisted video record as returned by POST /api/videos.
// Status carries the values of services.VideoStatus; the enums tag has to be
// widened by hand whenever that type gains a state.
type CompleteResponse struct {
ID string `json:"id" example:"0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f"`
Title string `json:"title" example:"Inside the Newsroom"`
Description string `json:"description" example:"A behind-the-scenes look at the evening bulletin."`
CategoryIDs []int16 `json:"categoryIds" example:"1,2"`
Tags string `json:"tags" example:"media, press, riyadh"`
FileName string `json:"fileName" example:"interview-cut.mov"`
StorageKey string `json:"storageKey" example:"videos/a1b2c3d4e5f6.mov"`
MediaConvertJobID string `json:"mediaConvertJobId" example:"1755300000000-abcdef"`
Status string `json:"status" enums:"processing,ready,failed" example:"processing"`
SizeBytes int64 `json:"sizeBytes" example:"60"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
// VideoResponse is a persisted video record as returned by
// GET /api/videos/{id}. It carries the same members as CompleteResponse —
// deliberately a separate type, so the shape POST publishes and the shape GET
// publishes can diverge without one silently dragging the other with it.
type VideoResponse struct {
ID string `json:"id" example:"0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f"`
Title string `json:"title" example:"Inside the Newsroom"`
Description string `json:"description" example:"A behind-the-scenes look at the evening bulletin."`
CategoryIDs []int16 `json:"categoryIds" example:"1,2"`
Tags string `json:"tags" example:"media, press, riyadh"`
FileName string `json:"fileName" example:"interview-cut.mov"`
StorageKey string `json:"storageKey" example:"videos/a1b2c3d4e5f6.mov"`
MediaConvertJobID string `json:"mediaConvertJobId" example:"1755300000000-abcdef"`
Status string `json:"status" enums:"processing,ready,failed" example:"processing"`
// PlaybackURL is the HLS master playlist, ready for an m3u8-capable
// player. Empty until the transcoding job reports success, and for a job
// that failed.
PlaybackURL string `json:"playbackUrl" example:"https://d111111abcdef8.cloudfront.net/videos/a1b2c3d4e5f6/index.m3u8"`
SizeBytes int64 `json:"sizeBytes" example:"60"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
+230
View File
@@ -0,0 +1,230 @@
// Package consumers holds the queue-driven half of the service: work that
// arrives on a queue rather than as an HTTP request. It is to SQS what
// internal/handlers is to HTTP — it decodes a message, calls repositories, and
// decides what the message's fate is — so the same seams apply, and no SQL
// lives here.
package consumers
import (
"context"
"encoding/json"
"errors"
"log"
"strings"
"thamanyah/cms/v2/internal/db/repositories"
"thamanyah/cms/v2/internal/models"
"thamanyah/cms/v2/internal/services"
"time"
)
// receiveBackoff is how long the loop waits after a failed receive, so a queue
// that is unreachable produces a slow trickle of log lines rather than a spin.
const receiveBackoff = 5 * time.Second
// jobStateChange is the EventBridge event MediaConvert emits on every job
// state transition. The subscription delivers it raw, so this is the whole
// message body — there is no SNS envelope to unwrap. Only the members this
// service acts on are modelled; AWS sends a good deal more.
type jobStateChange struct {
Source string `json:"source"`
DetailType string `json:"detail-type"`
Detail struct {
JobID string `json:"jobId"`
Status string `json:"status"`
// Present on COMPLETE only, and only for output groups that produced
// something. It is how the service learns where the output landed
// rather than having to guess the path back from the job settings.
OutputGroupDetails []outputGroup `json:"outputGroupDetails"`
} `json:"detail"`
}
// outputGroup is one output group's result. An HLS group lists the manifests
// it wrote under playlistFilePaths, master first; the segments are not listed.
type outputGroup struct {
Type string `json:"type"`
PlaylistFilePaths []string `json:"playlistFilePaths"`
}
// MediaConvertEvents consumes MediaConvert job state changes.
type MediaConvertEvents struct {
// PlaybackBaseURL is the public host the encoded output is served from —
// the CloudFront distribution in front of the output bucket. MediaConvert
// reports s3:// paths into a bucket that blocks all public access, so a
// path is only usable to a player once it has been rewritten onto this.
PlaybackBaseURL string
}
// hlsPlaylist picks the master playlist out of a finished job's output groups.
// It returns "" when the job produced no HLS group, which is not an error: a
// job configured with only a file group legitimately has no playlist.
func hlsPlaylist(groups []outputGroup) string {
for _, group := range groups {
for _, playlist := range group.PlaylistFilePaths {
if strings.HasSuffix(playlist, ".m3u8") {
return playlist
}
}
}
return ""
}
// playbackURL rewrites an s3://bucket/key path onto the public delivery host.
// The bucket is dropped rather than checked: the job writes to exactly one
// output bucket, and that bucket is what PlaybackBaseURL fronts.
func (c MediaConvertEvents) playbackURL(s3URI string) (string, bool) {
const scheme = "s3://"
if !strings.HasPrefix(s3URI, scheme) {
return "", false
}
_, key, found := strings.Cut(strings.TrimPrefix(s3URI, scheme), "/")
if !found || key == "" {
return "", false
}
return strings.TrimRight(c.PlaybackBaseURL, "/") + "/" + key, true
}
// statusForJobState maps MediaConvert's job states onto the catalogue's. The
// second return is false for states that are not an outcome — the job is still
// running, and the record should stay where it is rather than being rewritten
// with what it already says.
func statusForJobState(state string) (models.VideoStatus, bool) {
switch state {
case "COMPLETE":
return models.VideoStatusReady, true
case "ERROR", "CANCELED":
return models.VideoStatusFailed, true
default:
return "", false
}
}
// Run consumes job state changes until ctx is cancelled. It is meant to be run
// in its own goroutine for the lifetime of the process.
func (c MediaConvertEvents) Run(ctx context.Context) {
log.Println("mediaconvert job events consumer started")
for {
if ctx.Err() != nil {
log.Println("mediaconvert job events consumer stopped")
return
}
messages, err := services.SQSClient.ReceiveMessages(ctx)
if err != nil {
if ctx.Err() != nil {
log.Println("mediaconvert job events consumer stopped")
return
}
log.Printf("Something Went Wrong Receiving Job Events: %s", err)
time.Sleep(receiveBackoff)
continue
}
for _, message := range messages {
if !c.handleJobEvent(ctx, message.Body) {
// Left on the queue on purpose: it becomes visible again when
// the visibility timeout expires, and reaches the dead-letter
// queue if it keeps failing.
continue
}
if err := services.SQSClient.DeleteMessage(ctx, message.ReceiptHandle); err != nil {
log.Printf("Something Went Wrong Acknowledging A Job Event: %s", err)
}
}
}
}
// handleJobEvent processes one message and reports whether it is finished
// with — that is, whether it should be deleted from the queue. It returns
// false only when a retry could plausibly succeed. Anything a retry cannot fix
// (a body that will never parse, an event for a job nobody has) is finished
// with, however little it accomplished: leaving it on the queue would only
// stall the events behind it.
func (c MediaConvertEvents) handleJobEvent(ctx context.Context, body string) bool {
var event jobStateChange
if err := json.Unmarshal([]byte(body), &event); err != nil {
log.Printf("Discarding A Job Event That Is Not JSON: %s", err)
return true
}
if event.Source != "aws.mediaconvert" || event.DetailType != "MediaConvert Job State Change" {
log.Printf("Discarding An Event That Is Not A MediaConvert Job State Change: source=%q detail-type=%q",
event.Source, event.DetailType)
return true
}
if event.Detail.JobID == "" {
log.Printf("Discarding A Job Event That Names No Job")
return true
}
status, isOutcome := statusForJobState(event.Detail.Status)
if !isOutcome {
// SUBMITTED, PROGRESSING and STATUS_UPDATE say the job is still
// running, which is what "processing" already records.
return true
}
// Only a finished job has output. A job that failed is recorded with no
// playback URL, which also clears one from an earlier attempt.
playbackURL := ""
if status == models.VideoStatusReady {
if playlist := hlsPlaylist(event.Detail.OutputGroupDetails); playlist != "" {
rewritten, ok := c.playbackURL(playlist)
if !ok {
log.Printf("Could Not Turn A Playlist Path Into A Playback URL: job=%q path=%q",
event.Detail.JobID, playlist)
}
playbackURL = rewritten
} else {
// Worth knowing about: the video is playable in principle but the
// catalogue has nothing to point a player at.
log.Printf("A Finished Job Reported No HLS Playlist: job=%q", event.Detail.JobID)
}
}
video, err := repositories.VideoRepo.UpdateVideoOutcomeByJobID(ctx, event.Detail.JobID, status, playbackURL)
if errors.Is(err, repositories.ErrVideoNotFound) {
// Not a fault: the topic carries every job in the account, including
// ones this service never submitted.
log.Printf("Ignoring A Job Event For A Job No Video Has: job=%q status=%q",
event.Detail.JobID, event.Detail.Status)
return true
}
if err != nil {
log.Printf("Something Went Wrong Recording A Job Outcome: job=%q status=%q: %s",
event.Detail.JobID, event.Detail.Status, err)
return false
}
log.Printf("Recorded A Job Outcome: job=%q status=%q playback=%q", event.Detail.JobID, status, playbackURL)
// Only a video that came out ready is worth showing, so only that one is
// announced. A failed job has already had its status recorded above, and
// the catalogue has no use for it.
if status != models.VideoStatusReady {
return true
}
if err := services.CatalogueClient.AnnounceVideo(ctx, services.CatalogueVideo{
VideoID: video.ID,
Title: video.Title,
PlaybackURL: video.PlaybackURL,
Categories: video.CategoryNames,
}); err != nil {
// Worth another attempt: the outcome is already recorded, and the
// update is idempotent, so a redelivery re-runs it and tries the
// announcement again. The cost is that a consumer can see the same
// video twice, which is what at-least-once delivery means anyway.
log.Printf("Something Went Wrong Announcing A Ready Video: video=%q job=%q: %s",
video.ID, event.Detail.JobID, err)
return false
}
log.Printf("Announced A Ready Video: video=%q playback=%q", video.ID, video.PlaybackURL)
return true
}
+30
View File
@@ -0,0 +1,30 @@
package db
import (
"context"
"database/sql"
"fmt"
"log"
)
func CreateDBConnection(connectionString string) *sql.DB {
sqlDB, err := sql.Open("postgres", connectionString)
if err != nil {
panic(fmt.Errorf("db: cannot open connection: %w", err))
}
return sqlDB
}
func CloseConnection(SQLDB *sql.DB) {
err := SQLDB.Close()
if err != nil {
log.Fatalf("Error Closing DB Connection: %s", err)
}
}
func AssertSuccessfulConnection(ctx context.Context, SQLDB *sql.DB) {
if err := SQLDB.PingContext(ctx); err != nil {
panic(fmt.Errorf("db: cannot connect: %w", err))
}
}
+38
View File
@@ -0,0 +1,38 @@
package db
import (
"database/sql"
"embed"
"errors"
"fmt"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
"github.com/golang-migrate/migrate/v4/source/iofs"
)
//go:embed migrations/*.sql
var migrationsFS embed.FS
func Migrate(sqlDB *sql.DB) error {
driver, err := postgres.WithInstance(sqlDB, &postgres.Config{})
if err != nil {
return fmt.Errorf("db: creating migration driver: %w", err)
}
source, err := iofs.New(migrationsFS, "migrations")
if err != nil {
return fmt.Errorf("db: loading embedded migrations: %w", err)
}
m, err := migrate.NewWithInstance("iofs", source, "postgres", driver)
if err != nil {
return fmt.Errorf("db: initializing migrator: %w", err)
}
if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return fmt.Errorf("db: applying migrations: %w", err)
}
return nil
}
@@ -0,0 +1,2 @@
DROP TABLE videos;
DROP TABLE categories;
@@ -0,0 +1,26 @@
CREATE TABLE categories (
id SMALLSERIAL PRIMARY KEY,
name TEXT NOT NULL UNIQUE
);
INSERT INTO categories (name) VALUES
('documentary'),
('news'),
('entertainment'),
('podcast'),
('other');
CREATE TABLE videos (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
title TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
category_id SMALLINT NOT NULL REFERENCES categories(id),
tags TEXT NOT NULL DEFAULT '',
file_name TEXT NOT NULL,
storage_key TEXT NOT NULL UNIQUE,
mediaconvert_job_id TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'processing',
size_bytes BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
@@ -0,0 +1,3 @@
DROP TABLE video_categories;
ALTER TABLE videos ADD COLUMN category_id SMALLINT REFERENCES categories(id);
@@ -0,0 +1,7 @@
ALTER TABLE videos DROP COLUMN category_id;
CREATE TABLE video_categories (
video_id UUID NOT NULL REFERENCES videos(id) ON DELETE CASCADE,
category_id SMALLINT NOT NULL REFERENCES categories(id),
PRIMARY KEY (video_id, category_id)
);
@@ -0,0 +1 @@
DROP INDEX videos_mediaconvert_job_id_idx;
@@ -0,0 +1,4 @@
-- Job state changes arrive keyed by MediaConvert's job id, so every event the
-- consumer handles is a lookup on this column. Nothing indexed it before: the
-- column was only ever written, never searched.
CREATE INDEX videos_mediaconvert_job_id_idx ON videos (mediaconvert_job_id);
@@ -0,0 +1 @@
ALTER TABLE videos DROP COLUMN playback_url;
@@ -0,0 +1,5 @@
-- Where the finished HLS playlist is served from, written by the job events
-- consumer when MediaConvert reports the job complete. Empty until then, and
-- for videos whose job failed — so it is DEFAULT '' rather than nullable: the
-- absence of a playback URL is "not ready yet", not "unknown".
ALTER TABLE videos ADD COLUMN playback_url TEXT NOT NULL DEFAULT '';
@@ -0,0 +1,49 @@
package repositories
import (
"context"
"database/sql"
"thamanyah/cms/v2/internal/models"
)
type CatagoriesRepository struct {
SQLDB *sql.DB
}
var CatagoriesRepo CatagoriesRepository
func (svc CatagoriesRepository) ListCategories(ctx context.Context) ([]models.Category, error) {
rows, err := svc.SQLDB.QueryContext(ctx, `SELECT id, name FROM categories ORDER BY name`)
if err != nil {
return nil, err
}
defer rows.Close()
var categories []models.Category
for rows.Next() {
var category models.Category
if err := rows.Scan(&category.ID, &category.Name); err != nil {
return nil, err
}
categories = append(categories, category)
}
return categories, rows.Err()
}
func (svc CatagoriesRepository) ListCategoriesIDs(ctx context.Context) ([]int16, error) {
rows, err := svc.SQLDB.QueryContext(ctx, `SELECT id FROM categories`)
if err != nil {
return nil, err
}
defer rows.Close()
var categories []int16
for rows.Next() {
var category int16
if err := rows.Scan(&category); err != nil {
return nil, err
}
categories = append(categories, category)
}
return categories, rows.Err()
}
+9
View File
@@ -0,0 +1,9 @@
package repositories
import "errors"
// Sentinel errors the repositories return so handlers can tell "there is no
// such row" from "the query failed", without importing database/sql to do it.
// ErrVideoNotFound reports that no video has the requested id.
var ErrVideoNotFound = errors.New("video not found")
+152
View File
@@ -0,0 +1,152 @@
package repositories
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"thamanyah/cms/v2/internal/models"
)
type VideoRepository struct {
SQLDB *sql.DB
}
var VideoRepo VideoRepository
// UpdateVideoOutcomeByJobID records what became of a transcoding job: the
// status it ended in, and the playback URL its output is served from. A job
// that failed has no output, so playbackURL is empty for those and the column
// is cleared along with the status. It returns ErrVideoNotFound when no video
// carries that job id, which the consumer treats as a message to drop rather
// than a fault.
//
// It returns the video it updated, carrying the members an announcement is
// built from — the row's id and title, and the names of the categories it is
// filed under. The consumer would otherwise have to read back what it just
// wrote, and reading it here keeps that SQL on this side of the seam.
func (svc VideoRepository) UpdateVideoOutcomeByJobID(ctx context.Context, jobID string, status models.VideoStatus, playbackURL string) (models.Video, error) {
updated := models.Video{
Status: status,
PlaybackURL: playbackURL,
MediaConvertJobID: jobID,
}
err := svc.SQLDB.QueryRowContext(ctx, `
UPDATE videos
SET status = $1, playback_url = $2, updated_at = now()
WHERE mediaconvert_job_id = $3
RETURNING id, title
`, string(status), playbackURL, jobID).Scan(&updated.ID, &updated.Title)
if errors.Is(err, sql.ErrNoRows) {
return models.Video{}, ErrVideoNotFound
}
if err != nil {
return models.Video{}, err
}
rows, err := svc.SQLDB.QueryContext(ctx, `
SELECT c.name
FROM video_categories vc
JOIN categories c ON c.id = vc.category_id
WHERE vc.video_id = $1
ORDER BY c.name
`, updated.ID)
if err != nil {
return models.Video{}, err
}
defer rows.Close()
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return models.Video{}, err
}
updated.CategoryNames = append(updated.CategoryNames, name)
}
return updated, rows.Err()
}
// GetVideoByID reads one video and the ids of the categories it is filed
// under. It returns ErrVideoNotFound when no video has that id.
func (svc VideoRepository) GetVideoByID(ctx context.Context, id string) (models.Video, error) {
var v models.Video
err := svc.SQLDB.QueryRowContext(ctx, `
SELECT id, title, description, tags, file_name, storage_key, mediaconvert_job_id, status, playback_url, size_bytes, created_at, updated_at
FROM videos
WHERE id = $1
`, id).Scan(&v.ID, &v.Title, &v.Description, &v.Tags, &v.FileName, &v.StorageKey,
&v.MediaConvertJobID, &v.Status, &v.PlaybackURL, &v.SizeBytes, &v.CreatedAt, &v.UpdatedAt)
if errors.Is(err, sql.ErrNoRows) {
return models.Video{}, ErrVideoNotFound
}
if err != nil {
return models.Video{}, err
}
rows, err := svc.SQLDB.QueryContext(ctx,
`SELECT category_id FROM video_categories WHERE video_id = $1 ORDER BY category_id`, v.ID)
if err != nil {
return models.Video{}, err
}
defer rows.Close()
for rows.Next() {
var categoryID int16
if err := rows.Scan(&categoryID); err != nil {
return models.Video{}, err
}
v.CategoryIDs = append(v.CategoryIDs, categoryID)
}
return v, rows.Err()
}
// VideoExistsWithStorageKey reports whether a video has already been
// registered under a storage key. It is served by videos_storage_key_key, the
// index Postgres builds for the column's UNIQUE constraint.
func (svc VideoRepository) VideoExistsWithStorageKey(ctx context.Context, storageKey string) (bool, error) {
var exists bool
err := svc.SQLDB.QueryRowContext(ctx,
`SELECT EXISTS (SELECT 1 FROM videos WHERE storage_key = $1)`, storageKey).Scan(&exists)
if err != nil {
return false, err
}
return exists, nil
}
func (svc VideoRepository) CreateVideo(ctx context.Context, v models.Video) (models.Video, error) {
tx, err := svc.SQLDB.BeginTx(ctx, nil)
if err != nil {
return models.Video{}, err
}
defer tx.Rollback()
row := tx.QueryRowContext(ctx, `
INSERT INTO videos (title, description, tags, file_name, storage_key, mediaconvert_job_id, status, size_bytes)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING id, created_at, updated_at
`, v.Title, v.Description, v.Tags, v.FileName, v.StorageKey, v.MediaConvertJobID, string(v.Status), v.SizeBytes)
if err := row.Scan(&v.ID, &v.CreatedAt, &v.UpdatedAt); err != nil {
return models.Video{}, err
}
rows := make([]string, 0, len(v.CategoryIDs))
for _, categoryID := range v.CategoryIDs {
rows = append(rows, fmt.Sprintf("($1, %d)", categoryID))
}
if _, err := tx.ExecContext(ctx, `INSERT INTO video_categories (video_id, category_id) VALUES `+strings.Join(rows, ", "), v.ID); err != nil {
return models.Video{}, fmt.Errorf("linking categories %v: %w", v.CategoryIDs, err)
}
if err := tx.Commit(); err != nil {
return models.Video{}, err
}
return v, nil
}
+16
View File
@@ -0,0 +1,16 @@
package handlers
import "errors"
// Sentinel errors the handlers branch on with errors.Is to decide which
// problem response a failure deserves. They exist to separate a client's
// mistake from a server fault where the code that detects both is the same
// code — the repositories return plain errors, so without these every failure
// out of a lookup would have to be answered as a 500.
// errUnknownCatagoryID reports that a submitted category id is not in the
// category table. It is kept distinct from the error the lookup itself
// returns because the two are not the same kind of failure: an id nobody has
// is the client's mistake and answers 404, while a lookup that could not run
// is the server's and answers 500.
var errUnknownCatagoryID = errors.New("unknown catagory id")
+41
View File
@@ -0,0 +1,41 @@
package handlers
import (
"encoding/json"
"log"
"net/http"
"thamanyah/cms/v2/internal/api"
)
// Health is the liveness probe the ALB target group polls.
//
// @Summary Health check
// @Description Reports that the service is up and serving. Used as the ALB target group health check; it does not verify the database or S3 connections.
// @Tags system
// @Produce json
// @Success 200 {object} api.HealthResponse
// @Router /health [get]
func Health(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, api.HealthResponse{Status: "ok"})
}
func writeJSON(w http.ResponseWriter, status int, body any) {
writeJSONContent(w, "application/json", status, body)
}
func writeProblem(w http.ResponseWriter, status int, title, detail string) {
writeJSONContent(w, "application/problem+json", status, api.ProblemDetails{
Type: "about:blank",
Title: title,
Status: status,
Detail: detail,
})
}
func writeJSONContent(w http.ResponseWriter, contentType string, status int, body any) {
w.Header().Set("Content-Type", contentType)
w.WriteHeader(status)
if err := json.NewEncoder(w).Encode(body); err != nil {
log.Printf("Something Went Wrong Writing The Response: %s", err)
}
}
+317
View File
@@ -0,0 +1,317 @@
package handlers
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"path/filepath"
"slices"
"strings"
"thamanyah/cms/v2/internal/api"
"thamanyah/cms/v2/internal/db/repositories"
"thamanyah/cms/v2/internal/models"
"thamanyah/cms/v2/internal/services"
"time"
"github.com/google/uuid"
)
const (
videoUploadPrefix = "videos"
uploadURLExpiry = 60 * time.Minute
maxJSONBodySize = 1 << 20 // 1 MiB
)
// ListCategories returns the categories a video may be assigned to.
//
// @Summary List video categories
// @Description Returns the fixed lookup set of categories a video can belong to, each with the id used to reference it. The `id` values from this list are the only ones accepted in the `categoryIds` field of POST /api/videos.
// @Tags categories
// @Produce json
// @Success 200 {object} api.CategoriesResponse
// @Failure 500 {object} api.ProblemDetails "Categories could not be read from the database"
// @Router /api/categories [get]
func ListCategories(w http.ResponseWriter, r *http.Request) {
categories, err := repositories.CatagoriesRepo.ListCategories(r.Context())
if err != nil {
log.Printf("Something Went Wrong Loading Categories: %s", err)
writeProblem(w, http.StatusInternalServerError, "Something Went Wrong Loading Categories",
"The list of video categories could not be read from the database. This is a server-side fault and the request was not processed; retrying in a few moments may succeed.")
return
}
body := make([]api.Category, 0, len(categories))
for _, category := range categories {
body = append(body, api.Category{ID: category.ID, Name: category.Name})
}
writeJSON(w, http.StatusOK, api.CategoriesResponse{Categories: body})
}
var allowedUploadContentTypes = map[string]struct{}{
"video/mp4": {}, // .mp4
"video/quicktime": {}, // .mov
}
// PresignVideoUpload issues a presigned S3 PUT URL for a video upload.
//
// @Summary Create a presigned upload URL
// @Description Step 1 of the upload flow. Returns a short-lived presigned S3 URL that the client PUTs the video file to directly, plus the storage key identifying it. The submitted contentType is signed into that URL, so the PUT must carry an identical Content-Type header or S3 rejects it as a signature mismatch. Once the PUT succeeds, pass the key to POST /api/videos to register the video and start transcoding. The file itself never passes through this API.
// @Tags videos
// @Accept json
// @Produce json
// @Param request body api.PresignRequest true "Name and media type of the file to be uploaded. contentType must be exactly 'video/mp4' or 'video/quicktime'."
// @Success 200 {object} api.PresignResponse
// @Failure 400 {object} api.ProblemDetails "Request body was not valid JSON"
// @Failure 422 {object} api.ProblemDetails "contentType was not 'video/mp4' or 'video/quicktime'"
// @Failure 500 {object} api.ProblemDetails "Upload URL could not be issued"
// @Router /api/videos/presign [post]
func PresignVideoUpload(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxJSONBodySize)
var req api.PresignRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeProblem(w, http.StatusBadRequest, "Invalid request.",
fmt.Sprintf("The request body could not be parsed as JSON (%s). Send an object with the string fields 'fileName' and 'contentType', and keep the body under 1 MiB.", err))
return
}
if _, ok := allowedUploadContentTypes[req.ContentType]; !ok {
writeProblem(w, http.StatusUnprocessableEntity, "Unsupported file type. Please upload an MP4 or MOV video.",
fmt.Sprintf("The 'contentType' field was %q, but only 'video/mp4' (.mp4) and 'video/quicktime' (.mov) are accepted. Send one of those two values exactly — it is signed into the upload URL, so the PUT must use the identical header.", req.ContentType))
return
}
filename, err := randomFilename(req.FileName)
if err != nil {
writeProblem(w, http.StatusInternalServerError, "Could not prepare upload.",
"A unique storage key could not be generated because the server's random source failed. This is a server-side fault; retry the request.")
return
}
key := videoUploadPrefix + "/" + filename
presignedURL, err := services.S3Client.GetPresignedURL(r.Context(), key, req.ContentType, uploadURLExpiry)
if err != nil {
writeProblem(w, http.StatusInternalServerError, "Could not prepare upload.",
"A presigned upload URL could not be issued for the storage bucket. This is a server-side fault; no upload slot was reserved, so retry the request.")
return
}
writeJSON(w, http.StatusOK, api.PresignResponse{UploadURL: presignedURL, Key: key})
}
// CompleteVideoUpload registers an uploaded video and queues it for transcoding.
//
// @Summary Register an uploaded video
// @Description Step 2 of the upload flow. Takes the storage key returned by POST /api/videos/presign — after the file has been PUT to the presigned URL — submits a MediaConvert transcoding job, and persists the video with its category links. The returned record has status "processing" — one of the states "processing", "ready" or "failed" — though nothing currently moves it on once transcoding finishes.
// @Tags videos
// @Accept json
// @Produce json
// @Param request body api.CompleteRequest true "Video metadata plus the storage key from the presign step. title, key and categoryIds are required; categoryIds must hold at least one id from GET /api/categories."
// @Success 201 {object} api.CompleteResponse
// @Failure 400 {object} api.ProblemDetails "Request body was not valid JSON"
// @Failure 404 {object} api.ProblemDetails "categoryIds referenced a category that does not exist"
// @Failure 409 {object} api.ProblemDetails "key has already been registered by an earlier request"
// @Failure 422 {object} api.ProblemDetails "title was empty, key was missing or not a key issued by this API, or categoryIds was empty"
// @Failure 500 {object} api.ProblemDetails "Categories could not be read, the key could not be checked, the transcoding job could not be queued, or the video record could not be saved"
// @Router /api/videos [post]
func CompleteVideoUpload(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxJSONBodySize)
var req api.CompleteRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeProblem(w, http.StatusBadRequest, "Invalid request.",
fmt.Sprintf("The request body could not be parsed as JSON (%s). Send an object containing at least 'title' and 'key', and keep the body under 1 MiB.", err))
return
}
title := strings.TrimSpace(req.Title)
if title == "" {
writeProblem(w, http.StatusUnprocessableEntity, "Title is required.",
"The 'title' field was absent or contained only whitespace. Every video needs a non-empty title.")
return
}
key := strings.TrimSpace(req.Key)
if key == "" || !strings.HasPrefix(key, videoUploadPrefix+"/") {
writeProblem(w, http.StatusUnprocessableEntity, "A video file is required.",
fmt.Sprintf("The 'key' field was %q, which is not a storage key issued by this API. Use the 'key' returned by POST /api/videos/presign, after the file has been PUT to the accompanying upload URL.", req.Key))
return
}
if len(req.CategoryIDs) == 0 {
writeProblem(w, http.StatusUnprocessableEntity, "At least one category is required.",
"The 'categoryIds' field was absent or empty. Every video belongs to at least one category; send ids taken from the 'id' field of GET /api/categories.")
return
}
exists := services.S3Client.DoesFileExist(r.Context(), key)
if exists {
writeProblem(w, http.StatusUnprocessableEntity, "Failed to find video in storage",
"The video file that was supposed to be uploaded via the url sent in the PresignVideoUpload endpoint was not uploaded before calling this endpoint")
return
}
unknownCatagoryID, err := validateCatagoryIDS(r.Context(), req.CategoryIDs)
if errors.Is(err, errUnknownCatagoryID) {
writeProblem(w, http.StatusNotFound, "Catagory Does Not Exist",
fmt.Sprintf("The 'categoryIds' field referenced the id %d, which no category has. Send only ids taken from the 'id' field of GET /api/categories. No video record was created and no transcoding job was queued; the uploaded file is still in storage, so retry this request with the same 'key'.", unknownCatagoryID))
return
}
if err != nil {
log.Printf("Something Went Wrong Loading Categories: %s", err)
writeProblem(w, http.StatusInternalServerError, "Something Went Wrong Loading Categories",
"The submitted category ids could not be checked against the database, so no video record was created. This is a server-side fault; the uploaded file is still in storage, so retry this request with the same 'key'.")
return
}
// Ahead of queueing, because a key that is already registered would
// otherwise get a second transcoding job before the insert failed on the
// storage_key UNIQUE constraint. That constraint is still the backstop:
// this check and the insert are not one atomic step, so two concurrent
// requests carrying the same key can both get past here.
alreadyRegistered, err := repositories.VideoRepo.VideoExistsWithStorageKey(r.Context(), key)
if err != nil {
log.Printf("Something Went Wrong Checking Whether The Video Was Already Registered: %s", err)
writeProblem(w, http.StatusInternalServerError, "Something Went Wrong Checking Whether The Video Was Already Registered",
"The submitted key could not be checked against the existing video records, so no video record was created and no transcoding job was queued. This is a server-side fault; the uploaded file is still in storage, so retry this request with the same 'key'.")
return
}
if alreadyRegistered {
writeProblem(w, http.StatusConflict, "Video Already Registered",
fmt.Sprintf("The 'key' %q has already been registered, so a video record and a transcoding job exist for that file. Registering it a second time would queue a duplicate job. If you meant to add a different video, request a fresh key from POST /api/videos/presign and upload the file to it.", key))
return
}
jobID, err := services.MediaConvertClient.QueueEncodingJob(r.Context(), key)
if err != nil {
log.Printf("Something Went Wrong On Creation Of Transcoding Job: %s", err)
writeProblem(w, http.StatusInternalServerError, "Something Went Wrong On Creation Of Transcoding Job",
"The transcoding job could not be queued, so no video record was created. The uploaded file is still in storage; retry this request with the same 'key' rather than uploading the file again.")
return
}
video, err := repositories.VideoRepo.CreateVideo(r.Context(), models.Video{
Title: title,
Description: strings.TrimSpace(req.Description),
CategoryIDs: req.CategoryIDs,
Tags: strings.TrimSpace(req.Tags),
FileName: strings.TrimSpace(req.FileName),
StorageKey: key,
MediaConvertJobID: jobID,
Status: models.VideoStatusProcessing,
SizeBytes: 60,
})
if err != nil {
log.Printf("Something Went Wrong Saving The Video Record: %s", err)
writeProblem(w, http.StatusInternalServerError, "Something Went Wrong Saving The Video Record",
"The transcoding job was queued but its video record could not be written to the database, so the video may finish transcoding without ever appearing in the catalogue. Retry this request or escalate to an operator.")
return
}
writeJSON(w, http.StatusCreated, api.CompleteResponse{
ID: video.ID,
Title: video.Title,
Description: video.Description,
CategoryIDs: video.CategoryIDs,
Tags: video.Tags,
FileName: video.FileName,
StorageKey: video.StorageKey,
MediaConvertJobID: video.MediaConvertJobID,
Status: video.Status.String(),
SizeBytes: video.SizeBytes,
CreatedAt: video.CreatedAt,
UpdatedAt: video.UpdatedAt,
})
}
// validateCatagoryIDS checks every submitted id against the category table. On
// the first id that has no category it returns that id alongside
// errUnknownCatagoryID, so the handler can name it in the response.
func validateCatagoryIDS(ctx context.Context, submitted []int16) (int16, error) {
categoryIDS, err := repositories.CatagoriesRepo.ListCategoriesIDs(ctx)
if err != nil {
return 0, err
}
for _, submittedID := range submitted {
if !slices.Contains(categoryIDS, submittedID) {
return submittedID, errUnknownCatagoryID
}
}
return 0, nil
}
func randomFilename(original string) (string, error) {
ext := filepath.Ext(filepath.Base(original))
buf := make([]byte, 16)
if _, err := rand.Read(buf); err != nil {
return "", err
}
return hex.EncodeToString(buf) + ext, nil
}
// GetVideo returns one video by its id.
//
// @Summary Read a video
// @Description Returns the catalogue record for one video, including the status its transcoding job has reached. This is how a client follows an upload after POST /api/videos: the record starts "processing" and moves to "ready" or "failed" once MediaConvert reports the job finished.
// @Tags videos
// @Produce json
// @Param id path string true "The video's id, as returned by POST /api/videos"
// @Success 200 {object} api.VideoResponse
// @Failure 404 {object} api.ProblemDetails "No video has that id"
// @Failure 500 {object} api.ProblemDetails "The video could not be read from the database"
// @Router /api/videos/{id} [get]
func GetVideo(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
// Parsed before the query, because Postgres rejects anything that is not a
// uuid the moment it is compared against the column rather than simply
// matching no row — which would surface as a 500. An id that cannot name a
// video is a video that does not exist.
//
// The parsed value, not the raw one, is what goes to the database: uuid
// accepts spellings Postgres will not take verbatim — notably the
// "urn:uuid:" form — and String() puts them all back into the canonical
// form the column holds.
videoID, err := uuid.Parse(id)
if err != nil {
writeProblem(w, http.StatusNotFound, "Video Not Found",
fmt.Sprintf("No video has the id %q. Use the 'id' returned by POST /api/videos.", id))
return
}
video, err := repositories.VideoRepo.GetVideoByID(r.Context(), videoID.String())
if errors.Is(err, repositories.ErrVideoNotFound) {
writeProblem(w, http.StatusNotFound, "Video Not Found",
fmt.Sprintf("No video has the id %q. Use the 'id' returned by POST /api/videos.", id))
return
}
if err != nil {
log.Printf("Something Went Wrong Loading The Video: %s", err)
writeProblem(w, http.StatusInternalServerError, "Something Went Wrong Loading The Video",
"The video could not be read from the database. This is a server-side fault and the request was not processed; retrying in a few moments may succeed.")
return
}
writeJSON(w, http.StatusOK, api.VideoResponse{
ID: video.ID,
Title: video.Title,
Description: video.Description,
CategoryIDs: video.CategoryIDs,
Tags: video.Tags,
FileName: video.FileName,
StorageKey: video.StorageKey,
MediaConvertJobID: video.MediaConvertJobID,
Status: video.Status.String(),
PlaybackURL: video.PlaybackURL,
SizeBytes: video.SizeBytes,
CreatedAt: video.CreatedAt,
UpdatedAt: video.UpdatedAt,
})
}
+6
View File
@@ -0,0 +1,6 @@
package models
type Category struct {
ID int16
Name string
}
+45
View File
@@ -0,0 +1,45 @@
package models
import "time"
type Video struct {
ID string
Title string
Description string
CategoryIDs []int16
// CategoryNames is the same filing read by name rather than id. It is
// filled only where a caller needs categories to be self-describing —
// the catalogue announcement — and is empty elsewhere.
CategoryNames []string
Tags string
FileName string
StorageKey string
MediaConvertJobID string
Status VideoStatus
PlaybackURL string
SizeBytes int64
CreatedAt time.Time
UpdatedAt time.Time
}
// VideoStatus is the transcoding lifecycle state of a video, stored verbatim
// in videos.status. The set is closed on both sides: the database rejects
// anything outside it via the videos_status_check constraint. Adding a state
// means adding a constant here, widening that constraint in a new migration,
// and extending the `enums` annotation on api.CompleteResponse.Status.
type VideoStatus string
const (
// VideoStatusProcessing is set when the MediaConvert job is queued. Every
// video starts here, and nothing moves it on yet — see Known gaps.
VideoStatusProcessing VideoStatus = "processing"
// VideoStatusReady means transcoding finished and the output is in the
// encoded bucket, reachable through CloudFront.
VideoStatusReady VideoStatus = "ready"
// VideoStatusFailed means the MediaConvert job ended in an error.
VideoStatusFailed VideoStatus = "failed"
)
func (s VideoStatus) String() string {
return string(s)
}
+75
View File
@@ -0,0 +1,75 @@
package services
import (
"context"
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/sns"
)
var CatalogueClient Catalogue
// CatalogueVideo is what cms announces when a video becomes ready. It is a
// published contract — every subscriber of the catalogue topic decodes it —
// so the JSON names here are as much an API as the ones in internal/api, and
// changing one is a change every subscriber sees.
//
// Categories are named rather than numbered on purpose: the ids are cms's
// own, and a consumer holding a name needs nothing from cms to make sense
// of it.
type CatalogueVideo struct {
VideoID string `json:"videoId"`
Title string `json:"title"`
PlaybackURL string `json:"playbackUrl"`
Categories []string `json:"categories"`
}
type Catalogue interface {
AnnounceVideo(ctx context.Context, video CatalogueVideo) error
}
type CatalogueConcrete struct {
SNSClient *sns.Client
TopicARN string
}
// AnnounceVideo publishes one video to the catalogue topic, which fans it out
// to whatever has subscribed. Delivery is at-least-once: the caller
// republishes when a later step fails, so a subscriber has to tolerate seeing
// the same video twice.
//
// A topic delivers only to the subscriptions that exist at the moment it
// publishes, so a subscriber that wants the backlog needs its queue in place
// before the announcement, not after.
func (svc CatalogueConcrete) AnnounceVideo(ctx context.Context, video CatalogueVideo) error {
// A nil slice marshals to null, which would make "has no categories" and
// "categories unknown" indistinguishable to a reader. An empty list says
// the first, which is what an unfiled video means.
if video.Categories == nil {
video.Categories = []string{}
}
body, err := json.Marshal(video)
if err != nil {
return err
}
_, err = svc.SNSClient.Publish(ctx, &sns.PublishInput{
TopicArn: aws.String(svc.TopicARN),
Message: aws.String(string(body)),
})
return err
}
// AssertSuccessfulConnection verifies the topic exists and is readable with
// the configured credentials. It panics on failure: a service that cannot
// announce would transcode videos the catalogue never hears about.
func (svc CatalogueConcrete) AssertSuccessfulConnection(ctx context.Context) {
if _, err := svc.SNSClient.GetTopicAttributes(ctx, &sns.GetTopicAttributesInput{
TopicArn: aws.String(svc.TopicARN),
}); err != nil {
panic(fmt.Errorf("sns: cannot connect to catalogue topic %q: %w", svc.TopicARN, err))
}
}
+121
View File
@@ -0,0 +1,121 @@
package services
import (
"context"
"fmt"
"path"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/mediaconvert"
"github.com/aws/aws-sdk-go-v2/service/mediaconvert/types"
)
var MediaConvertClient MediaConvert
type MediaConvert interface {
QueueEncodingJob(ctx context.Context, key string) (string, error)
}
// segmentLengthSeconds is how long each HLS segment runs. Six seconds is the
// common default: long enough to keep the playlist and the request count down,
// short enough that a player can switch rendition without a long stall.
const segmentLengthSeconds = 6
type MediaConvertConcrete struct {
MediaConvertClient *mediaconvert.Client
Role string
InputBucket string
OutputBucket string
}
func (svc MediaConvertConcrete) QueueEncodingJob(ctx context.Context, key string) (string, error) {
input := fmt.Sprintf("s3://%s/%s", svc.InputBucket, key)
// HlsGroupSettings.Destination is a *filename base*, not a directory and
// not a finished name: MediaConvert appends each output's NameModifier and
// the extensions it needs. Giving each video its own folder keeps the
// master playlist, the variant playlists and the segments together, and
// keeps two videos from interleaving their segments in one prefix.
base := strings.TrimSuffix(key, path.Ext(key))
destination := fmt.Sprintf("s3://%s/%s/index", svc.OutputBucket, base)
job, err := svc.MediaConvertClient.CreateJob(ctx, &mediaconvert.CreateJobInput{
Role: &svc.Role,
Settings: &types.JobSettings{
Inputs: []types.Input{
{
FileInput: &input,
AudioSelectors: map[string]types.AudioSelector{
"Audio Selector 1": {
DefaultSelection: types.AudioDefaultSelectionDefault,
},
},
VideoSelector: &types.VideoSelector{},
},
},
OutputGroups: []types.OutputGroup{
{
Name: aws.String("HLS Group"),
OutputGroupSettings: &types.OutputGroupSettings{
Type: types.OutputGroupTypeHlsGroupSettings,
HlsGroupSettings: &types.HlsGroupSettings{
Destination: &destination,
SegmentLength: aws.Int32(segmentLengthSeconds),
// 0 lets the last segment be as short as it needs to
// be rather than padding the ones before it.
MinSegmentLength: aws.Int32(0),
},
},
Outputs: []types.Output{
{
// One rendition. A ladder is another entry here per
// rendition, each with its own NameModifier and
// height — the master playlist lists whatever is
// present, so nothing downstream changes.
NameModifier: aws.String("_720p"),
ContainerSettings: &types.ContainerSettings{
Container: types.ContainerTypeM3u8,
M3u8Settings: &types.M3u8Settings{},
},
VideoDescription: &types.VideoDescription{
Height: aws.Int32(720),
CodecSettings: &types.VideoCodecSettings{
Codec: types.VideoCodecH264,
H264Settings: &types.H264Settings{
RateControlMode: types.H264RateControlModeQvbr,
MaxBitrate: aws.Int32(3000000),
// A segment has to start on a keyframe,
// so the GOP is pinned to a divisor of
// the segment length. Left to follow the
// source, segments come out uneven and
// players stall at the joins.
GopSizeUnits: types.H264GopSizeUnitsSeconds,
GopSize: aws.Float64(2),
},
},
},
AudioDescriptions: []types.AudioDescription{
{
CodecSettings: &types.AudioCodecSettings{
Codec: types.AudioCodecAac,
AacSettings: &types.AacSettings{
Bitrate: aws.Int32(96000),
CodingMode: types.AacCodingModeCodingMode20,
SampleRate: aws.Int32(48000),
},
},
},
},
},
},
},
},
},
})
if err != nil {
return "", err
}
return aws.ToString(job.Job.Id), nil
}
+82
View File
@@ -0,0 +1,82 @@
package services
import (
"context"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
var S3Client S3
type S3 interface {
GetPresignedURL(ctx context.Context, key, contentType string, expiry time.Duration) (string, error)
DoesFileExist(ctx context.Context, key string) bool
}
type S3Concrete struct {
S3Client *s3.Client
Bucket string
}
func (svc S3Concrete) GetPresignedURL(ctx context.Context, key, contentType string, expiry time.Duration) (string, error) {
presignClient := s3.NewPresignClient(svc.S3Client)
request, err := presignClient.PresignPutObject(ctx, &s3.PutObjectInput{
Bucket: &svc.Bucket,
Key: &key,
ContentType: &contentType,
}, s3.WithPresignExpires(expiry))
if err != nil {
return "", err
}
return request.URL, nil
}
func (svc S3Concrete) DoesFileExist(ctx context.Context, key string) bool {
_, error := svc.S3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &svc.Bucket,
Key: &key,
})
return error != nil
}
// AssertSuccessfulConnection verifies that the bucket is reachable and that
// the configured credentials can get, put, and presign objects in it. It
// panics with a descriptive message on the first check that fails, since the
// service cannot function without these permissions.
func (svc S3Concrete) AssertSuccessfulConnection(ctx context.Context) {
if _, err := svc.S3Client.HeadBucket(ctx, &s3.HeadBucketInput{
Bucket: &svc.Bucket,
}); err != nil {
panic(fmt.Errorf("s3: cannot connect to bucket %q: %w", svc.Bucket, err))
}
const probeKey = ".s3-connectivity-check"
if _, err := svc.S3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &svc.Bucket,
Key: aws.String(probeKey),
Body: strings.NewReader("ok"),
}); err != nil {
panic(fmt.Errorf("s3: missing permission to put objects in bucket %q: %w", svc.Bucket, err))
}
object, err := svc.S3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &svc.Bucket,
Key: aws.String(probeKey),
})
if err != nil {
panic(fmt.Errorf("s3: missing permission to get objects from bucket %q: %w", svc.Bucket, err))
}
object.Body.Close()
if _, err := svc.GetPresignedURL(ctx, probeKey, "text/plain", time.Minute); err != nil {
panic(fmt.Errorf("s3: cannot create presigned urls for bucket %q: %w", svc.Bucket, err))
}
}
+80
View File
@@ -0,0 +1,80 @@
package services
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/service/sqs"
)
var SQSClient SQS
// QueueMessage is one message taken off the queue. ReceiptHandle is what
// identifies it for deletion — it belongs to this delivery, not to the
// message, so it cannot be held across receives.
type QueueMessage struct {
Body string
ReceiptHandle string
}
type SQS interface {
ReceiveMessages(ctx context.Context) ([]QueueMessage, error)
DeleteMessage(ctx context.Context, receiptHandle string) error
}
type SQSConcrete struct {
SQSClient *sqs.Client
QueueURL string
}
// receiveWaitSeconds turns every receive into a long poll: the call parks on
// the server until a message arrives or this elapses, so an event is picked up
// within milliseconds of being published while an idle consumer costs one
// request every twenty seconds rather than spinning. 20 is the AWS maximum.
const receiveWaitSeconds = 20
func (svc SQSConcrete) ReceiveMessages(ctx context.Context) ([]QueueMessage, error) {
output, err := svc.SQSClient.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{
QueueUrl: &svc.QueueURL,
MaxNumberOfMessages: 10,
WaitTimeSeconds: receiveWaitSeconds,
})
if err != nil {
return nil, err
}
messages := make([]QueueMessage, 0, len(output.Messages))
for _, message := range output.Messages {
if message.Body == nil || message.ReceiptHandle == nil {
continue
}
messages = append(messages, QueueMessage{
Body: *message.Body,
ReceiptHandle: *message.ReceiptHandle,
})
}
return messages, nil
}
// DeleteMessage acknowledges a message. Until this is called the message is
// merely invisible, and it returns to the queue when its visibility timeout
// expires — which is how a handler that failed gets another attempt.
func (svc SQSConcrete) DeleteMessage(ctx context.Context, receiptHandle string) error {
_, err := svc.SQSClient.DeleteMessage(ctx, &sqs.DeleteMessageInput{
QueueUrl: &svc.QueueURL,
ReceiptHandle: &receiptHandle,
})
return err
}
// AssertSuccessfulConnection verifies the queue is reachable and readable with
// the configured credentials. It panics on failure, since a service whose
// consumer cannot start would silently leave every video stuck "processing".
func (svc SQSConcrete) AssertSuccessfulConnection(ctx context.Context) {
if _, err := svc.SQSClient.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{
QueueUrl: &svc.QueueURL,
}); err != nil {
panic(fmt.Errorf("sqs: cannot connect to queue %q: %w", svc.QueueURL, err))
}
}
+212
View File
@@ -0,0 +1,212 @@
package main
import (
"context"
"fmt"
"log"
"net/http"
"os"
"thamanyah/cms/v2/internal/consumers"
"thamanyah/cms/v2/internal/db"
"thamanyah/cms/v2/internal/db/repositories"
"thamanyah/cms/v2/internal/handlers"
"thamanyah/cms/v2/internal/services"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/mediaconvert"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sns"
"github.com/aws/aws-sdk-go-v2/service/sqs"
_ "github.com/lib/pq"
httpSwagger "github.com/swaggo/http-swagger/v2"
_ "thamanyah/cms/v2/docs"
)
// @title Thamanyah CMS API
// @version 1.0
// @description JSON API for ingesting videos into the Thamanyah catalogue. Uploads are two-step: presign, PUT the file straight to S3, then register the video to queue transcoding.
// @BasePath /
func main() {
if len(os.Args) > 1 && os.Args[1] == "migrate" {
runMigrate()
return
}
runServer()
}
func runMigrate() {
concreteDBClient := db.CreateDBConnection(requireDBConnectionString())
defer db.CloseConnection(concreteDBClient)
db.Migrate(concreteDBClient)
log.Println("migrations applied successfully")
}
func runServer() {
awsConfig, err := config.LoadDefaultConfig(context.Background())
if err != nil {
panic(fmt.Errorf("failed To Load S3 Config: %s", err))
}
bucketName := os.Getenv("S3_BUCKET")
if bucketName == "" {
panic(fmt.Errorf("missing required env var: S3_BUCKET"))
}
mediaConvertRole := os.Getenv("MEDIACONVERT_ROLE_ARN")
if mediaConvertRole == "" {
panic(fmt.Errorf("missing required env var: MEDIACONVERT_ROLE_ARN"))
}
mediaConvertInputBucket := os.Getenv("MEDIACONVERT_INPUT_BUCKET")
if mediaConvertInputBucket == "" {
panic(fmt.Errorf("missing required env var: MEDIACONVERT_INPUT_BUCKET"))
}
mediaConvertOutputBucket := os.Getenv("MEDIACONVERT_OUTPUT_BUCKET")
if mediaConvertOutputBucket == "" {
panic(fmt.Errorf("missing required env var: MEDIACONVERT_OUTPUT_BUCKET"))
}
mediaConvertEventsQueueURL := os.Getenv("MEDIACONVERT_EVENTS_QUEUE_URL")
if mediaConvertEventsQueueURL == "" {
panic(fmt.Errorf("missing required env var: MEDIACONVERT_EVENTS_QUEUE_URL"))
}
// The topic ready videos are announced on, which fans out to whatever has
// subscribed — the read side's queue, today.
catalogueEventsTopicARN := os.Getenv("CATALOGUE_EVENTS_TOPIC_ARN")
if catalogueEventsTopicARN == "" {
panic(fmt.Errorf("missing required env var: CATALOGUE_EVENTS_TOPIC_ARN"))
}
// The public host the encoded output is served from — the CloudFront
// distribution in front of the output bucket. MediaConvert reports where
// it wrote as an s3:// path into a bucket that blocks public access, so
// this is what makes a finished video reachable by a player.
playbackBaseURL := os.Getenv("PLAYBACK_BASE_URL")
if playbackBaseURL == "" {
panic(fmt.Errorf("missing required env var: PLAYBACK_BASE_URL"))
}
// AWS_ENDPOINT_URL is only ever set when pointing at something other than
// real S3 — LocalStack, in docker-compose. Virtual-host addressing would
// resolve <bucket>.<endpoint host> there, which neither Docker's DNS nor
// LocalStack's own wildcard domain serves from inside a container, so those
// deployments need path-style URLs. Left off against real S3, which has
// been steering away from path-style for new buckets.
s3Options := func(o *s3.Options) {
if os.Getenv("AWS_ENDPOINT_URL") != "" || os.Getenv("AWS_ENDPOINT_URL_S3") != "" {
o.UsePathStyle = true
}
}
concreteS3Client := &services.S3Concrete{
S3Client: s3.NewFromConfig(awsConfig, s3Options),
Bucket: bucketName,
}
concreteS3Client.AssertSuccessfulConnection(context.Background())
services.S3Client = concreteS3Client
services.MediaConvertClient = &services.MediaConvertConcrete{
MediaConvertClient: mediaconvert.NewFromConfig(awsConfig),
Role: mediaConvertRole,
InputBucket: mediaConvertInputBucket,
OutputBucket: mediaConvertOutputBucket,
}
concreteSQSClient := &services.SQSConcrete{
SQSClient: sqs.NewFromConfig(awsConfig),
QueueURL: mediaConvertEventsQueueURL,
}
concreteSQSClient.AssertSuccessfulConnection(context.Background())
services.SQSClient = concreteSQSClient
concreteCatalogueClient := &services.CatalogueConcrete{
SNSClient: sns.NewFromConfig(awsConfig),
TopicARN: catalogueEventsTopicARN,
}
concreteCatalogueClient.AssertSuccessfulConnection(context.Background())
services.CatalogueClient = concreteCatalogueClient
concreteDBClient := db.CreateDBConnection(requireDBConnectionString())
defer db.CloseConnection(concreteDBClient)
db.AssertSuccessfulConnection(context.Background(), concreteDBClient)
repositories.VideoRepo = repositories.VideoRepository{
SQLDB: concreteDBClient,
}
repositories.CatagoriesRepo = repositories.CatagoriesRepository{
SQLDB: concreteDBClient,
}
// MediaConvert reports job state changes to a topic that fans out to this
// service's queue; the consumer runs alongside the HTTP server for the
// lifetime of the process, so a video's status catches up with its
// transcoding job without anything calling back into this service.
consumerCtx, stopConsumer := context.WithCancel(context.Background())
defer stopConsumer()
go consumers.MediaConvertEvents{PlaybackBaseURL: playbackBaseURL}.Run(consumerCtx)
mux := http.NewServeMux()
mux.HandleFunc("GET /health", handlers.Health)
mux.HandleFunc("GET /api/categories", handlers.ListCategories)
mux.HandleFunc("POST /api/videos/presign", handlers.PresignVideoUpload)
mux.HandleFunc("POST /api/videos", handlers.CompleteVideoUpload)
mux.HandleFunc("GET /api/videos/{id}", handlers.GetVideo)
// Swagger UI and the generated spec. The UI assets are embedded in the
// binary by swaggo/files, so this needs no static directory on disk.
mux.Handle("GET /swagger/", httpSwagger.WrapHandler)
addr := ":8081"
log.Printf("listening on %s", addr)
if err := http.ListenAndServe(addr, mux); err != nil {
log.Fatal(err)
}
}
func requireDBConnectionString() string {
dbHost := os.Getenv("DB_HOST")
if dbHost == "" {
panic(fmt.Errorf("missing required env var: DB_HOST"))
}
dbPort := os.Getenv("DB_PORT")
if dbPort == "" {
panic(fmt.Errorf("missing required env var: DB_PORT"))
}
dbUser := os.Getenv("DB_USER")
if dbUser == "" {
panic(fmt.Errorf("missing required env var: DB_USER"))
}
dbPassword := os.Getenv("DB_PASSWORD")
if dbPassword == "" {
panic(fmt.Errorf("missing required env var: DB_PASSWORD"))
}
dbName := os.Getenv("DB_NAME")
if dbName == "" {
panic(fmt.Errorf("missing required env var: DB_NAME"))
}
// Defaults to require: RDS terminates TLS, and only a local Postgres that
// serves none (LocalStack's RDS emulation) has any business overriding it.
dbSSLMode := os.Getenv("DB_SSLMODE")
if dbSSLMode == "" {
dbSSLMode = "require"
}
return fmt.Sprintf(
"host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
dbHost, dbPort, dbUser, dbPassword, dbName, dbSSLMode,
)
}
+33
View File
@@ -0,0 +1,33 @@
# Binaries
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool
*.out
coverage.txt
# Dependency directories
vendor/
# Go workspace file
go.work
go.work.sum
# Env files
.env
.env.local
# Build output
/bin/
/dist/
# IDE
.idea/
.vscode/
*.swp
+19
View File
@@ -0,0 +1,19 @@
# syntax=docker/dockerfile:1
FROM golang:1.25-alpine AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o /out/discovery .
FROM alpine:3.20
RUN apk add --no-cache ca-certificates
WORKDIR /app
COPY --from=builder /out/discovery ./discovery
EXPOSE 8080
ENTRYPOINT ["./discovery"]
+238
View File
@@ -0,0 +1,238 @@
// Package docs Code generated by swaggo/swag. DO NOT EDIT
package docs
import "github.com/swaggo/swag"
const docTemplate = `{
"schemes": {{ marshal .Schemes }},
"swagger": "2.0",
"info": {
"description": "{{escape .Description}}",
"title": "{{.Title}}",
"contact": {},
"version": "{{.Version}}"
},
"host": "{{.Host}}",
"basePath": "{{.BasePath}}",
"paths": {
"/api/videos": {
"get": {
"description": "Finds catalogued videos by the words of their title and the categories they are filed under, most relevant first. Every parameter is optional — a request with none of them browses the whole catalogue, newest first. Paging is by cursor rather than by offset, so a page stays the same page while videos are being announced into the catalogue around it.",
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Search the catalogue",
"parameters": [
{
"type": "string",
"example": "desert fal",
"description": "Matched lexically against video titles: the words are all required, and the last is treated as a prefix so a part-typed word still finds something.",
"name": "title",
"in": "query"
},
{
"type": "array",
"items": {
"type": "string"
},
"collectionFormat": "multi",
"example": "documentary",
"description": "Narrows the search to videos filed under any one of these names — not all of them. Repeat the parameter per category. Omitted means every category.",
"name": "categories",
"in": "query"
},
{
"type": "integer",
"default": 20,
"description": "How many videos to return. Absent, zero or negative takes the default of 20; anything above the maximum of 100 is capped to it rather than refused, and the cursor still reaches the rest.",
"name": "limit",
"in": "query"
},
{
"type": "string",
"description": "Asks for the page after the one a previous search ended at. Send back the nextCursor from that search unchanged; it is opaque, and the only thing to do with it is return it.",
"name": "cursor",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.SearchResults"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/api/videos/{id}": {
"get": {
"description": "Returns the catalogue's copy of a video: what a reader needs to show and play it. A video appears here only after the CMS has announced it as ready, so a video that is still transcoding — or one the CMS never made ready — is a 404.",
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Get a catalogued video",
"parameters": [
{
"type": "string",
"description": "Video id, as issued by the CMS",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.Video"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/health": {
"get": {
"description": "Reports that the service is up and serving. Used as the ALB target group health check; it does not verify the database connection.",
"produces": [
"application/json"
],
"tags": [
"system"
],
"summary": "Health check",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.HealthResponse"
}
}
}
}
}
},
"definitions": {
"api.HealthResponse": {
"type": "object",
"properties": {
"status": {
"type": "string",
"example": "ok"
}
}
},
"api.ProblemDetails": {
"type": "object",
"properties": {
"detail": {
"type": "string",
"example": "The catalogue could not be read from the database. This is a server-side fault and the request was not processed; retrying in a few moments may succeed."
},
"status": {
"type": "integer",
"example": 500
},
"title": {
"type": "string",
"example": "Something Went Wrong Loading The Catalogue"
},
"type": {
"type": "string",
"example": "about:blank"
}
}
},
"api.SearchResults": {
"type": "object",
"properties": {
"nextCursor": {
"description": "NextCursor reaches the page after this one. Empty on the last page,\nwhich is how a reader knows there is no more to ask for.",
"type": "string"
},
"videos": {
"description": "Videos are the matches, most relevant first. Never null: a search that\nmatches nothing is an empty list.",
"type": "array",
"items": {
"$ref": "#/definitions/api.Video"
}
}
}
},
"api.Video": {
"type": "object",
"properties": {
"categories": {
"type": "array",
"items": {
"type": "string"
},
"example": [
"documentary",
"news"
]
},
"id": {
"type": "string",
"example": "0f8fad5b-d9cb-469f-a165-70867728950e"
},
"playbackUrl": {
"type": "string",
"example": "https://d111111abcdef8.cloudfront.net/videos/abc123/index.m3u8"
},
"title": {
"type": "string",
"example": "The Evening Bulletin"
}
}
}
}
}`
// SwaggerInfo holds exported Swagger Info so clients can modify it
var SwaggerInfo = &swag.Spec{
Version: "1.0",
Host: "",
BasePath: "/",
Schemes: []string{},
Title: "Thamanyah Discovery API",
Description: "JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.\n\nThe catalogue search is `GET /api/videos`, listed below: title is matched lexically with the last word as a prefix, categories narrow to videos filed under any one of them, limit defaults to 20 and is capped at 100, and cursor is the opaque `nextCursor` of a previous search.",
InfoInstanceName: "swagger",
SwaggerTemplate: docTemplate,
LeftDelim: "{{",
RightDelim: "}}",
}
func init() {
swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
}
+213
View File
@@ -0,0 +1,213 @@
{
"swagger": "2.0",
"info": {
"description": "JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.\n\nThe catalogue search is `GET /api/videos`, listed below: title is matched lexically with the last word as a prefix, categories narrow to videos filed under any one of them, limit defaults to 20 and is capped at 100, and cursor is the opaque `nextCursor` of a previous search.",
"title": "Thamanyah Discovery API",
"contact": {},
"version": "1.0"
},
"basePath": "/",
"paths": {
"/api/videos": {
"get": {
"description": "Finds catalogued videos by the words of their title and the categories they are filed under, most relevant first. Every parameter is optional — a request with none of them browses the whole catalogue, newest first. Paging is by cursor rather than by offset, so a page stays the same page while videos are being announced into the catalogue around it.",
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Search the catalogue",
"parameters": [
{
"type": "string",
"example": "desert fal",
"description": "Matched lexically against video titles: the words are all required, and the last is treated as a prefix so a part-typed word still finds something.",
"name": "title",
"in": "query"
},
{
"type": "array",
"items": {
"type": "string"
},
"collectionFormat": "multi",
"example": "documentary",
"description": "Narrows the search to videos filed under any one of these names — not all of them. Repeat the parameter per category. Omitted means every category.",
"name": "categories",
"in": "query"
},
{
"type": "integer",
"default": 20,
"description": "How many videos to return. Absent, zero or negative takes the default of 20; anything above the maximum of 100 is capped to it rather than refused, and the cursor still reaches the rest.",
"name": "limit",
"in": "query"
},
{
"type": "string",
"description": "Asks for the page after the one a previous search ended at. Send back the nextCursor from that search unchanged; it is opaque, and the only thing to do with it is return it.",
"name": "cursor",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.SearchResults"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/api/videos/{id}": {
"get": {
"description": "Returns the catalogue's copy of a video: what a reader needs to show and play it. A video appears here only after the CMS has announced it as ready, so a video that is still transcoding — or one the CMS never made ready — is a 404.",
"produces": [
"application/json"
],
"tags": [
"videos"
],
"summary": "Get a catalogued video",
"parameters": [
{
"type": "string",
"description": "Video id, as issued by the CMS",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.Video"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ProblemDetails"
}
}
}
}
},
"/health": {
"get": {
"description": "Reports that the service is up and serving. Used as the ALB target group health check; it does not verify the database connection.",
"produces": [
"application/json"
],
"tags": [
"system"
],
"summary": "Health check",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/api.HealthResponse"
}
}
}
}
}
},
"definitions": {
"api.HealthResponse": {
"type": "object",
"properties": {
"status": {
"type": "string",
"example": "ok"
}
}
},
"api.ProblemDetails": {
"type": "object",
"properties": {
"detail": {
"type": "string",
"example": "The catalogue could not be read from the database. This is a server-side fault and the request was not processed; retrying in a few moments may succeed."
},
"status": {
"type": "integer",
"example": 500
},
"title": {
"type": "string",
"example": "Something Went Wrong Loading The Catalogue"
},
"type": {
"type": "string",
"example": "about:blank"
}
}
},
"api.SearchResults": {
"type": "object",
"properties": {
"nextCursor": {
"description": "NextCursor reaches the page after this one. Empty on the last page,\nwhich is how a reader knows there is no more to ask for.",
"type": "string"
},
"videos": {
"description": "Videos are the matches, most relevant first. Never null: a search that\nmatches nothing is an empty list.",
"type": "array",
"items": {
"$ref": "#/definitions/api.Video"
}
}
}
},
"api.Video": {
"type": "object",
"properties": {
"categories": {
"type": "array",
"items": {
"type": "string"
},
"example": [
"documentary",
"news"
]
},
"id": {
"type": "string",
"example": "0f8fad5b-d9cb-469f-a165-70867728950e"
},
"playbackUrl": {
"type": "string",
"example": "https://d111111abcdef8.cloudfront.net/videos/abc123/index.m3u8"
},
"title": {
"type": "string",
"example": "The Evening Bulletin"
}
}
}
}
}
+167
View File
@@ -0,0 +1,167 @@
basePath: /
definitions:
api.HealthResponse:
properties:
status:
example: ok
type: string
type: object
api.ProblemDetails:
properties:
detail:
example: The catalogue could not be read from the database. This is a server-side
fault and the request was not processed; retrying in a few moments may succeed.
type: string
status:
example: 500
type: integer
title:
example: Something Went Wrong Loading The Catalogue
type: string
type:
example: about:blank
type: string
type: object
api.SearchResults:
properties:
nextCursor:
description: |-
NextCursor reaches the page after this one. Empty on the last page,
which is how a reader knows there is no more to ask for.
type: string
videos:
description: |-
Videos are the matches, most relevant first. Never null: a search that
matches nothing is an empty list.
items:
$ref: '#/definitions/api.Video'
type: array
type: object
api.Video:
properties:
categories:
example:
- documentary
- news
items:
type: string
type: array
id:
example: 0f8fad5b-d9cb-469f-a165-70867728950e
type: string
playbackUrl:
example: https://d111111abcdef8.cloudfront.net/videos/abc123/index.m3u8
type: string
title:
example: The Evening Bulletin
type: string
type: object
info:
contact: {}
description: |-
JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.
The catalogue search is `GET /api/videos`, listed below: title is matched lexically with the last word as a prefix, categories narrow to videos filed under any one of them, limit defaults to 20 and is capped at 100, and cursor is the opaque `nextCursor` of a previous search.
title: Thamanyah Discovery API
version: "1.0"
paths:
/api/videos:
get:
description: Finds catalogued videos by the words of their title and the categories
they are filed under, most relevant first. Every parameter is optional — a
request with none of them browses the whole catalogue, newest first. Paging
is by cursor rather than by offset, so a page stays the same page while videos
are being announced into the catalogue around it.
parameters:
- description: 'Matched lexically against video titles: the words are all required,
and the last is treated as a prefix so a part-typed word still finds something.'
example: desert fal
in: query
name: title
type: string
- collectionFormat: multi
description: Narrows the search to videos filed under any one of these names
— not all of them. Repeat the parameter per category. Omitted means every
category.
example: documentary
in: query
items:
type: string
name: categories
type: array
- default: 20
description: How many videos to return. Absent, zero or negative takes the
default of 20; anything above the maximum of 100 is capped to it rather
than refused, and the cursor still reaches the rest.
in: query
name: limit
type: integer
- description: Asks for the page after the one a previous search ended at. Send
back the nextCursor from that search unchanged; it is opaque, and the only
thing to do with it is return it.
in: query
name: cursor
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/api.SearchResults'
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.ProblemDetails'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/api.ProblemDetails'
summary: Search the catalogue
tags:
- videos
/api/videos/{id}:
get:
description: 'Returns the catalogue''s copy of a video: what a reader needs
to show and play it. A video appears here only after the CMS has announced
it as ready, so a video that is still transcoding — or one the CMS never made
ready — is a 404.'
parameters:
- description: Video id, as issued by the CMS
in: path
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/api.Video'
"404":
description: Not Found
schema:
$ref: '#/definitions/api.ProblemDetails'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/api.ProblemDetails'
summary: Get a catalogued video
tags:
- videos
/health:
get:
description: Reports that the service is up and serving. Used as the ALB target
group health check; it does not verify the database connection.
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/api.HealthResponse'
summary: Health check
tags:
- system
swagger: "2.0"
+44
View File
@@ -0,0 +1,44 @@
module thamanyah/discovery
go 1.25.12
require (
github.com/aws/aws-sdk-go-v2/config v1.33.1
github.com/aws/aws-sdk-go-v2/service/sqs v1.48.1
github.com/golang-migrate/migrate/v4 v4.19.1
github.com/lib/pq v1.12.3
github.com/redis/go-redis/v9 v9.22.0
github.com/swaggo/http-swagger/v2 v2.0.2
github.com/swaggo/swag v1.16.6
)
require (
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.45.1 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.7.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.35.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.40.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.47.1 // indirect
github.com/aws/smithy-go v1.28.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/spec v0.20.6 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/swaggo/files/v2 v2.0.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/tools v0.38.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
+161
View File
@@ -0,0 +1,161 @@
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/aws/aws-sdk-go-v2 v1.45.1 h1:iIoG3NaLhV6UZpPXyPXlDj2I9oS8tV/nMcMnITCC6Ks=
github.com/aws/aws-sdk-go-v2 v1.45.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
github.com/aws/aws-sdk-go-v2/config v1.33.1 h1:bq9jze1hQ5YTCLoVxNnbp0T7rglrlOE7N9YsHqjGkEw=
github.com/aws/aws-sdk-go-v2/config v1.33.1/go.mod h1:2A3HQwG4zaL5Tm80rc6RZj8LmWWv4WYT5v8raSz/L7A=
github.com/aws/aws-sdk-go-v2/credentials v1.20.1 h1:Z8GRNEx0u9sDkZOq4PUnN8mjGwbUQGRzMSXpvt3d8xQ=
github.com/aws/aws-sdk-go-v2/credentials v1.20.1/go.mod h1:uBIK00kFo95dnemqfFMTWx0X8YRqsh6ecIoCjjOkZqM=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1 h1:YIEBqcqRnpi4Pfv0YHImtgi6czGCwKHANC7SwmUAVD0=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1/go.mod h1:imEf0oufgAo8KAkCHhrOdqGEC0YWx1PPBQH82shSxGw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 h1:pc138gM1CW+XPc60rEwUlwwuwWFQK16CI1T7v1F9Oec=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1/go.mod h1:1+koxpPIbfBdfzP6vojm5/zTpTQ/micYwlxIiNB3TxI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 h1:K0JsbZQj+1h208Ro1zHeA4l7bMp0NvRffHQ91q8Ol1s=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1/go.mod h1:W3/vL6EtCIatICGy9ab29QhMuae+cOKPWcMxv02CO+Q=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1 h1:yhw5KD1phVyP9vijxOUzDfEtJx+bt+L63k+VfuiYFAA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1/go.mod h1:ZW2e0d7DYlRxlS9hEiMXE47gTdX5KRN4byUiNbUpG+Q=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1 h1:RmmWQPREQdk9U+PfqeHW3MqZaBaNK7TpV9W3RY+b+7g=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1/go.mod h1:0A3W4F+68ZnNk5XcNL/e9HFMwnP8RlEicFfy6eOEDyw=
github.com/aws/aws-sdk-go-v2/service/signin v1.7.1 h1:mdMtSVKdQ3+mzBh+l0ogrFYZVQUCg6pJZOirA2ARsYE=
github.com/aws/aws-sdk-go-v2/service/signin v1.7.1/go.mod h1:9IqUlsJDbUPcg6cgx3WEzXdjrbWzLDQrak0aaSqlTcI=
github.com/aws/aws-sdk-go-v2/service/sqs v1.48.1 h1:jXP3BdVenFa8RfLVH+D2gswrWZHJcgtygKCf22APFqo=
github.com/aws/aws-sdk-go-v2/service/sqs v1.48.1/go.mod h1:d4DToDhLnEofHKvFu4yCF0Be65pZW267COfKOztsZOQ=
github.com/aws/aws-sdk-go-v2/service/sso v1.35.1 h1:B6WFn91tobD6gG4724ONHaqrpKsoETGnv98LHe/yIGM=
github.com/aws/aws-sdk-go-v2/service/sso v1.35.1/go.mod h1:tWuiVBUtPBr8/rgRiYS8Uf85sHcAN+G7XS3D3CEoUh8=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.40.1 h1:6yeYCWFvgbI2TI3K6jr9LtBNhXgJ7g4xqD+DEiaDDmM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.40.1/go.mod h1:naFe83jSMuYkH+QjQPX8n1MLhBkeCFM5Lsnh5m5wz3c=
github.com/aws/aws-sdk-go-v2/service/sts v1.47.1 h1:Sv2xPnRHlThSUtVujYuUBPI/Il8si6UPHXL8DMiB/F0=
github.com/aws/aws-sdk-go-v2/service/sts v1.47.1/go.mod h1:mKo/CzaCz8qytGW70NG4vIIGAx1HXTlb5lHNkC5k3lk=
github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ=
github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0=
github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
github.com/swaggo/http-swagger/v2 v2.0.2 h1:FKCdLsl+sFCx60KFsyM0rDarwiUSZ8DqbfSyIKC9OBg=
github.com/swaggo/http-swagger/v2 v2.0.2/go.mod h1:r7/GBkAWIfK6E/OLnE8fXnviHiDeAHmgIyooa4xm3AQ=
github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+23
View File
@@ -0,0 +1,23 @@
// Package api holds the request and response bodies of the HTTP API — the
// wire contract, kept apart from the handlers that serve it. The swaggo
// annotations on the handlers reference these types by name
// (api.HealthResponse and so on), so renaming one changes the generated spec.
package api
// HealthResponse is the body of GET /health.
type HealthResponse struct {
Status string `json:"status" example:"ok"`
}
// ProblemDetails is an error body in the RFC 9457 "Problem Details for HTTP
// APIs" format. Type stays "about:blank" — the value RFC 9457 defines for
// problems with no dedicated documentation URI. Title is a short summary that
// stays identical for every occurrence of the same problem, so clients can
// branch on it; Detail explains this particular occurrence and is the only
// member that varies with request data.
type ProblemDetails struct {
Type string `json:"type" example:"about:blank"`
Title string `json:"title" example:"Something Went Wrong Loading The Catalogue"`
Status int `json:"status" example:"500"`
Detail string `json:"detail,omitempty" example:"The catalogue could not be read from the database. This is a server-side fault and the request was not processed; retrying in a few moments may succeed."`
}
+28
View File
@@ -0,0 +1,28 @@
package api
// Video is the catalogue's copy of a video, as GET /api/videos/{id} serves it.
// It is deliberately not cms's video record of the same name: the catalogue
// publishes what a reader needs to show and play a video, and none of the
// ingestion detail — no storage key, no transcoding job id, no status, since
// a video that is not ready never reaches the catalogue at all.
type Video struct {
ID string `json:"id" example:"0f8fad5b-d9cb-469f-a165-70867728950e"`
Title string `json:"title" example:"The Evening Bulletin"`
PlaybackURL string `json:"playbackUrl" example:"https://d111111abcdef8.cloudfront.net/videos/abc123/index.m3u8"`
Categories []string `json:"categories" example:"documentary,news"`
}
// The search itself takes no body type: GET /api/videos reads its parameters
// — title, categories, limit and cursor — from the query string, so there is
// no request document to publish a schema for. They are documented as
// parameters on handlers.SearchVideos, and appear in the generated spec there.
// SearchResults is one page of search results.
type SearchResults struct {
// Videos are the matches, most relevant first. Never null: a search that
// matches nothing is an empty list.
Videos []Video `json:"videos"`
// NextCursor reaches the page after this one. Empty on the last page,
// which is how a reader knows there is no more to ask for.
NextCursor string `json:"nextCursor"`
}
+110
View File
@@ -0,0 +1,110 @@
// Package consumers holds the queue-driven half of the service: work that
// arrives on a queue rather than as an HTTP request. It is to SQS what
// internal/handlers is to HTTP — it decodes a message, calls repositories, and
// decides what the message's fate is — so the same seams apply, and no SQL
// lives here.
package consumers
import (
"context"
"encoding/json"
"log"
"strings"
"thamanyah/discovery/internal/db/repositories"
"thamanyah/discovery/internal/models"
"thamanyah/discovery/internal/services"
"time"
)
// receiveBackoff is how long the loop waits after a failed receive, so a queue
// that is unreachable produces a slow trickle of log lines rather than a spin.
const receiveBackoff = 5 * time.Second
// announcement is what cms publishes to the catalogue topic when a video
// becomes ready. It is cms's published contract, mirrored here rather than
// imported: the two services share no code, and this struct is exactly the
// coupling between them — so a field renamed on one side has to be renamed
// here too, deliberately.
//
// The subscription delivers raw, so this is the whole message body; there is
// no SNS envelope to unwrap.
type announcement struct {
VideoID string `json:"videoId"`
Title string `json:"title"`
PlaybackURL string `json:"playbackUrl"`
Categories []string `json:"categories"`
}
// CatalogueEvents consumes the announcements cms publishes.
type CatalogueEvents struct{}
// Run consumes announcements until ctx is cancelled. It is meant to be run in
// its own goroutine for the lifetime of the process.
func (c CatalogueEvents) Run(ctx context.Context) {
log.Println("catalogue announcements consumer started")
for {
if ctx.Err() != nil {
log.Println("catalogue announcements consumer stopped")
return
}
messages, err := services.SQSClient.ReceiveMessages(ctx)
if err != nil {
if ctx.Err() != nil {
log.Println("catalogue announcements consumer stopped")
return
}
log.Printf("Something Went Wrong Receiving Announcements: %s", err)
time.Sleep(receiveBackoff)
continue
}
for _, message := range messages {
if !c.handleAnnouncement(ctx, message.Body) {
// Left on the queue on purpose: it becomes visible again when
// the visibility timeout expires, and reaches the dead-letter
// queue if it keeps failing.
continue
}
if err := services.SQSClient.DeleteMessage(ctx, message.ReceiptHandle); err != nil {
log.Printf("Something Went Wrong Acknowledging An Announcement: %s", err)
}
}
}
}
// handleAnnouncement processes one message and reports whether it is finished
// with — that is, whether it should be deleted from the queue. It returns
// false only when a retry could plausibly succeed. Anything a retry cannot fix
// (a body that will never parse, an announcement naming no video) is finished
// with, however little it accomplished: leaving it on the queue would only
// stall the announcements behind it.
func (c CatalogueEvents) handleAnnouncement(ctx context.Context, body string) bool {
var announced announcement
if err := json.Unmarshal([]byte(body), &announced); err != nil {
log.Printf("Discarding An Announcement That Is Not JSON: %s", err)
return true
}
if strings.TrimSpace(announced.VideoID) == "" {
log.Printf("Discarding An Announcement That Names No Video")
return true
}
if err := repositories.VideoRepo.SaveVideo(ctx, models.Video{
ID: announced.VideoID,
Title: announced.Title,
PlaybackURL: announced.PlaybackURL,
Categories: announced.Categories,
}); err != nil {
// Worth another attempt: the save is an upsert, so a redelivery
// re-runs it harmlessly.
log.Printf("Something Went Wrong Saving An Announced Video: video=%q: %s", announced.VideoID, err)
return false
}
log.Printf("Catalogued An Announced Video: video=%q title=%q", announced.VideoID, announced.Title)
return true
}
+30
View File
@@ -0,0 +1,30 @@
package db
import (
"context"
"database/sql"
"fmt"
"log"
)
func CreateDBConnection(connectionString string) *sql.DB {
sqlDB, err := sql.Open("postgres", connectionString)
if err != nil {
panic(fmt.Errorf("db: cannot open connection: %w", err))
}
return sqlDB
}
func CloseConnection(SQLDB *sql.DB) {
err := SQLDB.Close()
if err != nil {
log.Fatalf("Error Closing DB Connection: %s", err)
}
}
func AssertSuccessfulConnection(ctx context.Context, SQLDB *sql.DB) {
if err := SQLDB.PingContext(ctx); err != nil {
panic(fmt.Errorf("db: cannot connect: %w", err))
}
}
+38
View File
@@ -0,0 +1,38 @@
package db
import (
"database/sql"
"embed"
"errors"
"fmt"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
"github.com/golang-migrate/migrate/v4/source/iofs"
)
//go:embed migrations/*.sql
var migrationsFS embed.FS
func Migrate(sqlDB *sql.DB) error {
driver, err := postgres.WithInstance(sqlDB, &postgres.Config{})
if err != nil {
return fmt.Errorf("db: creating migration driver: %w", err)
}
source, err := iofs.New(migrationsFS, "migrations")
if err != nil {
return fmt.Errorf("db: loading embedded migrations: %w", err)
}
m, err := migrate.NewWithInstance("iofs", source, "postgres", driver)
if err != nil {
return fmt.Errorf("db: initializing migrator: %w", err)
}
if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return fmt.Errorf("db: applying migrations: %w", err)
}
return nil
}
@@ -0,0 +1,2 @@
-- Nothing to undo: 0001 creates no objects.
SELECT 1;
@@ -0,0 +1,6 @@
-- Discovery's schema starts here. The service owns its own database — the
-- `discovery` role and same-named database provisioned by newServiceDatabase
-- in infrastructure/main.go — and shares no tables with cms, so nothing is
-- created yet. The file exists because internal/db/migrate.go embeds
-- migrations/*.sql: with no migration at all the package would not compile.
SELECT 1;
@@ -0,0 +1 @@
DROP TABLE videos;
@@ -0,0 +1,19 @@
-- The catalogue's own copy of a video, built from the announcements cms
-- publishes on the catalogue topic. It is a read model, not a second source of
-- truth: every column here arrives in an announcement, and nothing in this
-- service ever writes a video of its own.
--
-- id is the id cms issued, carried in the announcement and reused verbatim, so
-- the two services can talk about the same video without a mapping table. It
-- is therefore NOT generated here: no DEFAULT, unlike cms's videos.id.
CREATE TABLE videos (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
playback_url TEXT NOT NULL DEFAULT '',
-- Denormalized on purpose. cms owns the category vocabulary and sends
-- names, so there is nothing here to join to and no id to keep in step;
-- an array is what the read side actually serves.
categories TEXT[] NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
@@ -0,0 +1,2 @@
DROP INDEX IF EXISTS videos_search_vector_idx;
ALTER TABLE videos DROP COLUMN IF EXISTS search_vector;
@@ -0,0 +1,25 @@
-- Lexical search over video titles.
--
-- The vector is a generated column rather than something the consumer writes:
-- the catalogue's only writer is SaveVideo, and a title that arrives in an
-- announcement should be searchable because it is stored, not because a second
-- statement remembered to index it. STORED because a GIN index needs the value
-- on disk to index it at all.
--
-- 'arabic' is the text search configuration, not 'english' or 'simple': it
-- stems Arabic (الوثائقي -> وثايق) while leaving Latin-script tokens as written,
-- which is the right trade for a mixed catalogue. The cost is that English
-- words are not stemmed, so "documentaries" does not find "documentary".
--
-- The configuration is named explicitly rather than left to default_text_search_config,
-- which is a per-session GUC — an expression that reads it is only STABLE, and
-- a generated column requires IMMUTABLE. Naming it is also what keeps the
-- column and every query in step: a query built with a different
-- configuration would stem its terms differently and silently match nothing.
ALTER TABLE videos
ADD COLUMN search_vector tsvector
GENERATED ALWAYS AS (to_tsvector('arabic', title)) STORED;
-- GIN, not GiST: the catalogue is read far more than it is written, and GIN
-- answers @@ faster at the cost of a slower update — the right way round here.
CREATE INDEX videos_search_vector_idx ON videos USING GIN (search_vector);
@@ -0,0 +1 @@
DROP INDEX IF EXISTS videos_categories_idx;
@@ -0,0 +1,7 @@
-- Narrowing a search by category.
--
-- categories is a TEXT[] of names, and the search overlaps it against the
-- names a reader asked for (&&). GIN is the index type that operator can use;
-- without it the overlap is a filter applied after the rows are read, which is
-- the whole table once the title term is broad.
CREATE INDEX videos_categories_idx ON videos USING GIN (categories);
@@ -0,0 +1 @@
DROP INDEX IF EXISTS videos_recent_idx;
@@ -0,0 +1,17 @@
-- Browsing the catalogue, and paging through it.
--
-- Every search orders by (rank, created_at, id). When there is no title term
-- the rank is a constant, and Postgres folds a constant sort key away — so the
-- order that remains is exactly this index, and a browse becomes an index-only
-- scan that stops as soon as the page is full instead of sorting the table.
--
-- Measured at 200k rows: a first page of a browse goes from a 15 ms parallel
-- sequential scan with a top-N sort to 0.03 ms. It matters more for the cursor
-- than for the first page — the keyset comparison (created_at, id) < (…)
-- becomes an index condition rather than a filter, so a deep page is a seek
-- to the right place in the index rather than a walk through everything
-- before it.
--
-- DESC on both columns to match the ORDER BY: a btree can be read backwards,
-- but only as a whole, so a mixed-direction ordering could not use it.
CREATE INDEX videos_recent_idx ON videos (created_at DESC, id DESC);
@@ -0,0 +1,14 @@
package repositories
import "errors"
// Sentinel errors the repositories return so handlers can tell "there is no
// such row" from "the query failed", without importing database/sql to do it.
// ErrVideoNotFound reports that the catalogue holds no video with that id.
var ErrVideoNotFound = errors.New("video not found")
// ErrInvalidCursor reports that a paging cursor could not be read. It is the
// caller's mistake, not a fault in the catalogue, so it must not reach the
// handler as an indistinguishable query failure.
var ErrInvalidCursor = errors.New("invalid cursor")
@@ -0,0 +1,7 @@
// Package repositories holds every SQL statement the service runs. A
// repository is a struct carrying the *sql.DB opened by db.CreateDBConnection
// and is assigned to a package-level var in main, the way cms wires VideoRepo
// and CatagoriesRepo, so handlers call through something substitutable.
//
// Empty until discovery has tables to read.
package repositories
@@ -0,0 +1,252 @@
package repositories
import (
"context"
"database/sql"
"encoding/base64"
"encoding/json"
"errors"
"strings"
"thamanyah/discovery/internal/models"
"time"
"unicode"
"github.com/lib/pq"
)
type VideoRepository struct {
SQLDB *sql.DB
}
var VideoRepo VideoRepository
// SaveVideo records an announced video, replacing what the catalogue already
// held for that id.
//
// The upsert is what makes the consumer safe to retry: the catalogue topic
// delivers at-least-once, so the same announcement can arrive twice, and a
// plain INSERT would fail the second time on the primary key. Re-announcing a
// video is also how it is corrected — a later announcement wins, which is why
// every column is overwritten rather than merged.
func (svc VideoRepository) SaveVideo(ctx context.Context, v models.Video) error {
_, err := svc.SQLDB.ExecContext(ctx, `
INSERT INTO videos (id, title, playback_url, categories)
VALUES ($1, $2, $3, $4)
ON CONFLICT (id) DO UPDATE
SET title = EXCLUDED.title,
playback_url = EXCLUDED.playback_url,
categories = EXCLUDED.categories,
updated_at = now()
`, v.ID, v.Title, v.PlaybackURL, pq.Array(v.Categories))
return err
}
// GetVideoByID reads the catalogue's copy of one video. It returns
// ErrVideoNotFound when the catalogue has not heard about it — which, for a
// video cms has only just made ready, is a matter of timing rather than a
// mistake.
func (svc VideoRepository) GetVideoByID(ctx context.Context, id string) (models.Video, error) {
var v models.Video
err := svc.SQLDB.QueryRowContext(ctx, `
SELECT id, title, playback_url, categories, created_at, updated_at
FROM videos
WHERE id = $1
`, id).Scan(&v.ID, &v.Title, &v.PlaybackURL, pq.Array(&v.Categories), &v.CreatedAt, &v.UpdatedAt)
if errors.Is(err, sql.ErrNoRows) {
return models.Video{}, ErrVideoNotFound
}
if err != nil {
return models.Video{}, err
}
return v, nil
}
// searchConfig is the text search configuration the search_vector column is
// generated with. Every query has to name the same one: a tsquery built under
// a different configuration stems its terms differently and matches nothing.
const searchConfig = "arabic"
// tsQueryFor turns what a reader typed into a tsquery.
//
// The words are ANDed, and the last one is given a :* prefix match — someone
// typing into a search box is usually part-way through their last word, so
// "desert fal" should still find "Desert Falcons". Earlier words are matched
// whole, because they have been finished.
//
// The input is split on everything that is not a letter or a digit, which
// drops every character tsquery gives a meaning to (&, |, !, :, *, parens) and
// leaves nothing that could change the shape of the query. That is what makes
// it safe to interpolate the result into to_tsquery: the alternative,
// websearch_to_tsquery, parses user syntax but cannot express a prefix match.
//
// It returns "" when there is nothing to search for, which callers read as
// "no title filter" rather than "match nothing".
func tsQueryFor(term string) string {
words := strings.FieldsFunc(term, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
})
if len(words) == 0 {
return ""
}
for i, word := range words {
words[i] = strings.ToLower(word)
}
return strings.Join(words, " & ") + ":*"
}
// searchCursor is where a page of results stopped: the sort key of its last
// row. Paging resumes by asking for the rows that sort after it.
//
// It carries the rank as well as the row's identity because the results are
// ordered by relevance first, and "the row after this one" is only a
// well-defined place if every term of the ordering is pinned.
type searchCursor struct {
Rank float64 `json:"r"`
CreatedAt time.Time `json:"t"`
ID string `json:"i"`
}
// encode renders a cursor as an opaque string. Opaque on purpose: it is a
// place in a result set, not a number a client should do arithmetic on, and
// making it look like one invites callers to guess a page.
func (c searchCursor) encode() (string, error) {
raw, err := json.Marshal(c)
if err != nil {
return "", err
}
return base64.RawURLEncoding.EncodeToString(raw), nil
}
func decodeCursor(encoded string) (searchCursor, error) {
raw, err := base64.RawURLEncoding.DecodeString(encoded)
if err != nil {
return searchCursor{}, ErrInvalidCursor
}
var c searchCursor
if err := json.Unmarshal(raw, &c); err != nil {
return searchCursor{}, ErrInvalidCursor
}
if strings.TrimSpace(c.ID) == "" {
return searchCursor{}, ErrInvalidCursor
}
return c, nil
}
// SearchVideos finds catalogued videos whose title matches the given term and
// which are filed under any of the given categories, most relevant first.
//
// It returns the page and the cursor that reaches the page after it; that
// cursor is empty on the last page, which is how a caller knows to stop.
//
// The title predicate is skipped entirely when the term yields no words, and
// the category predicate when no categories are named, so a search with
// neither browses the whole catalogue newest-first.
//
// Paging is by key rather than by offset. OFFSET makes the database walk and
// discard every row it skips, so deep pages get steadily more expensive, and
// it addresses a page by its position — which moves whenever a video is
// announced, repeating or skipping rows for a reader mid-browse. Comparing
// against the last row's sort key has neither problem: the work is the same at
// any depth, and the page after a given row is that same page however much has
// been added since.
func (svc VideoRepository) SearchVideos(ctx context.Context, term string, categories []string, limit int, cursor string) ([]models.Video, string, error) {
query := tsQueryFor(term)
// Never nil: a nil slice reaches Postgres as NULL, and cardinality(NULL)
// is NULL rather than 0, so the "no category filter" branch would not fire
// and every search would come back empty.
if categories == nil {
categories = []string{}
}
// A zero cursor is passed as NULL, which the query reads as "start at the
// beginning" rather than as a row to seek past.
var after *searchCursor
if strings.TrimSpace(cursor) != "" {
decoded, err := decodeCursor(cursor)
if err != nil {
return nil, "", err
}
after = &decoded
}
var (
afterRank *float64
afterTime *time.Time
afterID *string
)
if after != nil {
afterRank, afterTime, afterID = &after.Rank, &after.CreatedAt, &after.ID
}
// One row more than asked for: if it comes back, there is another page and
// the extra row is discarded. That is what lets the last page say so,
// rather than handing out a cursor to an empty page.
//
// The rank is computed in the subquery and referenced by name in both the
// cursor comparison and the ORDER BY, so the two cannot drift apart — if
// they did, "the row after this one" would mean something different from
// the order the rows are actually in, and pages would overlap.
rows, err := svc.SQLDB.QueryContext(ctx, `
SELECT id, title, playback_url, categories, created_at, rank
FROM (
SELECT id, title, playback_url, categories, created_at,
CASE WHEN $1 = '' THEN 0::float8
ELSE ts_rank(search_vector, to_tsquery('`+searchConfig+`', $1))::float8
END AS rank
FROM videos
WHERE ($1 = '' OR search_vector @@ to_tsquery('`+searchConfig+`', $1))
AND (cardinality($2::text[]) = 0 OR categories && $2)
) ranked
WHERE $4::float8 IS NULL
OR (rank, created_at, id) < ($4::float8, $5::timestamptz, $6::uuid)
ORDER BY rank DESC, created_at DESC, id DESC
LIMIT $3
`, query, pq.Array(categories), limit+1, afterRank, afterTime, afterID)
if err != nil {
return nil, "", err
}
defer rows.Close()
// Never nil: a search that matches nothing is an empty list on the wire,
// not null.
videos := []models.Video{}
ranks := []float64{}
for rows.Next() {
var (
v models.Video
rank float64
)
if err := rows.Scan(&v.ID, &v.Title, &v.PlaybackURL, pq.Array(&v.Categories), &v.CreatedAt, &rank); err != nil {
return nil, "", err
}
videos = append(videos, v)
ranks = append(ranks, rank)
}
if err := rows.Err(); err != nil {
return nil, "", err
}
if len(videos) <= limit {
return videos, "", nil
}
videos, ranks = videos[:limit], ranks[:limit]
last := videos[len(videos)-1]
next, err := searchCursor{
Rank: ranks[len(ranks)-1],
CreatedAt: last.CreatedAt,
ID: last.ID,
}.encode()
if err != nil {
return nil, "", err
}
return videos, next, nil
}
+41
View File
@@ -0,0 +1,41 @@
package handlers
import (
"encoding/json"
"log"
"net/http"
"thamanyah/discovery/internal/api"
)
// Health is the liveness probe the ALB target group polls.
//
// @Summary Health check
// @Description Reports that the service is up and serving. Used as the ALB target group health check; it does not verify the database connection.
// @Tags system
// @Produce json
// @Success 200 {object} api.HealthResponse
// @Router /health [get]
func Health(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, api.HealthResponse{Status: "ok"})
}
func writeJSON(w http.ResponseWriter, status int, body any) {
writeJSONContent(w, "application/json", status, body)
}
func writeProblem(w http.ResponseWriter, status int, title, detail string) {
writeJSONContent(w, "application/problem+json", status, api.ProblemDetails{
Type: "about:blank",
Title: title,
Status: status,
Detail: detail,
})
}
func writeJSONContent(w http.ResponseWriter, contentType string, status int, body any) {
w.Header().Set("Content-Type", contentType)
w.WriteHeader(status)
if err := json.NewEncoder(w).Encode(body); err != nil {
log.Printf("Something Went Wrong Writing The Response: %s", err)
}
}
+306
View File
@@ -0,0 +1,306 @@
package handlers
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"thamanyah/discovery/internal/api"
"thamanyah/discovery/internal/db/repositories"
"thamanyah/discovery/internal/services"
"time"
)
// GetVideo serves the catalogue's copy of one video.
//
// @Summary Get a catalogued video
// @Description Returns the catalogue's copy of a video: what a reader needs to show and play it. A video appears here only after the CMS has announced it as ready, so a video that is still transcoding — or one the CMS never made ready — is a 404.
// @Tags videos
// @Produce json
// @Param id path string true "Video id, as issued by the CMS"
// @Success 200 {object} api.Video
// @Failure 404 {object} api.ProblemDetails
// @Failure 500 {object} api.ProblemDetails
// @Router /api/videos/{id} [get]
func GetVideo(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
video, err := repositories.VideoRepo.GetVideoByID(r.Context(), id)
if errors.Is(err, repositories.ErrVideoNotFound) {
writeProblem(w, http.StatusNotFound, "Video Not In The Catalogue",
"No video with that id is in the catalogue. A video appears here once the CMS announces it as ready, so one that is still being transcoded is not here yet.")
return
}
if err != nil {
log.Printf("Something Went Wrong Loading A Catalogued Video: id=%q: %s", id, err)
writeProblem(w, http.StatusInternalServerError, "Something Went Wrong Loading The Catalogue",
"The video could not be read from the database. This is a server-side fault and the request was not processed; retrying in a few moments may succeed.")
return
}
// Translated field by field rather than returned as-is: models.Video is the
// row, api.Video is the published schema, and neither should drag the other
// along when it changes.
writeJSON(w, http.StatusOK, api.Video{
ID: video.ID,
Title: video.Title,
PlaybackURL: video.PlaybackURL,
// Never null on the wire: a video with no categories has an empty
// list, which is a different thing from "unknown".
Categories: append([]string{}, video.Categories...),
})
}
const (
// searchPageSize is how many videos a search returns when the caller does
// not say. Small enough to keep the common request cheap.
searchPageSize = 20
// maxSearchPageSize is the largest page the catalogue will build, however
// big a page the caller asks for. The endpoint is read constantly, and a
// page size is a request for work: without a ceiling, one client asking
// for the whole catalogue in a single response makes every other reader
// wait behind it.
//
// Asking for more is not an error — the page is simply capped, and the
// cursor still reaches the rest — because a client that wants everything
// is doing something legitimate, just not in one request.
maxSearchPageSize = 100
// searchCacheTTL is how long a page of results stays servable from the
// cache before it has to be built again.
//
// Nothing invalidates an entry when a video is announced: the consumer
// writes rows continuously and would have to know which of the cached
// pages a new title belongs on, which for a ranked search is every page it
// outranks. Expiry is the cheaper answer, and it bounds the staleness — a
// video is findable within a minute of being catalogued, which is the same
// order as the announcement's own delivery lag.
searchCacheTTL = time.Minute
)
// searchCacheKey is what one page of search results is stored under: the title
// searched for, the categories it was narrowed to, and the page within that
// result set.
//
// The page is named by the cursor, because with keyset paging that is what a
// page *is* — there is no page number to key on, and the cursor identifies the
// same rows whenever it is presented. The empty cursor is the first page.
//
// The limit is in the key too, even though a reader would not call it part of
// "which page". It has to be: the same title, categories and cursor with a
// different limit is a different set of rows, and sharing an entry between
// them would hand a client asking for 50 a page of 20.
//
// The parts are JSON-encoded rather than pasted together with separators so
// that no category name can be mistaken for the boundary between two of them —
// {"c":["a,b"]} and {"c":["a","b"]} stay distinguishable, which "a,b" and
// "a,b" would not. The key stays readable in redis-cli either way.
func searchCacheKey(title string, categories []string, limit int, cursor string) string {
// The title is lowercased because the search itself is: tsQueryFor folds
// case before it builds the tsquery, so two spellings that differ only in
// case are the same search and should be the same entry. The categories
// are not — they are compared to the stored array verbatim, so "News" and
// "news" really do ask different questions.
//
// Sorting the categories is safe for the same reason the SQL uses &&:
// they mean "any of these", so their order never changed the answer. Two
// readers naming the same categories in a different order now share one
// entry instead of building the same page twice.
//
// Never nil, so that a client sending no categories at all and one whose
// categories parsed away to none — the same search, since both mean "every
// category" — land on one entry rather than on `null` and `[]`.
sorted := append([]string{}, categories...)
slices.Sort(sorted)
sorted = slices.Compact(sorted)
parts := struct {
Title string `json:"t"`
Categories []string `json:"c"`
Limit int `json:"l"`
Cursor string `json:"p"`
}{
Title: strings.ToLower(strings.TrimSpace(title)),
Categories: sorted,
Limit: limit,
Cursor: cursor,
}
// Cannot fail: every field is a string, a string slice or an int.
encoded, _ := json.Marshal(parts)
return "discovery:search:v1:" + string(encoded)
}
// cachedSearch returns the page held under key, or false when there is none to
// serve.
//
// Every failure reads as "no cached page": a cache that is unreachable, slow
// or holding something unreadable must cost a reader nothing more than the
// database query they would have paid for anyway. Only faults worth acting on
// are logged — a miss is not one.
func cachedSearch(ctx context.Context, key string) (api.SearchResults, bool) {
encoded, err := services.CacheClient.Get(ctx, key)
if errors.Is(err, services.ErrCacheMiss) {
return api.SearchResults{}, false
}
if err != nil {
log.Printf("Could Not Read The Search Cache, Falling Back To The Database: key=%q: %s", key, err)
return api.SearchResults{}, false
}
var results api.SearchResults
if err := json.Unmarshal(encoded, &results); err != nil {
// Only reachable if something else wrote this key, or the shape
// changed without the v1 in the prefix changing with it.
log.Printf("Could Not Read A Cached Search Page, Falling Back To The Database: key=%q: %s", key, err)
return api.SearchResults{}, false
}
return results, true
}
func cacheSearch(ctx context.Context, key string, results api.SearchResults) {
encoded, err := json.Marshal(results)
if err != nil {
log.Printf("Could Not Encode A Search Page For The Cache: key=%q: %s", key, err)
return
}
if err := services.CacheClient.Set(ctx, key, encoded, searchCacheTTL); err != nil {
log.Printf("Could Not Write The Search Cache: key=%q: %s", key, err)
}
}
// searchQuery is a search as the query string spells it out. Every parameter
// is optional: a request carrying none of them browses the whole catalogue,
// newest first.
type searchQuery struct {
title string
categories []string
limit int
cursor string
}
// parseSearchQuery reads the search out of the URL, applying the default and
// the cap to the page size so that the rest of the handler works with a limit
// it can use as given.
//
// The only parameter that can be malformed is the limit: every other one is a
// string the search takes as written, so there is nothing to spell wrong.
func parseSearchQuery(values url.Values) (searchQuery, error) {
parsed := searchQuery{
title: values.Get("title"),
// Repeated — ?categories=news&categories=documentary — rather than one
// comma-separated value, for the reason searchCacheKey encodes them as
// JSON rather than joining them: a category name is a name, and one
// containing a comma must not read as two.
//
// Empty values are dropped so that a client sending ?categories= means
// "every category", as omitting it does, rather than narrowing the
// search to a category named "".
categories: nonEmpty(values["categories"]),
cursor: values.Get("cursor"),
limit: searchPageSize,
}
if raw := values.Get("limit"); raw != "" {
limit, err := strconv.Atoi(raw)
if err != nil {
return searchQuery{}, fmt.Errorf("limit %q is not a whole number", raw)
}
// Zero and negative are read as "no preference", the same as omitting
// the parameter. A page of nothing is not what anyone meant by it, and
// the request is still answerable, so answering it beats refusing it.
if limit > 0 {
parsed.limit = limit
}
}
if parsed.limit > maxSearchPageSize {
parsed.limit = maxSearchPageSize
}
return parsed, nil
}
func nonEmpty(values []string) []string {
kept := make([]string, 0, len(values))
for _, v := range values {
if v != "" {
kept = append(kept, v)
}
}
return kept
}
// SearchVideos serves the catalogue's lexical search.
//
// It is served on GET: a search is safe and idempotent, which POST is not, and
// its parameters are small enough to say in a query string — which keeps the
// endpoint reachable from anything that speaks HTTP, describable in the
// OpenAPI spec, and cacheable by the intermediaries between the reader and
// this service.
//
// @Summary Search the catalogue
// @Description Finds catalogued videos by the words of their title and the categories they are filed under, most relevant first. Every parameter is optional — a request with none of them browses the whole catalogue, newest first. Paging is by cursor rather than by offset, so a page stays the same page while videos are being announced into the catalogue around it.
// @Tags videos
// @Produce json
// @Param title query string false "Matched lexically against video titles: the words are all required, and the last is treated as a prefix so a part-typed word still finds something." example(desert fal)
// @Param categories query []string false "Narrows the search to videos filed under any one of these names — not all of them. Repeat the parameter per category. Omitted means every category." collectionFormat(multi) example(documentary)
// @Param limit query int false "How many videos to return. Absent, zero or negative takes the default of 20; anything above the maximum of 100 is capped to it rather than refused, and the cursor still reaches the rest." default(20)
// @Param cursor query string false "Asks for the page after the one a previous search ended at. Send back the nextCursor from that search unchanged; it is opaque, and the only thing to do with it is return it."
// @Success 200 {object} api.SearchResults
// @Failure 400 {object} api.ProblemDetails
// @Failure 500 {object} api.ProblemDetails
// @Router /api/videos [get]
func SearchVideos(w http.ResponseWriter, r *http.Request) {
request, err := parseSearchQuery(r.URL.Query())
if err != nil {
writeProblem(w, http.StatusBadRequest, "Malformed Search",
"The search parameters could not be read: "+err.Error()+". Every parameter is optional; a request with none of them browses the whole catalogue.")
return
}
cacheKey := searchCacheKey(request.title, request.categories, request.limit, request.cursor)
if cached, hit := cachedSearch(r.Context(), cacheKey); hit {
writeJSON(w, http.StatusOK, cached)
return
}
found, next, err := repositories.VideoRepo.SearchVideos(r.Context(), request.title, request.categories, request.limit, request.cursor)
if errors.Is(err, repositories.ErrInvalidCursor) {
writeProblem(w, http.StatusBadRequest, "Malformed Search Cursor",
"The cursor could not be read. Send back the nextCursor from a previous search unchanged, or omit it to start from the first page.")
return
}
if err != nil {
log.Printf("Something Went Wrong Searching The Catalogue: title=%q: %s", request.title, err)
writeProblem(w, http.StatusInternalServerError, "Something Went Wrong Searching The Catalogue",
"The catalogue could not be searched. This is a server-side fault and the request was not processed; retrying in a few moments may succeed.")
return
}
// Translated field by field, for the same reason GetVideo does it: the row
// and the published schema are different things.
videos := make([]api.Video, 0, len(found))
for _, v := range found {
videos = append(videos, api.Video{
ID: v.ID,
Title: v.Title,
PlaybackURL: v.PlaybackURL,
Categories: append([]string{}, v.Categories...),
})
}
results := api.SearchResults{Videos: videos, NextCursor: next}
cacheSearch(r.Context(), cacheKey, results)
writeJSON(w, http.StatusOK, results)
}
+8
View File
@@ -0,0 +1,8 @@
// Package models holds the domain types the rest of the service passes
// around, the way cms holds Video and Category: plain Go structs with no JSON
// tags and no SQL, so that renaming a field here never moves the published
// API (that lives in internal/api) and never rewrites a query (that lives in
// internal/db/repositories).
//
// Empty until discovery has a domain to model.
package models
+18
View File
@@ -0,0 +1,18 @@
package models
import "time"
// Video is the catalogue's copy of a video cms has announced as ready. Every
// member is filled from the announcement — discovery originates none of it.
type Video struct {
// ID is the id cms issued, reused verbatim so both services name the same
// video the same way.
ID string
Title string
PlaybackURL string
// Categories are names, not ids: cms owns the vocabulary and announces it
// by name, so nothing here has to track its numbering.
Categories []string
CreatedAt time.Time
UpdatedAt time.Time
}
+58
View File
@@ -0,0 +1,58 @@
package services
import (
"context"
"errors"
"fmt"
"log"
"time"
"github.com/redis/go-redis/v9"
)
var CacheClient Cache
var ErrCacheMiss = errors.New("cache: miss")
type Cache interface {
Get(ctx context.Context, key string) ([]byte, error)
Set(ctx context.Context, key string, value []byte, ttl time.Duration) error
}
type RedisConcrete struct {
Redis *redis.Client
}
const cacheOperationTimeout = 100 * time.Millisecond
func (svc RedisConcrete) Get(ctx context.Context, key string) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, cacheOperationTimeout)
defer cancel()
value, err := svc.Redis.Get(ctx, key).Bytes()
if errors.Is(err, redis.Nil) {
return nil, ErrCacheMiss
}
if err != nil {
return nil, err
}
return value, nil
}
func (svc RedisConcrete) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
ctx, cancel := context.WithTimeout(ctx, cacheOperationTimeout)
defer cancel()
return svc.Redis.Set(ctx, key, value, ttl).Err()
}
func (svc RedisConcrete) Close() error {
return svc.Redis.Close()
}
func (svc RedisConcrete) AssertSuccessfulConnection(ctx context.Context) {
if err := svc.Redis.Ping(ctx).Err(); err != nil {
log.Printf("WARNING: %s", fmt.Errorf("redis: cannot reach cache at %q, searches will be served from the database: %w",
svc.Redis.Options().Addr, err))
}
}
+83
View File
@@ -0,0 +1,83 @@
// Package services holds this service's AWS clients. Discovery reaches exactly
// one AWS API — the SQS queue subscribed to the catalogue topic — so unlike
// cms there is no S3 or MediaConvert here, and nothing database-related.
package services
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/service/sqs"
)
var SQSClient SQS
// QueueMessage is one message taken off the queue. ReceiptHandle is what
// identifies it for deletion — it belongs to this delivery, not to the
// message, so it cannot be held across receives.
type QueueMessage struct {
Body string
ReceiptHandle string
}
type SQS interface {
ReceiveMessages(ctx context.Context) ([]QueueMessage, error)
DeleteMessage(ctx context.Context, receiptHandle string) error
}
type SQSConcrete struct {
SQSClient *sqs.Client
QueueURL string
}
// receiveWaitSeconds turns every receive into a long poll: the call parks on
// the server until a message arrives or this elapses, so an event is picked up
// within milliseconds of being published while an idle consumer costs one
// request every twenty seconds rather than spinning. 20 is the AWS maximum.
const receiveWaitSeconds = 20
func (svc SQSConcrete) ReceiveMessages(ctx context.Context) ([]QueueMessage, error) {
output, err := svc.SQSClient.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{
QueueUrl: &svc.QueueURL,
MaxNumberOfMessages: 10,
WaitTimeSeconds: receiveWaitSeconds,
})
if err != nil {
return nil, err
}
messages := make([]QueueMessage, 0, len(output.Messages))
for _, message := range output.Messages {
if message.Body == nil || message.ReceiptHandle == nil {
continue
}
messages = append(messages, QueueMessage{
Body: *message.Body,
ReceiptHandle: *message.ReceiptHandle,
})
}
return messages, nil
}
// DeleteMessage acknowledges a message. Until this is called the message is
// merely invisible, and it returns to the queue when its visibility timeout
// expires — which is how a handler that failed gets another attempt.
func (svc SQSConcrete) DeleteMessage(ctx context.Context, receiptHandle string) error {
_, err := svc.SQSClient.DeleteMessage(ctx, &sqs.DeleteMessageInput{
QueueUrl: &svc.QueueURL,
ReceiptHandle: &receiptHandle,
})
return err
}
// AssertSuccessfulConnection verifies the queue is reachable and readable with
// the configured credentials. It panics on failure, since a service whose
// consumer cannot start would serve a catalogue that silently stops growing.
func (svc SQSConcrete) AssertSuccessfulConnection(ctx context.Context) {
if _, err := svc.SQSClient.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{
QueueUrl: &svc.QueueURL,
}); err != nil {
panic(fmt.Errorf("sqs: cannot connect to queue %q: %w", svc.QueueURL, err))
}
}
+174
View File
@@ -0,0 +1,174 @@
package main
import (
"context"
"fmt"
"log"
"net/http"
"os"
"thamanyah/discovery/internal/consumers"
"thamanyah/discovery/internal/db"
"thamanyah/discovery/internal/db/repositories"
"thamanyah/discovery/internal/handlers"
"thamanyah/discovery/internal/services"
"time"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/sqs"
_ "github.com/lib/pq"
"github.com/redis/go-redis/v9"
httpSwagger "github.com/swaggo/http-swagger/v2"
_ "thamanyah/discovery/docs"
)
// @title Thamanyah Discovery API
// @version 1.0
// @description JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.
// @description
// @description The catalogue search is `GET /api/videos`, listed below: title is matched lexically with the last word as a prefix, categories narrow to videos filed under any one of them, limit defaults to 20 and is capped at 100, and cursor is the opaque `nextCursor` of a previous search.
// @BasePath /
func main() {
if len(os.Args) > 1 && os.Args[1] == "migrate" {
runMigrate()
return
}
runServer()
}
func runMigrate() {
concreteDBClient := db.CreateDBConnection(requireDBConnectionString())
defer db.CloseConnection(concreteDBClient)
if err := db.Migrate(concreteDBClient); err != nil {
panic(err)
}
log.Println("migrations applied successfully")
}
func runServer() {
// The queue subscribed to cms's catalogue topic. This is the only AWS
// service discovery talks to.
catalogueQueueURL := os.Getenv("CATALOGUE_EVENTS_QUEUE_URL")
if catalogueQueueURL == "" {
panic(fmt.Errorf("missing required env var: CATALOGUE_EVENTS_QUEUE_URL"))
}
awsConfig, err := awsconfig.LoadDefaultConfig(context.Background())
if err != nil {
panic(err)
}
concreteSQSClient := &services.SQSConcrete{
SQSClient: sqs.NewFromConfig(awsConfig),
QueueURL: catalogueQueueURL,
}
concreteSQSClient.AssertSuccessfulConnection(context.Background())
services.SQSClient = concreteSQSClient
// The ElastiCache Redis node the catalogue search is answered from. Not an
// AWS API call — the address resolves inside the VPC and the security
// group is what grants access — so it needs nothing from the task role and
// nothing from awsConfig.
//
// The address is required because a missing one is a deployment mistake,
// not a choice; whether the node actually answers is a different question,
// and one the service deliberately survives getting "no" to.
redisAddress := os.Getenv("REDIS_ADDR")
if redisAddress == "" {
panic(fmt.Errorf("missing required env var: REDIS_ADDR"))
}
concreteCacheClient := &services.RedisConcrete{
Redis: redis.NewClient(&redis.Options{
Addr: redisAddress,
// Per-call deadlines on top of the budget RedisConcrete already
// applies, so a connection that hangs rather than refusing cannot
// tie up a goroutine past the request that opened it. Retries are
// off for the same reason the budget is small: a second attempt at
// a cache costs more than the query it is saving.
DialTimeout: 100 * time.Millisecond,
ReadTimeout: 100 * time.Millisecond,
WriteTimeout: 100 * time.Millisecond,
MaxRetries: -1,
}),
}
defer func() { _ = concreteCacheClient.Close() }()
concreteCacheClient.AssertSuccessfulConnection(context.Background())
services.CacheClient = concreteCacheClient
concreteDBClient := db.CreateDBConnection(requireDBConnectionString())
defer db.CloseConnection(concreteDBClient)
db.AssertSuccessfulConnection(context.Background(), concreteDBClient)
repositories.VideoRepo = repositories.VideoRepository{
SQLDB: concreteDBClient,
}
// Consumes announcements for the lifetime of the process. Cancelled when
// runServer returns, which today only happens if the server itself fails.
consumerCtx, stopConsumer := context.WithCancel(context.Background())
defer stopConsumer()
go consumers.CatalogueEvents{}.Run(consumerCtx)
mux := http.NewServeMux()
mux.HandleFunc("GET /health", handlers.Health)
mux.HandleFunc("GET /api/videos/{id}", handlers.GetVideo)
// The catalogue search. GET rather than POST because a search is safe and
// idempotent, and its parameters — title, categories, limit, cursor — fit
// a query string. ServeMux keeps this apart from the {id} pattern above:
// the more specific one wins.
mux.HandleFunc("GET /api/videos", handlers.SearchVideos)
// Swagger UI and the generated spec. The UI assets are embedded in the
// binary by swaggo/files, so this needs no static directory on disk.
mux.Handle("GET /swagger/", httpSwagger.WrapHandler)
addr := ":8080"
log.Printf("listening on %s", addr)
if err := http.ListenAndServe(addr, mux); err != nil {
log.Fatal(err)
}
}
func requireDBConnectionString() string {
dbHost := os.Getenv("DB_HOST")
if dbHost == "" {
panic(fmt.Errorf("missing required env var: DB_HOST"))
}
dbPort := os.Getenv("DB_PORT")
if dbPort == "" {
panic(fmt.Errorf("missing required env var: DB_PORT"))
}
dbUser := os.Getenv("DB_USER")
if dbUser == "" {
panic(fmt.Errorf("missing required env var: DB_USER"))
}
dbPassword := os.Getenv("DB_PASSWORD")
if dbPassword == "" {
panic(fmt.Errorf("missing required env var: DB_PASSWORD"))
}
dbName := os.Getenv("DB_NAME")
if dbName == "" {
panic(fmt.Errorf("missing required env var: DB_NAME"))
}
// Defaults to require: RDS terminates TLS, and only a local Postgres that
// serves none (LocalStack's RDS emulation) has any business overriding it.
dbSSLMode := os.Getenv("DB_SSLMODE")
if dbSSLMode == "" {
dbSSLMode = "require"
}
return fmt.Sprintf(
"host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
dbHost, dbPort, dbUser, dbPassword, dbName, dbSSLMode,
)
}
+190
View File
@@ -0,0 +1,190 @@
services:
localstack:
container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
image: localstack/localstack
ports:
- "127.0.0.1:4566:4566" # LocalStack Gateway
- "127.0.0.1:4510-4559:4510-4559" # external services port range
- "127.0.0.1:443:443" # LocalStack HTTPS Gateway
environment:
# Activate LocalStack for AWS: https://docs.localstack.cloud/getting-started/auth-token/
- LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN:?} # required for Pro
# LocalStack configuration: https://docs.localstack.cloud/references/configuration/
- DEBUG=${DEBUG:-0}
- PERSISTENCE=${PERSISTENCE:-0}
# Queue URLs are handed out by LocalStack and then connected to by the
# cms container. The default "standard" strategy builds them on
# sqs.<region>.localhost.localstack.cloud, which public DNS points at
# 127.0.0.1 and which this network has no alias for — so cms would fail
# to resolve its own queue. "off" keeps them on the gateway host, the
# one name that resolves from the host and from inside the network.
- SQS_ENDPOINT_STRATEGY=off
volumes:
- "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
- "/var/run/docker.sock:/var/run/docker.sock"
networks:
default:
aliases:
# Publicly this name resolves to 127.0.0.1, so the endpoint in
# infrastructure/Pulumi.local.yaml works from the host; the alias
# makes the same name resolve here from inside the network.
- localhost.localstack.cloud
# S3 Control prefixes its endpoint host with the caller's account id
# — the SDK does this even for a custom endpoint — so the bucket-tag
# read goes to <account>.localhost.localstack.cloud. The public
# wildcard points that at 127.0.0.1, which inside the network is the
# calling container, not this one. LocalStack's account is always
# 000000000000, so one more alias covers it.
- 000000000000.localhost.localstack.cloud
# Provisions the buckets, the Postgres instance and the IAM roles inside
# LocalStack, by running infrastructure/ — the same program that deploys
# production, on its "local" stack. Runs to completion before cms starts.
# Equivalent to `pulumi stack select local && pulumi up` from the host.
infra:
image: pulumi/pulumi-go:latest
working_dir: /infra
entrypoint: ["/bin/sh", "-c"]
# One list element on purpose: a string command is split on whitespace, and
# `sh -c` would then take just the first word as its script.
command:
- >
pulumi login file:///state &&
pulumi stack select local --create &&
pulumi up --yes --non-interactive
environment:
- PULUMI_CONFIG_PASSPHRASE=${PULUMI_CONFIG_PASSPHRASE:-local}
- PULUMI_SKIP_UPDATE_CHECK=true
- DB_PASSWORD=example_password
volumes:
- "./infrastructure:/infra"
- "pulumi-state:/state" # stack state, kept out of the repo
- "pulumi-home:/root/.pulumi" # provider plugins, downloaded once
- "go-mod-cache:/go/pkg/mod"
depends_on:
localstack:
condition: service_healthy
develop:
watch:
# The program is bind-mounted, not baked into the image, so there is
# nothing to rebuild — the restart is what re-runs `pulumi up`.
- action: sync+restart
path: ./infrastructure
target: /infra
ignore:
- README.md
cms:
container_name: "${CMS_DOCKER_NAME:-cms}"
build:
context: ./cms
dockerfile: Dockerfile
ports:
- "127.0.0.1:8081:8081" # JSON API + Swagger UI at /swagger/
environment:
# AWS: point the SDK at LocalStack instead of the real endpoints. This
# host is deliberately not "localstack": it is baked into the presigned
# upload URLs this service hands out, so it has to resolve for whoever
# PUTs the file — on the host as well as inside this network.
- AWS_REGION=${AWS_REGION:-us-east-1}
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-test}
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-test}
- AWS_ENDPOINT_URL=${AWS_ENDPOINT_URL:-http://localhost.localstack.cloud:4566}
# Bucket and role wiring — see main.go, which panics if any is empty. The
# bucket names are pinned by the localstack branch in infrastructure/main.go.
- S3_BUCKET=${S3_BUCKET:-raw-uploads-bucket}
- MEDIACONVERT_INPUT_BUCKET=${MEDIACONVERT_INPUT_BUCKET:-raw-uploads-bucket}
- MEDIACONVERT_OUTPUT_BUCKET=${MEDIACONVERT_OUTPUT_BUCKET:-encoded-bucket}
- MEDIACONVERT_ROLE_ARN=${MEDIACONVERT_ROLE_ARN:-arn:aws:iam::000000000000:role/mediaconvert-service-role}
# The queue the job-events topic fans out to, consumed for the lifetime
# of the process. Pinned by the localstack branch in infrastructure/main.go;
# LocalStack always uses account 000000000000.
- MEDIACONVERT_EVENTS_QUEUE_URL=${MEDIACONVERT_EVENTS_QUEUE_URL:-http://localhost.localstack.cloud:4566/000000000000/cms-mediaconvert-events}
# The topic ready videos are announced on, which fans out to the read
# side's queue. Pinned by the localstack branch in infrastructure/main.go.
- CATALOGUE_EVENTS_TOPIC_ARN=${CATALOGUE_EVENTS_TOPIC_ARN:-arn:aws:sns:us-east-1:000000000000:catalogue-events}
# Where finished HLS output is served from. In AWS this is the CloudFront
# distribution; there is none under LocalStack, so the encoded bucket is
# addressed directly — path-style, for the same reason S3 is elsewhere.
- PLAYBACK_BASE_URL=${PLAYBACK_BASE_URL:-http://localhost.localstack.cloud:4566/encoded-bucket}
# Postgres: the RDS instance LocalStack provisions, which runs inside the
# localstack container and speaks plain TCP — hence sslmode=disable.
- DB_HOST=${DB_HOST:-localhost.localstack.cloud}
- DB_PORT=${DB_PORT:-4510}
- DB_NAME=${DB_NAME:-cms}
- DB_USER=${DB_USER:-cms}
- DB_PASSWORD=example_password
- DB_SSLMODE=${DB_SSLMODE:-disable}
depends_on:
infra:
condition: service_completed_successfully
develop:
watch:
# cms is a compiled binary, so there is nothing useful to sync into the
# running container — every change means a new image. Needs
# `docker compose up --watch` (or `docker compose watch`) to take effect.
- action: rebuild
path: ./cms
# Stands in for the ElastiCache node infrastructure/main.go provisions on the
# AWS stacks. Not provisioned through LocalStack like the buckets and the
# database are: ElastiCache hands out an endpoint on a port it chooses, and
# the literal REDIS_ADDR below has to be known before anything is created.
# Nothing here is worth persisting — every entry is derivable from Postgres,
# which is the whole point of a cache — so there is no volume.
redis:
container_name: "${REDIS_DOCKER_NAME:-redis}"
image: redis:7-alpine
ports:
- "127.0.0.1:6379:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
discovery:
container_name: "${DISCOVERY_DOCKER_NAME:-discovery}"
build:
context: ./discovery
dockerfile: Dockerfile
ports:
- "127.0.0.1:8080:8080" # JSON API + Swagger UI at /swagger/
environment:
# AWS: discovery reaches exactly one AWS API — the queue subscribed to
# cms's catalogue topic, which it drains for the lifetime of the process.
# That is why it now has a task role in infrastructure/main.go, scoped to
# this queue and nothing else.
- AWS_REGION=${AWS_REGION:-us-east-1}
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-test}
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-test}
- AWS_ENDPOINT_URL=${AWS_ENDPOINT_URL:-http://localhost.localstack.cloud:4566}
- CATALOGUE_EVENTS_QUEUE_URL=${CATALOGUE_EVENTS_QUEUE_URL:-http://localhost.localstack.cloud:4566/000000000000/discovery-catalogue-events}
# The catalogue search's cache — the redis service below, standing in for
# ElastiCache. main.go panics without an address but only warns if the
# node does not answer: a cold cache means slower searches, not none.
- REDIS_ADDR=${REDIS_ADDR:-redis:6379}
# Postgres: its own role and database, provisioned alongside the cms ones
# by newServiceDatabase, so the two services share no credentials.
- DB_HOST=${DISCOVERY_DB_HOST:-localhost.localstack.cloud}
- DB_PORT=${DISCOVERY_DB_PORT:-4510}
- DB_NAME=${DISCOVERY_DB_NAME:-discovery}
- DB_USER=${DISCOVERY_DB_USER:-discovery}
- DB_PASSWORD=example_password
- DB_SSLMODE=${DISCOVERY_DB_SSLMODE:-disable}
depends_on:
infra:
condition: service_completed_successfully
redis:
condition: service_healthy
develop:
watch:
# Same as cms: a compiled binary, so every change means a new image.
# Needs `docker compose up --watch` (or `docker compose watch`).
- action: rebuild
path: ./discovery
volumes:
pulumi-state:
pulumi-home:
go-mod-cache:
+36
View File
@@ -0,0 +1,36 @@
# The LocalStack target for the same program that deploys production. Nothing
# here changes what the code does except thamanyah:localstack — every AWS
# endpoint override lives in the aws:* provider config, which the AWS provider
# reads on its own.
#
# localhost.localstack.cloud resolves to 127.0.0.1 in public DNS, and
# docker-compose.yml also publishes it as a network alias for the localstack
# container — so this one endpoint works whether pulumi runs on the host or as
# the compose "infra" service.
config:
aws:region: us-east-1
aws:accessKey: test
aws:secretKey: test
aws:skipCredentialsValidation: "true"
# SNS builds topic ARNs from the caller's account id, so unlike the other
# services it cannot work without one — "true" here makes the provider hand
# SNS an empty account and every topic call fails. sts is pointed at
# LocalStack just below, so asking is cheap and answers 000000000000.
aws:skipRequestingAccountId: "false"
aws:skipMetadataApiCheck: "true"
# Virtual-host addressing would resolve <bucket>.localhost.localstack.cloud,
# which points at 127.0.0.1 — the wrong container from inside compose.
aws:s3UsePathStyle: "true"
aws:endpoints:
- iam: http://localhost.localstack.cloud:4566
rds: http://localhost.localstack.cloud:4566
s3: http://localhost.localstack.cloud:4566
# aws@7 reads a bucket's tags back through S3 Control's
# ListTagsForResource, so without this override the create call
# goes to real AWS and 403s — even though no tags are set here.
s3control: http://localhost.localstack.cloud:4566
sns: http://localhost.localstack.cloud:4566
sqs: http://localhost.localstack.cloud:4566
sts: http://localhost.localstack.cloud:4566
thamanyah:localstack: "true"
encryptionsalt: v1:QKoqcSNBvDc=:v1:bTGZko5jHuGNRVoq:2oQ4pFb8MYwspEbxipYeO+rVqVnLBg==
+3 -1
View File
@@ -4,6 +4,8 @@ go 1.25.11
require ( require (
github.com/pulumi/pulumi-aws/sdk/v7 v7.40.0 github.com/pulumi/pulumi-aws/sdk/v7 v7.40.0
github.com/pulumi/pulumi-postgresql/sdk/v3 v3.18.0
github.com/pulumi/pulumi-random/sdk/v4 v4.21.1
github.com/pulumi/pulumi/sdk/v3 v3.256.0 github.com/pulumi/pulumi/sdk/v3 v3.256.0
) )
@@ -71,11 +73,11 @@ require (
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/term v1.1.0 // indirect github.com/pkg/term v1.1.0 // indirect
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
github.com/pulumi/pulumi-random/sdk/v4 v4.21.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
github.com/sergi/go-diff v1.4.0 // indirect github.com/sergi/go-diff v1.4.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/cobra v1.10.2 // indirect
github.com/spf13/pflag v1.0.10 // indirect github.com/spf13/pflag v1.0.10 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect
+6 -1
View File
@@ -172,6 +172,8 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE= github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE=
github.com/pulumi/pulumi-aws/sdk/v7 v7.40.0 h1:hCWhnZ4qaXECoxFrUGmDh1S4U6aCMuPghllRvZc+VqY= github.com/pulumi/pulumi-aws/sdk/v7 v7.40.0 h1:hCWhnZ4qaXECoxFrUGmDh1S4U6aCMuPghllRvZc+VqY=
github.com/pulumi/pulumi-aws/sdk/v7 v7.40.0/go.mod h1:jv6dSPiG6CDnje0YLo7MdzkvR91kWCJjXcnETXwJdrw= github.com/pulumi/pulumi-aws/sdk/v7 v7.40.0/go.mod h1:jv6dSPiG6CDnje0YLo7MdzkvR91kWCJjXcnETXwJdrw=
github.com/pulumi/pulumi-postgresql/sdk/v3 v3.18.0 h1:2MqXGrJCeGzpiECyETTKZv+Zt/4LPZiiePYSwkuggpM=
github.com/pulumi/pulumi-postgresql/sdk/v3 v3.18.0/go.mod h1:SturfYpqsgJ8lQg0R2qEpwe5MdeUKk8b/lUPh1OH+TE=
github.com/pulumi/pulumi-random/sdk/v4 v4.21.1 h1:pqvBaBMwFPP3DV8BYh/f45V4A62kFqJRQv171pfgObk= github.com/pulumi/pulumi-random/sdk/v4 v4.21.1 h1:pqvBaBMwFPP3DV8BYh/f45V4A62kFqJRQv171pfgObk=
github.com/pulumi/pulumi-random/sdk/v4 v4.21.1/go.mod h1:4Q2jFqgCimgOQxvWntZSnV6u8+JhCkPHewloJQfLoeQ= github.com/pulumi/pulumi-random/sdk/v4 v4.21.1/go.mod h1:4Q2jFqgCimgOQxvWntZSnV6u8+JhCkPHewloJQfLoeQ=
github.com/pulumi/pulumi/sdk/v3 v3.256.0 h1:OrkIu7Iw3HUcMtoEy2uzpD21C32B5EYBLBgBoh/uQPM= github.com/pulumi/pulumi/sdk/v3 v3.256.0 h1:OrkIu7Iw3HUcMtoEy2uzpD21C32B5EYBLBgBoh/uQPM=
@@ -185,14 +187,17 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9Am
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+1330 -22
View File
File diff suppressed because it is too large Load Diff
+120
View File
@@ -0,0 +1,120 @@
# tests
BDD scenarios for the video upload feature, written in Gherkin and run by
[gobdd](https://github.com/go-bdd/gobdd).
The suite is **black box**: it speaks nothing but HTTP to a running `cms`, so
the same scenarios run against the local docker-compose stack and against a
deployed environment. It is its own Go module (`thamanyah/tests`) and imports
nothing from `cms/` — the API is the contract under test, not the code.
The upload to storage is a plain `PUT` to the presigned URL the API hands out,
with no AWS SDK involved, because that is exactly what a real client does.
```
features/*.feature the scenarios, in Gherkin
suite_test.go the gobdd suite: step registration + per-scenario world
steps_test.go what each step does
client_test.go HTTP plumbing and the wire types
```
## Running
The suite needs a live `cms` with its database migrated and its buckets
provisioned. From the repo root:
```bash
docker compose up -d # localstack + infra + cms
docker exec cms ./cms migrate # see "Migrations" below
cd tests && go test ./...
```
If nothing is serving, the suite **skips** rather than fails, so `go test ./...`
in a fresh checkout stays green.
Point it at another environment with `CMS_BASE_URL` (default
`http://localhost:8081`):
```bash
CMS_BASE_URL=https://cms.example.com go test -v ./...
```
Use `-v` to see the Gherkin: gobdd nests a Go subtest per feature, scenario and
step, so the step text shows up in the test output.
### Migrations
`docker-compose.yml` starts `cms` in server mode only. The migrate container
that runs `./cms migrate` to completion before the service starts exists in the
ECS task definition (`runMigrations` in `infrastructure/main.go`), not in
compose — so a freshly created local database has no schema and every scenario
fails with `relation "categories" does not exist`. Run `docker exec cms ./cms
migrate` once after the stack first comes up.
## What the scenarios cover
Both steps of the upload flow, end to end:
- listing the categories a video can be filed under, and the shape of that list
- issuing an upload slot for MP4 and QuickTime, including that the storage key
is generated rather than taken from the client's file name
- refusing a slot for every other media type, and for a body that is not JSON
- uploading the file to the presigned URL and registering it: the record comes
back `processing`, under the key from the slot, with a transcoding job id and
the categories and metadata that were sent
- refusing to register: a file that was never uploaded, a missing title, no
category, and a storage key this API never issued
- answering 404 for a category id no category has, kept distinct from the 500
the category lookup itself failing produces
- answering 409 for a key that was already registered, rather than queueing a
second transcoding job for the same file
- announcing a video on the catalogue topic once its job comes back COMPLETE,
carrying its id, title, playback URL and the **names** of its categories —
and announcing nothing for a job that ended in an error
- the read side building its catalogue from those announcements: the video
turns up in discovery under the id cms issued, a video still transcoding is
not there at all, and a video announced a second time is updated rather than
colliding with its own row
- searching that catalogue over `GET /api/videos`: finding a video by a word
in its title and by a part-typed one, narrowing to a category and being left
out of another, several categories meaning "any of", paging by cursor so the
pages tile the results exactly once, refusing a cursor no search issued, and
capping a page size that asks for the whole catalogue
Search scenarios give every video they catalogue a nonce in its title, and
assert relative to the video they created ("returns that video") rather than on
absolute counts. Nothing cleans up between runs, so a fixed title accumulates a
copy per run and a count would stop meaning anything. The paging scenario shares
one nonce across its videos so that searching for it matches that run's cohort
and nothing else.
The publication scenarios read `tests-catalogue-events`, the suite's **own**
queue on the catalogue topic — not discovery's. A consumer destroys what it
reads, so sharing discovery's queue would have the suite and the running
service race for every announcement and each see about half. Delivery is raw,
so a message body is the announcement with no SNS envelope.
The ingestion scenarios need `discovery` up as well as `cms`; they read it over
HTTP at `DISCOVERY_BASE_URL` (default `http://localhost:8080`).
Like the job-state publisher they stand in for AWS, so they are the second
place the suite reaches for the AWS SDK; everything a real client does still
goes over plain HTTP.
## Adding a scenario
Write it in `features/`. If it needs a step that does not exist yet, add the
function to `steps_test.go` and register it in `TestVideoUpload`. Two things to
know about gobdd:
- Step patterns are matched **unanchored**, and the first registered pattern
that matches wins. Anchor every pattern with `^…$`, as the existing ones do,
or a new step can silently shadow an old one.
- State moves between steps through the `*world` in the gobdd context, not
package variables — gobdd clones the context between the `Background` and the
scenario, so the world is held by pointer.
Scenarios write real rows and real objects, and nothing cleans up after them.
That is fine against LocalStack, which is disposable; think twice before
pointing the suite at anything that is not.
+269
View File
@@ -0,0 +1,269 @@
package tests
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
)
// defaultBaseURL is where docker-compose publishes cms on the host.
const defaultBaseURL = "http://localhost:8081"
// baseURL is the CMS the scenarios are run against. Override it to point the
// suite at a deployed environment instead of the local compose stack.
func baseURL() string {
if v := strings.TrimSpace(os.Getenv("CMS_BASE_URL")); v != "" {
return strings.TrimRight(v, "/")
}
return defaultBaseURL
}
// defaultDiscoveryBaseURL is where docker-compose publishes discovery on the
// host.
const defaultDiscoveryBaseURL = "http://localhost:8080"
// discoveryBaseURL is the Discovery service the scenarios read the catalogue
// from. Override it the same way as CMS_BASE_URL.
func discoveryBaseURL() string {
if v := strings.TrimSpace(os.Getenv("DISCOVERY_BASE_URL")); v != "" {
return strings.TrimRight(v, "/")
}
return defaultDiscoveryBaseURL
}
// response is one HTTP exchange, kept whole so a failing assertion can print
// the body it actually got rather than just a status code.
type response struct {
status int
contentType string
body []byte
}
// json decodes the body into dest. It returns an error rather than failing the
// step so callers can report the raw body alongside the decode failure.
func (r response) json(dest any) error {
if err := json.Unmarshal(r.body, dest); err != nil {
return fmt.Errorf("%w (body: %s)", err, r.summary())
}
return nil
}
func (r response) summary() string {
const max = 512
body := strings.TrimSpace(string(r.body))
if len(body) > max {
body = body[:max] + "…"
}
if body == "" {
body = "<empty>"
}
return fmt.Sprintf("%d %s: %s", r.status, r.contentType, body)
}
// problem is the RFC 9457 body every error from the CMS carries.
type problem struct {
Type string `json:"type"`
Title string `json:"title"`
Status int `json:"status"`
Detail string `json:"detail"`
}
type category struct {
ID int16 `json:"id"`
Name string `json:"name"`
}
type categoriesBody struct {
Categories []category `json:"categories"`
}
type presignBody struct {
UploadURL string `json:"uploadUrl"`
Key string `json:"key"`
}
type videoBody struct {
ID string `json:"id"`
Title string `json:"title"`
Description string `json:"description"`
CategoryIDs []int16 `json:"categoryIds"`
Tags string `json:"tags"`
FileName string `json:"fileName"`
StorageKey string `json:"storageKey"`
MediaConvertJobID string `json:"mediaConvertJobId"`
Status string `json:"status"`
PlaybackURL string `json:"playbackUrl"`
SizeBytes int64 `json:"sizeBytes"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
// client talks to the CMS over plain HTTP. It deliberately does not use the
// AWS SDK: the presigned URL is handed out by the API and is a complete
// request on its own, so uploading is a plain PUT — exactly what a browser
// client does, and what these scenarios are meant to exercise.
type client struct {
base string
http *http.Client
}
func newClient() *client {
return &client{
base: baseURL(),
// Generous: the registration call queues a MediaConvert job inline.
http: &http.Client{Timeout: 30 * time.Second},
}
}
// newDiscoveryClient is newClient pointed at the read side. Same plumbing —
// only the host differs, since both services speak the same JSON dialect.
func newDiscoveryClient() *client {
return &client{
base: discoveryBaseURL(),
http: &http.Client{Timeout: 30 * time.Second},
}
}
// catalogueVideoBody is GET /api/videos/{id} on discovery: the catalogue's own
// copy of a video, which is a different published shape from the CMS record of
// the same name.
type catalogueVideoBody struct {
ID string `json:"id"`
Title string `json:"title"`
PlaybackURL string `json:"playbackUrl"`
Categories []string `json:"categories"`
}
// searchRequest is a search of GET /api/videos on discovery: what a reader is
// looking for, plus where in the results to carry on from.
//
// Every field is optional. A search with none of them set is the whole
// catalogue, newest first.
type searchRequest struct {
Title string
Categories []string
Limit int
Cursor string
}
// query renders the search as the query string discovery reads it from.
// Categories are repeated rather than joined, since that is how the endpoint
// takes several of them, and an unset field is left out entirely so that the
// service applies its own default.
func (r searchRequest) query() url.Values {
values := url.Values{}
if r.Title != "" {
values.Set("title", r.Title)
}
for _, category := range r.Categories {
values.Add("categories", category)
}
if r.Limit != 0 {
values.Set("limit", strconv.Itoa(r.Limit))
}
if r.Cursor != "" {
values.Set("cursor", r.Cursor)
}
return values
}
// searchResultsBody is what that query answers with: the page of videos, and
// the cursor that reaches the page after it. NextCursor is empty on the last
// page, which is how a client knows to stop.
type searchResultsBody struct {
Videos []catalogueVideoBody `json:"videos"`
NextCursor string `json:"nextCursor"`
}
// ids lists the video ids in the page, for asserting on which videos came back
// without caring about the rest of their fields.
func (b searchResultsBody) ids() []string {
found := make([]string, 0, len(b.Videos))
for _, v := range b.Videos {
found = append(found, v.ID)
}
return found
}
func (b searchResultsBody) holds(id string) bool {
for _, v := range b.Videos {
if v.ID == id {
return true
}
}
return false
}
// search sends the catalogue search: a plain GET with the search spelled out
// in the query string.
func (c *client) search(path string, request searchRequest) (response, error) {
query := request.query().Encode()
if query != "" {
path += "?" + query
}
return c.get(path)
}
// postJSON marshals payload and posts it to a path on the CMS.
func (c *client) postJSON(path string, payload any) (response, error) {
encoded, err := json.Marshal(payload)
if err != nil {
return response{}, err
}
return c.send(http.MethodPost, c.base+path, "application/json", encoded)
}
// postRaw posts a body the caller has already framed, so a scenario can send
// something that is not valid JSON.
func (c *client) postRaw(path, contentType string, body []byte) (response, error) {
return c.send(http.MethodPost, c.base+path, contentType, body)
}
func (c *client) get(path string) (response, error) {
return c.send(http.MethodGet, c.base+path, "", nil)
}
// put uploads bytes to an absolute URL — the presigned one, which points at
// storage rather than at the CMS.
func (c *client) put(url, contentType string, body []byte) (response, error) {
return c.send(http.MethodPut, url, contentType, body)
}
func (c *client) send(method, url, contentType string, body []byte) (response, error) {
var reader io.Reader
if body != nil {
reader = bytes.NewReader(body)
}
request, err := http.NewRequest(method, url, reader)
if err != nil {
return response{}, err
}
if contentType != "" {
request.Header.Set("Content-Type", contentType)
}
result, err := c.http.Do(request)
if err != nil {
return response{}, fmt.Errorf("%s %s: %w", method, url, err)
}
defer result.Body.Close()
responseBody, err := io.ReadAll(result.Body)
if err != nil {
return response{}, fmt.Errorf("%s %s: reading body: %w", method, url, err)
}
return response{
status: result.StatusCode,
contentType: result.Header.Get("Content-Type"),
body: responseBody,
}, nil
}
+259
View File
@@ -0,0 +1,259 @@
package tests
import (
"context"
"encoding/json"
"fmt"
"os"
"path"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/sns"
"github.com/aws/aws-sdk-go-v2/service/sqs"
)
// These scenarios stand in for EventBridge, not for a client: they publish the
// same "MediaConvert Job State Change" event AWS would put on the topic when a
// job changes state. That is the seam cms owns — the topic, the queue, the
// consumer and the record it updates. The EventBridge rule that feeds the
// topic in AWS is Pulumi configuration, and is not exercised here.
//
// This is the only place the suite reaches for the AWS SDK. Everything a real
// client does still goes over plain HTTP.
const (
defaultEndpointURL = "http://localhost.localstack.cloud:4566"
defaultTopicARN = "arn:aws:sns:us-east-1:000000000000:mediaconvert-job-events"
defaultRegion = "us-east-1"
defaultOutputBucket = "encoded-bucket"
defaultPlaybackBaseURL = "http://localhost.localstack.cloud:4566/encoded-bucket"
// The suite's own queue on the catalogue topic cms announces ready videos
// to. The suite reads a queue rather than the topic because that is what a
// subscriber sees — SNS has nothing to poll.
//
// Deliberately not discovery's queue: a consumer destroys what it reads, so
// sharing one would have the suite and the running service race for every
// announcement and each see about half. Pinned by the localstack branch in
// infrastructure/main.go; LocalStack always uses account 000000000000.
defaultCatalogueQueueURL = "http://localhost.localstack.cloud:4566/000000000000/tests-catalogue-events"
)
// playbackBaseURL is the public host the encoded output is served from — the
// CloudFront distribution in AWS, LocalStack's own S3 endpoint here. It has to
// match what cms is configured with, since the whole point of the rewrite is
// that the two agree.
func playbackBaseURL() string {
return strings.TrimRight(envOr("PLAYBACK_BASE_URL", defaultPlaybackBaseURL), "/")
}
// playlistPathFor is where the HLS output group writes the master playlist for
// a given source key: one folder per video, named after the source object.
// QueueEncodingJob derives the job's destination the same way.
func playlistPathFor(storageKey string) string {
base := strings.TrimSuffix(storageKey, path.Ext(storageKey))
return fmt.Sprintf("s3://%s/%s/index.m3u8", envOr("MEDIACONVERT_OUTPUT_BUCKET", defaultOutputBucket), base)
}
// playbackURLFor is what cms should end up storing for that same key.
func playbackURLFor(storageKey string) string {
base := strings.TrimSuffix(storageKey, path.Ext(storageKey))
return fmt.Sprintf("%s/%s/index.m3u8", playbackBaseURL(), base)
}
func envOr(name, fallback string) string {
if v := strings.TrimSpace(os.Getenv(name)); v != "" {
return v
}
return fallback
}
// jobStateChange is the EventBridge event MediaConvert emits. Only the members
// cms reads are modelled; AWS sends a good deal more.
type jobStateChange struct {
Version string `json:"version"`
ID string `json:"id"`
DetailType string `json:"detail-type"`
Source string `json:"source"`
Account string `json:"account"`
Time time.Time `json:"time"`
Region string `json:"region"`
Resources []string `json:"resources"`
Detail jobStateDetail `json:"detail"`
}
type jobStateDetail struct {
Timestamp int64 `json:"timestamp"`
AccountID string `json:"accountId"`
Queue string `json:"queue"`
JobID string `json:"jobId"`
Status string `json:"status"`
OutputGroupDetails []outputGroup `json:"outputGroupDetails,omitempty"`
}
// outputGroup is one output group's result. An HLS group reports the manifests
// it wrote under playlistFilePaths; the segments themselves are not listed.
type outputGroup struct {
Type string `json:"type"`
PlaylistFilePaths []string `json:"playlistFilePaths,omitempty"`
}
// eventPublisher puts job state changes on the topic cms's queue subscribes to.
type eventPublisher struct {
sns *sns.Client
topicARN string
}
// awsConfig is the SDK configuration both ends of the event seam use: the
// publisher that stands in for EventBridge, and the reader that stands in for
// whoever consumes the catalogue queue.
func awsConfig(ctx context.Context) (aws.Config, error) {
return awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(envOr("AWS_REGION", defaultRegion)),
// LocalStack accepts any credentials; these keep the SDK from hunting
// for a profile that a developer's machine may not have.
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
envOr("AWS_ACCESS_KEY_ID", "test"), envOr("AWS_SECRET_ACCESS_KEY", "test"), "")),
)
}
func newEventPublisher(ctx context.Context) (*eventPublisher, error) {
cfg, err := awsConfig(ctx)
if err != nil {
return nil, err
}
endpoint := envOr("AWS_ENDPOINT_URL", defaultEndpointURL)
return &eventPublisher{
sns: sns.NewFromConfig(cfg, func(o *sns.Options) { o.BaseEndpoint = aws.String(endpoint) }),
topicARN: envOr("MEDIACONVERT_EVENTS_TOPIC_ARN", defaultTopicARN),
}, nil
}
// publishJobState sends the event MediaConvert would send. playlistPath is the
// master playlist an HLS output group reports on COMPLETE; pass "" to send a
// COMPLETE that names no playlist, which is what a job with no HLS group does.
func (p *eventPublisher) publishJobState(ctx context.Context, jobID, state, playlistPath string) error {
event := jobStateChange{
Version: "0",
ID: fmt.Sprintf("test-%d", time.Now().UnixNano()),
DetailType: "MediaConvert Job State Change",
Source: "aws.mediaconvert",
Account: "000000000000",
Time: time.Now().UTC(),
Region: envOr("AWS_REGION", defaultRegion),
Resources: []string{fmt.Sprintf("arn:aws:mediaconvert:%s:000000000000:jobs/%s", envOr("AWS_REGION", defaultRegion), jobID)},
Detail: jobStateDetail{
Timestamp: time.Now().UnixMilli(),
AccountID: "000000000000",
JobID: jobID,
Status: state,
},
}
if playlistPath != "" {
event.Detail.OutputGroupDetails = []outputGroup{{
Type: "HLS_GROUP",
PlaylistFilePaths: []string{playlistPath},
}}
}
body, err := json.Marshal(event)
if err != nil {
return err
}
_, err = p.sns.Publish(ctx, &sns.PublishInput{
TopicArn: aws.String(p.topicARN),
Message: aws.String(string(body)),
})
if err != nil {
return fmt.Errorf("publishing to %s: %w", p.topicARN, err)
}
return nil
}
// --- the catalogue queue ----------------------------------------------------
// catalogueAnnouncement is what cms puts on the catalogue queue when a video
// becomes ready. This is the published contract, so the suite spells it out
// rather than importing it: a change here is a change other services see.
type catalogueAnnouncement struct {
VideoID string `json:"videoId"`
Title string `json:"title"`
PlaybackURL string `json:"playbackUrl"`
Categories []string `json:"categories"`
}
// catalogueReader stands in for the read side, draining its own queue on the
// catalogue topic. It consumes what it reads, exactly as a real subscriber
// would. The subscription delivers raw, so a message body is the announcement
// itself with no SNS envelope to unwrap.
type catalogueReader struct {
sqs *sqs.Client
queueURL string
}
func newCatalogueReader(ctx context.Context) (*catalogueReader, error) {
cfg, err := awsConfig(ctx)
if err != nil {
return nil, err
}
endpoint := envOr("AWS_ENDPOINT_URL", defaultEndpointURL)
return &catalogueReader{
sqs: sqs.NewFromConfig(cfg, func(o *sqs.Options) { o.BaseEndpoint = aws.String(endpoint) }),
queueURL: envOr("CATALOGUE_QUEUE_URL", defaultCatalogueQueueURL),
}, nil
}
// await waits for the announcement naming videoID and returns it. Messages for
// other videos are consumed and discarded on the way: scenarios run one at a
// time, so anything else on the queue is a leftover, and leaving it would only
// make the next scenario wade through it. It returns nil if nothing names that
// video before the deadline.
func (r *catalogueReader) await(ctx context.Context, videoID string, within time.Duration) (*catalogueAnnouncement, error) {
deadline := time.Now().Add(within)
for time.Now().Before(deadline) {
output, err := r.sqs.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{
QueueUrl: aws.String(r.queueURL),
MaxNumberOfMessages: 10,
WaitTimeSeconds: 2,
})
if err != nil {
return nil, fmt.Errorf("receiving from %s: %w", r.queueURL, err)
}
var found *catalogueAnnouncement
for _, message := range output.Messages {
if message.Body == nil || message.ReceiptHandle == nil {
continue
}
var announcement catalogueAnnouncement
if err := json.Unmarshal([]byte(*message.Body), &announcement); err == nil &&
announcement.VideoID == videoID {
found = &announcement
}
if _, err := r.sqs.DeleteMessage(ctx, &sqs.DeleteMessageInput{
QueueUrl: aws.String(r.queueURL),
ReceiptHandle: message.ReceiptHandle,
}); err != nil {
return nil, fmt.Errorf("acknowledging a message on %s: %w", r.queueURL, err)
}
}
if found != nil {
return found, nil
}
}
return nil, nil
}
@@ -0,0 +1,40 @@
Feature: Building the catalogue from announcements
Discovery is the read side. It owns no ingestion of its own: it learns what
exists by subscribing to the catalogue topic cms announces ready videos on,
and keeps its own copy in its own database. Nothing polls cms and neither
service reads the other's tables.
A video therefore reaches the catalogue only once its transcode has finished
— which is the point, since a catalogue entry nobody can play is no use.
Background:
Given the CMS API is available
And the Discovery API is available
Scenario: A video the CMS announces appears in the catalogue
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "COMPLETE"
Then the catalogue eventually holds that video
And the catalogue's copy carries its title, playback URL and categories
# The catalogue is built from announcements, and cms announces only what came
# out ready. A video part-way through its transcode is therefore genuinely
# absent rather than merely late, and saying so is the honest answer.
Scenario: A video that is still being transcoded is not in the catalogue
Given I have registered a video that is being transcoded
Then the catalogue does not hold that video
And the request is rejected with status 404
And the problem title is "Video Not In The Catalogue"
# The catalogue topic delivers at-least-once, and a job's outcome can be
# reported more than once, so the same video can be announced again. The
# later announcement is the truth: it lands on top of what is already there
# rather than colliding with it. Announcing it again with no playlist is what
# makes that visible — a second announcement carrying identical content is
# indistinguishable from never having arrived.
Scenario: A video announced again is updated rather than colliding
Given I have registered a video that is being transcoded
And MediaConvert reports that the job reached "COMPLETE"
And the catalogue eventually holds that video
When MediaConvert reports that the job reached "COMPLETE" without a playlist
Then the catalogue eventually holds that video with no playback URL
@@ -0,0 +1,43 @@
Feature: Announcing ready videos to the catalogue
A video is only worth showing once its transcode has finished. When
MediaConvert reports a job complete, cms announces that video on a queue of
its own — the seam the read side reads to build the public catalogue, so
nothing downstream has to poll cms or reach into its database.
The announcement is the published contract: the video's id, its title, the
URL a player can open, and the categories it is filed under, named rather
than numbered so a reader needs nothing from cms to make sense of it.
Background:
Given the CMS API is available
Scenario: A finished job announces the video to the catalogue
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "COMPLETE"
Then the video eventually has status "ready"
And the catalogue is told about the video
And the announcement carries the video's title, playback URL and categories
# The catalogue is a list of things worth showing. A job that ended in an
# error produced nothing to play, so it is recorded against the video and
# goes no further.
Scenario Outline: A job that did not finish is not announced
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "<state>"
Then the video eventually has status "failed"
And the catalogue is told nothing about the video
Examples:
| state |
| ERROR |
| CANCELED |
# A video can be filed under several categories, so the announcement carries
# all of them. Names, not the ids the register call took: what cms numbers
# them is its own business.
Scenario: The announcement lists every category the video is filed under
Given I have registered a video being transcoded under categories "documentary, news, podcast"
When MediaConvert reports that the job reached "COMPLETE"
Then the video eventually has status "ready"
And the catalogue is told about the video
And the announcement files it under "documentary, news, podcast"
+78
View File
@@ -0,0 +1,78 @@
Feature: Searching the catalogue
Discovery holds a copy of every video cms has announced as ready. Reaching
one by id is no use to a reader who does not have an id — finding something
to watch means searching for it by what it is called and what it is filed
under.
The search is lexical: it matches the words of a title, not its meaning. The
last word a reader types is treated as a prefix, because someone typing into
a search box is usually part-way through a word.
Background:
Given the CMS API is available
And the Discovery API is available
Scenario: Finding a catalogued video by a word in its title
Given the catalogue holds a video titled "Desert Falcons"
When I search the catalogue for that video's title
Then the search returns that video
# The last word is matched as a prefix, so the results keep up with someone
# still typing. Earlier words are matched whole — they have been finished.
Scenario: Finding a video from a part-typed word
Given the catalogue holds a video titled "Desert Falcons"
When I search the catalogue for "Desert Fal"
Then the search returns that video
# Categories narrow a search rather than widening it: a video filed elsewhere
# is not an answer to a reader who asked for one category in particular.
Scenario: A search narrowed to another category leaves the video out
Given the catalogue holds a video titled "Desert Falcons" filed under "documentary"
When I search the catalogue for that video's title in category "news"
Then the search does not return that video
Scenario: A search narrowed to the video's own category still finds it
Given the catalogue holds a video titled "Desert Falcons" filed under "documentary"
When I search the catalogue for that video's title in category "documentary"
Then the search returns that video
# Several categories mean "any of these", not "all of these": naming more of
# them offers the reader more, rather than demanding the video be filed under
# every one at once.
Scenario: Naming several categories matches a video filed under any of them
Given the catalogue holds a video titled "Desert Falcons" filed under "documentary"
When I search the catalogue for that video's title in categories "news, documentary"
Then the search returns that video
# Paging is by cursor rather than by offset: the catalogue is added to while
# people are reading it, and an offset silently repeats or skips a video when
# something is announced between one page and the next. A cursor names where
# the last page stopped, so the page after it is the same page whenever it is
# asked for.
Scenario: Paging through the results a page at a time
Given the catalogue holds 3 videos titled "Mountain Wolves"
When I search the catalogue for that title 2 at a time
Then the search returns 2 videos
And the search offers a cursor to the next page
When I follow the cursor
Then the search returns 1 video
And the search offers no further cursor
And the pages together hold every one of those videos exactly once
# A cursor is opaque, so a client that makes one up has made a mistake rather
# than found a fault. Saying 400 is what tells the two apart.
Scenario: Refusing a cursor no search issued
Given the catalogue holds a video titled "Desert Falcons"
When I search the catalogue for that video's title from the cursor "not-a-cursor"
Then the request is rejected with status 400
And the problem title is "Malformed Search Cursor"
# A page size is a request for work, and an uncapped one is a request for all
# of it — the catalogue is read constantly, and one client asking for every
# row at once should not be able to make everyone else wait. The cap applies
# silently rather than as a rejection: the reader still gets a page, and the
# cursor still reaches the rest.
Scenario: Capping a page size that asks for the whole catalogue
When I search the whole catalogue 5000 at a time
Then the search returns at most 100 videos
And the search offers a cursor to the next page
+65
View File
@@ -0,0 +1,65 @@
Feature: Transcoding status
A registered video starts life as "processing" and is handed to MediaConvert.
MediaConvert reports every job state change to an SNS topic, which fans the
event out to a queue cms consumes, so the catalogue record catches up with
the transcoding job on its own — nobody polls AWS and nothing is exposed for
AWS to call back into.
Background:
Given the CMS API is available
Scenario: A finished job marks the video ready
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "COMPLETE"
Then the video eventually has status "ready"
# The COMPLETE event carries the paths MediaConvert actually wrote. The
# playlist among them is an s3:// URI into a private bucket, so what the
# catalogue stores is that path rewritten onto the public delivery host.
Scenario: A finished job gives the video an HLS playback URL
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "COMPLETE"
Then the video eventually has status "ready"
And the video has a playback URL for its HLS playlist
Scenario: A finished job that reported no playlist still marks the video ready
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "COMPLETE" without a playlist
Then the video eventually has status "ready"
And the video has no playback URL
Scenario: A failed job leaves the video with no playback URL
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "ERROR"
Then the video eventually has status "failed"
And the video has no playback URL
Scenario Outline: A job that did not finish cleanly marks the video failed
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "<state>"
Then the video eventually has status "failed"
Examples:
| state |
| ERROR |
| CANCELED |
Scenario Outline: A state that is not an outcome leaves the video processing
Given I have registered a video that is being transcoded
When MediaConvert reports that the job reached "<state>"
Then the video keeps status "processing"
Examples:
| state |
| SUBMITTED |
| PROGRESSING |
| STATUS_UPDATE |
# A message naming a job nobody has must be consumed and dropped, not left to
# redeliver forever. The second event is the assertion that it was: it can
# only be handled if the first one did not wedge the consumer.
Scenario: An event for a job no video has does not stop the consumer
Given I have registered a video that is being transcoded
When MediaConvert reports that the job "1700000000000-nosuchjob" reached "COMPLETE"
And MediaConvert reports that the job reached "COMPLETE"
Then the video eventually has status "ready"
+52
View File
@@ -0,0 +1,52 @@
Feature: Video details
A registered video can be read back by its id. This is how a client learns
what became of an upload once it was handed to the transcoding pipeline —
without it, the status a video carries is written but never visible.
Background:
Given the CMS API is available
Scenario: Reading back a video that was registered
Given I have requested an upload slot for "detail.mp4" of type "video/mp4"
And I have uploaded the file to the upload URL
And I have registered the uploaded video titled "Inside the Newsroom" under categories "documentary, news"
When I ask the CMS for that video
Then the request succeeds with status 200
And the video has an id
And the video is registered with status "processing"
And the video is stored under the key from the upload slot
And the video is filed under categories "documentary, news"
And the video keeps the metadata I sent
# A uuid has several accepted spellings, and Postgres takes only some of them
# verbatim — the urn form it rejects outright. The handler parses the id and
# queries with the canonical form, so every spelling names the same video
# instead of some of them faulting.
Scenario Outline: Reading a video by another accepted spelling of its id
Given I have registered a video
When I ask the CMS for that video with its id written <form>
Then the request succeeds with status 200
And the video has an id
Examples:
| form |
| braced |
| unhyphenated |
| as a urn |
Scenario: Asking for a video that does not exist
When I ask the CMS for the video with id "0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6f"
Then the request is rejected with status 404
And the problem title is "Video Not Found"
Scenario Outline: Asking for a video with an id that is not a video id
When I ask the CMS for the video with id "<id>"
Then the request is rejected with status 404
And the problem title is "Video Not Found"
Examples:
| id |
| not-a-uuid |
| 12345 |
| 0199f3a1-7c2e-7b21-9f0d-1a2b3c4d5e6 |
| '; DROP TABLE videos; -- |
+115
View File
@@ -0,0 +1,115 @@
Feature: Video upload
Getting a video into the catalogue is a two-step exchange. The client asks
the CMS for an upload slot, PUTs the file straight to storage with the
short-lived URL it is handed, and then registers the video — which queues
the transcoding job and writes the catalogue record. The file itself never
passes through the API.
Background:
Given the CMS API is available
Scenario: Discovering the categories a video can be filed under
When I ask the CMS for the list of categories
Then the request succeeds with status 200
And the category list is not empty
And every category has an id and a name
Scenario: Asking for an upload slot for an MP4
When I request an upload slot for "interview-cut.mp4" of type "video/mp4"
Then the request succeeds with status 200
And I am given an upload URL
And the storage key is under "videos/" and ends with ".mp4"
Scenario: Asking for an upload slot for a QuickTime movie
When I request an upload slot for "interview-cut.mov" of type "video/quicktime"
Then the request succeeds with status 200
And I am given an upload URL
And the storage key is under "videos/" and ends with ".mov"
Scenario Outline: Refusing to hand out a slot for anything that is not an MP4 or MOV
When I request an upload slot for "<file>" of type "<type>"
Then the request is rejected with status 422
And the problem title is "Unsupported file type. Please upload an MP4 or MOV video."
Examples:
| file | type |
| clip.avi | video/avi |
| reel.mkv | video/x-matroska |
| poster.png | image/png |
| notes.txt | text/plain |
| blob.bin | application/octet-stream |
| clip.mp4 | |
Scenario: Refusing a request body that is not JSON
When I send malformed JSON to the upload slot endpoint
Then the request is rejected with status 400
And the problem title is "Invalid request."
Scenario: Registering a video that has been uploaded
Given I have requested an upload slot for "interview-cut.mp4" of type "video/mp4"
And I have uploaded the file to the upload URL
When I register the uploaded video titled "Inside the Newsroom" under categories "documentary, news"
Then the request succeeds with status 201
And the video is registered with status "processing"
And the video has an id
And the video is stored under the key from the upload slot
And the video has a transcoding job id
And the video is filed under categories "documentary, news"
And the video keeps the metadata I sent
Scenario: Registering a video under a single category
Given I have requested an upload slot for "short.mov" of type "video/quicktime"
And I have uploaded the file to the upload URL
When I register the uploaded video titled "A Short Film" under categories "documentary"
Then the request succeeds with status 201
And the video is filed under categories "documentary"
Scenario: Refusing to register a video whose file was never uploaded
Given I have requested an upload slot for "ghost.mp4" of type "video/mp4"
When I register the uploaded video titled "The One That Got Away" under categories "other"
Then the request is rejected with status 422
And the problem title is "Failed to find video in storage"
Scenario: Refusing to register a video without a title
Given I have requested an upload slot for "untitled.mp4" of type "video/mp4"
And I have uploaded the file to the upload URL
When I register the uploaded video titled " " under categories "other"
Then the request is rejected with status 422
And the problem title is "Title is required."
Scenario: Refusing to register a video with no category
Given I have requested an upload slot for "uncategorised.mp4" of type "video/mp4"
And I have uploaded the file to the upload URL
When I register the uploaded video titled "Uncategorised" under categories ""
Then the request is rejected with status 422
And the problem title is "At least one category is required."
Scenario Outline: Refusing to register a storage key this API never issued
When I register a video titled "Smuggled In" with the key "<key>" under categories "other"
Then the request is rejected with status 422
And the problem title is "A video file is required."
Examples:
| key |
| |
| not-mine.mp4 |
| uploads/not-mine.mp4 |
| ../videos/escape.mp4 |
| videos-of-someone-else/clip.mp4 |
Scenario: Registering a video under a category id that does not exist
Given I have requested an upload slot for "mystery.mp4" of type "video/mp4"
And I have uploaded the file to the upload URL
When I register the uploaded video titled "Mystery Genre" under category id 32767
Then the request is rejected with status 404
And the problem title is "Catagory Does Not Exist"
# A key is registered once. The second attempt is turned away before the
# transcoding job is queued, so it cannot leave a duplicate job behind.
Scenario: Registering the same upload twice
Given I have requested an upload slot for "twice.mp4" of type "video/mp4"
And I have uploaded the file to the upload URL
And I have registered the uploaded video titled "Filed Once" under categories "other"
When I register the uploaded video titled "Filed Twice" under categories "other"
Then the request is rejected with status 409
And the problem title is "Video Already Registered"
+29
View File
@@ -0,0 +1,29 @@
module thamanyah/tests
go 1.25.12
require (
github.com/aws/aws-sdk-go-v2 v1.45.1
github.com/aws/aws-sdk-go-v2/config v1.32.40
github.com/aws/aws-sdk-go-v2/credentials v1.19.39
github.com/aws/aws-sdk-go-v2/service/sns v1.43.0
github.com/aws/aws-sdk-go-v2/service/sqs v1.48.1
github.com/go-bdd/gobdd v1.1.4
)
require (
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 // indirect
github.com/aws/smithy-go v1.28.1 // indirect
github.com/cucumber/gherkin/go/v33 v33.0.0 // indirect
github.com/cucumber/messages/go/v28 v28.0.0 // indirect
github.com/google/uuid v1.6.0 // indirect
)
+50
View File
@@ -0,0 +1,50 @@
github.com/aws/aws-sdk-go-v2 v1.45.1 h1:iIoG3NaLhV6UZpPXyPXlDj2I9oS8tV/nMcMnITCC6Ks=
github.com/aws/aws-sdk-go-v2 v1.45.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
github.com/aws/aws-sdk-go-v2/config v1.32.40 h1:lAVC9gMmKusmqDRe32dPtgKl/BWvJmMJoWELKHCAObw=
github.com/aws/aws-sdk-go-v2/config v1.32.40/go.mod h1:8xOJLbe/hOj1g4PVsfJYV7O2byq+UGET1onDdUgbwqc=
github.com/aws/aws-sdk-go-v2/credentials v1.19.39 h1:XOg8LC3Kgnsa3WiPQjc7Bi8k5IBN92cPYfIV9XMFss0=
github.com/aws/aws-sdk-go-v2/credentials v1.19.39/go.mod h1:GonTDBQ+mTpCVNwaHjj0PagspfrYYMEqOx7FehoEP/I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 h1:r5aGipEVgI9aT/tAGjdrPbDQvIAKdTrS3rUPQtG4Rmo=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40/go.mod h1:vOD3CnPxAdkL6MWZeROkZsTlskklMFfgVFkHzx/oZpY=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 h1:pc138gM1CW+XPc60rEwUlwwuwWFQK16CI1T7v1F9Oec=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1/go.mod h1:1+koxpPIbfBdfzP6vojm5/zTpTQ/micYwlxIiNB3TxI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 h1:K0JsbZQj+1h208Ro1zHeA4l7bMp0NvRffHQ91q8Ol1s=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1/go.mod h1:W3/vL6EtCIatICGy9ab29QhMuae+cOKPWcMxv02CO+Q=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 h1:nv/ILuCY0yXACzMQwvtt/HbqDDjemZiI0AeDbxGQlnU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41/go.mod h1:dzvOSpxaPqQ3j0xS6Lc1vyVuWW0RBj7s/QqYpzu3Q/0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 h1:gr3Fw1cxZXNCdeo/lQ7isHEHzvHVM7z75qb2zW9aMjw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40/go.mod h1:8z/9CmfnQhiuXD7Ykbcg4a/whSWsniE0ODSx9uwVzfk=
github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 h1:agcr0j8YeFEzdXNo17Rg9MbbjLRjrimabwNtji4e+lU=
github.com/aws/aws-sdk-go-v2/service/signin v1.6.0/go.mod h1:qU5PxgQ4JiUOOMotzfO3+5oUda5W+8JDVKyLQqlrJik=
github.com/aws/aws-sdk-go-v2/service/sns v1.43.0 h1:VPYjwn0BoX34hb44OT8T+Ikgn4NzsN7fHetaHaevsDc=
github.com/aws/aws-sdk-go-v2/service/sns v1.43.0/go.mod h1:I1vnLPvvi9KBqxddu8nJ4vktoPJvaIG05UmjBD9sqm8=
github.com/aws/aws-sdk-go-v2/service/sqs v1.48.1 h1:jXP3BdVenFa8RfLVH+D2gswrWZHJcgtygKCf22APFqo=
github.com/aws/aws-sdk-go-v2/service/sqs v1.48.1/go.mod h1:d4DToDhLnEofHKvFu4yCF0Be65pZW267COfKOztsZOQ=
github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 h1:FxaN8/sn61DTXNI6Gt678tFJUY8iUsCchm6Y/F/RjaA=
github.com/aws/aws-sdk-go-v2/service/sso v1.34.0/go.mod h1:vu4OY6s8LJtT8BtYG2LD6BGSZMptkYn3o5hvCPB22jc=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 h1:crWKPeGYTBTuBxQ3p73kjfJvt4brUIsr+Fuypko8FxY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0/go.mod h1:HjjZVhaBz0JBR/kbWKThmNDhFKS7y6EURuk493tJk9Y=
github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 h1:IZ63JdogSNNjex/jsODNv7jGDcO/xJYd9FsgyfCsp1g=
github.com/aws/aws-sdk-go-v2/service/sts v1.46.0/go.mod h1:I+rwAf3spG5dITBaAo3xXRowk8kiOhtU1kYxfvCTC44=
github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/cucumber/gherkin/go/v33 v33.0.0 h1:PqQ81cHjD732/GZ7c/6k/sjlYG4LzLp51nggL6172Sg=
github.com/cucumber/gherkin/go/v33 v33.0.0/go.mod h1:mnP4fdkoc+LmjSLi9Kq3M5D84GphRWLdgeKAlvUYp2c=
github.com/cucumber/messages/go/v28 v28.0.0 h1:BOJmy8LKSbdKxM6Ba1v9ZmpZk7j5cyH+LTAaGxMeflc=
github.com/cucumber/messages/go/v28 v28.0.0/go.mod h1:2njGQBk+mu0aqzQgN8xcXvnrfWNcP7UjEyi1YauQ5Lc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-bdd/assert v0.0.0-20200713105154-236f01430281 h1:jfonqldVSNVekjmLvHqelFzBba7SwylNIZ3fbM/lKVs=
github.com/go-bdd/assert v0.0.0-20200713105154-236f01430281/go.mod h1:dOoqt7g2I/fpR7/Pyz0P19J3xjDj5lsHn3v9EaFLRjM=
github.com/go-bdd/gobdd v1.1.4 h1:HCECNRNcqEBqbOVCCT6CP/K0iBkqIxfUwX6e47rI1Lk=
github.com/go-bdd/gobdd v1.1.4/go.mod h1:CRd+r+YXUzK44n8GxagSEFsIYhd8fl0reKg7AoArUMw=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+1219
View File
File diff suppressed because it is too large Load Diff
+245
View File
@@ -0,0 +1,245 @@
package tests
import (
"strings"
"testing"
"github.com/go-bdd/gobdd"
)
// world is the state one scenario accumulates as its steps run: the exchange
// the last step performed, and the upload slot it is working with.
//
// It is stored in the gobdd context behind worldKey and handed out as a
// pointer, because gobdd clones the context between the Background steps and
// the scenario steps — a value would leave the Background's work behind.
type world struct {
client *client
// discovery is the read side, set by the Background step that checks it is
// up. Nil in scenarios that never mention the catalogue.
discovery *client
// last is the most recent exchange with the CMS. Every "Then the request
// …" step reads it.
last response
// The upload slot handed out by POST /api/videos/presign, and whether the
// file has been PUT to it.
slot presignBody
slotType string
slotFile string
uploaded bool
// sent is the last registration body posted to POST /api/videos, and video
// is the record that came back from it.
sent registerRequest
video videoBody
// categoriesByName caches GET /api/categories so scenarios can name a
// category instead of hard-coding the id the seed migration happened to
// give it.
categoriesByName map[string]int16
// announcement is the message the catalogue topic carried for this
// scenario's video, once a step has waited for it.
announcement *catalogueAnnouncement
// catalogued is the read side's own copy of the video, once a step has
// waited for it to arrive.
catalogued *catalogueVideoBody
// searchTitle is the exact title the scenario's video was catalogued
// under. Scenarios name a plain title like "Desert Falcons"; the step
// registering it appends a nonce, because nothing cleans up between runs
// and a title reused across runs would make a result count meaningless.
searchTitle string
// results is the last page the catalogue search answered with.
results searchResultsBody
// pageSeen accumulates the ids from every page a paging scenario has
// walked, so it can assert the pages tile the results rather than
// repeating or dropping one.
pageSeen []string
// cohort is the ids of the videos a scenario catalogued together under one
// shared title, in the order they were created.
cohort []string
}
type worldKey struct{}
func worldOf(t gobdd.StepTest, ctx gobdd.Context) *world {
value, err := ctx.Get(worldKey{})
if err != nil {
t.Fatalf("no world in the scenario context: %s", err)
return nil
}
w, ok := value.(*world)
if !ok {
t.Fatalf("the scenario context holds a %T, not a *world", value)
return nil
}
return w
}
// categoryID resolves a category name to the id the API issues for it, so the
// feature file can say "documentary" instead of "1".
func (w *world) categoryID(t gobdd.StepTest, name string) int16 {
if w.categoriesByName == nil {
result, err := w.client.get("/api/categories")
if err != nil {
t.Fatalf("could not load the category list: %s", err)
return 0
}
if result.status != 200 {
t.Fatalf("could not load the category list: %s", result.summary())
return 0
}
var body categoriesBody
if err := result.json(&body); err != nil {
t.Fatalf("could not decode the category list: %s", err)
return 0
}
w.categoriesByName = make(map[string]int16, len(body.Categories))
for _, c := range body.Categories {
w.categoriesByName[c.Name] = c.ID
}
}
id, ok := w.categoriesByName[name]
if !ok {
t.Fatalf("the CMS knows no category named %q; it offers %v", name, w.categoriesByName)
return 0
}
return id
}
// categoryName resolves a category id back to its name, for asserting on a
// message that names categories rather than numbering them.
func (w *world) categoryName(t gobdd.StepTest, id int16) string {
w.categoryID(t, "other") // ensures the list is loaded
for name, known := range w.categoriesByName {
if known == id {
return name
}
}
t.Fatalf("the CMS knows no category with id %d; it offers %v", id, w.categoriesByName)
return ""
}
// categoryIDs turns a comma-separated list of category names into the ids the
// register endpoint expects. An empty list stays empty — that is a scenario in
// its own right.
func (w *world) categoryIDs(t gobdd.StepTest, names string) []int16 {
ids := []int16{}
for _, name := range strings.Split(names, ",") {
name = strings.TrimSpace(name)
if name == "" {
continue
}
ids = append(ids, w.categoryID(t, name))
}
return ids
}
// TestVideoUpload runs features/*.feature against a live CMS.
//
// The suite is black-box on purpose: it speaks only HTTP, so the same
// scenarios run against the docker-compose stack and against a deployed
// environment. Point it somewhere else with CMS_BASE_URL.
func TestVideoUpload(t *testing.T) {
c := newClient()
// Skip rather than fail when nothing is serving: a bare `go test ./...`
// in a fresh checkout should not go red because the stack is down.
if _, err := c.get("/health"); err != nil {
t.Skipf("no CMS at %s (%s) — start one with `docker compose up` from the repo root, "+
"or set CMS_BASE_URL to point at a running instance", c.base, err)
}
suite := gobdd.NewSuite(t,
gobdd.WithFeaturesPath("features/*.feature"),
gobdd.WithBeforeScenario(func(ctx gobdd.Context) {
ctx.Set(worldKey{}, &world{client: newClient()})
}),
)
// Given — the stack is up, and steps that arrange state a later When acts on.
suite.AddStep(`^the CMS API is available$`, theAPIIsAvailable)
suite.AddStep(`^the Discovery API is available$`, theDiscoveryAPIIsAvailable)
suite.AddStep(`^I have requested an upload slot for "(.*)" of type "(.*)"$`, requestUploadSlot)
suite.AddStep(`^I have uploaded the file to the upload URL$`, uploadTheFile)
suite.AddStep(`^I have registered the uploaded video titled "(.*)" under categories "(.*)"$`, registerUploadedVideo)
// When — the action under test.
suite.AddStep(`^I ask the CMS for the list of categories$`, askForCategories)
suite.AddStep(`^I request an upload slot for "(.*)" of type "(.*)"$`, requestUploadSlot)
suite.AddStep(`^I upload the file to the upload URL$`, uploadTheFile)
suite.AddStep(`^I send malformed JSON to the upload slot endpoint$`, sendMalformedJSON)
suite.AddStep(`^I ask the CMS for that video$`, askForThatVideo)
suite.AddStep(`^I ask the CMS for the video with id "(.*)"$`, askForVideoWithID)
suite.AddStep(`^I have registered a video that is being transcoded$`, registerVideoBeingTranscoded)
suite.AddStep(`^I have registered a video being transcoded under categories "(.*)"$`, registerVideoBeingTranscodedUnder)
suite.AddStep(`^I have registered a video$`, registerVideoBeingTranscoded)
suite.AddStep(`^I ask the CMS for that video with its id written (.*)$`, askForThatVideoWrittenAs)
suite.AddStep(`^MediaConvert reports that the job reached "(.*)"$`, mediaConvertReportsState)
suite.AddStep(`^MediaConvert reports that the job reached "(.*)" without a playlist$`, mediaConvertReportsStateWithoutPlaylist)
suite.AddStep(`^MediaConvert reports that the job "(.*)" reached "(.*)"$`, mediaConvertReportsStateForJob)
suite.AddStep(`^the video eventually has status "(.*)"$`, theVideoEventuallyHasStatus)
suite.AddStep(`^the video keeps status "(.*)"$`, theVideoKeepsStatus)
suite.AddStep(`^the video has a playback URL for its HLS playlist$`, theVideoHasAPlaybackURL)
suite.AddStep(`^the video has no playback URL$`, theVideoHasNoPlaybackURL)
suite.AddStep(`^the catalogue is told about the video$`, theCatalogueIsToldAboutTheVideo)
suite.AddStep(`^the catalogue is told nothing about the video$`, theCatalogueIsToldNothingAboutTheVideo)
suite.AddStep(`^the announcement carries the video's title, playback URL and categories$`, theAnnouncementCarriesTheVideosDetails)
suite.AddStep(`^the announcement files it under "(.*)"$`, theAnnouncementFilesItUnder)
suite.AddStep(`^the catalogue eventually holds that video$`, theCatalogueEventuallyHoldsThatVideo)
suite.AddStep(`^the catalogue does not hold that video$`, theCatalogueDoesNotHoldThatVideo)
suite.AddStep(`^the catalogue eventually holds that video with no playback URL$`, theCatalogueEventuallyHoldsThatVideoWithNoPlaybackURL)
suite.AddStep(`^the catalogue's copy carries its title, playback URL and categories$`, theCataloguesCopyCarriesTheDetails)
suite.AddStep(`^the catalogue holds a video titled "([^"]*)"$`, theCatalogueHoldsAVideoTitled)
suite.AddStep(`^I search the catalogue for that video's title$`, searchTheCatalogueForThatTitle)
suite.AddStep(`^I search the catalogue for "([^"]*)"$`, searchTheCatalogueFor)
suite.AddStep(`^the catalogue holds a video titled "([^"]*)" filed under "([^"]*)"$`, theCatalogueHoldsAVideoTitledUnder)
suite.AddStep(`^I search the catalogue for that video's title in category "([^"]*)"$`, searchTheCatalogueForThatTitleInCategory)
suite.AddStep(`^I search the catalogue for that video's title in categories "([^"]*)"$`, searchTheCatalogueForThatTitleInCategories)
suite.AddStep(`^the catalogue holds (\d+) videos titled "([^"]*)"$`, theCatalogueHoldsVideosTitled)
suite.AddStep(`^I search the catalogue for that title (\d+) at a time$`, searchTheCatalogueForThatTitleAPageAtATime)
suite.AddStep(`^I search the whole catalogue (\d+) at a time$`, searchTheWholeCatalogue)
suite.AddStep(`^I follow the cursor$`, followTheCursor)
suite.AddStep(`^I search the catalogue for that video's title from the cursor "([^"]*)"$`, searchTheCatalogueFromCursor)
suite.AddStep(`^the search returns (\d+) videos?$`, theSearchReturnsNVideos)
suite.AddStep(`^the search returns at most (\d+) videos$`, theSearchReturnsAtMostNVideos)
suite.AddStep(`^the search offers a cursor to the next page$`, theSearchOffersACursor)
suite.AddStep(`^the search offers no further cursor$`, theSearchOffersNoFurtherCursor)
suite.AddStep(`^the pages together hold every one of those videos exactly once$`, thePagesTileTheCohort)
suite.AddStep(`^the search does not return that video$`, theSearchDoesNotReturnThatVideo)
suite.AddStep(`^the search returns that video$`, theSearchReturnsThatVideo)
suite.AddStep(`^I register the uploaded video titled "(.*)" under categories "(.*)"$`, registerUploadedVideo)
suite.AddStep(`^I register the uploaded video titled "(.*)" under category id (\d+)$`, registerUnderCategoryID)
suite.AddStep(`^I register a video titled "(.*)" with the key "(.*)" under categories "(.*)"$`, registerWithKey)
// Then — assertions on the last exchange.
suite.AddStep(`^the request succeeds with status (\d+)$`, theRequestSucceedsWith)
suite.AddStep(`^the request is rejected with status (\d+)$`, theRequestIsRejectedWith)
suite.AddStep(`^the problem title is "(.*)"$`, theProblemTitleIs)
suite.AddStep(`^the category list is not empty$`, theCategoryListIsNotEmpty)
suite.AddStep(`^every category has an id and a name$`, everyCategoryHasAnIDAndAName)
suite.AddStep(`^I am given an upload URL$`, iAmGivenAnUploadURL)
suite.AddStep(`^the storage key is under "(.*)" and ends with "(.*)"$`, theStorageKeyIs)
suite.AddStep(`^the video is registered with status "(.*)"$`, theVideoIsRegisteredWithStatus)
suite.AddStep(`^the video has an id$`, theVideoHasAnID)
suite.AddStep(`^the video is stored under the key from the upload slot$`, theVideoIsStoredUnderTheSlotKey)
suite.AddStep(`^the video has a transcoding job id$`, theVideoHasATranscodingJobID)
suite.AddStep(`^the video is filed under categories "(.*)"$`, theVideoIsFiledUnder)
suite.AddStep(`^the video keeps the metadata I sent$`, theVideoKeepsTheMetadataISent)
suite.Run()
}
+357
View File
@@ -0,0 +1,357 @@
#!/usr/bin/env bash
#
# upload-video.sh — drive the cms upload flow end to end from a local file.
#
# 1. GET /api/categories pick what the video is filed under
# 2. POST /api/videos/presign ask for an upload slot
# 3. PUT <presigned url> send the file straight to S3, not through cms
# 4. POST /api/videos register it, which queues the transcode
# 5. GET /api/videos/{id} poll until playbackUrl appears
#
# The playback URL is empty until the MediaConvert job reports success and
# cms's consumer records it, so step 5 is a wait, not a formality. A job that
# comes back "failed" never gets a URL — the script stops rather than polls on.
#
# Usage: scripts/upload-video.sh [options]
# Run with --help for the full list. With no options it asks for everything.
set -euo pipefail
readonly DEFAULT_BASE_URL="http://cms-alb-d478b02-648162889.us-east-1.elb.amazonaws.com"
readonly DEFAULT_POLL_INTERVAL=5
readonly DEFAULT_POLL_TIMEOUT=900
base_url="${CMS_BASE_URL:-$DEFAULT_BASE_URL}"
file_path=""
title=""
description=""
tags=""
categories_arg=""
poll_interval="$DEFAULT_POLL_INTERVAL"
poll_timeout="$DEFAULT_POLL_TIMEOUT"
skip_poll=false
usage() {
cat <<'USAGE'
Upload a video to the Thamanyah cms and wait for its playback URL.
Usage: upload-video.sh [options]
Options:
-f, --file PATH Video file to upload (.mp4 or .mov). Prompted for if omitted.
-t, --title TITLE Video title. Prompted for if omitted.
-d, --description TEXT Video description. Optional.
--tags TAGS Free-text comma-separated tags. Optional.
-c, --categories LIST Comma-separated category names or ids (e.g. "news,podcast"
or "2,4"). Prompted for if omitted.
-u, --url URL cms base URL. Default: $CMS_BASE_URL or http://localhost:8081
-i, --interval SECONDS Seconds between status polls. Default: 5
--timeout SECONDS Give up waiting after this long. Default: 900
--no-poll Register the video and exit without waiting.
-h, --help Show this help.
Requires: curl, jq.
Exit codes: 0 ready, 1 usage/upload error, 2 transcode failed, 3 poll timed out.
USAGE
}
die() {
printf '\nerror: %s\n' "$*" >&2
exit 1
}
info() { printf '%s\n' "$*" >&2; }
# The poll writes over one line on a terminal and one line per check when
# redirected — an escape-littered log file helps nobody.
progress() {
if [[ -t 2 ]]; then
printf '\r\033[K %s' "$*" >&2
else
printf ' %s\n' "$*" >&2
fi
}
clear_progress() { [[ -t 2 ]] && printf '\r\033[K' >&2 || true; }
require_tools() {
local missing=()
for tool in curl jq; do
command -v "$tool" >/dev/null 2>&1 || missing+=("$tool")
done
[[ ${#missing[@]} -eq 0 ]] || die "missing required command(s): ${missing[*]}"
}
# The API reports failures as RFC 9457 problem details; surface the human parts
# of that rather than dumping the raw body.
report_api_error() {
local context="$1" status="$2" body="$3" problem_title problem_detail
problem_title=$(jq -r '.title // empty' <<<"$body" 2>/dev/null || true)
problem_detail=$(jq -r '.detail // empty' <<<"$body" 2>/dev/null || true)
if [[ -n "$problem_title" ]]; then
printf '\nerror: %s (HTTP %s)\n %s\n' "$context" "$status" "$problem_title" >&2
[[ -n "$problem_detail" ]] && printf ' %s\n' "$problem_detail" >&2
else
printf '\nerror: %s (HTTP %s)\n %s\n' "$context" "$status" "${body:-<empty response>}" >&2
fi
exit 1
}
# Splits curl's "body + trailing status line" into the two globals the callers
# read, so a request needs one subshell rather than two round trips.
http_status=""
http_body=""
request() {
local response
response=$(curl --silent --show-error --location --write-out $'\n%{http_code}' "$@") ||
die "request to cms failed — is it running at $base_url ?"
http_status="${response##*$'\n'}"
http_body="${response%$'\n'*}"
}
content_type_for() {
case "${1,,}" in
*.mp4) printf 'video/mp4' ;;
*.mov) printf 'video/quicktime' ;;
*) die "unsupported file type: $1 — cms accepts only .mp4 and .mov" ;;
esac
}
prompt_required() {
local varname="$1" prompt="$2" value=""
[[ -t 0 ]] || die "$varname not given and stdin is not a terminal — pass it as an option"
while [[ -z "$value" ]]; do
read -r -e -p "$prompt" value
value="${value#"${value%%[![:space:]]*}"}"
done
printf '%s' "$value"
}
while [[ $# -gt 0 ]]; do
case "$1" in
-f | --file)
file_path="${2:-}"
shift 2
;;
-t | --title)
title="${2:-}"
shift 2
;;
-d | --description)
description="${2:-}"
shift 2
;;
--tags)
tags="${2:-}"
shift 2
;;
-c | --categories)
categories_arg="${2:-}"
shift 2
;;
-u | --url)
base_url="${2:-}"
shift 2
;;
-i | --interval)
poll_interval="${2:-}"
shift 2
;;
--timeout)
poll_timeout="${2:-}"
shift 2
;;
--no-poll)
skip_poll=true
shift
;;
-h | --help)
usage
exit 0
;;
*)
usage >&2
die "unknown option: $1"
;;
esac
done
require_tools
base_url="${base_url%/}"
[[ "$poll_interval" =~ ^[0-9]+$ && "$poll_interval" -gt 0 ]] || die "--interval must be a positive integer"
[[ "$poll_timeout" =~ ^[0-9]+$ && "$poll_timeout" -gt 0 ]] || die "--timeout must be a positive integer"
# ---------------------------------------------------------------- the file ---
if [[ -z "$file_path" ]]; then
# -e gives readline's filename completion, which is the whole point of
# asking here rather than making the flag mandatory.
file_path=$(prompt_required "--file" "Video file to upload: ")
fi
file_path="${file_path/#\~/$HOME}"
[[ -f "$file_path" ]] || die "no such file: $file_path"
[[ -r "$file_path" ]] || die "file is not readable: $file_path"
[[ -s "$file_path" ]] || die "file is empty: $file_path"
file_name=$(basename -- "$file_path")
content_type=$(content_type_for "$file_name")
info "Checking cms at $base_url ..."
request "$base_url/health"
[[ "$http_status" == "200" ]] || report_api_error "cms is not healthy" "$http_status" "$http_body"
# ---------------------------------------------------------- the categories ---
request "$base_url/api/categories"
[[ "$http_status" == "200" ]] || report_api_error "could not list categories" "$http_status" "$http_body"
categories_json="$http_body"
if [[ -z "$categories_arg" ]]; then
info ""
info "Available categories:"
jq -r '.categories[] | " \(.id)) \(.name)"' <<<"$categories_json" >&2
info ""
categories_arg=$(prompt_required "--categories" "Categories (comma-separated ids or names): ")
fi
# Accept either spelling — an id straight from the list, or the name it goes by
# — and resolve both to the ids POST /api/videos wants. Unmatched entries come
# back in their own member rather than as a jq error, so they can be named.
resolved=$(jq -c --arg raw "$categories_arg" '
[$raw | split(",") | .[] | gsub("^\\s+|\\s+$"; "") | select(length > 0)] as $wanted
| {
ids: [ $wanted[] as $w
| $categories.categories[]
| select((.id | tostring) == $w or (.name | ascii_downcase) == ($w | ascii_downcase))
| .id ] | unique,
unknown: [ $wanted[] as $w
| select([ $categories.categories[]
| select((.id | tostring) == $w or (.name | ascii_downcase) == ($w | ascii_downcase)) ] | length == 0)
| $w ]
}
' --argjson categories "$categories_json" -n)
unknown_categories=$(jq -r '.unknown | join(", ")' <<<"$resolved")
[[ -z "$unknown_categories" ]] || die "no category is called \"$unknown_categories\" — pick from the list with GET $base_url/api/categories"
category_ids_json=$(jq -c '.ids' <<<"$resolved")
[[ "$category_ids_json" != "[]" ]] || die "at least one category is required"
category_names=$(jq -r --argjson ids "$category_ids_json" \
'[.categories[] | select(.id as $i | $ids | index($i)) | .name] | join(", ")' <<<"$categories_json")
# -------------------------------------------------------------- the metadata ---
[[ -n "$title" ]] || title=$(prompt_required "--title" "Title: ")
info ""
info "Uploading : $file_path ($content_type)"
info "Title : $title"
info "Categories : $category_names"
info ""
# ------------------------------------------------------------- 1. presign ---
info "Requesting an upload URL ..."
presign_body=$(jq -n --arg fileName "$file_name" --arg contentType "$content_type" \
'{fileName: $fileName, contentType: $contentType}')
request -X POST "$base_url/api/videos/presign" \
-H 'Content-Type: application/json' \
--data-binary "$presign_body"
[[ "$http_status" == "200" ]] || report_api_error "could not get an upload URL" "$http_status" "$http_body"
upload_url=$(jq -r '.uploadUrl' <<<"$http_body")
storage_key=$(jq -r '.key' <<<"$http_body")
[[ -n "$upload_url" && "$upload_url" != "null" ]] || die "cms returned no upload URL"
# ----------------------------------------------------- 2. PUT the file to S3 ---
# The content type is signed into the URL, so this header has to match the one
# sent to /presign exactly or S3 rejects the PUT as a signature mismatch.
info "Uploading $(du -h -- "$file_path" | cut -f1) to storage ..."
upload_status=$(curl --silent --show-error --progress-bar \
--request PUT \
--header "Content-Type: $content_type" \
--upload-file "$file_path" \
--write-out '%{http_code}' \
--output /dev/null \
"$upload_url") || die "upload to storage failed"
[[ "$upload_status" =~ ^2 ]] || die "storage rejected the upload (HTTP $upload_status)"
info "Upload complete: $storage_key"
# ------------------------------------------- 3. register + queue transcoding ---
info "Registering the video ..."
complete_body=$(jq -n \
--arg title "$title" \
--arg description "$description" \
--arg tags "$tags" \
--arg fileName "$file_name" \
--arg key "$storage_key" \
--argjson categoryIds "$category_ids_json" \
'{title: $title, description: $description, categoryIds: $categoryIds, tags: $tags, fileName: $fileName, key: $key}')
request -X POST "$base_url/api/videos" \
-H 'Content-Type: application/json' \
--data-binary "$complete_body"
[[ "$http_status" == "201" ]] || report_api_error "could not register the video" "$http_status" "$http_body"
video_id=$(jq -r '.id' <<<"$http_body")
info "Registered as $video_id (status: $(jq -r '.status' <<<"$http_body"))"
if [[ "$skip_poll" == true ]]; then
info ""
info "Not waiting for transcoding (--no-poll). Follow it with:"
info " curl -s $base_url/api/videos/$video_id | jq"
printf '%s\n' "$video_id"
exit 0
fi
# ------------------------------------------ 4. poll until the URL is there ---
info ""
info "Waiting for transcoding to finish (timeout ${poll_timeout}s, checking every ${poll_interval}s) ..."
started_at=$SECONDS
while :; do
request "$base_url/api/videos/$video_id"
[[ "$http_status" == "200" ]] || report_api_error "could not read the video" "$http_status" "$http_body"
status=$(jq -r '.status' <<<"$http_body")
playback_url=$(jq -r '.playbackUrl // empty' <<<"$http_body")
elapsed=$((SECONDS - started_at))
# The URL is what the caller is actually waiting for, and it is written
# alongside the status — so wait for the URL, not merely for "ready".
if [[ -n "$playback_url" ]]; then
clear_progress
info "Ready after ${elapsed}s."
info ""
info "Playback URL:"
printf '%s\n' "$playback_url"
exit 0
fi
if [[ "$status" == "failed" ]]; then
clear_progress
printf '\nerror: transcoding failed for video %s after %ss — no playback URL will be produced.\n' "$video_id" "$elapsed" >&2
exit 2
fi
if ((elapsed >= poll_timeout)); then
clear_progress
printf '\nerror: timed out after %ss with status %q and no playback URL.\n Check again with: curl -s %s/api/videos/%s | jq\n' \
"$elapsed" "$status" "$base_url" "$video_id" >&2
exit 3
fi
progress "status: $status — ${elapsed}s elapsed"
sleep "$poll_interval"
done