From 3cba235723f69cc3089a6d5589494263985b8cde Mon Sep 17 00:00:00 2001 From: fahd Date: Sun, 16 Aug 2026 18:57:40 +0300 Subject: [PATCH] FEAT: Media Convert + s3 Integration --- .gitea/workflows/cms-deploy.yml | 55 +++++++++++++ cms/internal/handlers/videos.go | 7 ++ cms/internal/services/mediaconvert.go | 88 +++++++++++++++++++++ cms/internal/services/s3.go | 38 +++++++++ cms/internal/views/types.go | 1 + cms/internal/views/video.templ | 2 + cms/main.go | 14 +++- infrastructure/main.go | 106 ++++++++++++++++++++++---- 8 files changed, 295 insertions(+), 16 deletions(-) create mode 100644 .gitea/workflows/cms-deploy.yml create mode 100644 cms/internal/services/mediaconvert.go diff --git a/.gitea/workflows/cms-deploy.yml b/.gitea/workflows/cms-deploy.yml new file mode 100644 index 0000000..333cb9e --- /dev/null +++ b/.gitea/workflows/cms-deploy.yml @@ -0,0 +1,55 @@ +name: Build, Push and Deploy CMS + +on: + push: + branches: [main] + paths: + - "cms/**" + - ".gitea/workflows/cms-deploy.yml" + +env: + AWS_REGION: us-east-1 + ECR_REPOSITORY: cms + +jobs: + build-push-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push image + uses: docker/build-push-action@v6 + with: + context: ./cms + file: ./cms/Dockerfile + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:latest + ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ gitea.sha }} + + # cluster/service ARNs come from `pulumi stack output ecsClusterArn` / + # `cmsServiceArn` in the infrastructure project — set once as repo + # variables (Settings > Actions > Variables), not secrets, since ARNs + # aren't sensitive. + - name: Deploy to ECS + run: | + aws ecs update-service \ + --cluster "${{ vars.ECS_CLUSTER_ARN }}" \ + --service "${{ vars.CMS_ECS_SERVICE_ARN }}" \ + --force-new-deployment \ + --region "${{ env.AWS_REGION }}" diff --git a/cms/internal/handlers/videos.go b/cms/internal/handlers/videos.go index 3d0967b..577b606 100644 --- a/cms/internal/handlers/videos.go +++ b/cms/internal/handlers/videos.go @@ -95,12 +95,19 @@ func CompleteVideoUpload(w http.ResponseWriter, r *http.Request) { return } + jobID, err := services.MediaConvertClient.QueueEncodingJob(r.Context(), key) + if err != nil { + renderUploadError(w, r, "Something Went Wrong On Creation Of Transcoding Job") + return + } + views.VideoUploadSuccess(views.VideoMetadata{ Title: title, Description: strings.TrimSpace(req.Description), Category: strings.TrimSpace(req.Category), Tags: strings.TrimSpace(req.Tags), FileName: strings.TrimSpace(req.FileName), + JobID: jobID, StoredAs: key, SizeBytes: 60, }).Render(r.Context(), w) diff --git a/cms/internal/services/mediaconvert.go b/cms/internal/services/mediaconvert.go new file mode 100644 index 0000000..a89708e --- /dev/null +++ b/cms/internal/services/mediaconvert.go @@ -0,0 +1,88 @@ +package services + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/mediaconvert" + "github.com/aws/aws-sdk-go-v2/service/mediaconvert/types" +) + +var MediaConvertClient MediaConvert + +type MediaConvert interface { + QueueEncodingJob(ctx context.Context, key string) (string, error) +} + +type MediaConvertConcrete struct { + MediaConvertClient *mediaconvert.Client + Role string + InputBucket string + OutputBucket string +} + +func (svc MediaConvertConcrete) QueueEncodingJob(ctx context.Context, key string) (string, error) { + input := fmt.Sprintf("s3://%s/%s", svc.InputBucket, key) + destination := fmt.Sprintf("s3://%s/%s", svc.OutputBucket, key) + + job, err := svc.MediaConvertClient.CreateJob(ctx, &mediaconvert.CreateJobInput{ + Role: &svc.Role, + Settings: &types.JobSettings{ + Inputs: []types.Input{ + { + FileInput: &input, + AudioSelectors: map[string]types.AudioSelector{ + "Audio Selector 1": { + DefaultSelection: types.AudioDefaultSelectionDefault, + }, + }, + VideoSelector: &types.VideoSelector{}, + }, + }, + OutputGroups: []types.OutputGroup{ + { + Name: aws.String("File Group"), + OutputGroupSettings: &types.OutputGroupSettings{ + Type: types.OutputGroupTypeFileGroupSettings, + FileGroupSettings: &types.FileGroupSettings{ + Destination: &destination, + }, + }, + Outputs: []types.Output{ + { + ContainerSettings: &types.ContainerSettings{ + Container: types.ContainerTypeMp4, + }, + VideoDescription: &types.VideoDescription{ + CodecSettings: &types.VideoCodecSettings{ + Codec: types.VideoCodecH264, + H264Settings: &types.H264Settings{ + RateControlMode: types.H264RateControlModeQvbr, + }, + }, + }, + AudioDescriptions: []types.AudioDescription{ + { + CodecSettings: &types.AudioCodecSettings{ + Codec: types.AudioCodecAac, + AacSettings: &types.AacSettings{ + Bitrate: aws.Int32(96000), + CodingMode: types.AacCodingModeCodingMode20, + SampleRate: aws.Int32(48000), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }) + if err != nil { + return "", err + } + + return aws.ToString(job.Job.Id), nil +} diff --git a/cms/internal/services/s3.go b/cms/internal/services/s3.go index 9767df0..7f8f67b 100644 --- a/cms/internal/services/s3.go +++ b/cms/internal/services/s3.go @@ -2,8 +2,11 @@ package services import ( "context" + "fmt" + "strings" "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" ) @@ -46,3 +49,38 @@ func (svc S3Concrete) DoesFileExist(ctx context.Context, key string) (bool, erro return true, nil } + +// AssertSuccessfulConnection verifies that the bucket is reachable and that +// the configured credentials can get, put, and presign objects in it. It +// panics with a descriptive message on the first check that fails, since the +// service cannot function without these permissions. +func (svc S3Concrete) AssertSuccessfulConnection(ctx context.Context) { + if _, err := svc.S3Client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: &svc.Bucket, + }); err != nil { + panic(fmt.Errorf("s3: cannot connect to bucket %q: %w", svc.Bucket, err)) + } + + const probeKey = ".s3-connectivity-check" + + if _, err := svc.S3Client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &svc.Bucket, + Key: aws.String(probeKey), + Body: strings.NewReader("ok"), + }); err != nil { + panic(fmt.Errorf("s3: missing permission to put objects in bucket %q: %w", svc.Bucket, err)) + } + + object, err := svc.S3Client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &svc.Bucket, + Key: aws.String(probeKey), + }) + if err != nil { + panic(fmt.Errorf("s3: missing permission to get objects from bucket %q: %w", svc.Bucket, err)) + } + object.Body.Close() + + if _, err := svc.GetPresignedURL(ctx, probeKey, "text/plain", time.Minute); err != nil { + panic(fmt.Errorf("s3: cannot create presigned urls for bucket %q: %w", svc.Bucket, err)) + } +} diff --git a/cms/internal/views/types.go b/cms/internal/views/types.go index 722c350..53e925f 100644 --- a/cms/internal/views/types.go +++ b/cms/internal/views/types.go @@ -9,6 +9,7 @@ type VideoMetadata struct { Tags string FileName string StoredAs string + JobID string SizeBytes int64 } diff --git a/cms/internal/views/video.templ b/cms/internal/views/video.templ index 5262cc6..1f33abb 100644 --- a/cms/internal/views/video.templ +++ b/cms/internal/views/video.templ @@ -135,6 +135,8 @@ templ VideoUploadSuccess(v VideoMetadata) { }
Category
{ v.Category }
+
Enqueue Job ID
+
{ v.JobID }
if v.Tags != "" {
Tags
{ v.Tags }
diff --git a/cms/main.go b/cms/main.go index c5e696e..8da4cb3 100644 --- a/cms/main.go +++ b/cms/main.go @@ -10,6 +10,7 @@ import ( "thamanyah/cms/v2/internal/services" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/mediaconvert" "github.com/aws/aws-sdk-go-v2/service/s3" ) @@ -19,11 +20,22 @@ func main() { panic(fmt.Errorf("failed To Load S3 Config: %s", err)) } - services.S3Client = &services.S3Concrete{ + concreteS3Client := &services.S3Concrete{ S3Client: s3.NewFromConfig(config), Bucket: "", } + concreteS3Client.AssertSuccessfulConnection(context.Background()) + + services.S3Client = concreteS3Client + + services.MediaConvertClient = &services.MediaConvertConcrete{ + MediaConvertClient: mediaconvert.NewFromConfig(config), + Role: "", + InputBucket: "", + OutputBucket: "", + } + mux := http.NewServeMux() mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static")))) diff --git a/infrastructure/main.go b/infrastructure/main.go index cc69b7b..6ee7d62 100644 --- a/infrastructure/main.go +++ b/infrastructure/main.go @@ -98,7 +98,7 @@ func deployFargateService( dbAddress pulumi.StringOutput, dbPort pulumi.IntOutput, dbPassword *random.RandomPassword, -) (*ecr.Repository, *lb.LoadBalancer, error) { +) (*ecr.Repository, *ecs.Service, *lb.LoadBalancer, error) { alb, err := lb.NewLoadBalancer(ctx, name+"-alb", &lb.LoadBalancerArgs{ LoadBalancerType: pulumi.String("application"), Internal: pulumi.Bool(false), @@ -106,7 +106,7 @@ func deployFargateService( Subnets: pulumi.ToStringArray(subnetIDs), }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } repo, err := ecr.NewRepository(ctx, name+"-repo", &ecr.RepositoryArgs{ @@ -115,7 +115,7 @@ func deployFargateService( ForceDelete: pulumi.Bool(true), }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } logGroup, err := cloudwatch.NewLogGroup(ctx, name+"-logs", &cloudwatch.LogGroupArgs{ @@ -123,7 +123,7 @@ func deployFargateService( RetentionInDays: pulumi.Int(7), }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } secret, err := secretsmanager.NewSecret(ctx, name+"-db-secret", &secretsmanager.SecretArgs{ @@ -131,7 +131,7 @@ func deployFargateService( RecoveryWindowInDays: pulumi.Int(0), }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } _, err = secretsmanager.NewSecretVersion(ctx, name+"-db-secret-version", &secretsmanager.SecretVersionArgs{ @@ -139,7 +139,7 @@ func deployFargateService( SecretString: dbPassword.Result, }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } secretPolicy := secret.Arn.ApplyT(func(secretArn string) (string, error) { @@ -162,7 +162,7 @@ func deployFargateService( Policy: secretPolicy, }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } targetGroup, err := lb.NewTargetGroup(ctx, name+"-tg", &lb.TargetGroupArgs{ @@ -176,7 +176,7 @@ func deployFargateService( }, }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } listener, err := lb.NewListener(ctx, name+"-listener", &lb.ListenerArgs{ @@ -191,7 +191,7 @@ func deployFargateService( }, }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } containerDefinitions := pulumi.All(repo.RepositoryUrl, logGroup.Name, secret.Arn, dbAddress, dbPort).ApplyT( @@ -243,10 +243,10 @@ func deployFargateService( ContainerDefinitions: containerDefinitions, }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - _, err = ecs.NewService(ctx, name+"-service", &ecs.ServiceArgs{ + service, err := ecs.NewService(ctx, name+"-service", &ecs.ServiceArgs{ Cluster: cluster.Arn, TaskDefinition: taskDefinition.Arn, DesiredCount: pulumi.Int(1), @@ -265,10 +265,10 @@ func deployFargateService( }, }, pulumi.DependsOn([]pulumi.Resource{listener})) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return repo, alb, nil + return repo, service, alb, nil } func main() { @@ -579,20 +579,91 @@ func main() { return err } - cmsRepo, cmsAlb, err := deployFargateService(ctx, "cms", 8081, + cmsRepo, cmsService, cmsAlb, err := deployFargateService(ctx, "cms", 8081, cluster, execRole, vpc.Id, subnets.Ids, albSecurityGroup, serviceSecurityGroup, db.Address, db.Port, cmsPassword) if err != nil { return err } - discoveryRepo, discoveryAlb, err := deployFargateService(ctx, "discovery", 8080, + discoveryRepo, discoveryService, discoveryAlb, err := deployFargateService(ctx, "discovery", 8080, cluster, execRole, vpc.Id, subnets.Ids, albSecurityGroup, serviceSecurityGroup, db.Address, db.Port, discoveryPassword) if err != nil { return err } + // Gitea Actions CI: one IAM user, scoped to just what the build/deploy + // pipeline needs — push images to the two ECR repos and force a new + // deployment on the two ECS services. Credentials are exported as + // stack secrets for the pipeline to consume (e.g. as Gitea Actions + // secrets); nothing broader like ecr:*, ecs:*, or task-def changes. + giteaCIPolicyDocument := pulumi.All(cmsRepo.Arn, discoveryRepo.Arn, cmsService.Arn, discoveryService.Arn).ApplyT( + func(args []any) (string, error) { + cmsRepoArn := args[0].(string) + discoveryRepoArn := args[1].(string) + cmsServiceArn := args[2].(string) + discoveryServiceArn := args[3].(string) + + doc := map[string]any{ + "Version": "2012-10-17", + "Statement": []map[string]any{ + { + "Sid": "ECRAuth", + "Effect": "Allow", + "Action": "ecr:GetAuthorizationToken", + "Resource": "*", + }, + { + "Sid": "ECRPush", + "Effect": "Allow", + "Action": []string{ + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + }, + "Resource": []string{cmsRepoArn, discoveryRepoArn}, + }, + { + "Sid": "ECSDeploy", + "Effect": "Allow", + "Action": []string{ + "ecs:UpdateService", + "ecs:DescribeServices", + }, + "Resource": []string{cmsServiceArn, discoveryServiceArn}, + }, + }, + } + b, err := json.Marshal(doc) + return string(b), err + }, + ).(pulumi.StringOutput) + + giteaCIUser, err := iam.NewUser(ctx, "gitea-ci-user", &iam.UserArgs{ + Name: pulumi.String("gitea-ci"), + }) + if err != nil { + return err + } + + _, err = iam.NewUserPolicy(ctx, "gitea-ci-policy", &iam.UserPolicyArgs{ + User: giteaCIUser.Name, + Policy: giteaCIPolicyDocument, + }) + if err != nil { + return err + } + + giteaCIAccessKey, err := iam.NewAccessKey(ctx, "gitea-ci-access-key", &iam.AccessKeyArgs{ + User: giteaCIUser.Name, + }) + if err != nil { + return err + } + // Export the name of the bucket ctx.Export("bucketName", bucket.ID()) ctx.Export("cdnDomainName", distribution.DomainName) @@ -602,6 +673,11 @@ func main() { ctx.Export("discoveryUrl", pulumi.Sprintf("http://%s", discoveryAlb.DnsName)) ctx.Export("cmsRepoUrl", cmsRepo.RepositoryUrl) ctx.Export("discoveryRepoUrl", discoveryRepo.RepositoryUrl) + ctx.Export("ecsClusterArn", cluster.Arn) + ctx.Export("cmsServiceArn", cmsService.Arn) + ctx.Export("discoveryServiceArn", discoveryService.Arn) + ctx.Export("giteaCiAccessKeyId", giteaCIAccessKey.ID()) + ctx.Export("giteaCiSecretAccessKey", giteaCIAccessKey.Secret) return nil }) }