FahdShalhoub
1527 lines
52 KiB
Go
1527 lines
52 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudfront"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudwatch"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/ec2"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/ecr"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/ecs"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/elasticache"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/iam"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/lb"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/rds"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/s3"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/secretsmanager"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/sns"
|
|
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/sqs"
|
|
"github.com/pulumi/pulumi-postgresql/sdk/v3/go/postgresql"
|
|
"github.com/pulumi/pulumi-random/sdk/v4/go/random"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
|
|
)
|
|
|
|
// Matches the region configured in Pulumi.main.yaml (aws:region).
|
|
const awsRegion = "us-east-1"
|
|
|
|
// Port the Postgres instance listens on under LocalStack. LocalStack backs an
|
|
// RDS instance with a real Postgres process inside its own container, and must
|
|
// stay inside the 4510-4559 range docker-compose.yml publishes so the instance
|
|
// is reachable both from the cms container and from the host.
|
|
const localstackDBPort = 4510
|
|
|
|
// currentPublicIP returns the caller's public IP, used to scope the RDS
|
|
// security group to the machine that will actually run `pulumi up`.
|
|
func currentPublicIP() (string, error) {
|
|
resp, err := http.Get("https://checkip.amazonaws.com")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer func() { _ = resp.Body.Close() }()
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return strings.TrimSpace(string(body)), nil
|
|
}
|
|
|
|
const originId = "encoded-bucket-origin"
|
|
|
|
// AWS managed "CachingOptimized" cache policy (same ID in every account/region).
|
|
const cachingOptimizedPolicyId = "658327ea-f89d-4fab-a63d-7e88639e58f6"
|
|
|
|
// newServiceDatabase creates a login role and a same-named database owned by
|
|
// that role, so the service only ever touches its own database. The password is
|
|
// generated and kept in stack state, except on the LocalStack stack, where
|
|
// fixedPassword carries the value docker-compose already hands the container as
|
|
// DB_PASSWORD — compose reads a static .env and cannot consume a stack output.
|
|
func newServiceDatabase(ctx *pulumi.Context, provider *postgresql.Provider, service, fixedPassword string) (pulumi.StringOutput, error) {
|
|
password := pulumi.String(fixedPassword).ToStringOutput()
|
|
if fixedPassword == "" {
|
|
generated, err := random.NewRandomPassword(ctx, service+"-db-password", &random.RandomPasswordArgs{
|
|
Length: pulumi.Int(24),
|
|
Special: pulumi.Bool(true),
|
|
MinUpper: pulumi.Int(1),
|
|
MinLower: pulumi.Int(1),
|
|
MinNumeric: pulumi.Int(1),
|
|
MinSpecial: pulumi.Int(1),
|
|
})
|
|
if err != nil {
|
|
return pulumi.StringOutput{}, err
|
|
}
|
|
password = generated.Result
|
|
}
|
|
|
|
role, err := postgresql.NewRole(ctx, service+"-db-role", &postgresql.RoleArgs{
|
|
Name: pulumi.String(service),
|
|
Login: pulumi.Bool(true),
|
|
Password: password,
|
|
}, pulumi.Provider(provider))
|
|
if err != nil {
|
|
return pulumi.StringOutput{}, err
|
|
}
|
|
|
|
_, err = postgresql.NewDatabase(ctx, service+"-database", &postgresql.DatabaseArgs{
|
|
Name: pulumi.String(service),
|
|
Owner: role.Name,
|
|
}, pulumi.Provider(provider))
|
|
if err != nil {
|
|
return pulumi.StringOutput{}, err
|
|
}
|
|
|
|
return password, nil
|
|
}
|
|
|
|
// envVar is a name/value pair for a container's extra environment variables,
|
|
// where the value is only known after other resources are provisioned.
|
|
type envVar struct {
|
|
Name string
|
|
Value pulumi.StringOutput
|
|
}
|
|
|
|
// deployFargateService gives a service its own ECR repo, log group, task
|
|
// definition, ECS service, and its own ALB — so it gets its own DNS name
|
|
// rather than sharing one load balancer with the other service on a
|
|
// different port. The service's DB password is stored in Secrets Manager and
|
|
// injected into the container as a secret rather than a plaintext env var.
|
|
// taskRole is optional (nil means no TaskRoleArn is set, i.e. the container
|
|
// gets no AWS identity of its own beyond execRole's pull/logs permissions);
|
|
// extraEnv is appended to the container's environment on top of the DB_* vars.
|
|
// When runMigrations is true, the task also gets a non-essential
|
|
// "<name>-migrate" container running the same image with `command: ["migrate"]`,
|
|
// and the main container depends on it with condition COMPLETE — the ECS
|
|
// equivalent of a Kubernetes init container, ensuring DB migrations finish
|
|
// before the service starts accepting traffic.
|
|
func deployFargateService(
|
|
ctx *pulumi.Context,
|
|
name string,
|
|
containerPort int,
|
|
cluster *ecs.Cluster,
|
|
execRole *iam.Role,
|
|
taskRole *iam.Role,
|
|
extraEnv []envVar,
|
|
vpcID string,
|
|
subnetIDs []string,
|
|
albSecurityGroup *ec2.SecurityGroup,
|
|
serviceSecurityGroup *ec2.SecurityGroup,
|
|
dbAddress pulumi.StringOutput,
|
|
dbPort pulumi.IntOutput,
|
|
dbPassword pulumi.StringOutput,
|
|
runMigrations bool,
|
|
) (*ecr.Repository, *ecs.Service, *lb.LoadBalancer, error) {
|
|
alb, err := lb.NewLoadBalancer(ctx, name+"-alb", &lb.LoadBalancerArgs{
|
|
LoadBalancerType: pulumi.String("application"),
|
|
Internal: pulumi.Bool(false),
|
|
SecurityGroups: pulumi.StringArray{albSecurityGroup.ID()},
|
|
Subnets: pulumi.ToStringArray(subnetIDs),
|
|
})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
repo, err := ecr.NewRepository(ctx, name+"-repo", &ecr.RepositoryArgs{
|
|
Name: pulumi.String(name),
|
|
ImageTagMutability: pulumi.String("MUTABLE"),
|
|
ForceDelete: pulumi.Bool(true),
|
|
})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
logGroup, err := cloudwatch.NewLogGroup(ctx, name+"-logs", &cloudwatch.LogGroupArgs{
|
|
Name: pulumi.String("/ecs/" + name),
|
|
RetentionInDays: pulumi.Int(7),
|
|
})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
secret, err := secretsmanager.NewSecret(ctx, name+"-db-secret", &secretsmanager.SecretArgs{
|
|
Name: pulumi.String(name + "-db-password"),
|
|
RecoveryWindowInDays: pulumi.Int(0),
|
|
})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
_, err = secretsmanager.NewSecretVersion(ctx, name+"-db-secret-version", &secretsmanager.SecretVersionArgs{
|
|
SecretId: secret.ID(),
|
|
SecretString: dbPassword,
|
|
})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
secretPolicy := secret.Arn.ApplyT(func(secretArn string) (string, error) {
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Effect": "Allow",
|
|
"Action": "secretsmanager:GetSecretValue",
|
|
"Resource": secretArn,
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
}).(pulumi.StringOutput)
|
|
|
|
_, err = iam.NewRolePolicy(ctx, name+"-secret-access", &iam.RolePolicyArgs{
|
|
Role: execRole.ID(),
|
|
Policy: secretPolicy,
|
|
})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
targetGroup, err := lb.NewTargetGroup(ctx, name+"-tg", &lb.TargetGroupArgs{
|
|
Port: pulumi.Int(containerPort),
|
|
Protocol: pulumi.String("HTTP"),
|
|
TargetType: pulumi.String("ip"),
|
|
VpcId: pulumi.String(vpcID),
|
|
HealthCheck: &lb.TargetGroupHealthCheckArgs{
|
|
Path: pulumi.String("/health"),
|
|
Matcher: pulumi.String("200"),
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
listener, err := lb.NewListener(ctx, name+"-listener", &lb.ListenerArgs{
|
|
LoadBalancerArn: alb.Arn,
|
|
Port: pulumi.Int(80),
|
|
Protocol: pulumi.String("HTTP"),
|
|
DefaultActions: lb.ListenerDefaultActionArray{
|
|
&lb.ListenerDefaultActionArgs{
|
|
Type: pulumi.String("forward"),
|
|
TargetGroupArn: targetGroup.Arn,
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
baseArgs := []any{repo.RepositoryUrl, logGroup.Name, secret.Arn, dbAddress, dbPort}
|
|
for _, ev := range extraEnv {
|
|
baseArgs = append(baseArgs, ev.Value)
|
|
}
|
|
|
|
containerDefinitions := pulumi.All(baseArgs...).ApplyT(
|
|
func(args []any) (string, error) {
|
|
image := args[0].(string) + ":latest"
|
|
logGroupName := args[1].(string)
|
|
secretArn := args[2].(string)
|
|
dbHost := args[3].(string)
|
|
dbPort := args[4].(int)
|
|
|
|
dbEnvironment := []map[string]string{
|
|
{"name": "DB_HOST", "value": dbHost},
|
|
{"name": "DB_PORT", "value": fmt.Sprintf("%d", dbPort)},
|
|
{"name": "DB_NAME", "value": name},
|
|
{"name": "DB_USER", "value": name},
|
|
}
|
|
dbSecrets := []map[string]string{
|
|
{"name": "DB_PASSWORD", "valueFrom": secretArn},
|
|
}
|
|
|
|
environment := append([]map[string]string{}, dbEnvironment...)
|
|
for i, ev := range extraEnv {
|
|
environment = append(environment, map[string]string{"name": ev.Name, "value": args[5+i].(string)})
|
|
}
|
|
|
|
mainContainer := map[string]any{
|
|
"name": name,
|
|
"image": image,
|
|
"portMappings": []map[string]any{
|
|
{"containerPort": containerPort, "protocol": "tcp"},
|
|
},
|
|
"environment": environment,
|
|
"secrets": dbSecrets,
|
|
"logConfiguration": map[string]any{
|
|
"logDriver": "awslogs",
|
|
"options": map[string]string{
|
|
"awslogs-group": logGroupName,
|
|
"awslogs-region": awsRegion,
|
|
"awslogs-stream-prefix": name,
|
|
},
|
|
},
|
|
}
|
|
|
|
def := []map[string]any{}
|
|
|
|
if runMigrations {
|
|
migrateContainerName := name + "-migrate"
|
|
def = append(def, map[string]any{
|
|
"name": migrateContainerName,
|
|
"image": image,
|
|
"essential": false,
|
|
"command": []string{"migrate"},
|
|
"environment": dbEnvironment,
|
|
"secrets": dbSecrets,
|
|
"logConfiguration": map[string]any{
|
|
"logDriver": "awslogs",
|
|
"options": map[string]string{
|
|
"awslogs-group": logGroupName,
|
|
"awslogs-region": awsRegion,
|
|
"awslogs-stream-prefix": migrateContainerName,
|
|
},
|
|
},
|
|
})
|
|
mainContainer["dependsOn"] = []map[string]string{
|
|
{"containerName": migrateContainerName, "condition": "COMPLETE"},
|
|
}
|
|
}
|
|
|
|
def = append(def, mainContainer)
|
|
|
|
b, err := json.Marshal(def)
|
|
return string(b), err
|
|
},
|
|
).(pulumi.StringOutput)
|
|
|
|
taskDefinitionArgs := &ecs.TaskDefinitionArgs{
|
|
Family: pulumi.String(name),
|
|
Cpu: pulumi.String("256"),
|
|
Memory: pulumi.String("512"),
|
|
NetworkMode: pulumi.String("awsvpc"),
|
|
RequiresCompatibilities: pulumi.StringArray{pulumi.String("FARGATE")},
|
|
ExecutionRoleArn: execRole.Arn,
|
|
ContainerDefinitions: containerDefinitions,
|
|
}
|
|
if taskRole != nil {
|
|
taskDefinitionArgs.TaskRoleArn = taskRole.Arn
|
|
}
|
|
|
|
taskDefinition, err := ecs.NewTaskDefinition(ctx, name+"-task", taskDefinitionArgs)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
service, err := ecs.NewService(ctx, name+"-service", &ecs.ServiceArgs{
|
|
Cluster: cluster.Arn,
|
|
TaskDefinition: taskDefinition.Arn,
|
|
DesiredCount: pulumi.Int(1),
|
|
LaunchType: pulumi.String("FARGATE"),
|
|
NetworkConfiguration: &ecs.ServiceNetworkConfigurationArgs{
|
|
Subnets: pulumi.ToStringArray(subnetIDs),
|
|
SecurityGroups: pulumi.StringArray{serviceSecurityGroup.ID()},
|
|
AssignPublicIp: pulumi.Bool(true),
|
|
},
|
|
LoadBalancers: ecs.ServiceLoadBalancerArray{
|
|
&ecs.ServiceLoadBalancerArgs{
|
|
TargetGroupArn: targetGroup.Arn,
|
|
ContainerName: pulumi.String(name),
|
|
ContainerPort: pulumi.Int(containerPort),
|
|
},
|
|
},
|
|
}, pulumi.DependsOn([]pulumi.Resource{listener}))
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
return repo, service, alb, nil
|
|
}
|
|
|
|
func main() {
|
|
pulumi.Run(func(ctx *pulumi.Context) error {
|
|
// The LocalStack stack (Pulumi.local.yaml) runs this same program
|
|
// against docker-compose. Everything AWS-endpoint-related is handled by
|
|
// that file's aws:* config, so the only thing the code has to know is
|
|
// which resources are meaningful locally: LocalStack stands in for S3,
|
|
// RDS and IAM, while docker-compose — not ECS — runs the containers, so
|
|
// the VPC, the load balancers, the ECS services, the CDN and the CI
|
|
// user are all skipped.
|
|
cfg := config.New(ctx, "")
|
|
localstack := cfg.GetBool("localstack")
|
|
|
|
// docker-compose passes this to the cms container as DB_PASSWORD; the
|
|
// service login roles below are created with the same value.
|
|
servicePassword := ""
|
|
if localstack {
|
|
servicePassword = os.Getenv("DB_PASSWORD")
|
|
if servicePassword == "" {
|
|
return fmt.Errorf("missing required env var: DB_PASSWORD (needed by the %q stack)", ctx.Stack())
|
|
}
|
|
}
|
|
|
|
var (
|
|
distribution *cloudfront.Distribution
|
|
vpcID string
|
|
subnetIDs []string
|
|
dbSubnetGroup *rds.SubnetGroup
|
|
dbSecurityGroup *ec2.SecurityGroup
|
|
cluster *ecs.Cluster
|
|
albSecurityGroup, serviceSecurityGroup *ec2.SecurityGroup
|
|
execRole *iam.Role
|
|
cmsRepo, discoveryRepo *ecr.Repository
|
|
cmsService, discoveryService *ecs.Service
|
|
cmsAlb, discoveryAlb *lb.LoadBalancer
|
|
searchCacheAddress pulumi.StringOutput
|
|
)
|
|
|
|
// Bucket names are left to Pulumi's auto-naming in AWS, but pinned under
|
|
// LocalStack because docker-compose.yml refers to them literally.
|
|
encodedBucketArgs := &s3.BucketArgs{}
|
|
rawUploadsBucketArgs := &s3.BucketArgs{}
|
|
if localstack {
|
|
encodedBucketArgs.Bucket = pulumi.String("encoded-bucket")
|
|
rawUploadsBucketArgs.Bucket = pulumi.String("raw-uploads-bucket")
|
|
}
|
|
|
|
// Create an AWS resource (S3 Bucket) that holds encoded assets
|
|
bucket, err := s3.NewBucket(ctx, "encoded-bucket", encodedBucketArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Keep the bucket private; CloudFront reaches it via Origin Access Control.
|
|
_, err = s3.NewBucketPublicAccessBlock(ctx, "encoded-bucket-public-access-block", &s3.BucketPublicAccessBlockArgs{
|
|
Bucket: bucket.ID(),
|
|
BlockPublicAcls: pulumi.Bool(true),
|
|
BlockPublicPolicy: pulumi.Bool(true),
|
|
IgnorePublicAcls: pulumi.Bool(true),
|
|
RestrictPublicBuckets: pulumi.Bool(true),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// CloudFront: public delivery of the finished renditions. Skipped under
|
|
// LocalStack, which reads the encoded bucket directly.
|
|
if !localstack {
|
|
oac, err := cloudfront.NewOriginAccessControl(ctx, "encoded-bucket-oac", &cloudfront.OriginAccessControlArgs{
|
|
Description: pulumi.String("OAC for encoded-bucket assets"),
|
|
OriginAccessControlOriginType: pulumi.String("s3"),
|
|
SigningBehavior: pulumi.String("always"),
|
|
SigningProtocol: pulumi.String("sigv4"),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// HLS is played by JavaScript (hls.js fetches the .m3u8 playlist
|
|
// and every .ts/.m4s segment with XHR), so the CDN has to answer
|
|
// with CORS headers or the browser drops the response — the
|
|
// "CORS Missing Allow Origin" failure. The S3 origin sends none of
|
|
// its own, so CloudFront adds them here, at the edge, for cached
|
|
// and uncached responses alike.
|
|
corsHeaders, err := cloudfront.NewResponseHeadersPolicy(ctx, "encoded-bucket-cors-headers", &cloudfront.ResponseHeadersPolicyArgs{
|
|
Comment: pulumi.String("CORS headers so browsers can fetch HLS playlists and segments"),
|
|
CorsConfig: &cloudfront.ResponseHeadersPolicyCorsConfigArgs{
|
|
AccessControlAllowCredentials: pulumi.Bool(false),
|
|
AccessControlAllowHeaders: &cloudfront.ResponseHeadersPolicyCorsConfigAccessControlAllowHeadersArgs{
|
|
Items: pulumi.ToStringArray([]string{"*"}),
|
|
},
|
|
AccessControlAllowMethods: &cloudfront.ResponseHeadersPolicyCorsConfigAccessControlAllowMethodsArgs{
|
|
Items: pulumi.ToStringArray([]string{"GET", "HEAD", "OPTIONS"}),
|
|
},
|
|
// Playback is public and unauthenticated, so any origin may
|
|
// read it. A wildcard also keeps Origin out of the cache
|
|
// key: with a fixed list CloudFront would have to vary the
|
|
// cached response on the request's Origin header.
|
|
AccessControlAllowOrigins: &cloudfront.ResponseHeadersPolicyCorsConfigAccessControlAllowOriginsArgs{
|
|
Items: pulumi.ToStringArray([]string{"*"}),
|
|
},
|
|
// Content-Range/Content-Length are what a player reads back
|
|
// when it seeks with a Range request.
|
|
AccessControlExposeHeaders: &cloudfront.ResponseHeadersPolicyCorsConfigAccessControlExposeHeadersArgs{
|
|
Items: pulumi.ToStringArray([]string{"Content-Length", "Content-Range", "Date", "ETag"}),
|
|
},
|
|
AccessControlMaxAgeSec: pulumi.Int(3000),
|
|
OriginOverride: pulumi.Bool(true),
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
distribution, err = cloudfront.NewDistribution(ctx, "encoded-bucket-cdn", &cloudfront.DistributionArgs{
|
|
Enabled: pulumi.Bool(true),
|
|
Comment: pulumi.String("Edge caching for encoded-bucket assets"),
|
|
Origins: cloudfront.DistributionOriginArray{
|
|
&cloudfront.DistributionOriginArgs{
|
|
DomainName: bucket.BucketRegionalDomainName,
|
|
OriginId: pulumi.String(originId),
|
|
OriginAccessControlId: oac.ID(),
|
|
S3OriginConfig: &cloudfront.DistributionOriginS3OriginConfigArgs{
|
|
OriginAccessIdentity: pulumi.String(""),
|
|
},
|
|
},
|
|
},
|
|
DefaultCacheBehavior: &cloudfront.DistributionDefaultCacheBehaviorArgs{
|
|
TargetOriginId: pulumi.String(originId),
|
|
ViewerProtocolPolicy: pulumi.String("redirect-to-https"),
|
|
// OPTIONS is listed so CloudFront answers CORS preflights
|
|
// itself from the response headers policy below; without it
|
|
// a preflight is rejected with 403 before the policy runs.
|
|
AllowedMethods: pulumi.ToStringArray([]string{"GET", "HEAD", "OPTIONS"}),
|
|
CachedMethods: pulumi.ToStringArray([]string{"GET", "HEAD", "OPTIONS"}),
|
|
Compress: pulumi.Bool(true),
|
|
CachePolicyId: pulumi.String(cachingOptimizedPolicyId),
|
|
ResponseHeadersPolicyId: corsHeaders.ID(),
|
|
},
|
|
Restrictions: &cloudfront.DistributionRestrictionsArgs{
|
|
GeoRestriction: &cloudfront.DistributionRestrictionsGeoRestrictionArgs{
|
|
RestrictionType: pulumi.String("none"),
|
|
},
|
|
},
|
|
ViewerCertificate: &cloudfront.DistributionViewerCertificateArgs{
|
|
CloudfrontDefaultCertificate: pulumi.Bool(true),
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Allow only this distribution to read objects from the bucket.
|
|
policy := pulumi.All(bucket.Arn, distribution.Arn).ApplyT(func(args []interface{}) (string, error) {
|
|
bucketArn := args[0].(string)
|
|
distributionArn := args[1].(string)
|
|
|
|
doc := map[string]interface{}{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]interface{}{
|
|
{
|
|
"Sid": "AllowCloudFrontServicePrincipal",
|
|
"Effect": "Allow",
|
|
"Principal": map[string]string{"Service": "cloudfront.amazonaws.com"},
|
|
"Action": "s3:GetObject",
|
|
"Resource": bucketArn + "/*",
|
|
"Condition": map[string]interface{}{
|
|
"StringEquals": map[string]string{"AWS:SourceArn": distributionArn},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
}).(pulumi.StringOutput)
|
|
|
|
_, err = s3.NewBucketPolicy(ctx, "encoded-bucket-policy", &s3.BucketPolicyArgs{
|
|
Bucket: bucket.ID(),
|
|
Policy: policy,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Networking for the RDS instance and the ECS services. Skipped under
|
|
// LocalStack: its RDS emulation is a Postgres process inside the
|
|
// LocalStack container, reached over the docker-compose network rather
|
|
// than through a VPC, and nothing else here runs in one.
|
|
if !localstack {
|
|
// Small, single-AZ Postgres instance for the assignment: default VPC,
|
|
// no public access, no Multi-AZ, no backups, destroyable without a snapshot.
|
|
vpc, err := ec2.LookupVpc(ctx, &ec2.LookupVpcArgs{Default: pulumi.BoolRef(true)})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
vpcID = vpc.Id
|
|
|
|
subnets, err := ec2.GetSubnets(ctx, &ec2.GetSubnetsArgs{
|
|
Filters: []ec2.GetSubnetsFilter{
|
|
{Name: "vpc-id", Values: []string{vpc.Id}},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
subnetIDs = subnets.Ids
|
|
|
|
dbSubnetGroup, err = rds.NewSubnetGroup(ctx, "postgres-subnet-group", &rds.SubnetGroupArgs{
|
|
SubnetIds: pulumi.ToStringArray(subnets.Ids),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
myIP, err := currentPublicIP()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
myIPCidr := fmt.Sprintf("%s/32", myIP)
|
|
|
|
dbSecurityGroup, err = ec2.NewSecurityGroup(ctx, "postgres-sg", &ec2.SecurityGroupArgs{
|
|
Description: pulumi.String("Allow Postgres access from within the VPC and the deployers IP"),
|
|
VpcId: pulumi.String(vpc.Id),
|
|
Ingress: ec2.SecurityGroupIngressArray{
|
|
&ec2.SecurityGroupIngressArgs{
|
|
Protocol: pulumi.String("tcp"),
|
|
FromPort: pulumi.Int(5432),
|
|
ToPort: pulumi.Int(5432),
|
|
CidrBlocks: pulumi.ToStringArray([]string{vpc.CidrBlock}),
|
|
},
|
|
&ec2.SecurityGroupIngressArgs{
|
|
Protocol: pulumi.String("tcp"),
|
|
FromPort: pulumi.Int(5432),
|
|
ToPort: pulumi.Int(5432),
|
|
CidrBlocks: pulumi.ToStringArray([]string{myIPCidr}),
|
|
Description: pulumi.String("Deployer IP, for the postgresql provider to create databases"),
|
|
},
|
|
},
|
|
Egress: ec2.SecurityGroupEgressArray{
|
|
&ec2.SecurityGroupEgressArgs{
|
|
Protocol: pulumi.String("-1"),
|
|
FromPort: pulumi.Int(0),
|
|
ToPort: pulumi.Int(0),
|
|
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Generated once and kept in the stack state, so it stays stable across deploys
|
|
// unless explicitly replaced. RDS master passwords can't contain '/', '@', '"', or spaces.
|
|
dbPassword, err := random.NewRandomPassword(ctx, "postgres-db-password", &random.RandomPasswordArgs{
|
|
Length: pulumi.Int(24),
|
|
Special: pulumi.Bool(true),
|
|
OverrideSpecial: pulumi.String("!#$%&*()-_=+[]{}<>:?"),
|
|
MinUpper: pulumi.Int(1),
|
|
MinLower: pulumi.Int(1),
|
|
MinNumeric: pulumi.Int(1),
|
|
MinSpecial: pulumi.Int(1),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
dbArgs := &rds.InstanceArgs{
|
|
Engine: pulumi.String("postgres"),
|
|
EngineVersion: pulumi.String("16"),
|
|
InstanceClass: pulumi.String("db.t3.micro"),
|
|
AllocatedStorage: pulumi.Int(20),
|
|
StorageType: pulumi.String("gp3"),
|
|
StorageEncrypted: pulumi.Bool(true),
|
|
DbName: pulumi.String("appdb"),
|
|
Username: pulumi.String("postgres"),
|
|
Password: dbPassword.Result,
|
|
PubliclyAccessible: pulumi.Bool(true),
|
|
MultiAz: pulumi.Bool(false),
|
|
BackupRetentionPeriod: pulumi.Int(0),
|
|
DeletionProtection: pulumi.Bool(false),
|
|
SkipFinalSnapshot: pulumi.Bool(true),
|
|
ApplyImmediately: pulumi.Bool(true),
|
|
}
|
|
if localstack {
|
|
dbArgs.Port = pulumi.Int(localstackDBPort)
|
|
} else {
|
|
dbArgs.DbSubnetGroupName = dbSubnetGroup.Name
|
|
dbArgs.VpcSecurityGroupIds = pulumi.StringArray{dbSecurityGroup.ID()}
|
|
}
|
|
|
|
db, err := rds.NewInstance(ctx, "postgres-db", dbArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// RDS terminates TLS; the Postgres process LocalStack runs does not.
|
|
sslMode := "require"
|
|
if localstack {
|
|
sslMode = "disable"
|
|
}
|
|
|
|
pgProvider, err := postgresql.NewProvider(ctx, "postgres-provider", &postgresql.ProviderArgs{
|
|
Host: db.Address,
|
|
Port: db.Port,
|
|
Username: pulumi.String("postgres"),
|
|
Password: dbPassword.Result,
|
|
Sslmode: pulumi.String(sslMode),
|
|
Superuser: pulumi.Bool(false),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cmsPassword, err := newServiceDatabase(ctx, pgProvider, "cms", servicePassword)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
discoveryPassword, err := newServiceDatabase(ctx, pgProvider, "discovery", servicePassword)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Trust policy shared by every ECS task role.
|
|
execRoleAssumePolicy, err := json.Marshal(map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Effect": "Allow",
|
|
"Action": "sts:AssumeRole",
|
|
"Principal": map[string]string{"Service": "ecs-tasks.amazonaws.com"},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// ECS Fargate: one cluster, one internet-facing ALB, one service per app.
|
|
// docker-compose runs the containers under LocalStack, so none of this
|
|
// — cluster, load balancer security groups, or the ECS agent's
|
|
// execution role — has a local counterpart.
|
|
if !localstack {
|
|
cluster, err = ecs.NewCluster(ctx, "app-cluster", nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = ecs.NewClusterCapacityProviders(ctx, "app-cluster-capacity-providers", &ecs.ClusterCapacityProvidersArgs{
|
|
ClusterName: cluster.Name,
|
|
CapacityProviders: pulumi.ToStringArray([]string{"FARGATE", "FARGATE_SPOT"}),
|
|
DefaultCapacityProviderStrategies: ecs.ClusterCapacityProvidersDefaultCapacityProviderStrategyArray{
|
|
&ecs.ClusterCapacityProvidersDefaultCapacityProviderStrategyArgs{
|
|
CapacityProvider: pulumi.String("FARGATE"),
|
|
Weight: pulumi.Int(1),
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
albSecurityGroup, err = ec2.NewSecurityGroup(ctx, "alb-sg", &ec2.SecurityGroupArgs{
|
|
Description: pulumi.String("Allow inbound HTTP to the app load balancers"),
|
|
VpcId: pulumi.String(vpcID),
|
|
Ingress: ec2.SecurityGroupIngressArray{
|
|
&ec2.SecurityGroupIngressArgs{
|
|
Protocol: pulumi.String("tcp"),
|
|
FromPort: pulumi.Int(80),
|
|
ToPort: pulumi.Int(80),
|
|
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
|
|
},
|
|
},
|
|
Egress: ec2.SecurityGroupEgressArray{
|
|
&ec2.SecurityGroupEgressArgs{
|
|
Protocol: pulumi.String("-1"),
|
|
FromPort: pulumi.Int(0),
|
|
ToPort: pulumi.Int(0),
|
|
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
serviceSecurityGroup, err = ec2.NewSecurityGroup(ctx, "ecs-service-sg", &ec2.SecurityGroupArgs{
|
|
Description: pulumi.String("Allow the ALB to reach cms and discovery tasks"),
|
|
VpcId: pulumi.String(vpcID),
|
|
Ingress: ec2.SecurityGroupIngressArray{
|
|
&ec2.SecurityGroupIngressArgs{
|
|
Protocol: pulumi.String("tcp"),
|
|
FromPort: pulumi.Int(0),
|
|
ToPort: pulumi.Int(65535),
|
|
SecurityGroups: pulumi.StringArray{albSecurityGroup.ID()},
|
|
},
|
|
},
|
|
Egress: ec2.SecurityGroupEgressArray{
|
|
&ec2.SecurityGroupEgressArgs{
|
|
Protocol: pulumi.String("-1"),
|
|
FromPort: pulumi.Int(0),
|
|
ToPort: pulumi.Int(0),
|
|
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
execRole, err = iam.NewRole(ctx, "ecs-task-execution-role", &iam.RoleArgs{
|
|
AssumeRolePolicy: pulumi.String(execRoleAssumePolicy),
|
|
ManagedPolicyArns: pulumi.ToStringArray([]string{
|
|
"arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
|
|
}),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if !localstack {
|
|
cacheSubnetGroup, err := elasticache.NewSubnetGroup(ctx, "search-cache-subnet-group", &elasticache.SubnetGroupArgs{
|
|
SubnetIds: pulumi.ToStringArray(subnetIDs),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cacheSecurityGroup, err := ec2.NewSecurityGroup(ctx, "search-cache-sg", &ec2.SecurityGroupArgs{
|
|
Description: pulumi.String("Allow the ECS tasks to reach the search cache"),
|
|
VpcId: pulumi.String(vpcID),
|
|
Ingress: ec2.SecurityGroupIngressArray{
|
|
&ec2.SecurityGroupIngressArgs{
|
|
Protocol: pulumi.String("tcp"),
|
|
FromPort: pulumi.Int(6379),
|
|
ToPort: pulumi.Int(6379),
|
|
SecurityGroups: pulumi.StringArray{serviceSecurityGroup.ID()},
|
|
Description: pulumi.String("Redis, from the service tasks only"),
|
|
},
|
|
},
|
|
Egress: ec2.SecurityGroupEgressArray{
|
|
&ec2.SecurityGroupEgressArgs{
|
|
Protocol: pulumi.String("-1"),
|
|
FromPort: pulumi.Int(0),
|
|
ToPort: pulumi.Int(0),
|
|
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
searchCache, err := elasticache.NewCluster(ctx, "search-cache", &elasticache.ClusterArgs{
|
|
Engine: pulumi.String("redis"),
|
|
EngineVersion: pulumi.String("7.1"),
|
|
NodeType: pulumi.String("cache.t4g.micro"),
|
|
NumCacheNodes: pulumi.Int(1),
|
|
ParameterGroupName: pulumi.String("default.redis7"),
|
|
Port: pulumi.Int(6379),
|
|
SubnetGroupName: cacheSubnetGroup.Name,
|
|
SecurityGroupIds: pulumi.StringArray{cacheSecurityGroup.ID()},
|
|
ApplyImmediately: pulumi.Bool(true),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// A single-node Redis cluster has no configuration endpoint — that
|
|
// is a Memcached thing — so the address is the one cache node's.
|
|
searchCacheAddress = pulumi.All(searchCache.CacheNodes, searchCache.Port).ApplyT(
|
|
func(args []any) (string, error) {
|
|
nodes := args[0].([]elasticache.ClusterCacheNode)
|
|
if len(nodes) == 0 || nodes[0].Address == nil {
|
|
return "", fmt.Errorf("search cache reported no node address")
|
|
}
|
|
return fmt.Sprintf("%s:%d", *nodes[0].Address, args[1].(int)), nil
|
|
},
|
|
).(pulumi.StringOutput)
|
|
}
|
|
|
|
// Private bucket for raw video uploads (pre-transcode). Kept separate
|
|
// from encoded-bucket, which is fronted by CloudFront/OAC for public
|
|
// delivery of finished renditions — raw source video must not be
|
|
// reachable through that CDN.
|
|
rawUploadsBucket, err := s3.NewBucket(ctx, "raw-uploads-bucket", rawUploadsBucketArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = s3.NewBucketPublicAccessBlock(ctx, "raw-uploads-bucket-public-access-block", &s3.BucketPublicAccessBlockArgs{
|
|
Bucket: rawUploadsBucket.ID(),
|
|
BlockPublicAcls: pulumi.Bool(true),
|
|
BlockPublicPolicy: pulumi.Bool(true),
|
|
IgnorePublicAcls: pulumi.Bool(true),
|
|
RestrictPublicBuckets: pulumi.Bool(true),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Role MediaConvert itself assumes (passed as CreateJobInput.Role by
|
|
// the cms service) to read the raw upload and write the encoded
|
|
// output — distinct from the ECS task role below.
|
|
mediaConvertAssumePolicy, err := json.Marshal(map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Effect": "Allow",
|
|
"Action": "sts:AssumeRole",
|
|
"Principal": map[string]string{"Service": "mediaconvert.amazonaws.com"},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
mediaConvertRole, err := iam.NewRole(ctx, "mediaconvert-service-role", &iam.RoleArgs{
|
|
AssumeRolePolicy: pulumi.String(mediaConvertAssumePolicy),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
mediaConvertS3Policy := pulumi.All(rawUploadsBucket.Arn, bucket.Arn).ApplyT(
|
|
func(args []any) (string, error) {
|
|
rawUploadsArn := args[0].(string)
|
|
encodedBucketArn := args[1].(string)
|
|
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Sid": "ReadRawUploads",
|
|
"Effect": "Allow",
|
|
"Action": "s3:GetObject",
|
|
"Resource": rawUploadsArn + "/*",
|
|
},
|
|
{
|
|
"Sid": "WriteEncodedOutput",
|
|
"Effect": "Allow",
|
|
"Action": "s3:PutObject",
|
|
"Resource": encodedBucketArn + "/*",
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
},
|
|
).(pulumi.StringOutput)
|
|
|
|
_, err = iam.NewRolePolicy(ctx, "mediaconvert-s3-access", &iam.RolePolicyArgs{
|
|
Role: mediaConvertRole.ID(),
|
|
Policy: mediaConvertS3Policy,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// MediaConvert job events: MediaConvert -> EventBridge -> SNS -> SQS ->
|
|
// cms. cms *consumes a queue* rather than being called back on an HTTPS
|
|
// endpoint, so nothing new is exposed on the ALB, authenticity comes
|
|
// from IAM instead of SNS message signatures, and a failed handler
|
|
// leaves the message on the queue to be retried instead of dropping the
|
|
// update. The topic sits between the rule and the queue so a second
|
|
// consumer — discovery, when it has code — can subscribe its own queue
|
|
// without touching this wiring or MediaConvert.
|
|
jobEventsTopicArgs := &sns.TopicArgs{}
|
|
jobEventsQueueArgs := &sqs.QueueArgs{}
|
|
jobEventsDeadLetterArgs := &sqs.QueueArgs{}
|
|
if localstack {
|
|
// docker-compose.yml and the tests/ suite refer to these literally.
|
|
jobEventsTopicArgs.Name = pulumi.String("mediaconvert-job-events")
|
|
jobEventsQueueArgs.Name = pulumi.String("cms-mediaconvert-events")
|
|
jobEventsDeadLetterArgs.Name = pulumi.String("cms-mediaconvert-events-dlq")
|
|
}
|
|
|
|
jobEventsTopic, err := sns.NewTopic(ctx, "mediaconvert-job-events", jobEventsTopicArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Where a message lands after maxReceiveCount failed handlings, so one
|
|
// event cms cannot process never blocks the ones behind it.
|
|
jobEventsDeadLetter, err := sqs.NewQueue(ctx, "cms-mediaconvert-events-dlq", jobEventsDeadLetterArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
jobEventsQueueArgs.VisibilityTimeoutSeconds = pulumi.Int(60)
|
|
jobEventsQueueArgs.RedrivePolicy = jobEventsDeadLetter.Arn.ApplyT(func(arn string) (string, error) {
|
|
b, err := json.Marshal(map[string]any{
|
|
"deadLetterTargetArn": arn,
|
|
"maxReceiveCount": 5,
|
|
})
|
|
return string(b), err
|
|
}).(pulumi.StringOutput)
|
|
|
|
jobEventsQueue, err := sqs.NewQueue(ctx, "cms-mediaconvert-events", jobEventsQueueArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// The catalogue topic: where cms announces a video once its transcode
|
|
// has finished. A topic rather than a queue so the announcement fans
|
|
// out — the read side subscribes the queue below, and a second reader
|
|
// (search indexing, notifications) can subscribe its own without cms
|
|
// knowing it exists or this wiring changing.
|
|
catalogueTopicArgs := &sns.TopicArgs{}
|
|
catalogueQueueArgs := &sqs.QueueArgs{}
|
|
catalogueDeadLetterArgs := &sqs.QueueArgs{}
|
|
if localstack {
|
|
// docker-compose.yml and the tests/ suite refer to these literally.
|
|
catalogueTopicArgs.Name = pulumi.String("catalogue-events")
|
|
catalogueQueueArgs.Name = pulumi.String("discovery-catalogue-events")
|
|
catalogueDeadLetterArgs.Name = pulumi.String("discovery-catalogue-events-dlq")
|
|
}
|
|
|
|
catalogueTopic, err := sns.NewTopic(ctx, "catalogue-events", catalogueTopicArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
catalogueDeadLetter, err := sqs.NewQueue(ctx, "discovery-catalogue-events-dlq", catalogueDeadLetterArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
catalogueQueueArgs.VisibilityTimeoutSeconds = pulumi.Int(60)
|
|
catalogueQueueArgs.RedrivePolicy = catalogueDeadLetter.Arn.ApplyT(func(arn string) (string, error) {
|
|
b, err := json.Marshal(map[string]any{
|
|
"deadLetterTargetArn": arn,
|
|
"maxReceiveCount": 5,
|
|
})
|
|
return string(b), err
|
|
}).(pulumi.StringOutput)
|
|
|
|
// The read side's own queue on the catalogue topic. It exists before
|
|
// discovery has code to drain it so the announcements pile up rather
|
|
// than being dropped on the floor — a topic delivers only to the
|
|
// subscriptions that exist when it publishes.
|
|
catalogueQueue, err := sqs.NewQueue(ctx, "discovery-catalogue-events", catalogueQueueArgs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// SNS is not an IAM principal the queue trusts by default; without this
|
|
// the subscription is created and silently delivers nothing.
|
|
catalogueQueuePolicy := pulumi.All(catalogueQueue.Arn, catalogueTopic.Arn).ApplyT(
|
|
func(args []any) (string, error) {
|
|
queueArn := args[0].(string)
|
|
topicArn := args[1].(string)
|
|
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Sid": "AllowCatalogueTopicToSend",
|
|
"Effect": "Allow",
|
|
"Principal": map[string]string{"Service": "sns.amazonaws.com"},
|
|
"Action": "sqs:SendMessage",
|
|
"Resource": queueArn,
|
|
"Condition": map[string]any{
|
|
"ArnEquals": map[string]string{"aws:SourceArn": topicArn},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
},
|
|
).(pulumi.StringOutput)
|
|
|
|
_, err = sqs.NewQueuePolicy(ctx, "discovery-catalogue-events-queue-policy", &sqs.QueuePolicyArgs{
|
|
QueueUrl: catalogueQueue.ID(),
|
|
Policy: catalogueQueuePolicy,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = sns.NewTopicSubscription(ctx, "discovery-catalogue-events-subscription", &sns.TopicSubscriptionArgs{
|
|
Topic: catalogueTopic.Arn,
|
|
Protocol: pulumi.String("sqs"),
|
|
Endpoint: catalogueQueue.Arn,
|
|
// The queue receives the announcement itself rather than an SNS
|
|
// envelope carrying it as a JSON string, so a consumer parses one
|
|
// document instead of unwrapping two.
|
|
RawMessageDelivery: pulumi.Bool(true),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// A second subscriber, for the tests/ suite only. The BDD scenarios
|
|
// assert on the announcement itself — the field names cms publishes,
|
|
// which no assertion through discovery's API can pin — and a consumer
|
|
// destroys what it reads. Sharing discovery's queue would mean the two
|
|
// racing for every message and each seeing about half, so the suite
|
|
// gets its own subscription. This is what the topic is for.
|
|
//
|
|
// LocalStack only: in AWS this queue would fill up unread, since
|
|
// nothing runs the suite there.
|
|
if localstack {
|
|
testsCatalogueQueue, err := sqs.NewQueue(ctx, "tests-catalogue-events", &sqs.QueueArgs{
|
|
Name: pulumi.String("tests-catalogue-events"),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
testsCatalogueQueuePolicy := pulumi.All(testsCatalogueQueue.Arn, catalogueTopic.Arn).ApplyT(
|
|
func(args []any) (string, error) {
|
|
queueArn := args[0].(string)
|
|
topicArn := args[1].(string)
|
|
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Sid": "AllowCatalogueTopicToSend",
|
|
"Effect": "Allow",
|
|
"Principal": map[string]string{"Service": "sns.amazonaws.com"},
|
|
"Action": "sqs:SendMessage",
|
|
"Resource": queueArn,
|
|
"Condition": map[string]any{
|
|
"ArnEquals": map[string]string{"aws:SourceArn": topicArn},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
},
|
|
).(pulumi.StringOutput)
|
|
|
|
_, err = sqs.NewQueuePolicy(ctx, "tests-catalogue-events-queue-policy", &sqs.QueuePolicyArgs{
|
|
QueueUrl: testsCatalogueQueue.ID(),
|
|
Policy: testsCatalogueQueuePolicy,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = sns.NewTopicSubscription(ctx, "tests-catalogue-events-subscription", &sns.TopicSubscriptionArgs{
|
|
Topic: catalogueTopic.Arn,
|
|
Protocol: pulumi.String("sqs"),
|
|
Endpoint: testsCatalogueQueue.Arn,
|
|
RawMessageDelivery: pulumi.Bool(true),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// SNS is not an IAM principal the queue trusts by default; without this
|
|
// the subscription is created and silently delivers nothing.
|
|
jobEventsQueuePolicy := pulumi.All(jobEventsQueue.Arn, jobEventsTopic.Arn).ApplyT(
|
|
func(args []any) (string, error) {
|
|
queueArn := args[0].(string)
|
|
topicArn := args[1].(string)
|
|
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Sid": "AllowJobEventsTopicToSend",
|
|
"Effect": "Allow",
|
|
"Principal": map[string]string{"Service": "sns.amazonaws.com"},
|
|
"Action": "sqs:SendMessage",
|
|
"Resource": queueArn,
|
|
"Condition": map[string]any{
|
|
"ArnEquals": map[string]string{"aws:SourceArn": topicArn},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
},
|
|
).(pulumi.StringOutput)
|
|
|
|
_, err = sqs.NewQueuePolicy(ctx, "cms-mediaconvert-events-queue-policy", &sqs.QueuePolicyArgs{
|
|
QueueUrl: jobEventsQueue.ID(),
|
|
Policy: jobEventsQueuePolicy,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = sns.NewTopicSubscription(ctx, "cms-mediaconvert-events-subscription", &sns.TopicSubscriptionArgs{
|
|
Topic: jobEventsTopic.Arn,
|
|
Protocol: pulumi.String("sqs"),
|
|
Endpoint: jobEventsQueue.Arn,
|
|
// The queue receives the EventBridge event itself rather than an SNS
|
|
// envelope carrying it as a JSON string, so the consumer parses one
|
|
// document instead of unwrapping two.
|
|
RawMessageDelivery: pulumi.Bool(true),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
jobEventsTopicPolicy := jobEventsTopic.Arn.ApplyT(func(topicArn string) (string, error) {
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Sid": "AllowEventBridgeToPublish",
|
|
"Effect": "Allow",
|
|
"Principal": map[string]string{"Service": "events.amazonaws.com"},
|
|
"Action": "sns:Publish",
|
|
"Resource": topicArn,
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
}).(pulumi.StringOutput)
|
|
|
|
_, err = sns.NewTopicPolicy(ctx, "mediaconvert-job-events-topic-policy", &sns.TopicPolicyArgs{
|
|
Arn: jobEventsTopic.Arn,
|
|
Policy: jobEventsTopicPolicy,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// MediaConvert does not publish to SNS itself: it reports state changes
|
|
// to EventBridge, and this rule is what forwards them to the topic.
|
|
// Skipped under LocalStack, whose MediaConvert emulation does not emit
|
|
// these — the suite in tests/ publishes to the topic directly instead,
|
|
// which is the same seam from cms's side.
|
|
if !localstack {
|
|
jobStateChangePattern, err := json.Marshal(map[string]any{
|
|
"source": []string{"aws.mediaconvert"},
|
|
"detail-type": []string{"MediaConvert Job State Change"},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
jobEventsRule, err := cloudwatch.NewEventRule(ctx, "mediaconvert-job-state-change", &cloudwatch.EventRuleArgs{
|
|
Description: pulumi.String("MediaConvert job state changes, forwarded to the cms job events topic"),
|
|
EventPattern: pulumi.String(string(jobStateChangePattern)),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = cloudwatch.NewEventTarget(ctx, "mediaconvert-job-state-change-to-topic", &cloudwatch.EventTargetArgs{
|
|
Rule: jobEventsRule.Name,
|
|
Arn: jobEventsTopic.Arn,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// cms's own ECS task role (distinct from the shared execRole, which
|
|
// is only for the ECS agent's pull/logs/secrets access): lets the
|
|
// running container call S3 and MediaConvert directly. discovery
|
|
// gets none of this since it never touches either API.
|
|
cmsTaskRole, err := iam.NewRole(ctx, "cms-task-role", &iam.RoleArgs{
|
|
AssumeRolePolicy: pulumi.String(execRoleAssumePolicy),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cmsTaskPolicy := pulumi.All(rawUploadsBucket.Arn, mediaConvertRole.Arn, jobEventsQueue.Arn, catalogueTopic.Arn).ApplyT(
|
|
func(args []any) (string, error) {
|
|
rawUploadsArn := args[0].(string)
|
|
mediaConvertRoleArn := args[1].(string)
|
|
jobEventsQueueArn := args[2].(string)
|
|
catalogueTopicArn := args[3].(string)
|
|
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Sid": "ListRawUploadsBucket",
|
|
"Effect": "Allow",
|
|
"Action": "s3:ListBucket",
|
|
"Resource": rawUploadsArn,
|
|
},
|
|
{
|
|
"Sid": "RawUploadsObjectAccess",
|
|
"Effect": "Allow",
|
|
"Action": []string{
|
|
"s3:PutObject",
|
|
"s3:GetObject",
|
|
},
|
|
"Resource": rawUploadsArn + "/*",
|
|
},
|
|
{
|
|
"Sid": "CreateTranscodeJob",
|
|
"Effect": "Allow",
|
|
"Action": "mediaconvert:CreateJob",
|
|
"Resource": "*",
|
|
},
|
|
{
|
|
"Sid": "ConsumeJobEvents",
|
|
"Effect": "Allow",
|
|
"Action": []string{
|
|
"sqs:ReceiveMessage",
|
|
"sqs:DeleteMessage",
|
|
"sqs:GetQueueAttributes",
|
|
},
|
|
"Resource": jobEventsQueueArn,
|
|
},
|
|
{
|
|
"Sid": "AnnounceReadyVideos",
|
|
"Effect": "Allow",
|
|
"Action": []string{
|
|
"sns:Publish",
|
|
"sns:GetTopicAttributes",
|
|
},
|
|
"Resource": catalogueTopicArn,
|
|
},
|
|
{
|
|
"Sid": "PassMediaConvertRole",
|
|
"Effect": "Allow",
|
|
"Action": "iam:PassRole",
|
|
"Resource": mediaConvertRoleArn,
|
|
"Condition": map[string]any{
|
|
"StringEquals": map[string]string{
|
|
"iam:PassedToService": "mediaconvert.amazonaws.com",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
},
|
|
).(pulumi.StringOutput)
|
|
|
|
_, err = iam.NewRolePolicy(ctx, "cms-task-role-policy", &iam.RolePolicyArgs{
|
|
Role: cmsTaskRole.ID(),
|
|
Policy: cmsTaskPolicy,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// discovery's own ECS task role. It used to have none — it touched no
|
|
// AWS API at all — and it still gets nothing but the one queue it
|
|
// drains: no S3, no MediaConvert, and no ability to publish back onto
|
|
// the catalogue topic. It is a subscriber, not a participant.
|
|
discoveryTaskRole, err := iam.NewRole(ctx, "discovery-task-role", &iam.RoleArgs{
|
|
AssumeRolePolicy: pulumi.String(execRoleAssumePolicy),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
discoveryTaskPolicy := catalogueQueue.Arn.ApplyT(func(catalogueQueueArn string) (string, error) {
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Sid": "ConsumeCatalogueAnnouncements",
|
|
"Effect": "Allow",
|
|
"Action": []string{
|
|
"sqs:ReceiveMessage",
|
|
"sqs:DeleteMessage",
|
|
"sqs:GetQueueAttributes",
|
|
},
|
|
"Resource": catalogueQueueArn,
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
}).(pulumi.StringOutput)
|
|
|
|
_, err = iam.NewRolePolicy(ctx, "discovery-task-role-policy", &iam.RolePolicyArgs{
|
|
Role: discoveryTaskRole.ID(),
|
|
Policy: discoveryTaskPolicy,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// The two ECS services and the CI user that deploys them. Under
|
|
// LocalStack the equivalent of all this is the cms service in
|
|
// docker-compose.yml, which reads the same env vars from .env.
|
|
if !localstack {
|
|
cmsExtraEnv := []envVar{
|
|
{Name: "AWS_REGION", Value: pulumi.String(awsRegion).ToStringOutput()},
|
|
{Name: "S3_BUCKET", Value: rawUploadsBucket.ID().ToStringOutput()},
|
|
{Name: "MEDIACONVERT_INPUT_BUCKET", Value: rawUploadsBucket.ID().ToStringOutput()},
|
|
{Name: "MEDIACONVERT_OUTPUT_BUCKET", Value: bucket.ID().ToStringOutput()},
|
|
{Name: "MEDIACONVERT_ROLE_ARN", Value: mediaConvertRole.Arn},
|
|
{Name: "MEDIACONVERT_EVENTS_QUEUE_URL", Value: jobEventsQueue.Url},
|
|
{Name: "CATALOGUE_EVENTS_TOPIC_ARN", Value: catalogueTopic.Arn},
|
|
// The CDN in front of the encoded bucket, which is the only way
|
|
// that bucket is readable — it blocks public access and grants
|
|
// only CloudFront's OAC principal. cms rewrites the s3:// paths
|
|
// MediaConvert reports onto this host.
|
|
{Name: "PLAYBACK_BASE_URL", Value: pulumi.Sprintf("https://%s", distribution.DomainName).ToStringOutput()},
|
|
}
|
|
|
|
cmsRepo, cmsService, cmsAlb, err = deployFargateService(ctx, "cms", 8081,
|
|
cluster, execRole, cmsTaskRole, cmsExtraEnv, vpcID, subnetIDs, albSecurityGroup, serviceSecurityGroup,
|
|
db.Address, db.Port, cmsPassword, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// The browser PUTs directly to raw-uploads-bucket using the presigned
|
|
// URL from PresignVideoUpload, so the bucket (not the cms app) is
|
|
// what needs to answer the CORS preflight — scoped to the cms
|
|
// origin the upload page is actually served from. Under LocalStack
|
|
// there is no ALB to name, and the caller is whatever the developer
|
|
// happens to be running.
|
|
var corsOrigins pulumi.StringArrayInput = pulumi.ToStringArray([]string{"*"})
|
|
if !localstack {
|
|
corsOrigins = pulumi.StringArray{pulumi.Sprintf("http://%s", cmsAlb.DnsName)}
|
|
}
|
|
|
|
// The encoded bucket needs its own CORS rule for the LocalStack stack,
|
|
// where there is no CloudFront in front of it and PLAYBACK_BASE_URL
|
|
// points the player straight at S3. Behind CloudFront the response
|
|
// headers policy already covers playback, but the rule is harmless
|
|
// there and keeps a direct-to-bucket player working either way.
|
|
_, err = s3.NewBucketCorsConfigurationV2(ctx, "encoded-bucket-cors", &s3.BucketCorsConfigurationV2Args{
|
|
Bucket: bucket.ID(),
|
|
CorsRules: s3.BucketCorsConfigurationV2CorsRuleArray{
|
|
&s3.BucketCorsConfigurationV2CorsRuleArgs{
|
|
AllowedMethods: pulumi.ToStringArray([]string{"GET", "HEAD"}),
|
|
AllowedOrigins: pulumi.ToStringArray([]string{"*"}),
|
|
AllowedHeaders: pulumi.ToStringArray([]string{"*"}),
|
|
ExposeHeaders: pulumi.ToStringArray([]string{"Content-Length", "Content-Range", "Date", "ETag"}),
|
|
MaxAgeSeconds: pulumi.Int(3000),
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = s3.NewBucketCorsConfigurationV2(ctx, "raw-uploads-bucket-cors", &s3.BucketCorsConfigurationV2Args{
|
|
Bucket: rawUploadsBucket.ID(),
|
|
CorsRules: s3.BucketCorsConfigurationV2CorsRuleArray{
|
|
&s3.BucketCorsConfigurationV2CorsRuleArgs{
|
|
AllowedMethods: pulumi.ToStringArray([]string{"PUT"}),
|
|
AllowedOrigins: corsOrigins,
|
|
AllowedHeaders: pulumi.ToStringArray([]string{"*"}),
|
|
ExposeHeaders: pulumi.ToStringArray([]string{"ETag"}),
|
|
MaxAgeSeconds: pulumi.Int(3000),
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !localstack {
|
|
// runMigrations is true for the same reason it is for cms:
|
|
// discovery embeds its own golang-migrate migrations and applies
|
|
// them through `./discovery migrate`, so ECS runs that container to
|
|
// completion before the service accepts traffic.
|
|
discoveryExtraEnv := []envVar{
|
|
{Name: "AWS_REGION", Value: pulumi.String(awsRegion).ToStringOutput()},
|
|
{Name: "CATALOGUE_EVENTS_QUEUE_URL", Value: catalogueQueue.Url},
|
|
// The search cache. Reached over the Redis protocol on the
|
|
// private network, so unlike the queue above it needs no
|
|
// matching grant on discovery-task-role.
|
|
{Name: "REDIS_ADDR", Value: searchCacheAddress},
|
|
}
|
|
|
|
discoveryRepo, discoveryService, discoveryAlb, err = deployFargateService(ctx, "discovery", 8080,
|
|
cluster, execRole, discoveryTaskRole, discoveryExtraEnv, vpcID, subnetIDs, albSecurityGroup, serviceSecurityGroup,
|
|
db.Address, db.Port, discoveryPassword, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Gitea Actions CI: one IAM user, scoped to just what the build/deploy
|
|
// pipeline needs — push images to the two ECR repos and force a new
|
|
// deployment on the two ECS services. Credentials are exported as
|
|
// stack secrets for the pipeline to consume (e.g. as Gitea Actions
|
|
// secrets); nothing broader like ecr:*, ecs:*, or task-def changes.
|
|
giteaCIPolicyDocument := pulumi.All(cmsRepo.Arn, discoveryRepo.Arn, cmsService.Arn, discoveryService.Arn).ApplyT(
|
|
func(args []any) (string, error) {
|
|
cmsRepoArn := args[0].(string)
|
|
discoveryRepoArn := args[1].(string)
|
|
cmsServiceArn := args[2].(string)
|
|
discoveryServiceArn := args[3].(string)
|
|
|
|
doc := map[string]any{
|
|
"Version": "2012-10-17",
|
|
"Statement": []map[string]any{
|
|
{
|
|
"Sid": "ECRAuth",
|
|
"Effect": "Allow",
|
|
"Action": "ecr:GetAuthorizationToken",
|
|
"Resource": "*",
|
|
},
|
|
{
|
|
"Sid": "ECRPush",
|
|
"Effect": "Allow",
|
|
"Action": []string{
|
|
"ecr:BatchCheckLayerAvailability",
|
|
"ecr:BatchGetImage",
|
|
"ecr:PutImage",
|
|
"ecr:InitiateLayerUpload",
|
|
"ecr:UploadLayerPart",
|
|
"ecr:CompleteLayerUpload",
|
|
},
|
|
"Resource": []string{cmsRepoArn, discoveryRepoArn},
|
|
},
|
|
{
|
|
"Sid": "ECSDeploy",
|
|
"Effect": "Allow",
|
|
"Action": []string{
|
|
"ecs:UpdateService",
|
|
"ecs:DescribeServices",
|
|
},
|
|
"Resource": []string{cmsServiceArn, discoveryServiceArn},
|
|
},
|
|
},
|
|
}
|
|
b, err := json.Marshal(doc)
|
|
return string(b), err
|
|
},
|
|
).(pulumi.StringOutput)
|
|
|
|
giteaCIUser, err := iam.NewUser(ctx, "gitea-ci-user", &iam.UserArgs{
|
|
Name: pulumi.String("gitea-ci"),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = iam.NewUserPolicy(ctx, "gitea-ci-policy", &iam.UserPolicyArgs{
|
|
User: giteaCIUser.Name,
|
|
Policy: giteaCIPolicyDocument,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Export the name of the bucket
|
|
ctx.Export("bucketName", bucket.ID())
|
|
ctx.Export("rawUploadsBucketName", rawUploadsBucket.ID())
|
|
ctx.Export("dbEndpoint", db.Endpoint)
|
|
ctx.Export("dbPassword", dbPassword.Result)
|
|
ctx.Export("mediaConvertRoleArn", mediaConvertRole.Arn)
|
|
|
|
// Everything below exists only on the AWS stacks. The CI workflows read
|
|
// ecsClusterArn and cmsServiceArn from here.
|
|
if !localstack {
|
|
ctx.Export("cdnDomainName", distribution.DomainName)
|
|
ctx.Export("cmsUrl", pulumi.Sprintf("http://%s", cmsAlb.DnsName))
|
|
ctx.Export("discoveryUrl", pulumi.Sprintf("http://%s", discoveryAlb.DnsName))
|
|
ctx.Export("cmsRepoUrl", cmsRepo.RepositoryUrl)
|
|
ctx.Export("discoveryRepoUrl", discoveryRepo.RepositoryUrl)
|
|
ctx.Export("ecsClusterArn", cluster.Arn)
|
|
ctx.Export("cmsServiceArn", cmsService.Arn)
|
|
ctx.Export("discoveryServiceArn", discoveryService.Arn)
|
|
ctx.Export("searchCacheAddress", searchCacheAddress)
|
|
}
|
|
return nil
|
|
})
|
|
}
|