FahdShalhoub/thamanyah
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
3f5301afb6437e9b5ce18b3385fd2314f8d74908
thamanyah/infrastructure/main.go
T

1182 lines
39 KiB
Go
Raw Blame History

package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strings"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudfront"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudwatch"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/ec2"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/ecr"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/ecs"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/iam"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/lb"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/rds"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/s3"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/secretsmanager"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/sns"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/sqs"
"github.com/pulumi/pulumi-postgresql/sdk/v3/go/postgresql"
"github.com/pulumi/pulumi-random/sdk/v4/go/random"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
)
// Matches the region configured in Pulumi.main.yaml (aws:region).
const awsRegion = "us-east-1"
// Port the Postgres instance listens on under LocalStack. LocalStack backs an
// RDS instance with a real Postgres process inside its own container, and must
// stay inside the 4510-4559 range docker-compose.yml publishes so the instance
// is reachable both from the cms container and from the host.
const localstackDBPort = 4510
// currentPublicIP returns the caller's public IP, used to scope the RDS
// security group to the machine that will actually run `pulumi up`.
func currentPublicIP() (string, error) {
resp, err := http.Get("https://checkip.amazonaws.com")
if err != nil {
return "", err
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
return strings.TrimSpace(string(body)), nil
}
const originId = "encoded-bucket-origin"
// AWS managed "CachingOptimized" cache policy (same ID in every account/region).
const cachingOptimizedPolicyId = "658327ea-f89d-4fab-a63d-7e88639e58f6"
// newServiceDatabase creates a login role and a same-named database owned by
// that role, so the service only ever touches its own database. The password is
// generated and kept in stack state, except on the LocalStack stack, where
// fixedPassword carries the value docker-compose already hands the container as
// DB_PASSWORD — compose reads a static .env and cannot consume a stack output.
func newServiceDatabase(ctx *pulumi.Context, provider *postgresql.Provider, service, fixedPassword string) (pulumi.StringOutput, error) {
password := pulumi.String(fixedPassword).ToStringOutput()
if fixedPassword == "" {
generated, err := random.NewRandomPassword(ctx, service+"-db-password", &random.RandomPasswordArgs{
Length: pulumi.Int(24),
Special: pulumi.Bool(true),
MinUpper: pulumi.Int(1),
MinLower: pulumi.Int(1),
MinNumeric: pulumi.Int(1),
MinSpecial: pulumi.Int(1),
})
if err != nil {
return pulumi.StringOutput{}, err
}
password = generated.Result
}
role, err := postgresql.NewRole(ctx, service+"-db-role", &postgresql.RoleArgs{
Name: pulumi.String(service),
Login: pulumi.Bool(true),
Password: password,
}, pulumi.Provider(provider))
if err != nil {
return pulumi.StringOutput{}, err
}
_, err = postgresql.NewDatabase(ctx, service+"-database", &postgresql.DatabaseArgs{
Name: pulumi.String(service),
Owner: role.Name,
}, pulumi.Provider(provider))
if err != nil {
return pulumi.StringOutput{}, err
}
return password, nil
}
// envVar is a name/value pair for a container's extra environment variables,
// where the value is only known after other resources are provisioned.
type envVar struct {
Name string
Value pulumi.StringOutput
}
// deployFargateService gives a service its own ECR repo, log group, task
// definition, ECS service, and its own ALB — so it gets its own DNS name
// rather than sharing one load balancer with the other service on a
// different port. The service's DB password is stored in Secrets Manager and
// injected into the container as a secret rather than a plaintext env var.
// taskRole is optional (nil means no TaskRoleArn is set, i.e. the container
// gets no AWS identity of its own beyond execRole's pull/logs permissions);
// extraEnv is appended to the container's environment on top of the DB_* vars.
// When runMigrations is true, the task also gets a non-essential
// "<name>-migrate" container running the same image with `command: ["migrate"]`,
// and the main container depends on it with condition COMPLETE — the ECS
// equivalent of a Kubernetes init container, ensuring DB migrations finish
// before the service starts accepting traffic.
func deployFargateService(
ctx *pulumi.Context,
name string,
containerPort int,
cluster *ecs.Cluster,
execRole *iam.Role,
taskRole *iam.Role,
extraEnv []envVar,
vpcID string,
subnetIDs []string,
albSecurityGroup *ec2.SecurityGroup,
serviceSecurityGroup *ec2.SecurityGroup,
dbAddress pulumi.StringOutput,
dbPort pulumi.IntOutput,
dbPassword pulumi.StringOutput,
runMigrations bool,
) (*ecr.Repository, *ecs.Service, *lb.LoadBalancer, error) {
alb, err := lb.NewLoadBalancer(ctx, name+"-alb", &lb.LoadBalancerArgs{
LoadBalancerType: pulumi.String("application"),
Internal: pulumi.Bool(false),
SecurityGroups: pulumi.StringArray{albSecurityGroup.ID()},
Subnets: pulumi.ToStringArray(subnetIDs),
})
if err != nil {
return nil, nil, nil, err
}
repo, err := ecr.NewRepository(ctx, name+"-repo", &ecr.RepositoryArgs{
Name: pulumi.String(name),
ImageTagMutability: pulumi.String("MUTABLE"),
ForceDelete: pulumi.Bool(true),
})
if err != nil {
return nil, nil, nil, err
}
logGroup, err := cloudwatch.NewLogGroup(ctx, name+"-logs", &cloudwatch.LogGroupArgs{
Name: pulumi.String("/ecs/" + name),
RetentionInDays: pulumi.Int(7),
})
if err != nil {
return nil, nil, nil, err
}
secret, err := secretsmanager.NewSecret(ctx, name+"-db-secret", &secretsmanager.SecretArgs{
Name: pulumi.String(name + "-db-password"),
RecoveryWindowInDays: pulumi.Int(0),
})
if err != nil {
return nil, nil, nil, err
}
_, err = secretsmanager.NewSecretVersion(ctx, name+"-db-secret-version", &secretsmanager.SecretVersionArgs{
SecretId: secret.ID(),
SecretString: dbPassword,
})
if err != nil {
return nil, nil, nil, err
}
secretPolicy := secret.Arn.ApplyT(func(secretArn string) (string, error) {
doc := map[string]any{
"Version": "2012-10-17",
"Statement": []map[string]any{
{
"Effect": "Allow",
"Action": "secretsmanager:GetSecretValue",
"Resource": secretArn,
},
},
}
b, err := json.Marshal(doc)
return string(b), err
}).(pulumi.StringOutput)
_, err = iam.NewRolePolicy(ctx, name+"-secret-access", &iam.RolePolicyArgs{
Role: execRole.ID(),
Policy: secretPolicy,
})
if err != nil {
return nil, nil, nil, err
}
targetGroup, err := lb.NewTargetGroup(ctx, name+"-tg", &lb.TargetGroupArgs{
Port: pulumi.Int(containerPort),
Protocol: pulumi.String("HTTP"),
TargetType: pulumi.String("ip"),
VpcId: pulumi.String(vpcID),
HealthCheck: &lb.TargetGroupHealthCheckArgs{
Path: pulumi.String("/health"),
Matcher: pulumi.String("200"),
},
})
if err != nil {
return nil, nil, nil, err
}
listener, err := lb.NewListener(ctx, name+"-listener", &lb.ListenerArgs{
LoadBalancerArn: alb.Arn,
Port: pulumi.Int(80),
Protocol: pulumi.String("HTTP"),
DefaultActions: lb.ListenerDefaultActionArray{
&lb.ListenerDefaultActionArgs{
Type: pulumi.String("forward"),
TargetGroupArn: targetGroup.Arn,
},
},
})
if err != nil {
return nil, nil, nil, err
}
baseArgs := []any{repo.RepositoryUrl, logGroup.Name, secret.Arn, dbAddress, dbPort}
for _, ev := range extraEnv {
baseArgs = append(baseArgs, ev.Value)
}
containerDefinitions := pulumi.All(baseArgs...).ApplyT(
func(args []any) (string, error) {
image := args[0].(string) + ":latest"
logGroupName := args[1].(string)
secretArn := args[2].(string)
dbHost := args[3].(string)
dbPort := args[4].(int)
dbEnvironment := []map[string]string{
{"name": "DB_HOST", "value": dbHost},
{"name": "DB_PORT", "value": fmt.Sprintf("%d", dbPort)},
{"name": "DB_NAME", "value": name},
{"name": "DB_USER", "value": name},
}
dbSecrets := []map[string]string{
{"name": "DB_PASSWORD", "valueFrom": secretArn},
}
environment := append([]map[string]string{}, dbEnvironment...)
for i, ev := range extraEnv {
environment = append(environment, map[string]string{"name": ev.Name, "value": args[5+i].(string)})
}
mainContainer := map[string]any{
"name": name,
"image": image,
"portMappings": []map[string]any{
{"containerPort": containerPort, "protocol": "tcp"},
},
"environment": environment,
"secrets": dbSecrets,
"logConfiguration": map[string]any{
"logDriver": "awslogs",
"options": map[string]string{
"awslogs-group": logGroupName,
"awslogs-region": awsRegion,
"awslogs-stream-prefix": name,
},
},
}
def := []map[string]any{}
if runMigrations {
migrateContainerName := name + "-migrate"
def = append(def, map[string]any{
"name": migrateContainerName,
"image": image,
"essential": false,
"command": []string{"migrate"},
"environment": dbEnvironment,
"secrets": dbSecrets,
"logConfiguration": map[string]any{
"logDriver": "awslogs",
"options": map[string]string{
"awslogs-group": logGroupName,
"awslogs-region": awsRegion,
"awslogs-stream-prefix": migrateContainerName,
},
},
})
mainContainer["dependsOn"] = []map[string]string{
{"containerName": migrateContainerName, "condition": "COMPLETE"},
}
}
def = append(def, mainContainer)
b, err := json.Marshal(def)
return string(b), err
},
).(pulumi.StringOutput)
taskDefinitionArgs := &ecs.TaskDefinitionArgs{
Family: pulumi.String(name),
Cpu: pulumi.String("256"),
Memory: pulumi.String("512"),
NetworkMode: pulumi.String("awsvpc"),
RequiresCompatibilities: pulumi.StringArray{pulumi.String("FARGATE")},
ExecutionRoleArn: execRole.Arn,
ContainerDefinitions: containerDefinitions,
}
if taskRole != nil {
taskDefinitionArgs.TaskRoleArn = taskRole.Arn
}
taskDefinition, err := ecs.NewTaskDefinition(ctx, name+"-task", taskDefinitionArgs)
if err != nil {
return nil, nil, nil, err
}
service, err := ecs.NewService(ctx, name+"-service", &ecs.ServiceArgs{
Cluster: cluster.Arn,
TaskDefinition: taskDefinition.Arn,
DesiredCount: pulumi.Int(1),
LaunchType: pulumi.String("FARGATE"),
NetworkConfiguration: &ecs.ServiceNetworkConfigurationArgs{
Subnets: pulumi.ToStringArray(subnetIDs),
SecurityGroups: pulumi.StringArray{serviceSecurityGroup.ID()},
AssignPublicIp: pulumi.Bool(true),
},
LoadBalancers: ecs.ServiceLoadBalancerArray{
&ecs.ServiceLoadBalancerArgs{
TargetGroupArn: targetGroup.Arn,
ContainerName: pulumi.String(name),
ContainerPort: pulumi.Int(containerPort),
},
},
}, pulumi.DependsOn([]pulumi.Resource{listener}))
if err != nil {
return nil, nil, nil, err
}
return repo, service, alb, nil
}
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
// The LocalStack stack (Pulumi.local.yaml) runs this same program
// against docker-compose. Everything AWS-endpoint-related is handled by
// that file's aws:* config, so the only thing the code has to know is
// which resources are meaningful locally: LocalStack stands in for S3,
// RDS and IAM, while docker-compose — not ECS — runs the containers, so
// the VPC, the load balancers, the ECS services, the CDN and the CI
// user are all skipped.
cfg := config.New(ctx, "")
localstack := cfg.GetBool("localstack")
// docker-compose passes this to the cms container as DB_PASSWORD; the
// service login roles below are created with the same value.
servicePassword := ""
if localstack {
servicePassword = os.Getenv("DB_PASSWORD")
if servicePassword == "" {
return fmt.Errorf("missing required env var: DB_PASSWORD (needed by the %q stack)", ctx.Stack())
}
}
var (
distribution *cloudfront.Distribution
vpcID string
subnetIDs []string
dbSubnetGroup *rds.SubnetGroup
dbSecurityGroup *ec2.SecurityGroup
cluster *ecs.Cluster
albSecurityGroup, serviceSecurityGroup *ec2.SecurityGroup
execRole *iam.Role
cmsRepo, discoveryRepo *ecr.Repository
cmsService, discoveryService *ecs.Service
cmsAlb, discoveryAlb *lb.LoadBalancer
)
// Bucket names are left to Pulumi's auto-naming in AWS, but pinned under
// LocalStack because docker-compose.yml refers to them literally.
encodedBucketArgs := &s3.BucketArgs{}
rawUploadsBucketArgs := &s3.BucketArgs{}
if localstack {
encodedBucketArgs.Bucket = pulumi.String("encoded-bucket")
rawUploadsBucketArgs.Bucket = pulumi.String("raw-uploads-bucket")
}
// Create an AWS resource (S3 Bucket) that holds encoded assets
bucket, err := s3.NewBucket(ctx, "encoded-bucket", encodedBucketArgs)
if err != nil {
return err
}
// Keep the bucket private; CloudFront reaches it via Origin Access Control.
_, err = s3.NewBucketPublicAccessBlock(ctx, "encoded-bucket-public-access-block", &s3.BucketPublicAccessBlockArgs{
Bucket: bucket.ID(),
BlockPublicAcls: pulumi.Bool(true),
BlockPublicPolicy: pulumi.Bool(true),
IgnorePublicAcls: pulumi.Bool(true),
RestrictPublicBuckets: pulumi.Bool(true),
})
if err != nil {
return err
}
// CloudFront: public delivery of the finished renditions. Skipped under
// LocalStack, which reads the encoded bucket directly.
if !localstack {
oac, err := cloudfront.NewOriginAccessControl(ctx, "encoded-bucket-oac", &cloudfront.OriginAccessControlArgs{
Description: pulumi.String("OAC for encoded-bucket assets"),
OriginAccessControlOriginType: pulumi.String("s3"),
SigningBehavior: pulumi.String("always"),
SigningProtocol: pulumi.String("sigv4"),
})
if err != nil {
return err
}
distribution, err = cloudfront.NewDistribution(ctx, "encoded-bucket-cdn", &cloudfront.DistributionArgs{
Enabled: pulumi.Bool(true),
Comment: pulumi.String("Edge caching for encoded-bucket assets"),
Origins: cloudfront.DistributionOriginArray{
&cloudfront.DistributionOriginArgs{
DomainName: bucket.BucketRegionalDomainName,
OriginId: pulumi.String(originId),
OriginAccessControlId: oac.ID(),
S3OriginConfig: &cloudfront.DistributionOriginS3OriginConfigArgs{
OriginAccessIdentity: pulumi.String(""),
},
},
},
DefaultCacheBehavior: &cloudfront.DistributionDefaultCacheBehaviorArgs{
TargetOriginId: pulumi.String(originId),
ViewerProtocolPolicy: pulumi.String("redirect-to-https"),
AllowedMethods: pulumi.ToStringArray([]string{"GET", "HEAD"}),
CachedMethods: pulumi.ToStringArray([]string{"GET", "HEAD"}),
Compress: pulumi.Bool(true),
CachePolicyId: pulumi.String(cachingOptimizedPolicyId),
},
Restrictions: &cloudfront.DistributionRestrictionsArgs{
GeoRestriction: &cloudfront.DistributionRestrictionsGeoRestrictionArgs{
RestrictionType: pulumi.String("none"),
},
},
ViewerCertificate: &cloudfront.DistributionViewerCertificateArgs{
CloudfrontDefaultCertificate: pulumi.Bool(true),
},
})
if err != nil {
return err
}
// Allow only this distribution to read objects from the bucket.
policy := pulumi.All(bucket.Arn, distribution.Arn).ApplyT(func(args []interface{}) (string, error) {
bucketArn := args[0].(string)
distributionArn := args[1].(string)
doc := map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{
{
"Sid": "AllowCloudFrontServicePrincipal",
"Effect": "Allow",
"Principal": map[string]string{"Service": "cloudfront.amazonaws.com"},
"Action": "s3:GetObject",
"Resource": bucketArn + "/*",
"Condition": map[string]interface{}{
"StringEquals": map[string]string{"AWS:SourceArn": distributionArn},
},
},
},
}
b, err := json.Marshal(doc)
return string(b), err
}).(pulumi.StringOutput)
_, err = s3.NewBucketPolicy(ctx, "encoded-bucket-policy", &s3.BucketPolicyArgs{
Bucket: bucket.ID(),
Policy: policy,
})
if err != nil {
return err
}
}
// Networking for the RDS instance and the ECS services. Skipped under
// LocalStack: its RDS emulation is a Postgres process inside the
// LocalStack container, reached over the docker-compose network rather
// than through a VPC, and nothing else here runs in one.
if !localstack {
// Small, single-AZ Postgres instance for the assignment: default VPC,
// no public access, no Multi-AZ, no backups, destroyable without a snapshot.
vpc, err := ec2.LookupVpc(ctx, &ec2.LookupVpcArgs{Default: pulumi.BoolRef(true)})
if err != nil {
return err
}
vpcID = vpc.Id
subnets, err := ec2.GetSubnets(ctx, &ec2.GetSubnetsArgs{
Filters: []ec2.GetSubnetsFilter{
{Name: "vpc-id", Values: []string{vpc.Id}},
},
})
if err != nil {
return err
}
subnetIDs = subnets.Ids
dbSubnetGroup, err = rds.NewSubnetGroup(ctx, "postgres-subnet-group", &rds.SubnetGroupArgs{
SubnetIds: pulumi.ToStringArray(subnets.Ids),
})
if err != nil {
return err
}
myIP, err := currentPublicIP()
if err != nil {
return err
}
myIPCidr := fmt.Sprintf("%s/32", myIP)
dbSecurityGroup, err = ec2.NewSecurityGroup(ctx, "postgres-sg", &ec2.SecurityGroupArgs{
Description: pulumi.String("Allow Postgres access from within the VPC and the deployers IP"),
VpcId: pulumi.String(vpc.Id),
Ingress: ec2.SecurityGroupIngressArray{
&ec2.SecurityGroupIngressArgs{
Protocol: pulumi.String("tcp"),
FromPort: pulumi.Int(5432),
ToPort: pulumi.Int(5432),
CidrBlocks: pulumi.ToStringArray([]string{vpc.CidrBlock}),
},
&ec2.SecurityGroupIngressArgs{
Protocol: pulumi.String("tcp"),
FromPort: pulumi.Int(5432),
ToPort: pulumi.Int(5432),
CidrBlocks: pulumi.ToStringArray([]string{myIPCidr}),
Description: pulumi.String("Deployer IP, for the postgresql provider to create databases"),
},
},
Egress: ec2.SecurityGroupEgressArray{
&ec2.SecurityGroupEgressArgs{
Protocol: pulumi.String("-1"),
FromPort: pulumi.Int(0),
ToPort: pulumi.Int(0),
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
},
},
})
if err != nil {
return err
}
}
// Generated once and kept in the stack state, so it stays stable across deploys
// unless explicitly replaced. RDS master passwords can't contain '/', '@', '"', or spaces.
dbPassword, err := random.NewRandomPassword(ctx, "postgres-db-password", &random.RandomPasswordArgs{
Length: pulumi.Int(24),
Special: pulumi.Bool(true),
OverrideSpecial: pulumi.String("!#$%&*()-_=+[]{}<>:?"),
MinUpper: pulumi.Int(1),
MinLower: pulumi.Int(1),
MinNumeric: pulumi.Int(1),
MinSpecial: pulumi.Int(1),
})
if err != nil {
return err
}
dbArgs := &rds.InstanceArgs{
Engine: pulumi.String("postgres"),
EngineVersion: pulumi.String("16"),
InstanceClass: pulumi.String("db.t3.micro"),
AllocatedStorage: pulumi.Int(20),
StorageType: pulumi.String("gp3"),
StorageEncrypted: pulumi.Bool(true),
DbName: pulumi.String("appdb"),
Username: pulumi.String("postgres"),
Password: dbPassword.Result,
PubliclyAccessible: pulumi.Bool(true),
MultiAz: pulumi.Bool(false),
BackupRetentionPeriod: pulumi.Int(0),
DeletionProtection: pulumi.Bool(false),
SkipFinalSnapshot: pulumi.Bool(true),
ApplyImmediately: pulumi.Bool(true),
}
if localstack {
dbArgs.Port = pulumi.Int(localstackDBPort)
} else {
dbArgs.DbSubnetGroupName = dbSubnetGroup.Name
dbArgs.VpcSecurityGroupIds = pulumi.StringArray{dbSecurityGroup.ID()}
}
db, err := rds.NewInstance(ctx, "postgres-db", dbArgs)
if err != nil {
return err
}
// RDS terminates TLS; the Postgres process LocalStack runs does not.
sslMode := "require"
if localstack {
sslMode = "disable"
}
pgProvider, err := postgresql.NewProvider(ctx, "postgres-provider", &postgresql.ProviderArgs{
Host: db.Address,
Port: db.Port,
Username: pulumi.String("postgres"),
Password: dbPassword.Result,
Sslmode: pulumi.String(sslMode),
Superuser: pulumi.Bool(false),
})
if err != nil {
return err
}
cmsPassword, err := newServiceDatabase(ctx, pgProvider, "cms", servicePassword)
if err != nil {
return err
}
discoveryPassword, err := newServiceDatabase(ctx, pgProvider, "discovery", servicePassword)
if err != nil {
return err
}
// Trust policy shared by every ECS task role.
execRoleAssumePolicy, err := json.Marshal(map[string]any{
"Version": "2012-10-17",
"Statement": []map[string]any{
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": map[string]string{"Service": "ecs-tasks.amazonaws.com"},
},
},
})
if err != nil {
return err
}
// ECS Fargate: one cluster, one internet-facing ALB, one service per app.
// docker-compose runs the containers under LocalStack, so none of this
// — cluster, load balancer security groups, or the ECS agent's
// execution role — has a local counterpart.
if !localstack {
cluster, err = ecs.NewCluster(ctx, "app-cluster", nil)
if err != nil {
return err
}
_, err = ecs.NewClusterCapacityProviders(ctx, "app-cluster-capacity-providers", &ecs.ClusterCapacityProvidersArgs{
ClusterName: cluster.Name,
CapacityProviders: pulumi.ToStringArray([]string{"FARGATE", "FARGATE_SPOT"}),
DefaultCapacityProviderStrategies: ecs.ClusterCapacityProvidersDefaultCapacityProviderStrategyArray{
&ecs.ClusterCapacityProvidersDefaultCapacityProviderStrategyArgs{
CapacityProvider: pulumi.String("FARGATE"),
Weight: pulumi.Int(1),
},
},
})
if err != nil {
return err
}
albSecurityGroup, err = ec2.NewSecurityGroup(ctx, "alb-sg", &ec2.SecurityGroupArgs{
Description: pulumi.String("Allow inbound HTTP to the app load balancers"),
VpcId: pulumi.String(vpcID),
Ingress: ec2.SecurityGroupIngressArray{
&ec2.SecurityGroupIngressArgs{
Protocol: pulumi.String("tcp"),
FromPort: pulumi.Int(80),
ToPort: pulumi.Int(80),
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
},
},
Egress: ec2.SecurityGroupEgressArray{
&ec2.SecurityGroupEgressArgs{
Protocol: pulumi.String("-1"),
FromPort: pulumi.Int(0),
ToPort: pulumi.Int(0),
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
},
},
})
if err != nil {
return err
}
serviceSecurityGroup, err = ec2.NewSecurityGroup(ctx, "ecs-service-sg", &ec2.SecurityGroupArgs{
Description: pulumi.String("Allow the ALB to reach cms and discovery tasks"),
VpcId: pulumi.String(vpcID),
Ingress: ec2.SecurityGroupIngressArray{
&ec2.SecurityGroupIngressArgs{
Protocol: pulumi.String("tcp"),
FromPort: pulumi.Int(0),
ToPort: pulumi.Int(65535),
SecurityGroups: pulumi.StringArray{albSecurityGroup.ID()},
},
},
Egress: ec2.SecurityGroupEgressArray{
&ec2.SecurityGroupEgressArgs{
Protocol: pulumi.String("-1"),
FromPort: pulumi.Int(0),
ToPort: pulumi.Int(0),
CidrBlocks: pulumi.ToStringArray([]string{"0.0.0.0/0"}),
},
},
})
if err != nil {
return err
}
execRole, err = iam.NewRole(ctx, "ecs-task-execution-role", &iam.RoleArgs{
AssumeRolePolicy: pulumi.String(execRoleAssumePolicy),
ManagedPolicyArns: pulumi.ToStringArray([]string{
"arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
}),
})
if err != nil {
return err
}
}
// Private bucket for raw video uploads (pre-transcode). Kept separate
// from encoded-bucket, which is fronted by CloudFront/OAC for public
// delivery of finished renditions — raw source video must not be
// reachable through that CDN.
rawUploadsBucket, err := s3.NewBucket(ctx, "raw-uploads-bucket", rawUploadsBucketArgs)
if err != nil {
return err
}
_, err = s3.NewBucketPublicAccessBlock(ctx, "raw-uploads-bucket-public-access-block", &s3.BucketPublicAccessBlockArgs{
Bucket: rawUploadsBucket.ID(),
BlockPublicAcls: pulumi.Bool(true),
BlockPublicPolicy: pulumi.Bool(true),
IgnorePublicAcls: pulumi.Bool(true),
RestrictPublicBuckets: pulumi.Bool(true),
})
if err != nil {
return err
}
// Role MediaConvert itself assumes (passed as CreateJobInput.Role by
// the cms service) to read the raw upload and write the encoded
// output — distinct from the ECS task role below.
mediaConvertAssumePolicy, err := json.Marshal(map[string]any{
"Version": "2012-10-17",
"Statement": []map[string]any{
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": map[string]string{"Service": "mediaconvert.amazonaws.com"},
},
},
})
if err != nil {
return err
}
mediaConvertRole, err := iam.NewRole(ctx, "mediaconvert-service-role", &iam.RoleArgs{
AssumeRolePolicy: pulumi.String(mediaConvertAssumePolicy),
})
if err != nil {
return err
}
mediaConvertS3Policy := pulumi.All(rawUploadsBucket.Arn, bucket.Arn).ApplyT(
func(args []any) (string, error) {
rawUploadsArn := args[0].(string)
encodedBucketArn := args[1].(string)
doc := map[string]any{
"Version": "2012-10-17",
"Statement": []map[string]any{
{
"Sid": "ReadRawUploads",
"Effect": "Allow",
"Action": "s3:GetObject",
"Resource": rawUploadsArn + "/*",
},
{
"Sid": "WriteEncodedOutput",
"Effect": "Allow",
"Action": "s3:PutObject",
"Resource": encodedBucketArn + "/*",
},
},
}
b, err := json.Marshal(doc)
return string(b), err
},
).(pulumi.StringOutput)
_, err = iam.NewRolePolicy(ctx, "mediaconvert-s3-access", &iam.RolePolicyArgs{
Role: mediaConvertRole.ID(),
Policy: mediaConvertS3Policy,
})
if err != nil {
return err
}
// MediaConvert job events: MediaConvert -> EventBridge -> SNS -> SQS ->
// cms. cms *consumes a queue* rather than being called back on an HTTPS
// endpoint, so nothing new is exposed on the ALB, authenticity comes
// from IAM instead of SNS message signatures, and a failed handler
// leaves the message on the queue to be retried instead of dropping the
// update. The topic sits between the rule and the queue so a second
// consumer — discovery, when it has code — can subscribe its own queue
// without touching this wiring or MediaConvert.
jobEventsTopicArgs := &sns.TopicArgs{}
jobEventsQueueArgs := &sqs.QueueArgs{}
jobEventsDeadLetterArgs := &sqs.QueueArgs{}
if localstack {
// docker-compose.yml and the tests/ suite refer to these literally.
jobEventsTopicArgs.Name = pulumi.String("mediaconvert-job-events")
jobEventsQueueArgs.Name = pulumi.String("cms-mediaconvert-events")
jobEventsDeadLetterArgs.Name = pulumi.String("cms-mediaconvert-events-dlq")
}
jobEventsTopic, err := sns.NewTopic(ctx, "mediaconvert-job-events", jobEventsTopicArgs)
if err != nil {
return err
}
// Where a message lands after maxReceiveCount failed handlings, so one
// event cms cannot process never blocks the ones behind it.
jobEventsDeadLetter, err := sqs.NewQueue(ctx, "cms-mediaconvert-events-dlq", jobEventsDeadLetterArgs)
if err != nil {
return err
}
jobEventsQueueArgs.VisibilityTimeoutSeconds = pulumi.Int(60)
jobEventsQueueArgs.RedrivePolicy = jobEventsDeadLetter.Arn.ApplyT(func(arn string) (string, error) {
b, err := json.Marshal(map[string]any{
"deadLetterTargetArn": arn,
"maxReceiveCount": 5,
})
return string(b), err
}).(pulumi.StringOutput)
jobEventsQueue, err := sqs.NewQueue(ctx, "cms-mediaconvert-events", jobEventsQueueArgs)
if err != nil {
return err
}
// SNS is not an IAM principal the queue trusts by default; without this
// the subscription is created and silently delivers nothing.
jobEventsQueuePolicy := pulumi.All(jobEventsQueue.Arn, jobEventsTopic.Arn).ApplyT(
func(args []any) (string, error) {
queueArn := args[0].(string)
topicArn := args[1].(string)
doc := map[string]any{
"Version": "2012-10-17",
"Statement": []map[string]any{
{
"Sid": "AllowJobEventsTopicToSend",
"Effect": "Allow",
"Principal": map[string]string{"Service": "sns.amazonaws.com"},
"Action": "sqs:SendMessage",
"Resource": queueArn,
"Condition": map[string]any{
"ArnEquals": map[string]string{"aws:SourceArn": topicArn},
},
},
},
}
b, err := json.Marshal(doc)
return string(b), err
},
).(pulumi.StringOutput)
_, err = sqs.NewQueuePolicy(ctx, "cms-mediaconvert-events-queue-policy", &sqs.QueuePolicyArgs{
QueueUrl: jobEventsQueue.ID(),
Policy: jobEventsQueuePolicy,
})
if err != nil {
return err
}
_, err = sns.NewTopicSubscription(ctx, "cms-mediaconvert-events-subscription", &sns.TopicSubscriptionArgs{
Topic: jobEventsTopic.Arn,
Protocol: pulumi.String("sqs"),
Endpoint: jobEventsQueue.Arn,
// The queue receives the EventBridge event itself rather than an SNS
// envelope carrying it as a JSON string, so the consumer parses one
// document instead of unwrapping two.
RawMessageDelivery: pulumi.Bool(true),
})
if err != nil {
return err
}
jobEventsTopicPolicy := jobEventsTopic.Arn.ApplyT(func(topicArn string) (string, error) {
doc := map[string]any{
"Version": "2012-10-17",
"Statement": []map[string]any{
{
"Sid": "AllowEventBridgeToPublish",
"Effect": "Allow",
"Principal": map[string]string{"Service": "events.amazonaws.com"},
"Action": "sns:Publish",
"Resource": topicArn,
},
},
}
b, err := json.Marshal(doc)
return string(b), err
}).(pulumi.StringOutput)
_, err = sns.NewTopicPolicy(ctx, "mediaconvert-job-events-topic-policy", &sns.TopicPolicyArgs{
Arn: jobEventsTopic.Arn,
Policy: jobEventsTopicPolicy,
})
if err != nil {
return err
}
// MediaConvert does not publish to SNS itself: it reports state changes
// to EventBridge, and this rule is what forwards them to the topic.
// Skipped under LocalStack, whose MediaConvert emulation does not emit
// these — the suite in tests/ publishes to the topic directly instead,
// which is the same seam from cms's side.
if !localstack {
jobStateChangePattern, err := json.Marshal(map[string]any{
"source": []string{"aws.mediaconvert"},
"detail-type": []string{"MediaConvert Job State Change"},
})
if err != nil {
return err
}
jobEventsRule, err := cloudwatch.NewEventRule(ctx, "mediaconvert-job-state-change", &cloudwatch.EventRuleArgs{
Description: pulumi.String("MediaConvert job state changes, forwarded to the cms job events topic"),
EventPattern: pulumi.String(string(jobStateChangePattern)),
})
if err != nil {
return err
}
_, err = cloudwatch.NewEventTarget(ctx, "mediaconvert-job-state-change-to-topic", &cloudwatch.EventTargetArgs{
Rule: jobEventsRule.Name,
Arn: jobEventsTopic.Arn,
})
if err != nil {
return err
}
}
// cms's own ECS task role (distinct from the shared execRole, which
// is only for the ECS agent's pull/logs/secrets access): lets the
// running container call S3 and MediaConvert directly. discovery
// gets none of this since it never touches either API.
cmsTaskRole, err := iam.NewRole(ctx, "cms-task-role", &iam.RoleArgs{
AssumeRolePolicy: pulumi.String(execRoleAssumePolicy),
})
if err != nil {
return err
}
cmsTaskPolicy := pulumi.All(rawUploadsBucket.Arn, mediaConvertRole.Arn, jobEventsQueue.Arn).ApplyT(
func(args []any) (string, error) {
rawUploadsArn := args[0].(string)
mediaConvertRoleArn := args[1].(string)
jobEventsQueueArn := args[2].(string)
doc := map[string]any{
"Version": "2012-10-17",
"Statement": []map[string]any{
{
"Sid": "ListRawUploadsBucket",
"Effect": "Allow",
"Action": "s3:ListBucket",
"Resource": rawUploadsArn,
},
{
"Sid": "RawUploadsObjectAccess",
"Effect": "Allow",
"Action": []string{
"s3:PutObject",
"s3:GetObject",
},
"Resource": rawUploadsArn + "/*",
},
{
"Sid": "CreateTranscodeJob",
"Effect": "Allow",
"Action": "mediaconvert:CreateJob",
"Resource": "*",
},
{
"Sid": "ConsumeJobEvents",
"Effect": "Allow",
"Action": []string{
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes",
},
"Resource": jobEventsQueueArn,
},
{
"Sid": "PassMediaConvertRole",
"Effect": "Allow",
"Action": "iam:PassRole",
"Resource": mediaConvertRoleArn,
"Condition": map[string]any{
"StringEquals": map[string]string{
"iam:PassedToService": "mediaconvert.amazonaws.com",
},
},
},
},
}
b, err := json.Marshal(doc)
return string(b), err
},
).(pulumi.StringOutput)
_, err = iam.NewRolePolicy(ctx, "cms-task-role-policy", &iam.RolePolicyArgs{
Role: cmsTaskRole.ID(),
Policy: cmsTaskPolicy,
})
if err != nil {
return err
}
// The two ECS services and the CI user that deploys them. Under
// LocalStack the equivalent of all this is the cms service in
// docker-compose.yml, which reads the same env vars from .env.
if !localstack {
cmsExtraEnv := []envVar{
{Name: "AWS_REGION", Value: pulumi.String(awsRegion).ToStringOutput()},
{Name: "S3_BUCKET", Value: rawUploadsBucket.ID().ToStringOutput()},
{Name: "MEDIACONVERT_INPUT_BUCKET", Value: rawUploadsBucket.ID().ToStringOutput()},
{Name: "MEDIACONVERT_OUTPUT_BUCKET", Value: bucket.ID().ToStringOutput()},
{Name: "MEDIACONVERT_ROLE_ARN", Value: mediaConvertRole.Arn},
{Name: "MEDIACONVERT_EVENTS_QUEUE_URL", Value: jobEventsQueue.Url},
}
cmsRepo, cmsService, cmsAlb, err = deployFargateService(ctx, "cms", 8081,
cluster, execRole, cmsTaskRole, cmsExtraEnv, vpcID, subnetIDs, albSecurityGroup, serviceSecurityGroup,
db.Address, db.Port, cmsPassword, true)
if err != nil {
return err
}
}
// The browser PUTs directly to raw-uploads-bucket using the presigned
// URL from PresignVideoUpload, so the bucket (not the cms app) is
// what needs to answer the CORS preflight — scoped to the cms
// origin the upload page is actually served from. Under LocalStack
// there is no ALB to name, and the caller is whatever the developer
// happens to be running.
var corsOrigins pulumi.StringArrayInput = pulumi.ToStringArray([]string{"*"})
if !localstack {
corsOrigins = pulumi.StringArray{pulumi.Sprintf("http://%s", cmsAlb.DnsName)}
}
_, err = s3.NewBucketCorsConfigurationV2(ctx, "raw-uploads-bucket-cors", &s3.BucketCorsConfigurationV2Args{
Bucket: rawUploadsBucket.ID(),
CorsRules: s3.BucketCorsConfigurationV2CorsRuleArray{
&s3.BucketCorsConfigurationV2CorsRuleArgs{
AllowedMethods: pulumi.ToStringArray([]string{"PUT"}),
AllowedOrigins: corsOrigins,
AllowedHeaders: pulumi.ToStringArray([]string{"*"}),
ExposeHeaders: pulumi.ToStringArray([]string{"ETag"}),
MaxAgeSeconds: pulumi.Int(3000),
},
},
})
if err != nil {
return err
}
if !localstack {
discoveryRepo, discoveryService, discoveryAlb, err = deployFargateService(ctx, "discovery", 8080,
cluster, execRole, nil, nil, vpcID, subnetIDs, albSecurityGroup, serviceSecurityGroup,
db.Address, db.Port, discoveryPassword, false)
if err != nil {
return err
}
// Gitea Actions CI: one IAM user, scoped to just what the build/deploy
// pipeline needs — push images to the two ECR repos and force a new
// deployment on the two ECS services. Credentials are exported as
// stack secrets for the pipeline to consume (e.g. as Gitea Actions
// secrets); nothing broader like ecr:*, ecs:*, or task-def changes.
giteaCIPolicyDocument := pulumi.All(cmsRepo.Arn, discoveryRepo.Arn, cmsService.Arn, discoveryService.Arn).ApplyT(
func(args []any) (string, error) {
cmsRepoArn := args[0].(string)
discoveryRepoArn := args[1].(string)
cmsServiceArn := args[2].(string)
discoveryServiceArn := args[3].(string)
doc := map[string]any{
"Version": "2012-10-17",
"Statement": []map[string]any{
{
"Sid": "ECRAuth",
"Effect": "Allow",
"Action": "ecr:GetAuthorizationToken",
"Resource": "*",
},
{
"Sid": "ECRPush",
"Effect": "Allow",
"Action": []string{
"ecr:BatchCheckLayerAvailability",
"ecr:BatchGetImage",
"ecr:PutImage",
"ecr:InitiateLayerUpload",
"ecr:UploadLayerPart",
"ecr:CompleteLayerUpload",
},
"Resource": []string{cmsRepoArn, discoveryRepoArn},
},
{
"Sid": "ECSDeploy",
"Effect": "Allow",
"Action": []string{
"ecs:UpdateService",
"ecs:DescribeServices",
},
"Resource": []string{cmsServiceArn, discoveryServiceArn},
},
},
}
b, err := json.Marshal(doc)
return string(b), err
},
).(pulumi.StringOutput)
giteaCIUser, err := iam.NewUser(ctx, "gitea-ci-user", &iam.UserArgs{
Name: pulumi.String("gitea-ci"),
})
if err != nil {
return err
}
_, err = iam.NewUserPolicy(ctx, "gitea-ci-policy", &iam.UserPolicyArgs{
User: giteaCIUser.Name,
Policy: giteaCIPolicyDocument,
})
if err != nil {
return err
}
}
// Export the name of the bucket
ctx.Export("bucketName", bucket.ID())
ctx.Export("rawUploadsBucketName", rawUploadsBucket.ID())
ctx.Export("dbEndpoint", db.Endpoint)
ctx.Export("dbPassword", dbPassword.Result)
ctx.Export("mediaConvertRoleArn", mediaConvertRole.Arn)
// Everything below exists only on the AWS stacks. The CI workflows read
// ecsClusterArn and cmsServiceArn from here.
if !localstack {
ctx.Export("cdnDomainName", distribution.DomainName)
ctx.Export("cmsUrl", pulumi.Sprintf("http://%s", cmsAlb.DnsName))
ctx.Export("discoveryUrl", pulumi.Sprintf("http://%s", discoveryAlb.DnsName))
ctx.Export("cmsRepoUrl", cmsRepo.RepositoryUrl)
ctx.Export("discoveryRepoUrl", discoveryRepo.RepositoryUrl)
ctx.Export("ecsClusterArn", cluster.Arn)
ctx.Export("cmsServiceArn", cmsService.Arn)
ctx.Export("discoveryServiceArn", discoveryService.Arn)
}
return nil
})
}
Reference in New Issue View Git Blame Copy Permalink