From 9410c098ed7e3f19fc1f9c5cd8eb4c54558f55db Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Mon, 16 Nov 2026 14:00:23 -0800 Subject: [PATCH] feat(aws): optional per-user EC2 capacity in the onboarding module Phase 7 Task 3 (E1-E4, E11; onboarding module version 2): - ec2.tf: with ec2_instance_type set, the user's own cluster triple-c---u- with exactly one capacity provider (managed termination protection, managed scaling 100% one instance at a time, 60 s warm-up), an Auto Scaling group of 0..ec2_max_instances (desired left to ECS, protect_from_scale_in, force_delete off so the provider drains before deleting, fixed launch-template version, default tags + AmazonECSManaged), a launch template on the ECS-optimized AL2023 AMI by architecture (IMDSv2 hop 1, 50 GiB gp3 encrypted root, the environment's tasks group, public IP as the environment's tasks get one, ECS_CLUSTER + ECS_AWSVPC_BLOCK_IMDS), the instance role triple-c---instance (bounded, tagged triple-c:cluster, ECS agent on its cluster only, ECR pull, logs; no SSM core) and its instance profile, and the user record /triple-c/users//. Empty type (the default): count 0 everywhere, Fargate plan unchanged. - Variables ec2_instance_type (the app's 30 types), ec2_max_instances (1-4), ec2_user_tag (as validate_ec2_user_tag, required with a type); onboarding_vars takes Option<&Ec2Compute> and validates the same. - The environment record's assign_public_ip is now format-checked. - tofu_modules.rs: ONBOARD_MODULE_VERSION 2, user_record_parameter_name, USER_RECORD_FIELDS, Ec2Compute, EC2_MAX_INSTANCES, ec2.tf embedded. - Tests: EC2 calls merged into MODULE_CALLS under real types (all_module_calls removed); gating + version-1 resource fingerprint; launch template, group, capacity provider, instance role, AMI by arch, record fields, names, variable rules; the instance role inside the boundary with its cluster principal tag. Co-Authored-By: Claude Opus 5.5 --- app/src-tauri/src/aws/live_tests.rs | 6 +- app/src-tauri/src/aws/tofu_modules.rs | 89 +- app/src-tauri/src/aws/tofu_modules/tests.rs | 1121 ++++++++++++++--- .../src/commands/environment_commands.rs | 4 +- .../commands/environment_commands/tests.rs | 2 +- aws/terraform/user-onboard/ec2.tf | 314 +++++ aws/terraform/user-onboard/main.tf | 12 +- aws/terraform/user-onboard/outputs.tf | 23 + aws/terraform/user-onboard/variables.tf | 55 + 9 files changed, 1430 insertions(+), 196 deletions(-) create mode 100644 aws/terraform/user-onboard/ec2.tf diff --git a/app/src-tauri/src/aws/live_tests.rs b/app/src-tauri/src/aws/live_tests.rs index 10156bf..8d5a502 100644 --- a/app/src-tauri/src/aws/live_tests.rs +++ b/app/src-tauri/src/aws/live_tests.rs @@ -1592,7 +1592,7 @@ mod environment { .run( ModuleKind::UserOnboard, Deployment::User { env: &name, owner_key: &okey }, - &onboarding_vars(&name, &t.e.region, &owner), + &onboarding_vars(&name, &t.e.region, &owner, None).unwrap(), TofuOp::Apply, t.keys_env(&keys), ) @@ -1643,7 +1643,7 @@ mod environment { .run( ModuleKind::UserOnboard, Deployment::User { env: &name, owner_key: &okey }, - &onboarding_vars(&name, &t.e.region, &owner), + &onboarding_vars(&name, &t.e.region, &owner, None).unwrap(), TofuOp::Destroy, t.keys_env(&keys), ) @@ -2003,7 +2003,7 @@ mod environment { .run( ModuleKind::UserOnboard, Deployment::User { env: &name, owner_key: &okey }, - &onboarding_vars(&name, ®ion, &owner), + &onboarding_vars(&name, ®ion, &owner, None).unwrap(), TofuOp::Apply, t.keys_env(&keys), ) diff --git a/app/src-tauri/src/aws/tofu_modules.rs b/app/src-tauri/src/aws/tofu_modules.rs index 067386d..293fb0f 100644 --- a/app/src-tauri/src/aws/tofu_modules.rs +++ b/app/src-tauri/src/aws/tofu_modules.rs @@ -16,9 +16,11 @@ use crate::models::host_connection::validate_supported_region; /// `locals.template_version` (and the `triple-c:template-version` default /// tag) of `aws/terraform/environment`. pub const ENVIRONMENT_MODULE_VERSION: u32 = 2; -/// The same for the user-onboarding module. +/// The same for the user-onboarding module. 2: optional per-user EC2 +/// capacity (phase 7); an onboarding without it plans as version 1 did, +/// apart from this tag. #[allow(dead_code)] // the onboarding module's version; checked against its HCL by a test -pub const ONBOARD_MODULE_VERSION: u32 = 1; +pub const ONBOARD_MODULE_VERSION: u32 = 2; /// The same for the permissions-boundary module. 2: the instance role's ECS /// agent and Session Manager actions (phase 7, E4); an account admin /// re-applies the boundary once. @@ -57,6 +59,7 @@ const USER_ONBOARD_FILES: &[(&str, &str)] = module!("user-onboard": "main.tf", "iam.tf", "efs.tf", + "ec2.tf", "outputs.tf", ".terraform.lock.hcl", ); @@ -148,7 +151,6 @@ pub const EC2_USER_TAG_KEY: &str = "triple-c-user"; /// `[A-Za-z0-9-]`, not starting or ending with a hyphen and without `--` /// (the separator in [`user_ec2_stem`], which environment names can't hold /// either, so a per-user name can be read back only one way). -#[allow(dead_code)] // the onboarding module (Task 3) uses it pub fn validate_ec2_user_tag(tag: &str) -> Result { let ok = (1..=32).contains(&tag.len()) && tag.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'-') @@ -171,7 +173,7 @@ pub fn validate_ec2_user_tag(tag: &str) -> Result { /// characters). Environment names and user tags never contain `--`, so the /// workspace policy's `cluster/triple-c-*--u-*` matches exactly the /// per-user clusters, and `triple-c-*--u-` exactly that user's. -#[allow(dead_code)] // the onboarding module (Task 3) uses it +#[allow(dead_code)] // the onboarding module builds it; the app's EC2 runs (Task 5) will pub fn user_ec2_stem(env: &str, user_tag: &str) -> String { format!("triple-c-{env}--u-{user_tag}") } @@ -180,11 +182,48 @@ pub fn user_ec2_stem(env: &str, user_tag: &str) -> String { /// not the user tag, because role names are account-global and at most 64 /// characters (this is at most 51). The admin's PassRole to EC2 names /// `role/triple-c-*-instance`. -#[allow(dead_code)] // the onboarding module (Task 3) uses it +#[allow(dead_code)] // the onboarding module builds it; pinned by tests pub fn user_instance_role_name(env: &str, owner_key: &str) -> String { format!("triple-c-{env}-{owner_key}-instance") } +/// The SSM parameter holding a user's EC2 record (E11), written by their +/// onboarding module only while it has EC2 capacity: a String of JSON with +/// exactly [`USER_RECORD_FIELDS`]. Workspace users can read it +/// (`parameter/triple-c/*`) though not the state bucket; it lives outside +/// `/triple-c/environments/`, which environment detection lists. +#[allow(dead_code)] // the app reads it from Task 4 on +pub fn user_record_parameter_name(env: &str, owner_key: &str) -> String { + format!("/triple-c/users/{env}/{owner_key}") +} + +/// The fields of a user's EC2 record, schema 1: `instance_type`, +/// `cluster_arn` (the user's cluster, [`user_ec2_stem`]), +/// `capacity_provider` (its one provider, the same name) and `user_tag` +/// (the `triple-c-user` value the cluster is named after). +#[allow(dead_code)] // the app reads it from Task 4 on +pub const USER_RECORD_FIELDS: [&str; 5] = [ + "schema", + "instance_type", + "cluster_arn", + "capacity_provider", + "user_tag", +]; + +/// The most instances a user's EC2 capacity may run at once. +pub const EC2_MAX_INSTANCES: u32 = 4; + +/// A user's optional EC2 capacity, as the onboarding module takes it. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Ec2Compute { + /// One of the types `ec2_instance_spec` knows. + pub instance_type: String, + /// 1 to [`EC2_MAX_INSTANCES`]. + pub max_instances: u32, + /// The user's `triple-c-user` principal tag value. + pub user_tag: String, +} + /// Whether a project id is used in names as it is: 1 to 64 characters of /// `[A-Za-z0-9-]`. Every UUID (how the app makes ids) is. fn is_clean_project_id(id: &str) -> bool { @@ -400,16 +439,46 @@ pub fn environment_vars( } /// The onboarding module's `terraform.tfvars.json` for the user whose -/// `aws:userid` is `owner_id`. Bedrock stays on the workspace user's own -/// credentials unless a later setting opts the task role in. -pub fn onboarding_vars(env: &str, region: &str, owner_id: &str) -> serde_json::Value { - json!({ +/// `aws:userid` is `owner_id`, with their EC2 capacity or none (empty +/// `ec2_instance_type`: the module makes nothing for EC2). Applies the same +/// rules as the module's EC2 variable validations. Bedrock stays on the +/// workspace user's own credentials unless a later setting opts the task +/// role in. +pub fn onboarding_vars( + env: &str, + region: &str, + owner_id: &str, + ec2: Option<&Ec2Compute>, +) -> Result { + let (instance_type, max_instances, user_tag) = match ec2 { + None => (String::new(), 1, String::new()), + Some(c) => { + if crate::models::project::ec2_instance_spec(&c.instance_type).is_none() { + return Err(format!( + "\"{}\" is not an instance type Triple-C supports (m7i, m7g, c7i, c7g, r7i, \ + r7g from large to 4xlarge; t3, t4g from large to 2xlarge)", + c.instance_type.escape_debug() + )); + } + if !(1..=EC2_MAX_INSTANCES).contains(&c.max_instances) { + return Err(format!( + "The number of instances must be from 1 to {EC2_MAX_INSTANCES}" + )); + } + let tag = validate_ec2_user_tag(&c.user_tag)?; + (c.instance_type.clone(), c.max_instances, tag) + } + }; + Ok(json!({ "environment_name": env, "region": region, "owner_id": owner_id, "owner_key": owner_key(region, owner_id), "bedrock_on_task_role": false, - }) + "ec2_instance_type": instance_type, + "ec2_max_instances": max_instances, + "ec2_user_tag": user_tag, + })) } /// The boundary module's `terraform.tfvars.json`: the shipped boundary diff --git a/app/src-tauri/src/aws/tofu_modules/tests.rs b/app/src-tauri/src/aws/tofu_modules/tests.rs index 4dcb3d3..fc6ec0b 100644 --- a/app/src-tauri/src/aws/tofu_modules/tests.rs +++ b/app/src-tauri/src/aws/tofu_modules/tests.rs @@ -242,8 +242,14 @@ fn same_user_and_environment_in_two_regions_do_not_collide() { assert_ne!(wt, et); assert_ne!(we, ee); assert_ne!( - state_key(Deployment::User { env: "dev", owner_key: &west }), - state_key(Deployment::User { env: "dev", owner_key: &east }) + state_key(Deployment::User { + env: "dev", + owner_key: &west + }), + state_key(Deployment::User { + env: "dev", + owner_key: &east + }) ); } @@ -361,7 +367,10 @@ fn an_existing_network_is_checked_before_anything_is_built() { let lookup = lookups[0]; assert_eq!(label(lookup, 1), Some("existing")); let count = render(attr(&lookup.body, "count").expect("count")); - assert!(count.contains("var.network_mode") && count.contains("\"existing\""), "{count}"); + assert!( + count.contains("var.network_mode") && count.contains("\"existing\""), + "{count}" + ); assert!(count.contains('2'), "{count}"); let id = render(attr(&lookup.body, "id").expect("id")); assert!(id.contains("var.subnet_ids[count.index]"), "{id}"); @@ -396,7 +405,10 @@ fn the_onboarding_completion_output_waits_for_every_resource() { let mut n = 0; for r in top_blocks(&body, "resource") { let addr = format!("{}.{}", label(r, 0).unwrap(), label(r, 1).unwrap()); - assert!(deps.contains(&addr), "onboarding_complete doesn't wait for {addr}: {deps}"); + assert!( + deps.contains(&addr), + "onboarding_complete doesn't wait for {addr}: {deps}" + ); n += 1; } assert!(n >= 9, "only {n} resources"); @@ -405,7 +417,7 @@ fn the_onboarding_completion_output_waits_for_every_resource() { #[test] fn the_module_versions_agree() { assert_eq!(ENVIRONMENT_MODULE_VERSION, 2); - assert_eq!(ONBOARD_MODULE_VERSION, 1); + assert_eq!(ONBOARD_MODULE_VERSION, 2); let body = parsed(ModuleKind::Environment); let locals: Vec<_> = top_blocks(&body, "locals") .filter_map(|b| attr(&b.body, "template_version")) @@ -626,7 +638,7 @@ fn every_variable_is_validated() { fn module_files_ship_the_lock_file() { for (kind, extra) in [ (ModuleKind::Environment, &["network.tf"][..]), - (ModuleKind::UserOnboard, &["iam.tf", "efs.tf"][..]), + (ModuleKind::UserOnboard, &["iam.tf", "efs.tf", "ec2.tf"][..]), (ModuleKind::PermissionsBoundary, &[][..]), ] { let names: Vec<_> = module_files(kind).iter().map(|(n, _)| *n).collect(); @@ -737,12 +749,15 @@ fn environment_vars_cover_every_network_mode() { // The onboarding and permissions-boundary modules. // --------------------------------------------------------------------------- -const ONBOARD_VARIABLES: [&str; 5] = [ +const ONBOARD_VARIABLES: [&str; 8] = [ "environment_name", "region", "owner_id", "owner_key", "bedrock_on_task_role", + "ec2_instance_type", + "ec2_max_instances", + "ec2_user_tag", ]; /// The boundary ARN every module-created role carries, as the module writes it. @@ -894,15 +909,38 @@ fn every_onboarding_role_carries_the_boundary_and_owner_tag() { .count(), 1 ); + // What default tags don't reach (a launch's instances and volumes, the + // group) gets them whole from `aws_default_tags`; the one other + // `triple-c:` tag is the instance role's cluster. for block in all_blocks(&body) { if block.identifier.as_str() == "default_tags" { continue; } for key in ["tags", "tags_all"] { if let Some(t) = attr(&block.body, key) { + if !matches!(t, Expression::Object(_)) { + assert_eq!( + render(t), + "data.aws_default_tags.current[0].tags", + "{:?}", + block.labels + ); + continue; + } let keys = object_keys(t); + let allowed: &[&str] = if block + .labels + .iter() + .map(|l| l.as_str()) + .eq(["aws_iam_role", "instance"]) + { + &["triple-c:cluster"] + } else { + &[] + }; assert!( - keys.iter().all(|k| !k.starts_with("triple-c:")), + keys.iter() + .all(|k| !k.starts_with("triple-c:") || allowed.contains(&k.as_str())), "{:?} sets triple-c tags {keys:?}", block.labels ); @@ -910,14 +948,15 @@ fn every_onboarding_role_carries_the_boundary_and_owner_tag() { } } - // Exactly the two per-user roles, named as the app names them, each + // Exactly the per-user roles, named as the app names them, each // bounded; their permissions are inline only (nothing attached that the // module doesn't spell out). let roles = resources(&body, "aws_iam_role"); - assert_eq!(roles.len(), 2, "task and exec roles"); + assert_eq!(roles.len(), 3, "task, exec and instance roles"); let key = owner_key("us-west-2", "AROAEXAMPLE:alice"); let (task, exec) = user_role_names("dev", &key); - for (name, want) in [("task", task), ("exec", exec)] { + let instance = user_instance_role_name("dev", &key); + for (name, want) in [("task", task), ("exec", exec), ("instance", instance)] { let role = resource(&body, "aws_iam_role", name); let named = expand_locals(&body, &template_text(attr(&role.body, "name").unwrap())) .replace("${var.environment_name}", "dev") @@ -938,7 +977,6 @@ fn every_onboarding_role_carries_the_boundary_and_owner_tag() { "aws_iam_policy_attachment", "aws_iam_policy", "aws_iam_user", - "aws_iam_instance_profile", ] { assert!(resources(&body, ty).is_empty(), "{ty} present"); } @@ -955,7 +993,12 @@ fn every_onboarding_role_carries_the_boundary_and_owner_tag() { #[test] fn role_trust_is_ecs_tasks_in_this_account_only() { let body = parsed(ModuleKind::UserOnboard); + let mut seen = 0; for role in resources(&body, "aws_iam_role") { + if label(role, 1) == Some("instance") { + continue; // EC2's: `the_instance_role_is_ec2s_and_runs_only_the_ecs_agent` + } + seen += 1; let trust = jsonencoded(role, "assume_role_policy"); assert_eq!( statements(&trust), @@ -973,6 +1016,7 @@ fn role_trust_is_ecs_tasks_in_this_account_only() { role.labels ); } + assert_eq!(seen, 2); } #[test] @@ -1154,13 +1198,14 @@ fn the_environment_security_group_gains_no_ingress() { assert_eq!(attr(&sg.body, "egress"), Some(&Expression::Array(vec![]))); assert!(top_blocks(&sg.body, "egress").next().is_none()); - // The environment's group is referenced only as that rule's source. + // The environment's group is referenced only as that rule's source and + // as the EC2 instances' group (which adds no rule to it). let text = tf_text(ModuleKind::UserOnboard); - assert_eq!(text.matches("local.env.security_group_id").count(), 1); + assert_eq!(text.matches("local.env.security_group_id").count(), 2); assert_eq!( text.matches("security_group_id").count(), - 4, - "the rule, the record check and its message, and the efs group output" + 5, + "the rule, the launch template, the record check and its message, and the efs group output" ); } @@ -1274,8 +1319,9 @@ fn exec_role_reads_only_its_owners_secrets() { #[test] fn onboarding_reads_the_environment_only_from_its_record() { let body = parsed(ModuleKind::UserOnboard); + // The other parameter read is the public AMI id (EC2 capacity). let params: Vec<_> = top_blocks(&body, "data") - .filter(|b| label(b, 0) == Some("aws_ssm_parameter")) + .filter(|b| label(b, 0) == Some("aws_ssm_parameter") && label(b, 1) != Some("ecs_ami")) .collect(); assert_eq!(params.len(), 1); assert_eq!( @@ -1286,13 +1332,19 @@ fn onboarding_reads_the_environment_only_from_its_record() { // No remote state, no other reads of the environment's deployment. let text = tf_text(ModuleKind::UserOnboard); assert!(!text.contains("terraform_remote_state")); - assert!(resources(&body, "aws_ssm_parameter").is_empty()); + // The one parameter it writes is the user's own record, not the + // environment's. + let written: Vec<_> = resources(&body, "aws_ssm_parameter") + .into_iter() + .filter_map(|b| label(b, 1)) + .collect(); + assert_eq!(written, ["user"]); } #[test] fn onboarding_vars_name_the_owner() { let owner = "AROAEXAMPLEID:alice@example.com"; - let vars = onboarding_vars("dev", "us-west-2", owner); + let vars = onboarding_vars("dev", "us-west-2", owner, None).unwrap(); assert_eq!( vars, json!({ @@ -1301,9 +1353,82 @@ fn onboarding_vars_name_the_owner() { "owner_id": owner, "owner_key": owner_key("us-west-2", owner), "bedrock_on_task_role": false, + "ec2_instance_type": "", + "ec2_max_instances": 1, + "ec2_user_tag": "", }) ); + // Without EC2 capacity the EC2 variables are the module's defaults, so + // the plan is the Fargate-only one. let body = parsed(ModuleKind::UserOnboard); + for (name, value) in [ + ("ec2_instance_type", json!("")), + ("ec2_max_instances", json!(1)), + ("ec2_user_tag", json!("")), + ] { + let v = top_blocks(&body, "variable") + .find(|b| label(b, 0) == Some(name)) + .unwrap(); + assert_eq!(to_json(attr(&v.body, "default").unwrap()), value, "{name}"); + assert_eq!(vars[name], value, "{name}"); + } + + let ec2 = Ec2Compute { + instance_type: "m7g.xlarge".into(), + max_instances: 3, + user_tag: "alice-1".into(), + }; + let vars = onboarding_vars("dev", "us-west-2", owner, Some(&ec2)).unwrap(); + assert_eq!(vars["ec2_instance_type"], "m7g.xlarge"); + assert_eq!(vars["ec2_max_instances"], 3); + assert_eq!(vars["ec2_user_tag"], "alice-1"); + for bad in [ + Ec2Compute { + instance_type: "".into(), + ..ec2.clone() + }, + Ec2Compute { + instance_type: "p5.48xlarge".into(), + ..ec2.clone() + }, + Ec2Compute { + instance_type: "m7i.8xlarge".into(), + ..ec2.clone() + }, + Ec2Compute { + instance_type: "t3.4xlarge".into(), + ..ec2.clone() + }, + Ec2Compute { + max_instances: 0, + ..ec2.clone() + }, + Ec2Compute { + max_instances: EC2_MAX_INSTANCES + 1, + ..ec2.clone() + }, + Ec2Compute { + user_tag: "".into(), + ..ec2.clone() + }, + Ec2Compute { + user_tag: "a--b".into(), + ..ec2.clone() + }, + Ec2Compute { + user_tag: "-a".into(), + ..ec2.clone() + }, + Ec2Compute { + user_tag: "x".repeat(33), + ..ec2.clone() + }, + ] { + assert!( + onboarding_vars("dev", "us-west-2", owner, Some(&bad)).is_err(), + "{bad:?} accepted" + ); + } let declared: BTreeSet<_> = top_blocks(&body, "variable") .filter_map(|b| label(b, 0)) .collect(); @@ -1612,6 +1737,82 @@ const BACKEND: &[Check] = &[ call("s3:DeleteObject", USER_LOCK, IN_ACCOUNT), ]; +// The onboarding module's EC2 capacity (phase 7, E1–E5, E11) for the user +// whose `triple-c-user` tag is `alice`: cluster, capacity provider, group and +// launch template are `triple-c---u-`, the instance role and +// profile `triple-c---instance` (pinned to the naming +// functions by `the_ec2_call_table_uses_the_shared_names`). +const USER_CLUSTER: &str = "arn:aws:ecs:{region}:{account}:cluster/triple-c-probe--u-alice"; +const CAPACITY_PROVIDER: &str = + "arn:aws:ecs:{region}:{account}:capacity-provider/triple-c-probe--u-alice"; +const LAUNCH_TEMPLATE: &str = "arn:aws:ec2:{region}:{account}:launch-template/lt-00000000000000000"; +const AUTO_SCALING_GROUP: &str = "arn:aws:autoscaling:{region}:{account}:autoScalingGroup:\ + 00000000-0000-0000-0000-000000000000:autoScalingGroupName/triple-c-probe--u-alice"; +const INSTANCE: &str = "arn:aws:ec2:{region}:{account}:instance/i-00000000000000000"; +const VOLUME: &str = "arn:aws:ec2:{region}:{account}:volume/vol-00000000000000000"; +const IMAGE: &str = "arn:aws:ec2:{region}::image/ami-00000000000000000"; +const INSTANCE_ROLE: &str = "arn:aws:iam::{account}:role/triple-c-probe-0123456789ab-instance"; +const INSTANCE_PROFILE: &str = + "arn:aws:iam::{account}:instance-profile/triple-c-probe-0123456789ab-instance"; +const AUTO_SCALING_SERVICE_ROLE: &str = "arn:aws:iam::{account}:role/aws-service-role/\ + autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling"; +const ECS_SERVICE_ROLE: &str = + "arn:aws:iam::{account}:role/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS"; +const AMI_PARAMETER: &str = "arn:aws:ssm:{region}::parameter/aws/service/ecs/optimized-ami/\ + amazon-linux-2023/recommended/image_id"; +const ARM64_AMI_PARAMETER: &str = "arn:aws:ssm:{region}::parameter/aws/service/ecs/optimized-ami/\ + amazon-linux-2023/arm64/recommended/image_id"; +const USER_RECORD: &str = + "arn:aws:ssm:{region}:{account}:parameter/triple-c/users/probe/0123456789ab"; +/// A launch from the user's launch template (what creating the group +/// checks the caller for). +const FROM_TEMPLATE: Context = &[( + "ec2:LaunchTemplate", + Literal("arn:aws:ec2:{region}:{account}:launch-template/lt-00000000000000000"), +)]; +const LT_ARN: ContextValue = + Literal("arn:aws:ec2:{region}:{account}:launch-template/lt-00000000000000000"); +/// The template's tag specifications tag the instance and its volume. +const TAGGED_INSTANCE: Context = &[ + (REQUEST_COMPONENT, Literal("user-onboarding")), + ("ec2:LaunchTemplate", LT_ARN), + ("ec2:InstanceType", Literal("m7i.large")), +]; +const TAGGED_VOLUME: Context = &[ + (REQUEST_COMPONENT, Literal("user-onboarding")), + ("ec2:LaunchTemplate", LT_ARN), +]; +/// The environment's tasks group, named in the template. +const TEMPLATE_GROUP: Context = &[ + ("ec2:LaunchTemplate", LT_ARN), + ("ec2:IsLaunchTemplateResource", Literal("true")), + (RESOURCE_COMPONENT, Literal("environment")), +]; +const AMAZON_IMAGE: Context = &[ + ("ec2:Owner", Literal("amazon")), + ("ec2:LaunchTemplate", LT_ARN), + ("ec2:IsLaunchTemplateResource", Literal("true")), +]; +/// The group pins the template to a version (`latest_version`, never +/// `$Latest`/`$Default`) and has no load balancer. +const GROUP_CREATE: Context = &[ + (REQUEST_COMPONENT, Literal("user-onboarding")), + ( + "autoscaling:LaunchTemplateVersionSpecified", + Literal("true"), + ), + ("autoscaling:InstanceTypes", Literal("m7i.large")), +]; +const GROUP_UPDATE: Context = &[ + (RESOURCE_COMPONENT, Literal("user-onboarding")), + ( + "autoscaling:LaunchTemplateVersionSpecified", + Literal("true"), + ), + ("autoscaling:InstanceTypes", Literal("m7i.large")), +]; +const TO_EC2: Context = &[("iam:PassedToService", Literal("ec2.amazonaws.com"))]; + const MODULE_CALLS: &[(&str, &[Check])] = &[ ( "aws_ecs_cluster", @@ -1628,14 +1829,35 @@ const MODULE_CALLS: &[(&str, &[Check])] = &[ call("ecs:TagResource", CLUSTER, ENV_TAGGED), call("ecs:UntagResource", CLUSTER, ENV_TAGGED), call("ecs:DeleteCluster", CLUSTER, ENV_TAGGED), + // The user's own cluster (onboarding, EC2 capacity). + call("ecs:CreateCluster", USER_CLUSTER, USER_CREATE), + call( + "ecs:TagResource", + USER_CLUSTER, + &[("ecs:CreateAction", Literal("CreateCluster"))], + ), + call("ecs:DescribeClusters", USER_CLUSTER, NONE), + call("ecs:ListTagsForResource", USER_CLUSTER, NONE), + call("ecs:UpdateCluster", USER_CLUSTER, USER_TAGGED), + call("ecs:TagResource", USER_CLUSTER, USER_TAGGED), + call("ecs:UntagResource", USER_CLUSTER, USER_TAGGED), + call("ecs:DeleteCluster", USER_CLUSTER, USER_TAGGED), ], ), ( - // Untaggable: authorized against the (named) cluster. + // Untaggable: authorized against the (named) cluster, and (a + // hypothesis) the user's capacity provider. "aws_ecs_cluster_capacity_providers", &[ call("ecs:PutClusterCapacityProviders", CLUSTER, ENV_TAGGED), call("ecs:DescribeClusters", CLUSTER, NONE), + call("ecs:PutClusterCapacityProviders", USER_CLUSTER, USER_TAGGED), + call( + "ecs:PutClusterCapacityProviders", + CAPACITY_PROVIDER, + USER_TAGGED, + ), + call("ecs:DescribeClusters", USER_CLUSTER, NONE), ], ), ( @@ -1731,6 +1953,13 @@ const MODULE_CALLS: &[(&str, &[Check])] = &[ call("ssm:ListTagsForResource", PARAMETER, NONE), call("ssm:RemoveTagsFromResource", PARAMETER, NONE), call("ssm:DeleteParameter", PARAMETER, NONE), + // The user's EC2 record (onboarding, E11). + call("ssm:PutParameter", USER_RECORD, USER_CREATE), + call("ssm:AddTagsToResource", USER_RECORD, USER_CREATE), + call("ssm:GetParameter", USER_RECORD, NONE), + call("ssm:ListTagsForResource", USER_RECORD, NONE), + call("ssm:RemoveTagsFromResource", USER_RECORD, NONE), + call("ssm:DeleteParameter", USER_RECORD, NONE), ], ), ( @@ -1921,6 +2150,14 @@ const MODULE_CALLS: &[(&str, &[Check])] = &[ call("iam:UpdateRoleDescription", TASK_ROLE, NONE), call("iam:UntagRole", TASK_ROLE, NONE), call("iam:DeleteRole", EXEC_ROLE, NONE), + // The EC2 instance role, tagged `triple-c:cluster = `: the boundary keeps the ECS agent on that cluster. + call("iam:CreateRole", INSTANCE_ROLE, BOUNDED), + call("iam:TagRole", INSTANCE_ROLE, NONE), + call("iam:GetRole", INSTANCE_ROLE, NONE), + call("iam:ListInstanceProfilesForRole", INSTANCE_ROLE, NONE), + call("iam:UntagRole", INSTANCE_ROLE, NONE), + call("iam:DeleteRole", INSTANCE_ROLE, NONE), ], ), ( @@ -1930,128 +2167,9 @@ const MODULE_CALLS: &[(&str, &[Check])] = &[ call("iam:PutRolePolicy", TASK_ROLE, BOUNDED), call("iam:GetRolePolicy", TASK_ROLE, NONE), call("iam:DeleteRolePolicy", EXEC_ROLE, BOUNDED), - ], - ), - ( - "data.aws_caller_identity", - &[call("sts:GetCallerIdentity", "*", NONE)], - ), - ("data.aws_partition", &[]), - ( - "data.aws_availability_zones", - &[call("ec2:DescribeAvailabilityZones", "*", NONE)], - ), - ( - "data.aws_ssm_parameter", - &[call("ssm:GetParameter", PARAMETER, NONE)], - ), - ("data.aws_subnet", &[call("ec2:DescribeSubnets", "*", NONE)]), - ("backend.s3", BACKEND), -]; - -// Phase 7 (per-user EC2 capacity, E1–E5, E11; fix round 1): the calls the onboarding -// module's EC2 resources will make, written down with the policy (Task 2) -// before the resources exist. Task 3 moves each entry into MODULE_CALLS -// under its type (merging with the existing `aws_ecs_cluster`, -// `aws_ecs_cluster_capacity_providers`, `aws_iam_role`, `aws_ssm_parameter` -// and `data.aws_ssm_parameter` entries) and deletes this table. Names are -// the user's (`triple-c--[-instance]`). -const USER_CLUSTER: &str = "arn:aws:ecs:{region}:{account}:cluster/triple-c-probe--u-alice"; -const CAPACITY_PROVIDER: &str = - "arn:aws:ecs:{region}:{account}:capacity-provider/triple-c-probe--u-alice"; -const LAUNCH_TEMPLATE: &str = "arn:aws:ec2:{region}:{account}:launch-template/lt-00000000000000000"; -const AUTO_SCALING_GROUP: &str = "arn:aws:autoscaling:{region}:{account}:autoScalingGroup:\ - 00000000-0000-0000-0000-000000000000:autoScalingGroupName/triple-c-probe--u-alice"; -const INSTANCE: &str = "arn:aws:ec2:{region}:{account}:instance/i-00000000000000000"; -const VOLUME: &str = "arn:aws:ec2:{region}:{account}:volume/vol-00000000000000000"; -const IMAGE: &str = "arn:aws:ec2:{region}::image/ami-00000000000000000"; -const INSTANCE_ROLE: &str = "arn:aws:iam::{account}:role/triple-c-probe-0123456789ab-instance"; -const INSTANCE_PROFILE: &str = - "arn:aws:iam::{account}:instance-profile/triple-c-probe-0123456789ab-instance"; -const AUTO_SCALING_SERVICE_ROLE: &str = "arn:aws:iam::{account}:role/aws-service-role/\ - autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling"; -const ECS_SERVICE_ROLE: &str = - "arn:aws:iam::{account}:role/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS"; -const AMI_PARAMETER: &str = "arn:aws:ssm:{region}::parameter/aws/service/ecs/optimized-ami/\ - amazon-linux-2023/recommended/image_id"; -const USER_RECORD: &str = - "arn:aws:ssm:{region}:{account}:parameter/triple-c/users/probe/0123456789ab"; -/// A launch from the user's launch template (what creating the group -/// checks the caller for). -const FROM_TEMPLATE: Context = &[( - "ec2:LaunchTemplate", - Literal("arn:aws:ec2:{region}:{account}:launch-template/lt-00000000000000000"), -)]; -const LT_ARN: ContextValue = - Literal("arn:aws:ec2:{region}:{account}:launch-template/lt-00000000000000000"); -/// The template's tag specifications tag the instance and its volume. -const TAGGED_INSTANCE: Context = &[ - (REQUEST_COMPONENT, Literal("user-onboarding")), - ("ec2:LaunchTemplate", LT_ARN), - ("ec2:InstanceType", Literal("m7i.large")), -]; -const TAGGED_VOLUME: Context = &[ - (REQUEST_COMPONENT, Literal("user-onboarding")), - ("ec2:LaunchTemplate", LT_ARN), -]; -/// The environment's tasks group, named in the template. -const TEMPLATE_GROUP: Context = &[ - ("ec2:LaunchTemplate", LT_ARN), - ("ec2:IsLaunchTemplateResource", Literal("true")), - (RESOURCE_COMPONENT, Literal("environment")), -]; -const AMAZON_IMAGE: Context = &[ - ("ec2:Owner", Literal("amazon")), - ("ec2:LaunchTemplate", LT_ARN), - ("ec2:IsLaunchTemplateResource", Literal("true")), -]; -/// The group pins the template to a version (`latest_version`, never -/// `$Latest`/`$Default`) and has no load balancer. -const GROUP_CREATE: Context = &[ - (REQUEST_COMPONENT, Literal("user-onboarding")), - ( - "autoscaling:LaunchTemplateVersionSpecified", - Literal("true"), - ), - ("autoscaling:InstanceTypes", Literal("m7i.large")), -]; -const GROUP_UPDATE: Context = &[ - (RESOURCE_COMPONENT, Literal("user-onboarding")), - ( - "autoscaling:LaunchTemplateVersionSpecified", - Literal("true"), - ), - ("autoscaling:InstanceTypes", Literal("m7i.large")), -]; -const TO_EC2: Context = &[("iam:PassedToService", Literal("ec2.amazonaws.com"))]; - -const EC2_COMPUTE_CALLS: &[(&str, &[Check])] = &[ - ( - "aws_ecs_cluster (user)", - &[ - call("ecs:CreateCluster", USER_CLUSTER, USER_CREATE), - call( - "ecs:TagResource", - USER_CLUSTER, - &[("ecs:CreateAction", Literal("CreateCluster"))], - ), - call("ecs:DescribeClusters", USER_CLUSTER, NONE), - call("ecs:ListTagsForResource", USER_CLUSTER, NONE), - call("ecs:UpdateCluster", USER_CLUSTER, USER_TAGGED), - call("ecs:DeleteCluster", USER_CLUSTER, USER_TAGGED), - ], - ), - ( - // Hypothesis: authorized against the cluster and the provider. - "aws_ecs_cluster_capacity_providers (user)", - &[ - call("ecs:PutClusterCapacityProviders", USER_CLUSTER, USER_TAGGED), - call( - "ecs:PutClusterCapacityProviders", - CAPACITY_PROVIDER, - USER_TAGGED, - ), - call("ecs:DescribeClusters", USER_CLUSTER, NONE), + call("iam:PutRolePolicy", INSTANCE_ROLE, BOUNDED), + call("iam:GetRolePolicy", INSTANCE_ROLE, NONE), + call("iam:DeleteRolePolicy", INSTANCE_ROLE, BOUNDED), ], ), ( @@ -2105,7 +2223,9 @@ const EC2_COMPUTE_CALLS: &[(&str, &[Check])] = &[ // launch puts on, the instance role passed to EC2). The group's own // launches run as Auto Scaling's service-linked role. ASG tags are // not the provider's default_tags: the module sets them per group. - // SetInstanceProtection: hypothesis, for draining before destroy. + // Deleting first drains the group (`force_delete` off): an update to + // 0 naming no template, scale-in protection cleared on its + // instances, then the wait for them to terminate. "aws_autoscaling_group", &[ call( @@ -2136,7 +2256,6 @@ const EC2_COMPUTE_CALLS: &[(&str, &[Check])] = &[ AUTO_SCALING_GROUP, GROUP_UPDATE, ), - // The drain before destroy leaves the template alone. call( "autoscaling:UpdateAutoScalingGroup", AUTO_SCALING_GROUP, @@ -2160,20 +2279,6 @@ const EC2_COMPUTE_CALLS: &[(&str, &[Check])] = &[ ), ], ), - ( - "aws_iam_role (instance)", - &[ - call("iam:CreateRole", INSTANCE_ROLE, BOUNDED), - // `triple-c:cluster = `: the boundary keeps - // the ECS agent on that cluster. - call("iam:TagRole", INSTANCE_ROLE, NONE), - call("iam:PutRolePolicy", INSTANCE_ROLE, BOUNDED), - call("iam:GetRole", INSTANCE_ROLE, NONE), - call("iam:ListInstanceProfilesForRole", INSTANCE_ROLE, NONE), - call("iam:DeleteRolePolicy", INSTANCE_ROLE, BOUNDED), - call("iam:DeleteRole", INSTANCE_ROLE, NONE), - ], - ), ( // Hypothesis: adding the role needs it passable to EC2. "aws_iam_instance_profile", @@ -2190,26 +2295,29 @@ const EC2_COMPUTE_CALLS: &[(&str, &[Check])] = &[ ], ), ( - "data.aws_ssm_parameter (AMI)", - &[call("ssm:GetParameter", AMI_PARAMETER, NONE)], + "data.aws_caller_identity", + &[call("sts:GetCallerIdentity", "*", NONE)], + ), + ("data.aws_partition", &[]), + // The provider's own configuration: no call. + ("data.aws_default_tags", &[]), + ( + "data.aws_availability_zones", + &[call("ec2:DescribeAvailabilityZones", "*", NONE)], ), ( - "aws_ssm_parameter (user record)", + "data.aws_ssm_parameter", &[ - call("ssm:PutParameter", USER_RECORD, USER_CREATE), - call("ssm:AddTagsToResource", USER_RECORD, USER_CREATE), - call("ssm:GetParameter", USER_RECORD, NONE), - call("ssm:ListTagsForResource", USER_RECORD, NONE), - call("ssm:DeleteParameter", USER_RECORD, NONE), + call("ssm:GetParameter", PARAMETER, NONE), + // The public ECS-optimized image ids (EC2 capacity). + call("ssm:GetParameter", AMI_PARAMETER, NONE), + call("ssm:GetParameter", ARM64_AMI_PARAMETER, NONE), ], ), + ("data.aws_subnet", &[call("ec2:DescribeSubnets", "*", NONE)]), + ("backend.s3", BACKEND), ]; -/// MODULE_CALLS and the phase-7 calls Task 3's resources will make. -fn all_module_calls() -> impl Iterator { - MODULE_CALLS.iter().chain(EC2_COMPUTE_CALLS.iter()) -} - /// `resource.` and `data.` of the environment and onboarding /// modules, and `backend.s3`. fn module_types() -> BTreeSet { @@ -2302,7 +2410,7 @@ fn the_admin_tier_can_apply_every_module_resource() { let admin = tier_statements(&[Tier::EnvironmentAdmin]); let both = tier_statements(&[Tier::EnvironmentAdmin, Tier::WorkspaceUser]); let mut missing = Vec::new(); - for (ty, calls) in all_module_calls() { + for (ty, calls) in MODULE_CALLS { for c in *calls { for (which, stmts) in [("admin", &admin), ("admin + workspace", &both)] { if !allowed_by(stmts, c) { @@ -2331,6 +2439,8 @@ const ONBOARD_SAMPLES: &[(&str, &str)] = &[ ), ("${var.region}", "us-west-2"), ("${var.owner_id}", "AROAEXAMPLEME:alice@example.com"), + ("${var.environment_name}", "probe"), + ("${var.ec2_user_tag}", "alice"), ("${local.env.log_group}", "/triple-c/probe"), ( "${local.env.kms_key_arn}", @@ -2380,6 +2490,16 @@ fn every_onboarding_role_stays_inside_the_boundary() { } let mut context: Vec<(&'static str, ContextValue)> = vec![(RESOURCE_COMPONENT, Literal("user-onboarding"))]; + // The instance role's sessions carry its `triple-c:cluster` tag. + if render(attr(&policy.body, "role").unwrap()) == "aws_iam_role.instance[0].id" { + let role = resource(&body, "aws_iam_role", "instance"); + let tag = object_get(attr(&role.body, "tags").unwrap(), "triple-c:cluster") + .expect("the instance role's cluster tag"); + context.push(( + "aws:PrincipalTag/triple-c:cluster", + Literal(leak(render_sample(&template_text(tag)))), + )); + } if let Some(cond) = stmt.get("Condition") { for (op, entries) in cond.as_object().unwrap() { assert_eq!(op, "StringEquals", "{:?}: unexpected {op}", policy.labels); @@ -2410,7 +2530,7 @@ fn every_onboarding_role_stays_inside_the_boundary() { } } } - assert!(seen >= 15, "only {seen} role grants checked"); + assert!(seen >= 25, "only {seen} role grants checked"); } /// Admin write checks for calls the app makes itself, not the modules. @@ -2450,7 +2570,8 @@ fn every_module_write_has_an_admin_check_and_vice_versa() { .map(|c| c.action) .filter(|a| is_write(a)) .collect(); - let module: BTreeSet<&str> = all_module_calls() + let module: BTreeSet<&str> = MODULE_CALLS + .iter() .flat_map(|(_, calls)| calls.iter().map(|c| c.action)) .filter(|a| is_write(a)) .collect(); @@ -2483,7 +2604,9 @@ fn the_lifetime_cap_maximum_is_the_same_in_rust_typescript_and_hcl() { .find(|(f, _)| *f == "variables.tf") .unwrap() .1; - let block = &vars[vars.find("variable \"max_workspace_hours\"").expect("the variable")..]; + let block = &vars[vars + .find("variable \"max_workspace_hours\"") + .expect("the variable")..]; let block = &block[..block.find("\n}\n").unwrap()]; assert!( block.contains(&format!("var.max_workspace_hours <= {max}")), @@ -2499,3 +2622,651 @@ fn the_lifetime_cap_maximum_is_the_same_in_rust_typescript_and_hcl() { let envs = include_str!("../environments.rs"); assert!(envs.contains(&format!("from 0 to {max}"))); } + +// --------------------------------------------------------------------------- +// The onboarding module's optional EC2 capacity (phase 7). +// --------------------------------------------------------------------------- + +/// The onboarding module's EC2 file, parsed alone. +fn ec2_body() -> Body { + let text = module_files(ModuleKind::UserOnboard) + .iter() + .find(|(n, _)| *n == "ec2.tf") + .expect("ec2.tf") + .1; + hcl::parse(text).unwrap() +} + +/// The 30 instance types the app can size (`ec2_instance_spec`). +fn supported_instance_types() -> BTreeSet { + let mut known = BTreeSet::new(); + for family in [ + "m7i", "m7g", "c7i", "c7g", "r7i", "r7g", "t3", "t4g", "m7a", "p5", "g6", + ] { + for size in [ + "medium", "large", "xlarge", "2xlarge", "4xlarge", "8xlarge", "48xlarge", + ] { + let t = format!("{family}.{size}"); + if crate::models::project::ec2_instance_spec(&t).is_some() { + known.insert(t); + } + } + } + assert_eq!(known.len(), 30); + known +} + +/// `render` without whitespace, for comparing short lists. +fn flat(expr: &Expression) -> String { + render(expr).split_whitespace().collect() +} + +fn one_block<'a>(body: &'a Body, ident: &str) -> &'a Block { + let found: Vec<_> = body + .blocks() + .filter(|b| b.identifier.as_str() == ident) + .collect(); + assert_eq!(found.len(), 1, "one {ident} block"); + found[0] +} + +fn variable<'a>(body: &'a Body, name: &str) -> &'a Block { + top_blocks(body, "variable") + .find(|b| label(b, 0) == Some(name)) + .unwrap_or_else(|| panic!("variable {name}")) +} + +/// Without `ec2_instance_type` (the default) the module makes nothing for +/// EC2: every resource and data source of ec2.tf is counted by it, and +/// every other one is the version-1 set, unchanged (its rendered HCL is +/// pinned), so a Fargate-only onboarding plans as before apart from the +/// version tag. +#[test] +fn ec2_capacity_exists_only_with_an_instance_type() { + let body = parsed(ModuleKind::UserOnboard); + assert_eq!( + render(local_def(&body, "ec2")), + "var.ec2_instance_type != \"\"" + ); + assert_eq!(render(local_def(&body, "ec2_count")), "local.ec2 ? 1 : 0"); + assert_eq!( + attr(&variable(&body, "ec2_instance_type").body, "default"), + Some(&Expression::String(String::new())) + ); + + let ec2 = ec2_body(); + let mut gated = 0; + for b in ec2.blocks() { + if matches!(b.identifier.as_str(), "resource" | "data") { + assert_eq!( + attr(&b.body, "count").map(render).as_deref(), + Some("local.ec2_count"), + "{:?} is not counted by the instance type", + b.labels + ); + gated += 1; + } + } + assert_eq!(gated, 11); + + // Everything else: exactly version 1's resources, as they were, and its + // reads (the environment record's check gained `assign_public_ip`, + // which the launch template uses; checks change no plan). + let ec2_addrs: BTreeSet = ec2 + .blocks() + .filter(|b| matches!(b.identifier.as_str(), "resource" | "data")) + .map(|b| format!("{} {:?}", b.identifier.as_str(), b.labels)) + .collect(); + let mut rest = Vec::new(); + let mut reads = BTreeSet::new(); + for b in body.blocks() { + let addr = format!("{} {:?}", b.identifier.as_str(), b.labels); + if ec2_addrs.contains(&addr) { + continue; + } + match b.identifier.as_str() { + "resource" => rest.push(hcl::format::to_string(b).unwrap()), + "data" => { + reads.insert(format!("{}.{}", label(b, 0).unwrap(), label(b, 1).unwrap())); + } + _ => {} + } + } + assert_eq!(rest.len(), 9, "version 1's 9 resources"); + assert_eq!( + reads, + BTreeSet::from([ + "aws_caller_identity.current".to_string(), + "aws_partition.current".to_string(), + "aws_ssm_parameter.environment".to_string(), + "aws_subnet.task".to_string(), + ]) + ); + let fingerprint = sha256_prefix(&rest.join("\n")); + assert_eq!( + fingerprint, FARGATE_ONBOARDING_FINGERPRINT, + "a version-1 resource changed: Fargate onboardings would plan differently" + ); +} + +/// `sha256_prefix` of the onboarding module's non-EC2 resources as of +/// version 1 (rendered by hcl-rs, in file order). +const FARGATE_ONBOARDING_FINGERPRINT: &str = "ab948d6ab8be"; + +#[test] +fn ec2_names_are_the_shared_names() { + let body = parsed(ModuleKind::UserOnboard); + let env = "${var.environment_name}"; + assert_eq!( + template_text(local_def(&body, "ec2_name")), + user_ec2_stem(env, "${var.ec2_user_tag}") + ); + assert_eq!( + expand_locals( + &body, + &template_text(local_def(&body, "instance_role_name")) + ), + user_instance_role_name(env, "${var.owner_key}") + ); + let ec2 = ec2_body(); + for ty in [ + "aws_ecs_cluster", + "aws_ecs_capacity_provider", + "aws_launch_template", + "aws_autoscaling_group", + ] { + let r = resource(&ec2, ty, "ec2"); + assert_eq!( + render(attr(&r.body, "name").unwrap()), + "local.ec2_name", + "{ty}" + ); + } + let role = resource(&ec2, "aws_iam_role", "instance"); + assert_eq!( + render(attr(&role.body, "name").unwrap()), + "local.instance_role_name" + ); + let profile = resource(&ec2, "aws_iam_instance_profile", "instance"); + assert_eq!( + render(attr(&profile.body, "name").unwrap()), + "aws_iam_role.instance[0].name" + ); + assert_eq!( + render(attr(&profile.body, "role").unwrap()), + "aws_iam_role.instance[0].name" + ); +} + +/// The EC2 variables follow the app's rules: the instance type is one of +/// the 30 the app can size (the admin policy's lists, `policies.rs` pins +/// those), 1 to 4 instances, the user tag as `validate_ec2_user_tag` and +/// required with a type. +#[test] +fn ec2_variables_mirror_the_apps_rules() { + let body = parsed(ModuleKind::UserOnboard); + let ty = variable(&body, "ec2_instance_type"); + let cond = render(attr(&one_block(&ty.body, "validation").body, "condition").unwrap()); + let listed: BTreeSet = cond + .split('"') + .filter(|s| s.contains('.') && !s.contains(' ')) + .map(str::to_string) + .collect(); + assert_eq!(listed, supported_instance_types()); + assert!( + cond.starts_with("var.ec2_instance_type == \"\" || contains(["), + "{cond}" + ); + + let max = variable(&body, "ec2_max_instances"); + assert_eq!( + attr(&max.body, "default"), + Some(&Expression::Number(1.into())) + ); + let cond = render(attr(&one_block(&max.body, "validation").body, "condition").unwrap()); + assert!( + cond.contains(&format!("var.ec2_max_instances <= {EC2_MAX_INSTANCES}")), + "{cond}" + ); + assert!(cond.contains("var.ec2_max_instances >= 1"), "{cond}"); + + let tag = variable(&body, "ec2_user_tag"); + let conds: Vec = top_blocks(&tag.body, "validation") + .map(|v| render(attr(&v.body, "condition").unwrap())) + .collect(); + assert_eq!(conds.len(), 2); + for needle in [ + "\"^[A-Za-z0-9-]{1,32}$\"", + "!startswith(var.ec2_user_tag, \"-\")", + "!endswith(var.ec2_user_tag, \"-\")", + "!strcontains(var.ec2_user_tag, \"--\")", + ] { + assert!( + conds[0].contains(needle), + "{needle} missing from {}", + conds[0] + ); + } + assert_eq!( + conds[1], + "var.ec2_instance_type == \"\" || var.ec2_user_tag != \"\"" + ); +} + +/// The image is the ECS-optimized Amazon Linux 2023 one for the type's +/// architecture: arm64 for Graviton families (a `g` after the generation). +#[test] +fn the_ami_is_the_ecs_optimized_one_for_the_architecture() { + let body = parsed(ModuleKind::UserOnboard); + assert_eq!( + render(local_def(&body, "ec2_arm64")), + "can(regex(\"^[a-z]+[0-9]+g[a-z]*\\\\.\", var.ec2_instance_type))" + ); + assert_eq!( + render(local_def(&body, "ecs_ami_parameter")), + "(local.ec2_arm64 ? \"/aws/service/ecs/optimized-ami/amazon-linux-2023/arm64/recommended/\ + image_id\" : \"/aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id\")" + ); + // What that rule says for the supported types. + for t in supported_instance_types() { + let family = t.split('.').next().unwrap(); + let arm = matches!(family, "m7g" | "c7g" | "r7g" | "t4g"); + let rule = family + .trim_start_matches(|c: char| c.is_ascii_lowercase()) + .trim_start_matches(|c: char| c.is_ascii_digit()) + .starts_with('g'); + assert_eq!(rule, arm, "{t}"); + } + let ec2 = ec2_body(); + let ami = top_blocks(&ec2, "data") + .find(|b| label(b, 0) == Some("aws_ssm_parameter")) + .unwrap(); + assert_eq!( + render(attr(&ami.body, "name").unwrap()), + "local.ecs_ami_parameter" + ); + let lt = resource(&ec2, "aws_launch_template", "ec2"); + assert_eq!( + render(attr(<.body, "image_id").unwrap()), + "data.aws_ssm_parameter.ecs_ami[0].insecure_value" + ); +} + +/// The launch template: the variable's type, IMDSv2 at one hop, an +/// encrypted 50 GiB gp3 root volume, the environment's tasks group named in +/// the template, the instance profile, the agent pointed at the user's +/// cluster with task IMDS blocked, and the default tags on what a launch +/// creates. Never a default-version change (no `ModifyLaunchTemplate`). +#[test] +fn the_launch_template_is_pinned() { + let ec2 = ec2_body(); + let lt = resource(&ec2, "aws_launch_template", "ec2"); + let b = <.body; + assert_eq!( + render(attr(b, "instance_type").unwrap()), + "var.ec2_instance_type" + ); + for forbidden in [ + "update_default_version", + "default_version", + "vpc_security_group_ids", + ] { + assert!(attr(b, forbidden).is_none(), "{forbidden}"); + } + for forbidden in [ + "instance_market_options", + "instance_requirements", + "placement", + ] { + assert!(top_blocks(b, forbidden).next().is_none(), "{forbidden}"); + } + + let md = &one_block(b, "metadata_options").body; + assert_eq!( + attr(md, "http_tokens"), + Some(&Expression::String("required".into())) + ); + assert_eq!( + attr(md, "http_put_response_hop_limit"), + Some(&Expression::Number(1.into())) + ); + assert_eq!( + attr(md, "http_endpoint"), + Some(&Expression::String("enabled".into())) + ); + + let bdm = &one_block(b, "block_device_mappings").body; + assert_eq!( + attr(bdm, "device_name"), + Some(&Expression::String("/dev/xvda".into())) + ); + let ebs = &one_block(bdm, "ebs").body; + assert_eq!( + attr(ebs, "volume_size"), + Some(&Expression::Number(50.into())) + ); + assert_eq!( + attr(ebs, "volume_type"), + Some(&Expression::String("gp3".into())) + ); + assert_eq!(attr(ebs, "encrypted"), Some(&Expression::Bool(true))); + assert_eq!( + attr(ebs, "delete_on_termination"), + Some(&Expression::Bool(true)) + ); + + let nic = &one_block(b, "network_interfaces").body; + assert_eq!( + flat(attr(nic, "security_groups").unwrap()), + "[local.env.security_group_id]" + ); + assert_eq!( + render(attr(nic, "associate_public_ip_address").unwrap()), + "local.env.assign_public_ip" + ); + assert!( + attr(nic, "subnet_id").is_none(), + "the group picks the subnet" + ); + + assert_eq!( + render(attr(&one_block(b, "iam_instance_profile").body, "name").unwrap()), + "aws_iam_instance_profile.instance[0].name" + ); + + let user_data = render(attr(b, "user_data").unwrap()); + for needle in [ + "#!/bin/bash", + ">>/etc/ecs/ecs.config", + "ECS_CLUSTER=${aws_ecs_cluster.ec2[0].name}\n", + "ECS_AWSVPC_BLOCK_IMDS=true\n", + ] { + assert!( + user_data.contains(needle), + "{needle:?} missing from {user_data}" + ); + } + assert!( + !user_data.contains("ECS_ENABLE_TASK_ENI"), + "no trunking (E3)" + ); + + let specs: BTreeSet = top_blocks(b, "tag_specifications") + .map(|t| { + assert_eq!( + render(attr(&t.body, "tags").unwrap()), + "data.aws_default_tags.current[0].tags" + ); + match attr(&t.body, "resource_type") { + Some(Expression::String(s)) => s.clone(), + other => panic!("{other:?}"), + } + }) + .collect(); + assert_eq!( + specs, + BTreeSet::from([ + "instance".into(), + "volume".into(), + "network-interface".into() + ]) + ); +} + +/// The group: 0 to `ec2_max_instances`, desired left to ECS, protected from +/// scale-in (ECS's managed termination protection decides), drained before +/// deletion, on a fixed template version, in the environment's subnets, no +/// load balancers, every default tag plus the one ECS adds. +#[test] +fn the_auto_scaling_group_is_pinned() { + let ec2 = ec2_body(); + let asg = resource(&ec2, "aws_autoscaling_group", "ec2"); + let b = &asg.body; + assert_eq!(attr(b, "min_size"), Some(&Expression::Number(0.into()))); + assert_eq!( + render(attr(b, "max_size").unwrap()), + "var.ec2_max_instances" + ); + assert!(attr(b, "desired_capacity").is_none()); + assert_eq!( + attr(b, "protect_from_scale_in"), + Some(&Expression::Bool(true)) + ); + assert_eq!(attr(b, "force_delete"), Some(&Expression::Bool(false))); + assert_eq!( + render(attr(b, "vpc_zone_identifier").unwrap()), + "local.env.subnet_ids" + ); + for forbidden in [ + "target_group_arns", + "load_balancers", + "availability_zones", + "launch_configuration", + ] { + assert!(attr(b, forbidden).is_none(), "{forbidden}"); + } + for forbidden in ["mixed_instances_policy", "warm_pool", "instance_refresh"] { + assert!(top_blocks(b, forbidden).next().is_none(), "{forbidden}"); + } + let lt = &one_block(b, "launch_template").body; + assert_eq!( + render(attr(lt, "id").unwrap()), + "aws_launch_template.ec2[0].id" + ); + assert_eq!( + render(attr(lt, "version").unwrap()), + "aws_launch_template.ec2[0].latest_version" + ); + assert_eq!( + flat(attr(&one_block(b, "lifecycle").body, "ignore_changes").unwrap()), + "[desired_capacity]" + ); + + let dynamic = one_block(b, "dynamic"); + assert_eq!(label(dynamic, 0), Some("tag")); + assert_eq!( + render(attr(&dynamic.body, "for_each").unwrap()), + "data.aws_default_tags.current[0].tags" + ); + let content = &one_block(&dynamic.body, "content").body; + assert_eq!(render(attr(content, "key").unwrap()), "tag.key"); + assert_eq!(render(attr(content, "value").unwrap()), "tag.value"); + let tag = &one_block(b, "tag").body; + assert_eq!( + attr(tag, "key"), + Some(&Expression::String("AmazonECSManaged".into())) + ); +} + +/// The capacity provider and the cluster's provider list: exactly the +/// user's provider, the default strategy, managed scaling to a full fleet +/// one instance at a time. +#[test] +fn the_capacity_provider_scales_one_instance_at_a_time() { + let ec2 = ec2_body(); + let cp = resource(&ec2, "aws_ecs_capacity_provider", "ec2"); + let asgp = &one_block(&cp.body, "auto_scaling_group_provider").body; + assert_eq!( + render(attr(asgp, "auto_scaling_group_arn").unwrap()), + "aws_autoscaling_group.ec2[0].arn" + ); + assert_eq!( + attr(asgp, "managed_termination_protection"), + Some(&Expression::String("ENABLED".into())) + ); + let ms = &one_block(asgp, "managed_scaling").body; + assert_eq!( + attr(ms, "status"), + Some(&Expression::String("ENABLED".into())) + ); + for (k, v) in [ + ("target_capacity", 100), + ("minimum_scaling_step_size", 1), + ("maximum_scaling_step_size", 1), + ("instance_warmup_period", 60), + ] { + assert_eq!(attr(ms, k), Some(&Expression::Number(v.into())), "{k}"); + } + + let ccp = resource(&ec2, "aws_ecs_cluster_capacity_providers", "ec2"); + assert_eq!( + render(attr(&ccp.body, "cluster_name").unwrap()), + "aws_ecs_cluster.ec2[0].name" + ); + assert_eq!( + flat(attr(&ccp.body, "capacity_providers").unwrap()), + "[aws_ecs_capacity_provider.ec2[0].name]" + ); + let strategy = &one_block(&ccp.body, "default_capacity_provider_strategy").body; + assert_eq!( + render(attr(strategy, "capacity_provider").unwrap()), + "aws_ecs_capacity_provider.ec2[0].name" + ); + assert_eq!( + attr(strategy, "weight"), + Some(&Expression::Number(1.into())) + ); +} + +/// The instance role: trusted by EC2, bounded, tagged with the user's +/// cluster (the boundary's fence), and granting exactly the ECS agent's +/// calls on that cluster and its container instances, endpoint discovery, +/// image pulls and the environment's logs. No Session Manager core: ECS +/// Exec's channels run on the task role. +#[test] +fn the_instance_role_is_ec2s_and_runs_only_the_ecs_agent() { + let body = parsed(ModuleKind::UserOnboard); + let role = resource(&body, "aws_iam_role", "instance"); + let trust = jsonencoded(&role, "assume_role_policy"); + assert_eq!( + statements(&trust), + vec![json!({ + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com" }, + "Action": "sts:AssumeRole", + })] + ); + let tags = attr(&role.body, "tags").unwrap(); + assert_eq!( + object_keys(tags), + BTreeSet::from(["triple-c:cluster".to_string()]) + ); + assert_eq!( + render(object_get(tags, "triple-c:cluster").unwrap()), + "local.ec2_name" + ); + + let policies = role_policies(&body, "instance[0]"); + assert_eq!(policies.len(), 1); + let stmts = statements(&jsonencoded(&policies[0], "policy")); + let mut granted = BTreeSet::new(); + let cluster = + "arn:${local.partition}:ecs:${var.region}:${local.account}:cluster/${local.ec2_name}"; + let instances = "arn:${local.partition}:ecs:${var.region}:${local.account}:\ + container-instance/${local.ec2_name}/*"; + for stmt in &stmts { + assert!(stmt.get("Condition").is_none()); + let resources = eval::strings(&stmt["Resource"]); + for a in actions(stmt) { + if a.starts_with("ecs:") && a != "ecs:DiscoverPollEndpoint" { + assert_eq!( + resources, + vec![cluster.to_string(), instances.to_string()], + "{a}" + ); + } + if a == "ecs:DiscoverPollEndpoint" || a == "ecr:GetAuthorizationToken" { + assert_eq!(resources, vec!["*".to_string()], "{a}"); + } + if a.starts_with("logs:") { + assert_eq!(resources, vec!["${local.log_group_arn}".to_string()], "{a}"); + } + granted.insert(a); + } + } + let want: BTreeSet = [ + "ecs:RegisterContainerInstance", + "ecs:DeregisterContainerInstance", + "ecs:Poll", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "ecs:UpdateContainerInstancesState", + "ecs:DiscoverPollEndpoint", + "ecr:GetAuthorizationToken", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchCheckLayerAvailability", + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + .into_iter() + .map(String::from) + .collect(); + assert_eq!(granted, want); +} + +/// The user record (E11): `/triple-c/users//`, a String of +/// exactly the schema-1 fields, written only with EC2 capacity and after +/// the cluster has its provider. +#[test] +fn the_user_record_has_exactly_the_schema_fields() { + let ec2 = ec2_body(); + let param = resource(&ec2, "aws_ssm_parameter", "user"); + assert_eq!( + template_text(attr(¶m.body, "name").unwrap()), + user_record_parameter_name("${var.environment_name}", "${var.owner_key}") + ); + assert_eq!( + user_record_parameter_name("dev", "0123456789ab"), + "/triple-c/users/dev/0123456789ab" + ); + assert!(USER_RECORD.ends_with(&format!( + "parameter{}", + user_record_parameter_name(ENV, OWNER_KEY) + ))); + assert!(!user_record_parameter_name("dev", "x").starts_with(&environment_parameter_name(""))); + assert_eq!( + attr(¶m.body, "type"), + Some(&Expression::String("String".into())) + ); + let value = jsonencoded(¶m, "value"); + let fields: BTreeSet = value.as_object().unwrap().keys().cloned().collect(); + assert_eq!( + fields, + USER_RECORD_FIELDS.iter().map(|s| s.to_string()).collect() + ); + assert_eq!(value["schema"], 1); + assert_eq!(value["instance_type"], "${var.ec2_instance_type}"); + assert_eq!(value["cluster_arn"], "${aws_ecs_cluster.ec2[0].arn}"); + assert_eq!( + value["capacity_provider"], + "${aws_ecs_capacity_provider.ec2[0].name}" + ); + assert_eq!(value["user_tag"], "${var.ec2_user_tag}"); + assert_eq!( + flat(attr(¶m.body, "depends_on").unwrap()), + "[aws_ecs_cluster_capacity_providers.ec2]" + ); +} + +/// Removing EC2 capacity tears it down in a safe order (see ec2.tf): the +/// cluster outlives the launch template (its user data names the cluster), +/// so every instance is gone before the cluster is deleted. +#[test] +fn the_cluster_outlives_its_instances() { + let ec2 = ec2_body(); + let lt = resource(&ec2, "aws_launch_template", "ec2"); + assert!(render(attr(<.body, "user_data").unwrap()).contains("aws_ecs_cluster.ec2[0].name")); + let asg = resource(&ec2, "aws_autoscaling_group", "ec2"); + assert!(hcl::format::to_string(&asg) + .unwrap() + .contains("aws_launch_template.ec2[0].id")); + let cp = resource(&ec2, "aws_ecs_capacity_provider", "ec2"); + assert!(hcl::format::to_string(&cp) + .unwrap() + .contains("aws_autoscaling_group.ec2[0].arn")); + let ccp = resource(&ec2, "aws_ecs_cluster_capacity_providers", "ec2"); + assert!(hcl::format::to_string(&ccp) + .unwrap() + .contains("aws_ecs_capacity_provider.ec2[0].name")); +} diff --git a/app/src-tauri/src/commands/environment_commands.rs b/app/src-tauri/src/commands/environment_commands.rs index 3edc6a5..dd7484f 100644 --- a/app/src-tauri/src/commands/environment_commands.rs +++ b/app/src-tauri/src/commands/environment_commands.rs @@ -1331,7 +1331,7 @@ pub async fn onboard_aws_user( let owner = validate_owner_id(&ctx.who.user_id).map_err(other)?; let found = detect(&ctx).await?; onboard_refusal(&found, &env).map_err(other)?; - let vars = onboarding_vars(&env, &ctx.region, &owner); + let vars = onboarding_vars(&env, &ctx.region, &owner, None).map_err(other)?; let key = DeploymentKey::User { owner_key: owner_key(&ctx.region, &owner), env, @@ -1420,7 +1420,7 @@ pub async fn save_aws_admin_bundle( .map(str::trim) .ok_or_else(|| other("The user's owner id (aws:userid) is required")) .and_then(|o| validate_owner_id(o).map_err(other))?; - let vars = onboarding_vars(&env, ®ion_ok, &owner); + let vars = onboarding_vars(&env, ®ion_ok, &owner, None).map_err(other)?; ( DeploymentKey::User { owner_key: owner_key(®ion_ok, &owner), diff --git a/app/src-tauri/src/commands/environment_commands/tests.rs b/app/src-tauri/src/commands/environment_commands/tests.rs index b419554..b7829c7 100644 --- a/app/src-tauri/src/commands/environment_commands/tests.rs +++ b/app/src-tauri/src/commands/environment_commands/tests.rs @@ -195,7 +195,7 @@ fn bundles_contain_the_module_backend_vars_and_readme() { let dir = tempfile::tempdir().unwrap(); let ok = owner_key(REGION, USER_ID); let d = Deployment::User { env: "dev", owner_key: &ok }; - let vars = onboarding_vars("dev", REGION, USER_ID); + let vars = onboarding_vars("dev", REGION, USER_ID, None).unwrap(); let written = write_bundle(dir.path(), d, ACCOUNT, REGION, &vars).unwrap(); for (name, text) in module_files(ModuleKind::UserOnboard) { diff --git a/aws/terraform/user-onboard/ec2.tf b/aws/terraform/user-onboard/ec2.tf new file mode 100644 index 0000000..30d2578 --- /dev/null +++ b/aws/terraform/user-onboard/ec2.tf @@ -0,0 +1,314 @@ +# The user's own EC2 capacity (phase 7, E1–E4, E11), made only when +# `ec2_instance_type` is set: every resource and data source here has +# `count = local.ec2_count`, so an onboarding without it plans exactly as +# before. One cluster per user (`triple-c---u-`, the name the +# workspace policy lets only that `triple-c-user` principal tag run tasks on) +# with exactly one capacity provider: an Auto Scaling group of 0 to +# `ec2_max_instances` instances that ECS's managed scaling grows when a task +# needs one and shrinks back to zero when none does. +# +# Removing it (re-applying with an empty type, or destroying) goes in +# dependency order: the user record first, then the cluster's provider list +# (ECS refuses while a task still runs on the provider, so a running +# workspace stops the removal before anything is torn down), the capacity +# provider, the group (`force_delete` off: the provider first drains it to +# zero, clearing scale-in protection, and waits for the instances to +# terminate), the launch template, the instance profile and role, and last +# the cluster, which the launch template's user data references so that it +# outlives every instance registered into it. + +locals { + ec2 = var.ec2_instance_type != "" + ec2_count = local.ec2 ? 1 : 0 + + # `user_ec2_stem` and `user_instance_role_name` in tofu_modules.rs. + ec2_name = "triple-c-${var.environment_name}--u-${var.ec2_user_tag}" + instance_role_name = "${local.name}-instance" + + # Graviton families (m7g, c7g, r7g, t4g: a `g` after the generation) run + # the arm64 image. + ec2_arm64 = can(regex("^[a-z]+[0-9]+g[a-z]*\\.", var.ec2_instance_type)) + ecs_ami_parameter = ( + local.ec2_arm64 + ? "/aws/service/ecs/optimized-ami/amazon-linux-2023/arm64/recommended/image_id" + : "/aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id" + ) +} + +# The provider's default tags (component, environment, owner, version), for +# the places they don't reach by themselves: the launch template's tag +# specifications and the group's tags. +data "aws_default_tags" "current" { + count = local.ec2_count +} + +# The current ECS-optimized Amazon Linux 2023 image, resolved at apply time. +data "aws_ssm_parameter" "ecs_ami" { + count = local.ec2_count + name = local.ecs_ami_parameter +} + +resource "aws_ecs_cluster" "ec2" { + count = local.ec2_count + name = local.ec2_name + + setting { + name = "containerInsights" + value = "disabled" + } +} + +# The instance's role: the ECS agent on this user's cluster only (the +# boundary allows the agent's calls only on the cluster this role's +# `triple-c:cluster` tag names), image pulls and the environment's logs. +# ECS Exec needs nothing here: its channels run on the task role. +resource "aws_iam_role" "instance" { + count = local.ec2_count + name = local.instance_role_name + description = "Triple-C ${var.environment_name}: EC2 instances of ${var.owner_id}" + permissions_boundary = local.boundary_arn + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) + + tags = { "triple-c:cluster" = local.ec2_name } +} + +resource "aws_iam_role_policy" "instance" { + count = local.ec2_count + name = "triple-c-instance" + role = aws_iam_role.instance[0].id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "EcsAgentOnOwnCluster" + Effect = "Allow" + Action = [ + "ecs:RegisterContainerInstance", + "ecs:DeregisterContainerInstance", + "ecs:Poll", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "ecs:UpdateContainerInstancesState", + ] + Resource = [ + "arn:${local.partition}:ecs:${var.region}:${local.account}:cluster/${local.ec2_name}", + "arn:${local.partition}:ecs:${var.region}:${local.account}:container-instance/${local.ec2_name}/*", + ] + }, + { + Sid = "EcsAgentDiscovery" + Effect = "Allow" + Action = "ecs:DiscoverPollEndpoint" + Resource = "*" + }, + { + Sid = "EcrAuthorization" + Effect = "Allow" + Action = "ecr:GetAuthorizationToken" + Resource = "*" + }, + { + Sid = "PullTripleCImages" + Effect = "Allow" + Action = [ + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchCheckLayerAvailability", + ] + Resource = [ + "arn:${local.partition}:ecr:${var.region}:${local.account}:repository/triple-c-*", + "arn:${local.partition}:ecr:${var.region}:${local.account}:repository/triple-c/*", + ] + }, + { + # The awslogs driver runs in the instance's Docker daemon. + Sid = "WriteEnvironmentLogs" + Effect = "Allow" + Action = ["logs:CreateLogStream", "logs:PutLogEvents"] + Resource = local.log_group_arn + }, + ] + }) +} + +resource "aws_iam_instance_profile" "instance" { + count = local.ec2_count + name = aws_iam_role.instance[0].name + role = aws_iam_role.instance[0].name +} + +resource "aws_launch_template" "ec2" { + count = local.ec2_count + name = local.ec2_name + description = "Triple-C ${var.environment_name}: EC2 capacity of ${var.owner_id}" + + image_id = data.aws_ssm_parameter.ecs_ami[0].insecure_value + instance_type = var.ec2_instance_type + + iam_instance_profile { + name = aws_iam_instance_profile.instance[0].name + } + + # The environment's tasks group (outbound only), named in the template so + # the admin policy sees it as a launch-template resource. A public IP only + # where the environment's tasks get one: in a public-subnet environment + # the agent reaches ECS through the internet gateway. + network_interfaces { + device_index = 0 + security_groups = [local.env.security_group_id] + associate_public_ip_address = local.env.assign_public_ip + delete_on_termination = true + } + + # IMDSv2 only, one hop: the agent (host network) reaches it, bridge + # containers don't, and awsvpc tasks are blocked by the agent below. + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + instance_metadata_tags = "disabled" + } + + block_device_mappings { + device_name = "/dev/xvda" + + ebs { + volume_size = 50 + volume_type = "gp3" + encrypted = true + delete_on_termination = true + } + } + + user_data = base64encode(<<-EOT + #!/bin/bash + cat >>/etc/ecs/ecs.config <<'EOF' + ECS_CLUSTER=${aws_ecs_cluster.ec2[0].name} + ECS_AWSVPC_BLOCK_IMDS=true + EOF + EOT + ) + + # Default tags don't reach what a launch creates: the instance, its + # volume and its interface are tagged here (the admin policy lets a launch + # create only component-tagged instances and volumes). + tag_specifications { + resource_type = "instance" + tags = data.aws_default_tags.current[0].tags + } + + tag_specifications { + resource_type = "volume" + tags = data.aws_default_tags.current[0].tags + } + + tag_specifications { + resource_type = "network-interface" + tags = data.aws_default_tags.current[0].tags + } +} + +resource "aws_autoscaling_group" "ec2" { + count = local.ec2_count + name = local.ec2_name + vpc_zone_identifier = local.env.subnet_ids + + # Zero when idle; ECS's managed scaling sets the desired count. + min_size = 0 + max_size = var.ec2_max_instances + protect_from_scale_in = true + force_delete = false + + # A fixed version, never $Latest/$Default (the admin policy refuses a + # group that would follow later versions). Running instances keep theirs + # until they scale in. + launch_template { + id = aws_launch_template.ec2[0].id + version = aws_launch_template.ec2[0].latest_version + } + + # Provider default tags don't apply to groups. + dynamic "tag" { + for_each = data.aws_default_tags.current[0].tags + content { + key = tag.key + value = tag.value + propagate_at_launch = false + } + } + + # ECS adds this when the capacity provider takes the group over; declared + # so a re-apply doesn't remove it. + tag { + key = "AmazonECSManaged" + value = "true" + propagate_at_launch = true + } + + # The drain before deletion waits for the instances to terminate. + timeouts { + delete = "20m" + } + + lifecycle { + ignore_changes = [desired_capacity] + } +} + +resource "aws_ecs_capacity_provider" "ec2" { + count = local.ec2_count + name = local.ec2_name + + auto_scaling_group_provider { + auto_scaling_group_arn = aws_autoscaling_group.ec2[0].arn + + # ECS protects an instance from scale-in while a task runs on it. + managed_termination_protection = "ENABLED" + + managed_scaling { + status = "ENABLED" + target_capacity = 100 + minimum_scaling_step_size = 1 + maximum_scaling_step_size = 1 + instance_warmup_period = 60 + } + } +} + +resource "aws_ecs_cluster_capacity_providers" "ec2" { + count = local.ec2_count + cluster_name = aws_ecs_cluster.ec2[0].name + capacity_providers = [aws_ecs_capacity_provider.ec2[0].name] + + default_capacity_provider_strategy { + capacity_provider = aws_ecs_capacity_provider.ec2[0].name + weight = 1 + } +} + +# The user's record (E11): what the app reads to run EC2 projects, since +# workspace users can't read the state bucket. Outside +# /triple-c/environments/, so environment detection never sees it. Written +# last, removed first. +resource "aws_ssm_parameter" "user" { + count = local.ec2_count + name = "/triple-c/users/${var.environment_name}/${var.owner_key}" + description = "Triple-C ${var.environment_name}: EC2 capacity of ${var.owner_id}" + type = "String" + value = jsonencode({ + schema = 1 + instance_type = var.ec2_instance_type + cluster_arn = aws_ecs_cluster.ec2[0].arn + capacity_provider = aws_ecs_capacity_provider.ec2[0].name + user_tag = var.ec2_user_tag + }) + + depends_on = [aws_ecs_cluster_capacity_providers.ec2] +} diff --git a/aws/terraform/user-onboard/main.tf b/aws/terraform/user-onboard/main.tf index e100951..33edf21 100644 --- a/aws/terraform/user-onboard/main.tf +++ b/aws/terraform/user-onboard/main.tf @@ -1,10 +1,11 @@ # One user's place in an environment: their task and execution roles and -# their own EFS file system (D2, strict tenancy). The environment itself is -# read only from its record, never from its state. +# their own EFS file system (D2, strict tenancy), and optionally their own +# EC2 capacity (ec2.tf, phase 7). The environment itself is read only from +# its record, never from its state. locals { # ONBOARD_MODULE_VERSION in app/src-tauri/src/aws/tofu_modules.rs. - template_version = 1 + template_version = 2 name = "triple-c-${var.environment_name}-${var.owner_key}" partition = data.aws_partition.current.partition @@ -64,10 +65,11 @@ data "aws_ssm_parameter" "environment" { )) && jsondecode(self.insecure_value).log_group == "/triple-c/${var.environment_name}" && can(regex("^sg-[0-9a-f]{8,17}$", jsondecode(self.insecure_value).security_group_id)) - && alltrue([for s in jsondecode(self.insecure_value).subnet_ids : can(regex("^subnet-[0-9a-f]{8,17}$", s))]), + && alltrue([for s in jsondecode(self.insecure_value).subnet_ids : can(regex("^subnet-[0-9a-f]{8,17}$", s))]) + && (jsondecode(self.insecure_value).assign_public_ip == true || jsondecode(self.insecure_value).assign_public_ip == false), false ) - error_message = "The environment record has a malformed kms_key_arn (not a key in this region and account), log_group (not /triple-c/), security_group_id or subnet id." + error_message = "The environment record has a malformed kms_key_arn (not a key in this region and account), log_group (not /triple-c/), security_group_id, subnet id or assign_public_ip." } } } diff --git a/aws/terraform/user-onboard/outputs.tf b/aws/terraform/user-onboard/outputs.tf index a630c65..8926882 100644 --- a/aws/terraform/user-onboard/outputs.tf +++ b/aws/terraform/user-onboard/outputs.tf @@ -18,6 +18,20 @@ output "efs_security_group_id" { value = aws_security_group.efs.id } +# The user's EC2 capacity, null without it. The app reads these from the +# user record (/triple-c/users//), not from the state. +output "ec2_cluster_arn" { + value = one(aws_ecs_cluster.ec2[*].arn) +} + +output "ec2_capacity_provider" { + value = one(aws_ecs_capacity_provider.ec2[*].name) +} + +output "ec2_instance_type" { + value = local.ec2 ? var.ec2_instance_type : null +} + # Written only when every resource above was applied: a run that fails # part-way never evaluates it, so the app reads its absence from the state # as "the last onboarding didn't finish" (Incomplete, Join re-applies) @@ -34,5 +48,14 @@ output "onboarding_complete" { aws_iam_role_policy.task_bedrock, aws_iam_role.exec, aws_iam_role_policy.exec, + aws_ecs_cluster.ec2, + aws_iam_role.instance, + aws_iam_role_policy.instance, + aws_iam_instance_profile.instance, + aws_launch_template.ec2, + aws_autoscaling_group.ec2, + aws_ecs_capacity_provider.ec2, + aws_ecs_cluster_capacity_providers.ec2, + aws_ssm_parameter.user, ] } diff --git a/aws/terraform/user-onboard/variables.tf b/aws/terraform/user-onboard/variables.tf index eee1894..f9d8805 100644 --- a/aws/terraform/user-onboard/variables.tf +++ b/aws/terraform/user-onboard/variables.tf @@ -54,3 +54,58 @@ variable "bedrock_on_task_role" { type = bool default = false } + +# Optional per-user EC2 capacity (phase 7). Empty = none: the module then +# makes nothing for EC2 and the user's projects run on Fargate only. +variable "ec2_instance_type" { + description = "The instance type of the user's own EC2 capacity (e.g. m7i.large or m7g.large), or empty for none. Only the types the app can size are accepted." + type = string + default = "" + + # The app's table (`ec2_instance_spec`) and the admin policy's lists. + validation { + condition = var.ec2_instance_type == "" || contains([ + "m7i.large", "m7i.xlarge", "m7i.2xlarge", "m7i.4xlarge", + "m7g.large", "m7g.xlarge", "m7g.2xlarge", "m7g.4xlarge", + "c7i.large", "c7i.xlarge", "c7i.2xlarge", "c7i.4xlarge", + "c7g.large", "c7g.xlarge", "c7g.2xlarge", "c7g.4xlarge", + "r7i.large", "r7i.xlarge", "r7i.2xlarge", "r7i.4xlarge", + "r7g.large", "r7g.xlarge", "r7g.2xlarge", "r7g.4xlarge", + "t3.large", "t3.xlarge", "t3.2xlarge", + "t4g.large", "t4g.xlarge", "t4g.2xlarge", + ], var.ec2_instance_type) + error_message = "ec2_instance_type must be empty or one of the m7i, m7g, c7i, c7g, r7i, r7g (large to 4xlarge), t3 or t4g (large to 2xlarge) types." + } +} + +variable "ec2_max_instances" { + description = "How many of the user's instances may run at once (1 to 4); each holds one workspace as expected use." + type = number + default = 1 + + validation { + condition = var.ec2_max_instances >= 1 && var.ec2_max_instances <= 4 && floor(var.ec2_max_instances) == var.ec2_max_instances + error_message = "ec2_max_instances must be a whole number from 1 to 4." + } +} + +variable "ec2_user_tag" { + description = "The value of the triple-c-user principal tag the user signs in with; names their EC2 cluster (triple-c---u-). Required with ec2_instance_type." + type = string + default = "" + + validation { + condition = var.ec2_user_tag == "" || ( + can(regex("^[A-Za-z0-9-]{1,32}$", var.ec2_user_tag)) + && !startswith(var.ec2_user_tag, "-") + && !endswith(var.ec2_user_tag, "-") + && !strcontains(var.ec2_user_tag, "--") + ) + error_message = "ec2_user_tag must be 1 to 32 letters, digits and single hyphens, not starting or ending with a hyphen." + } + + validation { + condition = var.ec2_instance_type == "" || var.ec2_user_tag != "" + error_message = "ec2_user_tag (the user's triple-c-user principal tag) is required when ec2_instance_type is set." + } +}