diff --git a/.gitignore b/.gitignore index f84c3a2..63bc985 100644 --- a/.gitignore +++ b/.gitignore @@ -9,12 +9,3 @@ # Python bytecode from the work scraper __pycache__/ *.py[cod] - -# OpenTofu / Terraform -infra/.terraform/ -infra/*.tfstate -infra/*.tfstate.* -infra/*.tfplan -infra/crash.log -infra/override.tf -infra/override.tf.json diff --git a/HANDOFF.md b/HANDOFF.md deleted file mode 100644 index eaf8d7f..0000000 --- a/HANDOFF.md +++ /dev/null @@ -1,138 +0,0 @@ -# Handoff: integrate Tally into `permadeath-aws-org` - -An agent with access to the private `jmeickle/permadeath-aws-org` repo should -use this document to wire the Tally stack into `envs/devinfra` and add a -private S3 dead-drop bucket for integration results. - -## Source of truth - -- Public app repo: https://tangled.org/permadeath.com/dev.permadeath.com -- Work to integrate: **PR #2** — https://tangled.org/permadeath.com/dev.permadeath.com/pulls/2 -- Terraform: [`infra/`](infra/) -- Existing integration notes: [`infra/INTEGRATION.md`](infra/INTEGRATION.md) -- Latest commits: - - `f6b271d` — `main`, the merged dashboard service - - `da9213c` — private-repo hash default - - `a2c32b3` — local repo-salt file - - `de5d473` — local upload-token file - -Merge PR #2 first, then copy `infra/` from `main`. Do not copy the generated -`.terraform/` directory or any `*.tfstate` / `*.tfplan`. When vendoring it as a -module, remove `backend.tf` and the `provider "aws"` block from `versions.tf`; -the platform root owns both. - -## Do not do this - -**Do not apply `infra/` as a separate Terraform root.** It would create a -second state bucket, ECS cluster, CloudFront distribution and DNS records -alongside the platform root. Integrate it as a module, or paste the resources -into `envs/devinfra`. - -## Integration shape - -1. Vendor `infra/` into the private repo as `modules/tally/`, or reference the - Tangled repo at a pinned commit. Vendoring is simpler for the first pass. -2. Add a `module "tally"` call in `envs/devinfra`. -3. Supply the platform root's existing values instead of letting the module - create duplicates: - - Route 53 zone ID for `dev.permadeath.com` - - ECS cluster ARN, VPC ID, subnet IDs, security group IDs - - state bucket name - - ECR repository URL (or let the module create one) - - ACM certificate ARN if already issued - - SSM parameter name for the personal GitHub token - - any permissions boundary or standard task-role ARN -4. Run `tofu fmt -recursive`, `tofu validate`, and `tofu plan`. Do not apply - without reviewing the plan. -5. After apply, confirm: - - `https://dev.permadeath.com` serves the dashboard - - the hourly EventBridge schedule launches the ECS task - - the task can read its state and write the site bucket - - the work inbox object is readable at `inbox/work-activity.json` - -The module is not currently parameterized for an externally supplied ECS -cluster ARN; add that input if the platform root owns the cluster. - -## Inputs the module needs - -Read `infra/variables.tf` for the full list. The important ones: - -| Input | Notes | -|---|---| -| `region` | `us-east-1` for CloudFront/ACM | -| `domain_name` | `dev.permadeath.com` | -| `route53_zone_name` | `dev.permadeath.com` | -| `manage_dns` | `true` only if the module should write the A/AAAA aliases | -| `acm_certificate_arn` | Prefer the platform certificate if one exists | -| `state_bucket_name` | Dedicated app state bucket, or an approved prefix | -| `site_bucket_name` | Optional; otherwise derived | -| `vpc_id`, `subnet_ids`, `assign_public_ip` | Fargate networking | -| `github_token_parameter_name` | SSM SecureString name, not the token | -| `inbox_urls` | Comma-separated `s3://…/inbox/*.json` payloads | -| `personal_dids`, `personal_emails` | Your Tangled identities | -| `didbot_pds` | Default `https://pds.did.bot` | -| `schedule_expression` | Default hourly | -| `image_tag` | Pin the built image | - -## Dead-drop bucket - -Create one private bucket for handoff output, separate from the site and -state buckets: - -```text -s3://tally-handoff-/ - agent/ # written by the private-root agent - tally/ # written by the Tally task role -``` - -Requirements: - -- Block all public access; SSE-S3 or SSE-KMS; versioning on; 30-day - noncurrent-version expiry. -- The private-root agent role gets `s3:PutObject` on `agent/*` only. -- The Tally task role gets `s3:PutObject` on `tally/*` only. -- A human/root role gets read access to both prefixes. -- No presigned public URLs. No secrets, tokens, `.tfstate`, `.tfplan` or - private repo contents in this bucket. - -The agent should write: - -- `agent/terraform-plan.txt` — plan summary with secrets redacted -- `agent/terraform-apply.txt` — apply output, if applied -- `agent/outputs.json` — non-secret module outputs -- `agent/deployed-url.txt` — the live dashboard URL -- `agent/notes.md` — deviations, missing inputs, follow-ups - -The Tally task should write: - -- `tally/status.json` — last crawl summary -- `tally/latest.json` — rendered dashboard data, if useful for another service -- optional `tally/site/` — only if the platform root wants a second copy of - the static site - -## Secrets and data handling - -- Personal GitHub token: SSM SecureString; pass only the parameter name. -- Work GitHub activity: local scraper on the work machine; uploads only SHA, - author date, public repo name or salted private-repo hash, and optional - `Change-Id`. -- Repo-hash salt: stays on the work machine at `~/.config/tally/repo-salt`. - It is not an AWS secret and must not be uploaded. -- Upload token: local file `~/.config/tally/upload-token`, or - `TALLY_UPLOAD_TOKEN`; not in Terraform state if avoidable. -- Do not put tokens, SQLite state, Terraform state or plan files in the - handoff bucket. - -## Definition of done - -- [ ] `envs/devinfra` owns the Tally resources; no duplicate DNS, VPC, state - bucket or ECS cluster. -- [ ] `tofu plan` is reviewed and clean. -- [ ] `https://dev.permadeath.com` serves the dashboard. -- [ ] Hourly ECS task runs successfully. -- [ ] Work inbox is readable and updates the dashboard. -- [ ] Dead-drop bucket exists with the two write-only prefixes and no public - access. -- [ ] Integration results are posted under `agent/`. -- [ ] No secrets or private repo contents are in the bucket or the public app - repo. diff --git a/Makefile b/Makefile index 4b3dae1..2d0bcb5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: check test fmt clippy build docker site serve infra-check +.PHONY: check test fmt clippy build docker site serve check: cargo check @@ -24,6 +24,3 @@ site: serve: cargo run -- serve --dir dist --bind 127.0.0.1:8787 - -infra-check: - cd infra && tofu init -backend=false -input=false >/dev/null && tofu fmt -check -recursive && tofu validate diff --git a/README.md b/README.md index 9cf4fa7..b94628f 100644 --- a/README.md +++ b/README.md @@ -189,8 +189,8 @@ The worker lists `/user/repos` and also enumerates every owner named in returned by the authenticated account's repository list. Commits are filtered to `author=`. -Store the token in SSM SecureString and pass its parameter name to Terraform as -`github_token_parameter_name`. +Store the token in SSM SecureString and give its parameter name to the infra +stack as `github_token_parameter_name`. ### Either account: upload a report instead of a token @@ -270,160 +270,32 @@ The work machine's credential needs one permission: } ``` -`tofu output work_inbox_uri` prints the URI to give the script, and -`tofu output work_inbox_policy` prints the exact write-only policy for the -work machine's credential. Terraform sets `TALLY_WORK_INBOX` from the same key -when `enable_work_inbox` is true (the default). +The infrastructure repo owns the state bucket and the write-only policy. Point +the scraper at the `s3://…/inbox/work-activity.json` URI it provides. A direct `GITHUB_WORK_TOKEN` is also supported, but it requires the service to hold the token and see repository names. Prefer the inbox. -## AWS deployment +## Deployment -> **Integration note.** The platform Terraform root (`envs/devinfra`) already -> owns the AWS account, VPC, Route 53 delegation and Terraform state. Do not -> apply this directory as a separate root in that account. See -> [`infra/INTEGRATION.md`](infra/INTEGRATION.md) for the inputs to map from that -> root; the app resources should be wired as a `module "tally"` call (or copied -> into the root). The steps below describe the app stack in isolation and are -> useful once those inputs are known. +This repository is the application only. AWS infrastructure lives in the +private `jmeickle/permadeath-aws-org` repo (`envs/devinfra/site*.tf` and +`modules/site`). -The stack is an hourly EventBridge Scheduler run of a Fargate task. Each run: +The app side of the contract is: -1. restores `state/tally.sqlite3` from S3 if the task has no local index, -2. crawls and deduplicates, -3. writes the static site, -4. uploads the site and a fresh index snapshot, -5. exits. +- build the image from `Dockerfile` and push it to the platform ECR repo +- run the nightly task as `crawl --global`, with its own + `TALLY_STATE_KEY=state/global.sqlite3`, `TALLY_GLOBAL_SUMMARY_OUT`, no time + budget, and no GitHub/inbox configuration +- run the hourly task as `crawl --owned-only`, with `TALLY_STATE_KEY` pointing + at the hourly snapshot and `TALLY_GLOBAL_SUMMARY` pointing at the nightly + summary +- give the task roles the S3 state/site permissions, SSM token access and + networking described in the infra repo -CloudFront serves the site. No load balancer or always-on service is required. - -### Where `dev.permadeath.com` is delegated - -It is already an **AWS Route 53 hosted zone**, separate from the Cloudflare -zone that serves the `permadeath.com` apex: - -``` -dev.permadeath.com NS: - ns-1026.awsdns-00.org - ns-1889.awsdns-44.co.uk - ns-420.awsdns-52.com - ns-996.awsdns-60.net - -permadeath.com NS: - carol.ns.cloudflare.com - ivan.ns.cloudflare.com -``` - -The delegation is live, but the Route 53 zone currently has no A, AAAA, CNAME, -or certificate records. No registrar or Cloudflare change is needed: the -Terraform writes the ACM validation records and the CloudFront A/AAAA aliases -into that existing Route 53 zone. - -To identify the owning account and zone ID (needs credentials for that -account): - -```bash -aws route53 list-hosted-zones-by-name --dns-name dev.permadeath.com. -``` - -The atgc.codes account is not necessarily the same account; the zone has its -own nameservers. If the zone is in a different account from -the service, set `manage_dns=false` and create the A/AAAA aliases and ACM -validation records in that account, or add a cross-account Route 53 provider. - -### One-time setup - -- AWS credentials for the account that will run the ECS task. -- An S3 state bucket in that account. The org's convention is - `permadeath-tfstate-` in `us-east-1`. -- A GitHub token in SSM SecureString: - -```bash -aws ssm put-parameter \ - --name /tally/github-token \ - --type SecureString \ - --value "$GITHUB_TOKEN" -``` - -### 1. Bootstrap ECR and push the image - -The task definition needs an image, and the ECR repository is created by the -same Terraform. Apply the repository first: - -```bash -cd infra -tofu init \ - -backend-config="bucket=permadeath-tfstate-" \ - -backend-config="key=dev.permadeath.com/terraform.tfstate" \ - -backend-config="region=us-east-1" \ - -backend-config="encrypt=true" \ - -backend-config="use_lockfile=true" - -tofu apply -target=aws_ecr_repository.tally - -aws ecr get-login-password --region us-east-1 \ - | docker login --username AWS --password-stdin \ - .dkr.ecr.us-east-1.amazonaws.com -docker build -t tally . -docker tag tally .dkr.ecr.us-east-1.amazonaws.com/tally:latest -docker push .dkr.ecr.us-east-1.amazonaws.com/tally:latest -``` - -### 2. Apply the stack - -```bash -tofu apply \ - -var 'image_tag=latest' \ - -var 'github_token_parameter_name=/tally/github-token' -``` - -The defaults are already aimed at this host: - -- `domain_name = "dev.permadeath.com"` -- `route53_zone_name = "dev.permadeath.com."` -- `manage_dns = true` - -Terraform creates the ACM certificate in `us-east-1`, writes its validation -record into the Route 53 zone, waits for validation, creates CloudFront, and -writes the apex A/AAAA aliases for `dev.permadeath.com`. - -If the zone is in another account, pass `-var 'manage_dns=false'` and either -supply an existing certificate with `acm_certificate_arn` or create the aliases -manually. `tofu output dns_instructions` says exactly what is needed. - -### 3. Run the first crawl - -The scheduler fires at `:07` every hour. To start immediately, run one task: - -```bash -aws ecs run-task \ - --cluster tally \ - --task-definition tally \ - --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[...],securityGroups=[...],assignPublicIp=ENABLED}" -``` - -Then: - -```bash -curl -I https://dev.permadeath.com/ -tofu output site_url -``` - -The first run is a global backfill and is bounded by -`TALLY_CRAWL_BUDGET_SECS` (2700 by default); the dashboard shows coverage and -labels the share preliminary until the backfill reaches it. Subsequent hourly -runs resume from the SQLite snapshot in S3. - -If you are migrating an existing local database, run it once with `--full` so -the `Change-Id` trailers are populated from commit messages. - -The default VPC and its subnets are used unless `vpc_id`/`subnet_ids` are set. -`assign_public_ip` should be false when the subnets route through a NAT. - -This workspace has no AWS credentials, so the commands above were not run -here; the Terraform configuration validates with `tofu validate`. +The worker is a one-shot Fargate task. CloudFront serves the static site; +there is no load balancer or always-on service. ## Tests @@ -431,14 +303,13 @@ here; the Terraform configuration validates with `tofu validate`. cargo test cargo check --features aws cargo fmt --check -cargo clippy -- -D warnings -cd infra && tofu fmt -check -recursive && tofu validate +cargo clippy --all-targets --features aws -- -D warnings ``` The unit tests cover the de-duplication invariant (same SHA on many branches and in two repositories), identity upgrades without duplication, the Git SHA -validator, repository-hash sanitising, the palette's stable bot colours, and -the S3 prefix/object-key rules. +validator, repository-hash sanitising, the palette's stable bot colours, the +S3 prefix/object-key rules and the global-summary period maths. ## Limits diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl deleted file mode 100644 index eb1cd87..0000000 --- a/infra/.terraform.lock.hcl +++ /dev/null @@ -1,25 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/aws" { - version = "6.69.1" - constraints = ">= 5.0.0, < 7.0.0" - hashes = [ - "h1:nqCg6epkPJ6nmQSE9pixmqwXSQEfqaUaOCYDLFOWKKg=", - "zh:2c654ec9ec3ea7df88eec27dd0a6389bf99b50e085b07ab5c542c05795ab16bf", - "zh:3ab10525d38208c63b32de8e037c10bae6b36107668b5b366ca69ee0be0e715f", - "zh:3cb49ed424b650de880e31f2910c2a5f78c670365962dbfc85093b46bc7684c0", - "zh:432b9fd680abe0bf60d34ac81b0b886fc651f642b4f2ebf128856c0fa6d5d628", - "zh:4be20d9ae9365828a745da1ebc323d1f1e2b6427a219df559a7c8a1609e16ceb", - "zh:522e51e5bf8bdcca3b27aa3fe7ea7c5072322387fdfad0913d892cc8b715fc09", - "zh:5d8177725146736907d9a65869889a65d669520a2a061642dd416bf4286c7d53", - "zh:5f8cecfa3fd37cf8f3d6ceb885cf0b05b3768a0931511214f62655e2d53ec31b", - "zh:889bff04fe32c1498f60566ab84c8eb0b2fe23ef58a4ba089f0b1f7d63142b05", - "zh:99e2797ea0193af088882f0a3de451df380257205481a3cc67aa40446700a230", - "zh:a309d73f66b577f35c6412bd62e239e6fce1f59c167caeece0b40d20220f6b37", - "zh:a4449853cc0c767d702332c4af81d9796b4e55ea1cead1dd0cfff1ab604e458a", - "zh:da7390b14b3cd51fa429c6c6b8cc18fd8ab0f29197b0dc1a684d5bf59c9d5d85", - "zh:f0107fc5637804487fa320a10907c9a2cec650115816e6add0b4654ede603dfa", - "zh:f073a72a36c918dde37512aa93229719eba57fe55b9808731d2adeb699f6256d", - ] -} diff --git a/infra/INTEGRATION.md b/infra/INTEGRATION.md deleted file mode 100644 index 5ec53b2..0000000 --- a/infra/INTEGRATION.md +++ /dev/null @@ -1,71 +0,0 @@ -# Integrating tally with the platform Terraform root - -The platform Terraform root (`envs/devinfra`) owns the AWS account, VPC, -Route 53 delegation and Terraform state. **Do not apply `infra/` as a -separate root in that account.** It would try to create its own state bucket, -ECS cluster, and DNS/certificate records alongside the ones the root already -manages. - -The intended shape is a `module "tally"` call in that root, or the equivalent -resources added directly there. The private root was not readable from this -workspace, so the exact output names below need to be mapped from it. - -## What the app needs from the root - -| input | why | -|---|---| -| `aws_region` | Must be `us-east-1` for CloudFront's ACM certificate. | -| `route53_zone_id` for `dev.permadeath.com` | ACM validation records and the CloudFront A/AAAA aliases. The delegation already exists; no Cloudflare change is needed. | -| `ecs_cluster_arn` (or permission to create the app cluster) | The scheduled Fargate task and EventBridge Scheduler target. | -| `vpc_id`, `subnet_ids`, `security_group_ids` (or permission to create a task SG) | Fargate `awsvpc` networking. | -| `state_bucket_name` | The worker restores and snapshots `state/tally.sqlite3` here. The org convention is `permadeath-tfstate-`, but a dedicated app bucket is fine. | -| `ecr_repository_url` (or permission to create one) | The task image. If the root owns ECR, the module needs the repository URL and the image tag. | -| `acm_certificate_arn` (optional) | If the root already issues certificates, pass it. Otherwise the app can create and validate one in the existing zone. | -| `github_token_parameter_name` | SSM SecureString holding the personal GitHub token. | -| `work_inbox_url` | `s3://.../inbox/work-activity.json`; the write-only work-machine inbox. | -| IAM conventions / permission boundary | The app currently creates task, execution and scheduler roles. If the root has standard roles or a permissions boundary, pass those ARNs or attach the boundary. | - -## App-owned resources - -Unless the root already provides them, the app stack creates: - -- S3 site bucket (private, CloudFront OAC) and state bucket -- CloudFront distribution -- ECR repository -- ECS cluster (when no `ecs_cluster_arn` is supplied) -- task definition, EventBridge Scheduler schedule, IAM roles, CloudWatch log group -- ACM certificate and Route 53 validation/alias records when `manage_dns = true` - -## Current variables that map cleanly - -These already exist in `variables.tf`: - -- `region` -- `project` -- `image_tag` -- `site_bucket_name` -- `state_bucket_name` -- `domain_name` (default `dev.permadeath.com`) -- `route53_zone_name` -- `manage_dns` -- `acm_certificate_arn` -- `vpc_id` -- `subnet_ids` -- `assign_public_ip` -- `schedule_expression` -- `github_token_parameter_name` -- `work_inbox_url` - -## What to paste back - -The fastest path is the relevant part of `envs/devinfra` (or its outputs): - -1. the Route 53 zone ID for `dev.permadeath.com` -2. the ECS cluster ARN, VPC ID, subnet IDs and security group ID(s) -3. the state bucket name and whether the app should create the site bucket/ECR -4. whether ACM is already issued, and the SSM parameter naming convention -5. any permissions-boundary or role-ARN convention for task roles - -With those, `infra/` can be turned into a module whose inputs are exactly -the root's outputs, and the `module "tally"` block can be added to -`envs/devinfra` without duplicating the org's DNS, VPC, state, or cluster. diff --git a/infra/backend.tf b/infra/backend.tf deleted file mode 100644 index f9db50f..0000000 --- a/infra/backend.tf +++ /dev/null @@ -1,13 +0,0 @@ -# Partial S3 backend. The org creates one state bucket per AWS account, named -# permadeath-tfstate-, versioned and encrypted. Pass its details at -# init time so this file stays account-neutral: -# -# tofu init \ -# -backend-config="bucket=permadeath-tfstate-" \ -# -backend-config="key=dev.permadeath.com/terraform.tfstate" \ -# -backend-config="region=us-east-1" \ -# -backend-config="encrypt=true" \ -# -backend-config="use_lockfile=true" -terraform { - backend "s3" {} -} diff --git a/infra/main.tf b/infra/main.tf deleted file mode 100644 index 3b91d8f..0000000 --- a/infra/main.tf +++ /dev/null @@ -1,601 +0,0 @@ -data "aws_caller_identity" "current" {} -data "aws_region" "current" {} - -locals { - account_id = data.aws_caller_identity.current.account_id - site_bucket = ( - var.site_bucket_name != "" - ? var.site_bucket_name - : "${var.project}-site-${local.account_id}" - ) - state_bucket = ( - var.state_bucket_name != "" - ? var.state_bucket_name - : "${var.project}-state-${local.account_id}" - ) - work_inbox_key = "inbox/work-activity.json" - personal_inbox_key = "inbox/personal-activity.json" - work_inbox_url = ( - var.work_inbox_url != "" - ? var.work_inbox_url - : var.enable_work_inbox - ? "s3://${local.state_bucket}/${local.work_inbox_key}" - : "" - ) - - ssm_parameter_names = compact([ - var.github_token_parameter_name, - var.github_work_token_parameter_name, - ]) - ssm_parameter_arns = [ - for name in local.ssm_parameter_names : - "arn:aws:ssm:${var.region}:${local.account_id}:parameter/${trimprefix(name, "/")}" - ] - - certificate_arn = ( - var.acm_certificate_arn != "" - ? var.acm_certificate_arn - : try(aws_acm_certificate_validation.site[0].certificate_arn, "") - ) - use_custom_domain = ( - var.domain_name != "" && (var.acm_certificate_arn != "" || var.manage_dns) - ) - site_url = ( - local.use_custom_domain - ? "https://${var.domain_name}" - : "https://${aws_cloudfront_distribution.site.domain_name}" - ) - - vpc_id = var.vpc_id != "" ? var.vpc_id : data.aws_vpc.default[0].id - subnet_ids = ( - length(var.subnet_ids) > 0 - ? var.subnet_ids - : data.aws_subnets.default[0].ids - ) -} - -# --------------------------------------------------------------------------- -# Static site - -resource "aws_s3_bucket" "site" { - bucket = local.site_bucket - tags = var.tags -} - -resource "aws_s3_bucket_public_access_block" "site" { - bucket = aws_s3_bucket.site.id - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "site" { - bucket = aws_s3_bucket.site.id - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} - -# The dev.permadeath.com zone is already delegated from the Cloudflare-managed -# permadeath.com apex, so this reads the zone rather than creating it. No -# registrar or Cloudflare change is needed for a deploy. -data "aws_route53_zone" "site" { - count = var.manage_dns ? 1 : 0 - name = var.route53_zone_name - private_zone = false -} - -resource "aws_acm_certificate" "site" { - count = var.manage_dns && var.acm_certificate_arn == "" ? 1 : 0 - domain_name = var.domain_name - validation_method = "DNS" - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_route53_record" "cert_validation" { - for_each = var.manage_dns && var.acm_certificate_arn == "" ? { - for dvo in aws_acm_certificate.site[0].domain_validation_options : dvo.domain_name => { - name = dvo.resource_record_name - type = dvo.resource_record_type - record = dvo.resource_record_value - } - } : {} - - zone_id = data.aws_route53_zone.site[0].zone_id - name = each.value.name - type = each.value.type - records = [each.value.record] - ttl = 300 -} - -resource "aws_acm_certificate_validation" "site" { - count = var.manage_dns && var.acm_certificate_arn == "" ? 1 : 0 - certificate_arn = aws_acm_certificate.site[0].arn - validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn] -} - -resource "aws_cloudfront_origin_access_control" "site" { - name = "${var.project}-site" - description = "OAC for ${local.site_bucket}" - origin_access_control_origin_type = "s3" - signing_behavior = "always" - signing_protocol = "sigv4" -} - -resource "aws_cloudfront_distribution" "site" { - enabled = true - is_ipv6_enabled = true - comment = "${var.project} commit activity" - default_root_object = "index.html" - price_class = "PriceClass_100" - aliases = local.use_custom_domain ? [var.domain_name] : [] - - origin { - domain_name = aws_s3_bucket.site.bucket_regional_domain_name - origin_id = "s3-${aws_s3_bucket.site.id}" - origin_access_control_id = aws_cloudfront_origin_access_control.site.id - } - - default_cache_behavior { - target_origin_id = "s3-${aws_s3_bucket.site.id}" - viewer_protocol_policy = "redirect-to-https" - allowed_methods = ["GET", "HEAD"] - cached_methods = ["GET", "HEAD"] - compress = true - # The worker writes once an hour and the site is tiny. Revalidate with the - # origin so a refreshed dashboard is never a stale CDN hit. - cache_policy_id = "4135ea2d-6df8-44a3-9df3-4b5a84be39ad" - } - - custom_error_response { - error_code = 403 - response_code = 200 - response_page_path = "/index.html" - error_caching_min_ttl = 0 - } - - custom_error_response { - error_code = 404 - response_code = 200 - response_page_path = "/index.html" - error_caching_min_ttl = 0 - } - - dynamic "viewer_certificate" { - for_each = local.use_custom_domain ? [1] : [] - content { - acm_certificate_arn = local.certificate_arn - ssl_support_method = "sni-only" - minimum_protocol_version = "TLSv1.2_2021" - } - } - - dynamic "viewer_certificate" { - for_each = local.use_custom_domain ? [] : [1] - content { - cloudfront_default_certificate = true - } - } - - restrictions { - geo_restriction { - restriction_type = "none" - } - } -} - -resource "aws_route53_record" "site_a" { - count = var.manage_dns ? 1 : 0 - zone_id = data.aws_route53_zone.site[0].zone_id - name = var.domain_name - type = "A" - - alias { - name = aws_cloudfront_distribution.site.domain_name - zone_id = aws_cloudfront_distribution.site.hosted_zone_id - evaluate_target_health = false - } -} - -resource "aws_route53_record" "site_aaaa" { - count = var.manage_dns ? 1 : 0 - zone_id = data.aws_route53_zone.site[0].zone_id - name = var.domain_name - type = "AAAA" - - alias { - name = aws_cloudfront_distribution.site.domain_name - zone_id = aws_cloudfront_distribution.site.hosted_zone_id - evaluate_target_health = false - } -} - -data "aws_iam_policy_document" "site" { - statement { - sid = "CloudFrontRead" - effect = "Allow" - - actions = ["s3:GetObject"] - resources = ["${aws_s3_bucket.site.arn}/*"] - - principals { - type = "Service" - identifiers = ["cloudfront.amazonaws.com"] - } - - condition { - test = "StringEquals" - variable = "AWS:SourceArn" - values = [aws_cloudfront_distribution.site.arn] - } - } -} - -resource "aws_s3_bucket_policy" "site" { - bucket = aws_s3_bucket.site.id - policy = data.aws_iam_policy_document.site.json -} - -# --------------------------------------------------------------------------- -# State and the write-only work inbox - -resource "aws_s3_bucket" "state" { - bucket = local.state_bucket - tags = var.tags -} - -resource "aws_s3_bucket_public_access_block" "state" { - bucket = aws_s3_bucket.state.id - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "state" { - bucket = aws_s3_bucket.state.id - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} - -resource "aws_s3_bucket_versioning" "state" { - bucket = aws_s3_bucket.state.id - - versioning_configuration { - status = "Enabled" - } -} - -resource "aws_s3_bucket_lifecycle_configuration" "state" { - bucket = aws_s3_bucket.state.id - - rule { - id = "expire-old-index-snapshots" - status = "Enabled" - - noncurrent_version_expiration { - noncurrent_days = 14 - } - - abort_incomplete_multipart_upload { - days_after_initiation = 3 - } - } -} - -# --------------------------------------------------------------------------- -# Image, logs, roles - -resource "aws_ecr_repository" "tally" { - name = var.project - image_tag_mutability = "MUTABLE" - - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_lifecycle_policy" "tally" { - repository = aws_ecr_repository.tally.name - - policy = jsonencode({ - rules = [ - { - rulePriority = 1 - description = "Keep the last 20 images" - selection = { - tagStatus = "any" - countType = "imageCountMoreThan" - countNumber = 20 - } - action = { type = "expire" } - } - ] - }) -} - -resource "aws_cloudwatch_log_group" "tally" { - name = "/ecs/${var.project}" - retention_in_days = 30 -} - -data "aws_iam_policy_document" "ecs_assume" { - statement { - actions = ["sts:AssumeRole"] - principals { - type = "Service" - identifiers = ["ecs-tasks.amazonaws.com"] - } - } -} - -resource "aws_iam_role" "execution" { - name = "${var.project}-execution" - assume_role_policy = data.aws_iam_policy_document.ecs_assume.json -} - -resource "aws_iam_role_policy_attachment" "execution" { - role = aws_iam_role.execution.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" -} - -resource "aws_iam_role_policy" "execution_secrets" { - count = length(local.ssm_parameter_arns) > 0 ? 1 : 0 - name = "${var.project}-execution-secrets" - role = aws_iam_role.execution.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = ["ssm:GetParameters"] - Resource = local.ssm_parameter_arns - }, - { - Effect = "Allow" - Action = ["kms:Decrypt"] - Resource = "*" - Condition = { - StringEquals = { - "kms:ViaService" = "ssm.${var.region}.amazonaws.com" - } - } - } - ] - }) -} - -resource "aws_iam_role" "task" { - name = "${var.project}-task" - assume_role_policy = data.aws_iam_policy_document.ecs_assume.json -} - -resource "aws_iam_role_policy" "task" { - name = "${var.project}-task" - role = aws_iam_role.task.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "WriteSite" - Effect = "Allow" - Action = ["s3:PutObject", "s3:AbortMultipartUpload"] - Resource = ["${aws_s3_bucket.site.arn}/*"] - }, - { - Sid = "ReadWriteIndex" - Effect = "Allow" - Action = ["s3:GetObject", "s3:PutObject"] - Resource = ["${aws_s3_bucket.state.arn}/state/*"] - }, - { - Sid = "ReadWorkInbox" - Effect = "Allow" - Action = ["s3:GetObject"] - Resource = [ - "${aws_s3_bucket.state.arn}/${local.work_inbox_key}", - "${aws_s3_bucket.state.arn}/${local.personal_inbox_key}" - ] - } - ] - }) -} - -# --------------------------------------------------------------------------- -# Network, cluster, task - -data "aws_vpc" "default" { - count = var.vpc_id == "" ? 1 : 0 - default = true -} - -data "aws_subnets" "default" { - count = length(var.subnet_ids) == 0 ? 1 : 0 - - filter { - name = "vpc-id" - values = [local.vpc_id] - } -} - -resource "aws_security_group" "task" { - name = "${var.project}-task" - description = "Egress only for the ${var.project} crawler" - vpc_id = local.vpc_id - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -resource "aws_ecs_cluster" "tally" { - name = var.project - - setting { - name = "containerInsights" - value = "enabled" - } -} - -resource "aws_ecs_task_definition" "tally" { - family = var.project - requires_compatibilities = ["FARGATE"] - network_mode = "awsvpc" - cpu = tostring(var.task_cpu) - memory = tostring(var.task_memory) - execution_role_arn = aws_iam_role.execution.arn - task_role_arn = aws_iam_role.task.arn - - runtime_platform { - operating_system_family = "LINUX" - cpu_architecture = "X86_64" - } - - container_definitions = jsonencode([ - { - name = "tally" - image = "${aws_ecr_repository.tally.repository_url}:${var.image_tag}" - essential = true - command = ["crawl"] - stopTimeout = 120 - - environment = concat( - [ - { name = "TALLY_PUBLISH", value = "1" }, - { name = "TALLY_SITE_BUCKET", value = aws_s3_bucket.site.id }, - { name = "TALLY_STATE_BUCKET", value = aws_s3_bucket.state.id }, - { name = "TALLY_STATE_KEY", value = "state/tally.sqlite3" }, - { name = "TALLY_SITE_URL", value = local.site_url }, - { name = "TALLY_DIDBOT_PDS", value = var.didbot_pds }, - { name = "TALLY_GITHUB_OWNERS", value = var.github_owners }, - { name = "TALLY_CRAWL_BUDGET_SECS", value = tostring(var.crawl_budget_seconds) }, - { name = "TALLY_PER_HOST_RPS", value = "10" }, - { name = "RUST_LOG", value = "info" }, - ], - var.inbox_urls != "" ? [ - { name = "TALLY_INBOXES", value = var.inbox_urls }, - ] : local.work_inbox_url != "" ? [ - { name = "TALLY_WORK_INBOX", value = local.work_inbox_url }, - ] : [], - var.personal_dids != "" ? [ - { name = "TALLY_PERSONAL_DIDS", value = var.personal_dids }, - ] : [], - var.personal_emails != "" ? [ - { name = "TALLY_PERSONAL_EMAILS", value = var.personal_emails }, - ] : [], - var.max_repos_per_run > 0 ? [ - { name = "TALLY_MAX_REPOS_PER_RUN", value = tostring(var.max_repos_per_run) }, - ] : [], - ) - - secrets = concat( - var.github_token_parameter_name != "" ? [ - { - name = "GITHUB_TOKEN" - valueFrom = "arn:aws:ssm:${var.region}:${local.account_id}:parameter/${trimprefix(var.github_token_parameter_name, "/")}" - }, - ] : [], - var.github_work_token_parameter_name != "" ? [ - { - name = "GITHUB_WORK_TOKEN" - valueFrom = "arn:aws:ssm:${var.region}:${local.account_id}:parameter/${trimprefix(var.github_work_token_parameter_name, "/")}" - }, - ] : [], - ) - - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.tally.name - "awslogs-region" = var.region - "awslogs-stream-prefix" = "tally" - } - } - } - ]) - - tags = var.tags -} - -data "aws_iam_policy_document" "scheduler_assume" { - statement { - actions = ["sts:AssumeRole"] - principals { - type = "Service" - identifiers = ["scheduler.amazonaws.com"] - } - } -} - -resource "aws_iam_role" "scheduler" { - name = "${var.project}-scheduler" - assume_role_policy = data.aws_iam_policy_document.scheduler_assume.json -} - -resource "aws_iam_role_policy" "scheduler" { - name = "${var.project}-scheduler" - role = aws_iam_role.scheduler.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = ["ecs:RunTask"] - Resource = ["arn:aws:ecs:${var.region}:${local.account_id}:task-definition/${var.project}:*"] - }, - { - Effect = "Allow" - Action = ["iam:PassRole"] - Resource = [aws_iam_role.execution.arn, aws_iam_role.task.arn] - } - ] - }) -} - -resource "aws_scheduler_schedule" "hourly" { - name = "${var.project}-hourly" - schedule_expression = var.schedule_expression - state = "ENABLED" - - flexible_time_window { - mode = "OFF" - } - - target { - arn = aws_ecs_cluster.tally.arn - role_arn = aws_iam_role.scheduler.arn - - ecs_parameters { - task_definition_arn = aws_ecs_task_definition.tally.arn - launch_type = "FARGATE" - platform_version = "LATEST" - - network_configuration { - subnets = local.subnet_ids - security_groups = [aws_security_group.task.id] - assign_public_ip = var.assign_public_ip - } - } - - retry_policy { - maximum_retry_attempts = 2 - } - } -} diff --git a/infra/outputs.tf b/infra/outputs.tf deleted file mode 100644 index 34d0d2b..0000000 --- a/infra/outputs.tf +++ /dev/null @@ -1,94 +0,0 @@ -output "region" { - value = var.region -} - -output "site_bucket" { - value = aws_s3_bucket.site.id -} - -output "state_bucket" { - value = aws_s3_bucket.state.id -} - -output "cloudfront_domain_name" { - value = aws_cloudfront_distribution.site.domain_name -} - -output "cloudfront_distribution_id" { - value = aws_cloudfront_distribution.site.id -} - -output "site_url" { - value = local.site_url -} - -output "ecr_repository_url" { - value = aws_ecr_repository.tally.repository_url -} - -output "ecs_cluster_name" { - value = aws_ecs_cluster.tally.name -} - -output "task_definition_arn" { - value = aws_ecs_task_definition.tally.arn -} - -output "schedule_name" { - value = aws_scheduler_schedule.hourly.name -} - -output "work_inbox_key" { - value = local.work_inbox_key -} - -output "work_inbox_uri" { - value = "s3://${aws_s3_bucket.state.id}/${local.work_inbox_key}" -} - -# Give this policy to the work machine's credential. It can write exactly one -# object and read nothing. -output "work_inbox_policy" { - value = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Sid = "PutRedactedWorkActivity" - Effect = "Allow" - Action = ["s3:PutObject"] - Resource = [ - "${aws_s3_bucket.state.arn}/${local.work_inbox_key}", - "${aws_s3_bucket.state.arn}/${local.personal_inbox_key}" - ] - } - ] - }) -} - -output "route53_zone_id" { - value = var.manage_dns ? data.aws_route53_zone.site[0].zone_id : null -} - -output "route53_name_servers" { - value = var.manage_dns ? data.aws_route53_zone.site[0].name_servers : [] -} - -output "certificate_arn" { - value = local.certificate_arn -} - -output "dns_instructions" { - value = ( - var.manage_dns - ? "Terraform created the ACM validation records and the ${var.domain_name} A/AAAA aliases in the existing Route 53 zone. No registrar or Cloudflare change is required." - : ( - var.domain_name == "" - ? "Point a CNAME for your chosen hostname at ${aws_cloudfront_distribution.site.domain_name}." - : ( - var.acm_certificate_arn == "" - ? "Create and validate an ACM certificate in us-east-1 for ${var.domain_name}, set acm_certificate_arn, then create A/AAAA aliases for ${var.domain_name} to ${aws_cloudfront_distribution.site.domain_name}." - : "Create A/AAAA alias records for ${var.domain_name} to ${aws_cloudfront_distribution.site.domain_name} in the Route 53 zone." - ) - ) - ) -} diff --git a/infra/variables.tf b/infra/variables.tf deleted file mode 100644 index ac27aad..0000000 --- a/infra/variables.tf +++ /dev/null @@ -1,158 +0,0 @@ -variable "region" { - type = string - default = "us-east-1" - description = "AWS region for ECS, S3 and ECR. A CloudFront ACM certificate must be in us-east-1." -} - -variable "project" { - type = string - default = "tally" - description = "Prefix for IAM roles, ECS resources and default bucket names." -} - -variable "image_tag" { - type = string - default = "latest" - description = "Tag of the image pushed to ECR." -} - -variable "site_bucket_name" { - type = string - default = "" - description = "Optional explicit site bucket name; otherwise project-site-accountid." -} - -variable "state_bucket_name" { - type = string - default = "" - description = "Optional explicit state/inbox bucket name; otherwise project-state-accountid." -} - -variable "domain_name" { - type = string - default = "dev.permadeath.com" - description = "Custom domain to serve. Empty uses the CloudFront hostname." -} - -variable "route53_zone_name" { - type = string - default = "dev.permadeath.com." - description = "Route 53 hosted zone that contains domain_name. The dev.permadeath.com zone is already delegated, so no registrar change is needed." -} - -variable "manage_dns" { - type = bool - default = true - description = "Create the ACM certificate and Route 53 alias records. Set false when the zone is in another AWS account and records are managed separately." -} - -variable "acm_certificate_arn" { - type = string - default = "" - description = "Optional existing validated ACM certificate ARN in us-east-1. Empty creates and validates one in route53_zone_name when manage_dns is true." -} - -variable "vpc_id" { - type = string - default = "" - description = "VPC for the Fargate task. Empty uses the default VPC." -} - -variable "subnet_ids" { - type = list(string) - default = [] - description = "Subnets for the Fargate task. Empty uses the default VPC's subnets." -} - -variable "assign_public_ip" { - type = bool - default = true - description = "Set false when the subnets route through a NAT gateway." -} - -variable "task_cpu" { - type = number - default = 2048 -} - -variable "task_memory" { - type = number - default = 4096 -} - -variable "crawl_budget_seconds" { - type = number - default = 2700 - description = "Stop starting new repositories after this many seconds, leaving slack before the next hourly run." -} - -variable "max_repos_per_run" { - type = number - default = 0 - description = "Optional cap on repositories per run. Zero means no cap; the time budget still applies." -} - -variable "schedule_expression" { - type = string - default = "cron(7 * * * ? *)" - description = "EventBridge Scheduler expression. The default runs at :07 every hour." -} - -variable "didbot_pds" { - type = string - default = "https://pds.did.bot" - description = "Comma-separated didbot PDS base URLs to discover operated agents from." -} - -variable "personal_dids" { - type = string - default = "" - description = "Optional did|handle|series-N overrides for the personal Tangled identities." -} - -variable "personal_emails" { - type = string - default = "" - description = "Optional address=did aliases for older Tangled commits." -} - -variable "github_owners" { - type = string - default = "permadeath-games" - description = "Extra GitHub owners whose repositories should be listed (the personal token's own repos are always included)." -} - -variable "github_token_parameter_name" { - type = string - default = "" - description = "Optional SSM SecureString parameter holding the personal GitHub token. Leave empty to run without direct GitHub ingestion." -} - -variable "github_work_token_parameter_name" { - type = string - default = "" - description = "Optional SSM SecureString parameter for a direct work token. Prefer work_inbox_url instead." -} - -variable "enable_work_inbox" { - type = bool - default = true - description = "Read redacted work activity from the state bucket's inbox key." -} - -variable "work_inbox_url" { - type = string - default = "" - description = "Optional explicit s3:// URI for the redacted work-activity payload; overrides the derived state-bucket URI." -} - -variable "inbox_urls" { - type = string - default = "" - description = "Optional comma-separated TALLY_INBOXES list, e.g. s3://bucket/inbox/personal-activity.json,s3://bucket/inbox/work-activity.json. Overrides work_inbox_url." -} - -variable "tags" { - type = map(string) - default = {} -} diff --git a/infra/versions.tf b/infra/versions.tf deleted file mode 100644 index cd3e92a..0000000 --- a/infra/versions.tf +++ /dev/null @@ -1,14 +0,0 @@ -terraform { - required_version = ">= 1.11.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 6.0" - } - } -} - -provider "aws" { - region = var.region -} diff --git a/infra/work-inbox-policy.json b/infra/work-inbox-policy.json deleted file mode 100644 index bc8fac8..0000000 --- a/infra/work-inbox-policy.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "PutRedactedWorkActivity", - "Effect": "Allow", - "Action": ["s3:PutObject"], - "Resource": [ - "arn:aws:s3:::REPLACE_STATE_BUCKET/inbox/work-activity.json", - "arn:aws:s3:::REPLACE_STATE_BUCKET/inbox/personal-activity.json" - ] - } - ] -} diff --git a/scripts/github_activity.py b/scripts/github_activity.py index c1e494f..cacaa9a 100755 --- a/scripts/github_activity.py +++ b/scripts/github_activity.py @@ -32,8 +32,8 @@ Typical daily cron entries: /usr/bin/python3 github_activity.py --label work \ --s3 s3://my-tally-state/inbox/work-activity.json -The S3 credential only needs s3:PutObject on those keys. See -infra/work-inbox-policy.json. +The S3 credential only needs s3:PutObject on those keys; the private infra +repo defines the bucket and write-only policy. """ from __future__ import annotations