Some modifications (#121)

Prepare for new release Implement deprecation Add some tagging for bookkeeping Add utility to delete images
NixOS · May 22, 2024 · 55ed46d · 55ed46d
1 parent e0faf6e
commit 55ed46d
Show file tree

Hide file tree

Showing 8 changed files with 264 additions and 70 deletions.
diff --git a/.github/workflows/upload-legacy-ami.yml b/.github/workflows/upload-legacy-ami.yml
@@ -8,7 +8,7 @@ on:
   pull_request:
   workflow_dispatch:
   schedule:
-    - cron: '0 0 * * 0' 
+    - cron: "0 0 * * 0"
 jobs:
   upload-ami:
     name: Upload Legacy Amazon Image
@@ -19,6 +19,9 @@ jobs:
       id-token: write
     strategy:
       matrix:
+        release:
+          - release-23.11
+          # - unstable-small
         system:
           - x86_64-linux
           - aarch64-linux
@@ -33,7 +36,7 @@ jobs:
         id: download_ami
         run: |
           set -o pipefail
-          out=$(curl --location --silent --fail-with-body --header 'Accept: application/json' https://hydra.nixos.org/job/nixos/release-23.11/nixos.amazonImage.${{ matrix.system }}/latest-finished  | jq --raw-output '.buildoutputs.out.path')
+          out=$(curl --location --silent --fail-with-body --header 'Accept: application/json' https://hydra.nixos.org/job/nixos/${{ matrix.release }}/nixos.amazonImage.${{ matrix.system }}/latest-finished  | jq --raw-output '.buildoutputs.out.path')
           nix-store --realise "$out" --add-root ./result
           echo "image_info=$out/nix-support/image-info.json" >> "$GITHUB_OUTPUT"
 
@@ -69,14 +72,13 @@ jobs:
           image_id=$(echo "$image_ids" | jq -r '.["${{ vars.AWS_REGION }}"]')
           run_id='${{ github.run_id }}'
           nix run .#smoke-test -- --image-id "$image_id" --cancel
-      
 
       # NOTE: We do not pass run-id as we're not  building the image ourselves
       # and we thus need to poll hydra periodically.  Including the run-id would
       # cause us to register  the same snapshot as an image over and over again
       # for each run.
       - name: Upload AMIs to all available regions
-        if: github.ref == 'refs/heads/main' 
+        if: github.ref == 'refs/heads/main'
         run: |
           image_info='${{ steps.download_ami.outputs.image_info }}'
           images_bucket='${{ vars.IMAGES_BUCKET }}'
@@ -116,4 +118,4 @@ jobs:
       - name: Deploy pages
         uses: actions/deploy-pages@decdde0ac072f6dcbe43649d82d9c635fff5b4e4 # v4.0.4
         id: deployment
-        if: github.ref == 'refs/heads/main' 
+        if: github.ref == 'refs/heads/main'
diff --git a/site/index.html b/site/index.html
@@ -208,6 +208,15 @@
 
 <body>
     <h1>Amazon Images / AMIs</h1>
+    <p>
+        NixOS can be deployed to Amazon EC2 using our official AMI. We publish
+        AMIs to all AWS regions for both `x86_64` and `arm64` on a weekly basis.
+    </p>
+    <p>We will start deprecating and garbage collecting images older than 90 days
+        in the future.
+        This is why we suggest using a terraform data source or the AWS API to query
+        for the latest AMI.
+    </p>
     <p>NixOS images are published under AWS Account ID <span id="owner-id"></span></p>
 
     <h2>Terraform / OpenTofu</h2>
@@ -223,7 +232,7 @@ <h2>Terraform / OpenTofu</h2>
 
   filter {
     name   = "name"
-    values = ["nixos/23.11*"]
+    values = ["nixos/24.05*"]
   }
   filter {
     name   = "architecture"
@@ -239,13 +248,12 @@ <h2>Terraform / OpenTofu</h2>
     <h2>AWS CLI</h2>
     <pre id="awscli2">
 aws ec2 describe-images --owners _OWNER_ID_ \
-  --filter 'Name=name,Values=nixos/23.11*' \
+  --filter 'Name=name,Values=nixos/24.05*' \
   --filter 'Name=architecture,Values=arm64'
 </pre>
 
     <h2>AMI table</h2>
     <p>Here are the latest NixOS images available in the Amazon cloud.</p>
-    <p>The data this table is generated from can be found at <a href="https://nixos.github.io/amis/images.json">https://nixos.github.io/amis/images.json</a>.</p>
     <table id="images-table">
         <thead>
             <tr>
@@ -291,4 +299,4 @@ <h2>AMI table</h2>
     </table>
 </body>
 
-</html>
+</html>
diff --git a/tf/iam_github_actions.tf b/tf/iam_github_actions.tf
@@ -8,6 +8,7 @@ data "aws_iam_policy_document" "upload_ami" {
     actions = [
       "s3:GetObject",
       "s3:PutObject",
+      "s3:DeleteObject",
     ]
     resources = ["${aws_s3_bucket.images.arn}/*"]
   }
@@ -21,6 +22,15 @@ data "aws_iam_policy_document" "upload_ami" {
     ]
     resources = ["*"]
   }
+  statement {
+    effect = "Allow"
+    actions = ["ec2:CreateTags"]
+    resources = [
+      "arn:aws:ec2:*:*:snapshot/*",
+      "arn:aws:ec2:*:*:image/*",
+      "arn:aws:ec2:*:*:import-snapshot-task/*",
+    ]
+  }
   statement {
     effect = "Allow"
     actions = [
@@ -30,7 +40,8 @@ data "aws_iam_policy_document" "upload_ami" {
       "ec2:DescribeRegions",
       "ec2:CopyImage",
       "ec2:ModifyImageAttribute",
-      "ec2:DisableImageBlockPublicAccess"
+      "ec2:DisableImageBlockPublicAccess",
+      "ec2:EnableImageDeprecation"
     ]
     resources = ["*"]
   }

diff --git a/tf/main.tf b/tf/main.tf
@@ -57,7 +57,7 @@ data "aws_iam_policy_document" "vmimport" {
       "ec2:ModifySnapshotAttribute",
       "ec2:CopySnapshot",
       "ec2:RegisterImage",
-      "ec2:Describe*"
+      "ec2:Describe*",
     ]
     effect    = "Allow"
     resources = ["*"]

diff --git a/upload-ami/pyproject.toml b/upload-ami/pyproject.toml
@@ -17,5 +17,6 @@ disable-image-block-public-access = "upload_ami.disable_image_block_public_acces
 enable-regions = "upload_ami.enable_regions:main"
 request-public-ami-quota-increase  = "upload_ami.request_public_ami_quota_increase:main"
 describe-images = "upload_ami.describe_images:main"
+delete-images = "upload_ami.delete_images:main"
 [tool.mypy]
 strict=true
diff --git a/upload-ami/src/upload_ami/delete_images.py b/upload-ami/src/upload_ami/delete_images.py
@@ -0,0 +1,82 @@
+import logging
+import boto3
+from mypy_boto3_ec2 import EC2Client
+import argparse
+import botocore.exceptions
+
+logger = logging.getLogger(__name__)
+
+
+def delete_images_by_name(ec2: EC2Client, image_name: str, dry_run: bool) -> None:
+    """
+    Delete an image by its name.
+
+    Name can be a filter
+
+    Idempotent, unlike nuke
+    """
+    logger.info(f"Deleting image by name {image_name}")
+    snapshots = ec2.describe_snapshots(
+        OwnerIds=["self"], Filters=[{"Name": "tag:Name", "Values": [image_name]}]
+    )
+    logger.info(f"Deleting {len(snapshots['Snapshots'])} snapshots")
+
+    for snapshot in snapshots["Snapshots"]:
+        assert "SnapshotId" in snapshot
+        images = ec2.describe_images(
+            Owners=["self"],
+            Filters=[
+                {
+                    "Name": "block-device-mapping.snapshot-id",
+                    "Values": [snapshot["SnapshotId"]],
+                }
+            ],
+        )
+        logger.info(f"Deleting {len(images['Images'])} images")
+        for image in images["Images"]:
+            assert "ImageId" in image
+            logger.info(f"Deregistering {image['ImageId']}")
+            try:
+                ec2.deregister_image(ImageId=image["ImageId"], DryRun=dry_run)
+            except botocore.exceptions.ClientError as e:
+                if "DryRunOperation" not in str(e):
+                    raise e
+
+        logger.info(f"Deleting {snapshot['SnapshotId']}")
+        try:
+            ec2.delete_snapshot(SnapshotId=snapshot["SnapshotId"], DryRun=dry_run)
+        except botocore.exceptions.ClientError as e:
+            if "DryRunOperation" not in str(e):
+                raise e
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--image-name",
+        type=str,
+        required=True,
+        help="Name of the image to delete. Can be a filter.",
+    )
+
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Do not actually delete anything, just log what would be deleted",
+    )
+    logging.basicConfig(level=logging.INFO)
+    ec2: EC2Client = boto3.client("ec2")
+
+    args = parser.parse_args()
+    regions = ec2.describe_regions()["Regions"]
+    for region in regions:
+        assert "RegionName" in region
+        ec2r = boto3.client("ec2", region_name=region["RegionName"])
+        logger.info(
+            f"Deleting image by name {args.image_name} in {region['RegionName']}"
+        )
+        delete_images_by_name(ec2r, args.image_name, args.dry_run)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/upload-ami/src/upload_ami/nuke.py b/upload-ami/src/upload_ami/nuke.py
@@ -1,38 +1,53 @@
 import logging
 import boto3
 from mypy_boto3_ec2 import EC2Client
+import argparse
+import botocore.exceptions
 
 
 def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--image-name",
+        type=str,
+        required=True,
+        help="Name of the image to delete. Can be a filter.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--older-than",
+        type=str,
+    )
+    args = parser.parse_args()
     logging.basicConfig(level=logging.INFO)
-    ec2: EC2Client = boto3.client("ec2", region_name="us-east-1")
+    ec2: EC2Client = boto3.client("ec2")
 
     regions = ec2.describe_regions()["Regions"]
 
     for region in regions:
         assert "RegionName" in region
         ec2r = boto3.client("ec2", region_name=region["RegionName"])
         logging.info(f"Nuking {region['RegionName']}")
-        snapshots = ec2r.describe_snapshots(OwnerIds=["self"])
-        for snapshot in snapshots["Snapshots"]:
-
-            assert "SnapshotId" in snapshot
-            images = ec2r.describe_images(
-                Owners=["self"],
-                Filters=[
-                    {
-                        "Name": "block-device-mapping.snapshot-id",
-                        "Values": [snapshot["SnapshotId"]],
-                    }
-                ],
-            )
-            for image in images["Images"]:
-                assert "ImageId" in image
-                logging.info(f"Deregistering {image['ImageId']}")
-                ec2r.deregister_image(ImageId=image["ImageId"])
-
-            logging.info(f"Deleting {snapshot['SnapshotId']}")
-            ec2r.delete_snapshot(SnapshotId=snapshot["SnapshotId"])
+        images = ec2r.describe_images(
+            Owners=["self"], Filters=[{"Name": "name", "Values": [args.image_name]}]
+        )
+        for image in images["Images"]:
+            snapshot_id = image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"]
+            logging.info(f"Deregistering {image['ImageId']}")
+            try:
+                ec2r.deregister_image(ImageId=image["ImageId"], DryRun=args.dry_run)
+            except botocore.exceptions.ClientError as e:
+                if "DryRunOperation" not in str(e):
+                    raise
+            logging.info(f"Deleting {snapshot_id}")
+            try:
+                ec2r.delete_snapshot(SnapshotId=snapshot_id, DryRun=args.dry_run)
+            except botocore.exceptions.ClientError as e:
+                if "DryRunOperation" not in str(e):
+                    raise
 
 
 if __name__ == "__main__":