From e89a6944faa390ad80df5481bd62203e432f9cba Mon Sep 17 00:00:00 2001 From: Eric Beard Date: Thu, 18 Jul 2024 15:14:47 -0700 Subject: [PATCH 1/2] Added check for sagemaker notebook quota --- go.mod | 1 + go.sum | 2 + internal/aws/sagemaker/sagemaker.go | 45 + internal/cmd/forecast/README.md | 2 + internal/cmd/forecast/forecast.go | 1 + .../sagemaker-notebook-instance-codes.json | 812 ++++++++++++++++++ internal/cmd/forecast/sagemaker.go | 151 ++++ .../forecast/sagemaker-notebook.yaml | 36 + 8 files changed, 1050 insertions(+) create mode 100644 internal/aws/sagemaker/sagemaker.go create mode 100644 internal/cmd/forecast/sagemaker-notebook-instance-codes.json create mode 100644 internal/cmd/forecast/sagemaker.go create mode 100644 test/templates/forecast/sagemaker-notebook.yaml diff --git a/go.mod b/go.mod index 23c0ca24..1593d3d7 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( require ( github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sagemaker v1.151.0 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index 9b727281..014ea56a 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1 h1:wsg9Z/vNnCmxWikfGIoOlnExtEU45 github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1/go.mod h1:8rDw3mVwmvIWWX/+LWY3PPIMZuwnQdJMCt0iVFVT3qw= github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 h1:sZXIzO38GZOU+O0C+INqbH7C2yALwfMWpd64tONS/NE= github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE= +github.com/aws/aws-sdk-go-v2/service/sagemaker v1.151.0 h1:zH7b/d8vOrOWdgluKEux2TAECYKhprH2eMztkpak/mI= +github.com/aws/aws-sdk-go-v2/service/sagemaker v1.151.0/go.mod h1:lDmK3DHWV6Y6hpzeUAaXq4w+ks6fFYXdkjavIe8STCE= github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.9 h1:3o5zcwZYvte3CeaYpLaWafwCSkJpclPXI5KSH+lXB90= github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.9/go.mod h1:QZpGkzlec0TPr8CA2Td5zRUJBC5+104ib0MusH5UVfI= github.com/aws/aws-sdk-go-v2/service/servicequotas v1.22.1 h1:QsHvqtdy0mGzpg/A+1lZX1ilf05Vuh2rSBzNJ3f3T1I= diff --git a/internal/aws/sagemaker/sagemaker.go b/internal/aws/sagemaker/sagemaker.go new file mode 100644 index 00000000..b1080c18 --- /dev/null +++ b/internal/aws/sagemaker/sagemaker.go @@ -0,0 +1,45 @@ +package sagemaker + +import ( + "context" + + "github.com/aws-cloudformation/rain/internal/aws" + "github.com/aws/aws-sdk-go-v2/service/sagemaker" +) + +func getClient() *sagemaker.Client { + return sagemaker.NewFromConfig(aws.Config()) +} + +type NotebookInstance struct { + InstanceType string +} + +func GetNotebookInstances() ([]NotebookInstance, error) { + retval := make([]NotebookInstance, 0) + + client := getClient() + var nextToken *string + + for { + resp, err := client.ListNotebookInstances(context.Background(), + &sagemaker.ListNotebookInstancesInput{ + NextToken: nextToken, + }) + if err != nil { + return retval, err + } + + for _, inst := range resp.NotebookInstances { + retval = append(retval, NotebookInstance{InstanceType: string(inst.InstanceType)}) + } + + if resp.NextToken != nil { + nextToken = resp.NextToken + } else { + break + } + } + + return retval, nil +} diff --git a/internal/cmd/forecast/README.md b/internal/cmd/forecast/README.md index 97ff16e2..b7a07723 100644 --- a/internal/cmd/forecast/README.md +++ b/internal/cmd/forecast/README.md @@ -53,6 +53,8 @@ These can be ignored with the `--ignore` argument. | F0015 | ELB target groups must be of type instance if they are used by an ASG | | F0016 | Lambda function role exists | | F0017 | Lambda function role can be assumed | +| F0018 | SageMaker Notebook quota limit has not been reached | + ## Estimates The forecast command also tries to estimate how long it thinks your stack will diff --git a/internal/cmd/forecast/forecast.go b/internal/cmd/forecast/forecast.go index 30b05dd7..b8675d5e 100644 --- a/internal/cmd/forecast/forecast.go +++ b/internal/cmd/forecast/forecast.go @@ -506,6 +506,7 @@ func init() { forecasters["AWS::SNS::Topic"] = checkSNSTopic forecasters["AWS::ElasticLoadBalancingV2::TargetGroup"] = checkELBTargetGroup forecasters["AWS::Lambda::Function"] = checkLambdaFunction + forecasters["AWS::SageMaker::NotebookInstance"] = checkSageMakerNotebook // Initialize estimates map InitEstimates() diff --git a/internal/cmd/forecast/sagemaker-notebook-instance-codes.json b/internal/cmd/forecast/sagemaker-notebook-instance-codes.json new file mode 100644 index 00000000..600c30b1 --- /dev/null +++ b/internal/cmd/forecast/sagemaker-notebook-instance-codes.json @@ -0,0 +1,812 @@ +[ + { + "Code": "L-D0B132AA", + "Name": "ml.c4.8xlarge for notebook instance usage", + "InstanceType": "ml.c4.8xlarge" + }, + { + "Code": "L-7AA0FEE8", + "Name": "ml.m5d.xlarge for notebook instance usage", + "InstanceType": "ml.m5d.xlarge" + }, + { + "Code": "L-C177D66D", + "Name": "ml.r6id.24xlarge for notebook instance usage", + "InstanceType": "ml.r6id.24xlarge" + }, + { + "Code": "L-A1364089", + "Name": "ml.c6i.4xlarge for notebook instance usage", + "InstanceType": "ml.c6i.4xlarge" + }, + { + "Code": "L-6E02BD88", + "Name": "ml.r6id.xlarge for notebook instance usage", + "InstanceType": "ml.r6id.xlarge" + }, + { + "Code": "L-CFF1A7E1", + "Name": "ml.r6id.2xlarge for notebook instance usage", + "InstanceType": "ml.r6id.2xlarge" + }, + { + "Code": "L-BE78F29C", + "Name": "ml.m4.4xlarge for notebook instance usage", + "InstanceType": "ml.m4.4xlarge" + }, + { + "Code": "L-4E89F558", + "Name": "ml.m7i.24xlarge for notebook instance usage", + "InstanceType": "ml.m7i.24xlarge" + }, + { + "Code": "L-96B75525", + "Name": "ml.c5.4xlarge for notebook instance usage", + "InstanceType": "ml.c5.4xlarge" + }, + { + "Code": "L-73EF2FAD", + "Name": "ml.r7i.48xlarge for notebook instance usage", + "InstanceType": "ml.r7i.48xlarge" + }, + { + "Code": "L-86FD5133", + "Name": "ml.r7i.2xlarge for notebook instance usage", + "InstanceType": "ml.r7i.2xlarge" + }, + { + "Code": "L-72530F39", + "Name": "ml.r6i.2xlarge for notebook instance usage", + "InstanceType": "ml.r6i.2xlarge" + }, + { + "Code": "L-3523E641", + "Name": "ml.r6i.24xlarge for notebook instance usage", + "InstanceType": "ml.r6i.24xlarge" + }, + { + "Code": "L-94C3A7A1", + "Name": "ml.t2.large for notebook instance usage", + "InstanceType": "ml.t2.large" + }, + { + "Code": "L-66A9D66D", + "Name": "ml.m6id.24xlarge for notebook instance usage", + "InstanceType": "ml.m6id.24xlarge" + }, + { + "Code": "L-956703D9", + "Name": "ml.m5d.24xlarge for notebook instance usage", + "InstanceType": "ml.m5d.24xlarge" + }, + { + "Code": "L-2C505E76", + "Name": "ml.r6id.12xlarge for notebook instance usage", + "InstanceType": "ml.r6id.12xlarge" + }, + { + "Code": "L-8E454C05", + "Name": "ml.t3.large for notebook instance usage", + "InstanceType": "ml.t3.large" + }, + { + "Code": "L-41CDD438", + "Name": "ml.m7i.12xlarge for notebook instance usage", + "InstanceType": "ml.m7i.12xlarge" + }, + { + "Code": "L-A0C4F9CF", + "Name": "ml.c6id.16xlarge for notebook instance usage", + "InstanceType": "ml.c6id.16xlarge" + }, + { + "Code": "L-5F356922", + "Name": "ml.c5.2xlarge for notebook instance usage", + "InstanceType": "ml.c5.2xlarge" + }, + { + "Code": "L-FB715320", + "Name": "ml.t3.2xlarge for notebook instance usage", + "InstanceType": "ml.t3.2xlarge" + }, + { + "Code": "L-D9B5C786", + "Name": "ml.r7i.8xlarge for notebook instance usage", + "InstanceType": "ml.r7i.8xlarge" + }, + { + "Code": "L-660A18BE", + "Name": "ml.p5.48xlarge for notebook instance usage", + "InstanceType": "ml.p5.48xlarge" + }, + { + "Code": "L-AC9EC023", + "Name": "ml.m6i.4xlarge for notebook instance usage", + "InstanceType": "ml.m6i.4xlarge" + }, + { + "Code": "L-E5884D25", + "Name": "ml.t3.xlarge for notebook instance usage", + "InstanceType": "ml.t3.xlarge" + }, + { + "Code": "L-786E9B47", + "Name": "ml.c4.xlarge for notebook instance usage", + "InstanceType": "ml.c4.xlarge" + }, + { + "Code": "L-862C1E14", + "Name": "ml.m5.4xlarge for notebook instance usage", + "InstanceType": "ml.m5.4xlarge" + }, + { + "Code": "L-5869E902", + "Name": "ml.r5.12xlarge for notebook instance usage", + "InstanceType": "ml.r5.12xlarge" + }, + { + "Code": "L-82E81781", + "Name": "ml.g6.xlarge for notebook instance usage", + "InstanceType": "ml.g6.xlarge" + }, + { + "Code": "L-6C73443F", + "Name": "ml.c5d.18xlarge for notebook instance usage", + "InstanceType": "ml.c5d.18xlarge" + }, + { + "Code": "L-E17566B7", + "Name": "ml.t3.medium for notebook instance usage", + "InstanceType": "ml.t3.medium" + }, + { + "Code": "L-E1249695", + "Name": "ml.m4.xlarge for notebook instance usage", + "InstanceType": "ml.m4.xlarge" + }, + { + "Code": "L-28CF4381", + "Name": "ml.m6i.12xlarge for notebook instance usage", + "InstanceType": "ml.m6i.12xlarge" + }, + { + "Code": "L-9FEBBCCF", + "Name": "ml.r6i.12xlarge for notebook instance usage", + "InstanceType": "ml.r6i.12xlarge" + }, + { + "Code": "L-FF2BFCDC", + "Name": "ml.inf1.6xlarge for notebook instance usage", + "InstanceType": "ml.inf1.6xlarge" + }, + { + "Code": "L-95E39457", + "Name": "ml.m5.2xlarge for notebook instance usage", + "InstanceType": "ml.m5.2xlarge" + }, + { + "Code": "L-EC374A06", + "Name": "ml.g5.12xlarge for notebook instance usage", + "InstanceType": "ml.g5.12xlarge" + }, + { + "Code": "L-C2F8103D", + "Name": "ml.g4dn.8xlarge for notebook instance usage", + "InstanceType": "ml.g4dn.8xlarge" + }, + { + "Code": "L-0E9335FF", + "Name": "ml.g6.12xlarge for notebook instance usage", + "InstanceType": "ml.g6.12xlarge" + }, + { + "Code": "L-1F122E71", + "Name": "ml.r6i.8xlarge for notebook instance usage", + "InstanceType": "ml.r6i.8xlarge" + }, + { + "Code": "L-C22BC883", + "Name": "ml.r6id.32xlarge for notebook instance usage", + "InstanceType": "ml.r6id.32xlarge" + }, + { + "Code": "L-2E21A1FA", + "Name": "ml.p3dn.24xlarge for notebook instance usage", + "InstanceType": "ml.p3dn.24xlarge" + }, + { + "Code": "L-8CB23490", + "Name": "ml.r5.2xlarge for notebook instance usage", + "InstanceType": "ml.r5.2xlarge" + }, + { + "Code": "L-47461EBA", + "Name": "ml.g4dn.12xlarge for notebook instance usage", + "InstanceType": "ml.g4dn.12xlarge" + }, + { + "Code": "L-19973BE2", + "Name": "ml.g5.2xlarge for notebook instance usage", + "InstanceType": "ml.g5.2xlarge" + }, + { + "Code": "L-E4348ACD", + "Name": "ml.r6id.large for notebook instance usage", + "InstanceType": "ml.r6id.large" + }, + { + "Code": "L-195A207B", + "Name": "ml.m5d.large for notebook instance usage", + "InstanceType": "ml.m5d.large" + }, + { + "Code": "L-40A7DC79", + "Name": "ml.g6.8xlarge for notebook instance usage", + "InstanceType": "ml.g6.8xlarge" + }, + { + "Code": "L-4D6B5DFA", + "Name": "ml.c6id.4xlarge for notebook instance usage", + "InstanceType": "ml.c6id.4xlarge" + }, + { + "Code": "L-FF78806F", + "Name": "ml.c5d.2xlarge for notebook instance usage", + "InstanceType": "ml.c5d.2xlarge" + }, + { + "Code": "L-252AB1C1", + "Name": "ml.c6i.large for notebook instance usage", + "InstanceType": "ml.c6i.large" + }, + { + "Code": "L-B8378206", + "Name": "ml.m6id.12xlarge for notebook instance usage", + "InstanceType": "ml.m6id.12xlarge" + }, + { + "Code": "L-A2E4CEFE", + "Name": "ml.r6id.4xlarge for notebook instance usage", + "InstanceType": "ml.r6id.4xlarge" + }, + { + "Code": "L-16553FE9", + "Name": "ml.c7i.2xlarge for notebook instance usage", + "InstanceType": "ml.c7i.2xlarge" + }, + { + "Code": "L-E8917BB7", + "Name": "ml.g5.xlarge for notebook instance usage", + "InstanceType": "ml.g5.xlarge" + }, + { + "Code": "L-CE6894AA", + "Name": "ml.m4.2xlarge for notebook instance usage", + "InstanceType": "ml.m4.2xlarge" + }, + { + "Code": "L-6829423A", + "Name": "ml.m5d.2xlarge for notebook instance usage", + "InstanceType": "ml.m5d.2xlarge" + }, + { + "Code": "L-1C2E1B03", + "Name": "ml.g4dn.2xlarge for notebook instance usage", + "InstanceType": "ml.g4dn.2xlarge" + }, + { + "Code": "L-9AD5A106", + "Name": "ml.c6i.xlarge for notebook instance usage", + "InstanceType": "ml.c6i.xlarge" + }, + { + "Code": "L-F2CC767C", + "Name": "ml.c6id.32xlarge for notebook instance usage", + "InstanceType": "ml.c6id.32xlarge" + }, + { + "Code": "L-8BACBF19", + "Name": "ml.m7i.xlarge for notebook instance usage", + "InstanceType": "ml.m7i.xlarge" + }, + { + "Code": "L-0A6678C9", + "Name": "ml.m6i.2xlarge for notebook instance usage", + "InstanceType": "ml.m6i.2xlarge" + }, + { + "Code": "L-7B2FD69B", + "Name": "ml.t2.medium for notebook instance usage", + "InstanceType": "ml.t2.medium" + }, + { + "Code": "L-DFF9A17E", + "Name": "ml.c6id.8xlarge for notebook instance usage", + "InstanceType": "ml.c6id.8xlarge" + }, + { + "Code": "L-C7B212A3", + "Name": "ml.c7i.24xlarge for notebook instance usage", + "InstanceType": "ml.c7i.24xlarge" + }, + { + "Code": "L-39F5FD98", + "Name": "ml.c5.xlarge for notebook instance usage", + "InstanceType": "ml.c5.xlarge" + }, + { + "Code": "L-27768634", + "Name": "ml.c4.4xlarge for notebook instance usage", + "InstanceType": "ml.c4.4xlarge" + }, + { + "Code": "L-64067773", + "Name": "ml.g4dn.16xlarge for notebook instance usage", + "InstanceType": "ml.g4dn.16xlarge" + }, + { + "Code": "L-EDE09F63", + "Name": "ml.m5d.4xlarge for notebook instance usage", + "InstanceType": "ml.m5d.4xlarge" + }, + { + "Code": "L-4689E606", + "Name": "ml.p3.16xlarge for notebook instance usage", + "InstanceType": "ml.p3.16xlarge" + }, + { + "Code": "L-6F0C387D", + "Name": "ml.c6i.16xlarge for notebook instance usage", + "InstanceType": "ml.c6i.16xlarge" + }, + { + "Code": "L-AEB45880", + "Name": "ml.m5d.8xlarge for notebook instance usage", + "InstanceType": "ml.m5d.8xlarge" + }, + { + "Code": "L-5C59A967", + "Name": "ml.m6id.xlarge for notebook instance usage", + "InstanceType": "ml.m6id.xlarge" + }, + { + "Code": "L-B99D4EA0", + "Name": "ml.c6i.32xlarge for notebook instance usage", + "InstanceType": "ml.c6i.32xlarge" + }, + { + "Code": "L-DD1D4232", + "Name": "ml.m6i.xlarge for notebook instance usage", + "InstanceType": "ml.m6i.xlarge" + }, + { + "Code": "L-00C0E80E", + "Name": "ml.m6id.2xlarge for notebook instance usage", + "InstanceType": "ml.m6id.2xlarge" + }, + { + "Code": "L-76AB2C05", + "Name": "ml.m5.xlarge for notebook instance usage", + "InstanceType": "ml.m5.xlarge" + }, + { + "Code": "L-9AD04286", + "Name": "ml.m5.12xlarge for notebook instance usage", + "InstanceType": "ml.m5.12xlarge" + }, + { + "Code": "L-FD4BCA24", + "Name": "ml.c7i.16xlarge for notebook instance usage", + "InstanceType": "ml.c7i.16xlarge" + }, + { + "Code": "L-54232165", + "Name": "ml.m6i.large for notebook instance usage", + "InstanceType": "ml.m6i.large" + }, + { + "Code": "L-ACD9A4A4", + "Name": "ml.r7i.4xlarge for notebook instance usage", + "InstanceType": "ml.r7i.4xlarge" + }, + { + "Code": "L-70680787", + "Name": "ml.m7i.16xlarge for notebook instance usage", + "InstanceType": "ml.m7i.16xlarge" + }, + { + "Code": "L-4ACAC7A4", + "Name": "ml.g5.24xlarge for notebook instance usage", + "InstanceType": "ml.g5.24xlarge" + }, + { + "Code": "L-8BED04E5", + "Name": "ml.r5.large for notebook instance usage", + "InstanceType": "ml.r5.large" + }, + { + "Code": "L-26EB4EC7", + "Name": "ml.m5d.16xlarge for notebook instance usage", + "InstanceType": "ml.m5d.16xlarge" + }, + { + "Code": "L-127D4C65", + "Name": "ml.g5.4xlarge for notebook instance usage", + "InstanceType": "ml.g5.4xlarge" + }, + { + "Code": "L-2BAB3767", + "Name": "ml.c6i.12xlarge for notebook instance usage", + "InstanceType": "ml.c6i.12xlarge" + }, + { + "Code": "L-D371F6EE", + "Name": "ml.g6.2xlarge for notebook instance usage", + "InstanceType": "ml.g6.2xlarge" + }, + { + "Code": "L-3A549026", + "Name": "ml.c7i.48xlarge for notebook instance usage", + "InstanceType": "ml.c7i.48xlarge" + }, + { + "Code": "L-BE44390B", + "Name": "ml.t2.2xlarge for notebook instance usage", + "InstanceType": "ml.t2.2xlarge" + }, + { + "Code": "L-2AD4A712", + "Name": "ml.m7i.48xlarge for notebook instance usage", + "InstanceType": "ml.m7i.48xlarge" + }, + { + "Code": "L-5D8382CB", + "Name": "ml.g5.8xlarge for notebook instance usage", + "InstanceType": "ml.g5.8xlarge" + }, + { + "Code": "L-B74ED3C1", + "Name": "ml.r5.4xlarge for notebook instance usage", + "InstanceType": "ml.r5.4xlarge" + }, + { + "Code": "L-85E60595", + "Name": "ml.m4.16xlarge for notebook instance usage", + "InstanceType": "ml.m4.16xlarge" + }, + { + "Code": "L-55E494BC", + "Name": "ml.m6i.24xlarge for notebook instance usage", + "InstanceType": "ml.m6i.24xlarge" + }, + { + "Code": "L-B5D1E461", + "Name": "ml.r6i.16xlarge for notebook instance usage", + "InstanceType": "ml.r6i.16xlarge" + }, + { + "Code": "L-6ABF7180", + "Name": "ml.c6i.2xlarge for notebook instance usage", + "InstanceType": "ml.c6i.2xlarge" + }, + { + "Code": "L-3360E714", + "Name": "ml.r7i.12xlarge for notebook instance usage", + "InstanceType": "ml.r7i.12xlarge" + }, + { + "Code": "L-E4B23F6B", + "Name": "ml.c7i.xlarge for notebook instance usage", + "InstanceType": "ml.c7i.xlarge" + }, + { + "Code": "L-462E7E5F", + "Name": "ml.m6id.16xlarge for notebook instance usage", + "InstanceType": "ml.m6id.16xlarge" + }, + { + "Code": "L-DAE78A2A", + "Name": "ml.r5.8xlarge for notebook instance usage", + "InstanceType": "ml.r5.8xlarge" + }, + { + "Code": "L-50222063", + "Name": "ml.c6id.12xlarge for notebook instance usage", + "InstanceType": "ml.c6id.12xlarge" + }, + { + "Code": "L-1B60238B", + "Name": "ml.c7i.12xlarge for notebook instance usage", + "InstanceType": "ml.c7i.12xlarge" + }, + { + "Code": "L-619D6E43", + "Name": "ml.p3.8xlarge for notebook instance usage", + "InstanceType": "ml.p3.8xlarge" + }, + { + "Code": "L-CEEF9A6E", + "Name": "ml.m4.10xlarge for notebook instance usage", + "InstanceType": "ml.m4.10xlarge" + }, + { + "Code": "L-BF4FD612", + "Name": "ml.r6i.xlarge for notebook instance usage", + "InstanceType": "ml.r6i.xlarge" + }, + { + "Code": "L-F314A0DD", + "Name": "ml.r7i.xlarge for notebook instance usage", + "InstanceType": "ml.r7i.xlarge" + }, + { + "Code": "L-878E276D", + "Name": "ml.c7i.large for notebook instance usage", + "InstanceType": "ml.c7i.large" + }, + { + "Code": "L-19E4BAD9", + "Name": "ml.m7i.large for notebook instance usage", + "InstanceType": "ml.m7i.large" + }, + { + "Code": "L-BD97DD7F", + "Name": "ml.m5.24xlarge for notebook instance usage", + "InstanceType": "ml.m5.24xlarge" + }, + { + "Code": "L-204097CE", + "Name": "ml.m7i.4xlarge for notebook instance usage", + "InstanceType": "ml.m7i.4xlarge" + }, + { + "Code": "L-08F097C3", + "Name": "ml.m7i.2xlarge for notebook instance usage", + "InstanceType": "ml.m7i.2xlarge" + }, + { + "Code": "L-15A99B7E", + "Name": "ml.g6.16xlarge for notebook instance usage", + "InstanceType": "ml.g6.16xlarge" + }, + { + "Code": "L-85870296", + "Name": "ml.r5.xlarge for notebook instance usage", + "InstanceType": "ml.r5.xlarge" + }, + { + "Code": "L-001EDCEF", + "Name": "ml.g5.16xlarge for notebook instance usage", + "InstanceType": "ml.g5.16xlarge" + }, + { + "Code": "L-3221855A", + "Name": "ml.m6id.large for notebook instance usage", + "InstanceType": "ml.m6id.large" + }, + { + "Code": "L-529379E4", + "Name": "ml.g4dn.4xlarge for notebook instance usage", + "InstanceType": "ml.g4dn.4xlarge" + }, + { + "Code": "L-9EFE4FAD", + "Name": "ml.t2.xlarge for notebook instance usage", + "InstanceType": "ml.t2.xlarge" + }, + { + "Code": "L-1F4A5AAB", + "Name": "ml.p3.2xlarge for notebook instance usage", + "InstanceType": "ml.p3.2xlarge" + }, + { + "Code": "L-AA088DA9", + "Name": "ml.c6i.8xlarge for notebook instance usage", + "InstanceType": "ml.c6i.8xlarge" + }, + { + "Code": "L-61C1E88D", + "Name": "ml.g6.24xlarge for notebook instance usage", + "InstanceType": "ml.g6.24xlarge" + }, + { + "Code": "L-68CFD09E", + "Name": "ml.c6id.24xlarge for notebook instance usage", + "InstanceType": "ml.c6id.24xlarge" + }, + { + "Code": "L-212B4A4E", + "Name": "ml.r6i.32xlarge for notebook instance usage", + "InstanceType": "ml.r6i.32xlarge" + }, + { + "Code": "L-2B2F8466", + "Name": "ml.inf1.24xlarge for notebook instance usage", + "InstanceType": "ml.inf1.24xlarge" + }, + { + "Code": "L-1122D5DF", + "Name": "ml.c6i.24xlarge for notebook instance usage", + "InstanceType": "ml.c6i.24xlarge" + }, + { + "Code": "L-51BF3F2F", + "Name": "ml.m6i.32xlarge for notebook instance usage", + "InstanceType": "ml.m6i.32xlarge" + }, + { + "Code": "L-A9CD2FA4", + "Name": "ml.m7i.8xlarge for notebook instance usage", + "InstanceType": "ml.m7i.8xlarge" + }, + { + "Code": "L-434233AD", + "Name": "ml.inf1.xlarge for notebook instance usage", + "InstanceType": "ml.inf1.xlarge" + }, + { + "Code": "L-8F818E82", + "Name": "ml.m6id.4xlarge for notebook instance usage", + "InstanceType": "ml.m6id.4xlarge" + }, + { + "Code": "L-9944ECC4", + "Name": "ml.c5d.9xlarge for notebook instance usage", + "InstanceType": "ml.c5d.9xlarge" + }, + { + "Code": "L-CC5B8B01", + "Name": "ml.g6.48xlarge for notebook instance usage", + "InstanceType": "ml.g6.48xlarge" + }, + { + "Code": "L-54E3D0FA", + "Name": "ml.r7i.16xlarge for notebook instance usage", + "InstanceType": "ml.r7i.16xlarge" + }, + { + "Code": "L-3F25BA48", + "Name": "ml.r7i.large for notebook instance usage", + "InstanceType": "ml.r7i.large" + }, + { + "Code": "L-CEED8B6B", + "Name": "ml.c7i.8xlarge for notebook instance usage", + "InstanceType": "ml.c7i.8xlarge" + }, + { + "Code": "L-4089485C", + "Name": "ml.p4d.24xlarge for notebook instance usage", + "InstanceType": "ml.p4d.24xlarge" + }, + { + "Code": "L-2D83F796", + "Name": "ml.m6i.8xlarge for notebook instance usage", + "InstanceType": "ml.m6i.8xlarge" + }, + { + "Code": "L-924D392D", + "Name": "ml.r5.16xlarge for notebook instance usage", + "InstanceType": "ml.r5.16xlarge" + }, + { + "Code": "L-0549E35C", + "Name": "ml.c6id.xlarge for notebook instance usage", + "InstanceType": "ml.c6id.xlarge" + }, + { + "Code": "L-EF662387", + "Name": "ml.c5.18xlarge for notebook instance usage", + "InstanceType": "ml.c5.18xlarge" + }, + { + "Code": "L-1DD0FB59", + "Name": "ml.m5d.12xlarge for notebook instance usage", + "InstanceType": "ml.m5d.12xlarge" + }, + { + "Code": "L-EC351EF7", + "Name": "ml.c6id.large for notebook instance usage", + "InstanceType": "ml.c6id.large" + }, + { + "Code": "L-4E9EE949", + "Name": "ml.c5d.xlarge for notebook instance usage", + "InstanceType": "ml.c5d.xlarge" + }, + { + "Code": "L-21DCDF63", + "Name": "ml.m6id.32xlarge for notebook instance usage", + "InstanceType": "ml.m6id.32xlarge" + }, + { + "Code": "L-6B0B7E9C", + "Name": "ml.p4de.24xlarge for notebook instance usage", + "InstanceType": "ml.p4de.24xlarge" + }, + { + "Code": "L-3B90E5BF", + "Name": "ml.r7i.24xlarge for notebook instance usage", + "InstanceType": "ml.r7i.24xlarge" + }, + { + "Code": "L-B06EA6BA", + "Name": "ml.m6id.8xlarge for notebook instance usage", + "InstanceType": "ml.m6id.8xlarge" + }, + { + "Code": "L-AED8308D", + "Name": "ml.r6i.large for notebook instance usage", + "InstanceType": "ml.r6i.large" + }, + { + "Code": "L-484388A9", + "Name": "ml.c5d.4xlarge for notebook instance usage", + "InstanceType": "ml.c5d.4xlarge" + }, + { + "Code": "L-D6B1EB0D", + "Name": "ml.r6id.16xlarge for notebook instance usage", + "InstanceType": "ml.r6id.16xlarge" + }, + { + "Code": "L-98560DD6", + "Name": "ml.m6i.16xlarge for notebook instance usage", + "InstanceType": "ml.m6i.16xlarge" + }, + { + "Code": "L-3C3B7CCF", + "Name": "ml.c6id.2xlarge for notebook instance usage", + "InstanceType": "ml.c6id.2xlarge" + }, + { + "Code": "L-B5F303BE", + "Name": "ml.c5.9xlarge for notebook instance usage", + "InstanceType": "ml.c5.9xlarge" + }, + { + "Code": "L-23F5AC7E", + "Name": "ml.inf1.2xlarge for notebook instance usage", + "InstanceType": "ml.inf1.2xlarge" + }, + { + "Code": "L-0F109211", + "Name": "ml.r6i.4xlarge for notebook instance usage", + "InstanceType": "ml.r6i.4xlarge" + }, + { + "Code": "L-A028E7A2", + "Name": "ml.c4.2xlarge for notebook instance usage", + "InstanceType": "ml.c4.2xlarge" + }, + { + "Code": "L-D8B97089", + "Name": "ml.g4dn.xlarge for notebook instance usage", + "InstanceType": "ml.g4dn.xlarge" + }, + { + "Code": "L-D7FF3362", + "Name": "ml.r5.24xlarge for notebook instance usage", + "InstanceType": "ml.r5.24xlarge" + }, + { + "Code": "L-C691FBAD", + "Name": "ml.r6id.8xlarge for notebook instance usage", + "InstanceType": "ml.r6id.8xlarge" + }, + { + "Code": "L-FE7F0B12", + "Name": "ml.g6.4xlarge for notebook instance usage", + "InstanceType": "ml.g6.4xlarge" + }, + { + "Code": "L-64211729", + "Name": "ml.c7i.4xlarge for notebook instance usage", + "InstanceType": "ml.c7i.4xlarge" + }, + { + "Code": "L-D266139B", + "Name": "ml.g5.48xlarge for notebook instance usage", + "InstanceType": "ml.g5.48xlarge" + } +] diff --git a/internal/cmd/forecast/sagemaker.go b/internal/cmd/forecast/sagemaker.go new file mode 100644 index 00000000..94a03fe8 --- /dev/null +++ b/internal/cmd/forecast/sagemaker.go @@ -0,0 +1,151 @@ +package forecast + +import ( + _ "embed" + "encoding/json" + "math" + + "github.com/aws-cloudformation/rain/internal/aws/sagemaker" + "github.com/aws-cloudformation/rain/internal/aws/servicequotas" + "github.com/aws-cloudformation/rain/internal/config" + "github.com/aws-cloudformation/rain/internal/console/spinner" + "github.com/aws-cloudformation/rain/internal/s11n" +) + +//go:embed sagemaker-notebook-instance-codes.json +var notebookCodes string + +// Created with this: +// +// aws service-quotas list-service-quotas --service-code sagemaker --no-cli-pager > ~/Desktop/sagemaker-quotas.json +// +// cat ~/Desktop/sagemaker-quotas.json| jq '[ .Quotas.[] | select(.QuotaName|test(".*notebook instance usage.*")) | { Code: .QuotaCode, Name: .QuotaName, InstanceType: .QuotaName|sub(" for notebook instance usage";"") } ]' + +type NotebookCode struct { + Code string + Name string + InstanceType string + InstanceCount int +} + +func ParseNotebookCodes(jsonData string) ([]NotebookCode, error) { + var retval []NotebookCode + err := json.Unmarshal([]byte(jsonData), &retval) + if err != nil { + return retval, err + } + return retval, nil +} + +func checkSageMakerNotebook(input PredictionInput) Forecast { + // AWS::SageMaker::NotebookInstance + + forecast := makeForecast(input.typeName, input.logicalId) + + checkNotebookLimit(&input, &forecast) + + return forecast +} + +func checkNotebookLimit(input *PredictionInput, forecast *Forecast) { + + // The account- service limit 'Total number of notebook instances' is 8 + // NotebookInstances, with current utilization of 16 NotebookInstances and + // a request delta of 1 NotebookInstances. Please contact AWS support to + // request an increase for this limit. + + spin(input.typeName, input.logicalId, "SageMaker notebook quota ok?") + defer spinner.Pop() + + atLimit := false + + serviceCode := "sagemaker" + + // Total number of notebook instances + quotaCode := "L-04CE2E67" + + // First check the overall limit + quota, err := servicequotas.GetQuota(serviceCode, quotaCode) + if err != nil { + config.Debugf("Unable to get quota %s %s: %v", serviceCode, quotaCode, err) + return + } + + config.Debugf("Quota for %s %s: %v", serviceCode, quotaCode, quota) + + // Query the SageMaker API to check the current number of instances + instances, err := sagemaker.GetNotebookInstances() + if err != nil { + config.Debugf("Unable to get notebook instances") + return + } + + if len(instances) >= int(math.Round(quota)) { + atLimit = true + } else { + config.Debugf("Overall notebook quota ok with %v current instances", len(instances)) + + // Then query the ListServiceQuotas API to get all codes for each individual + // instance type, and check that specific quota. + // We should probably cache this data, since it's a slow API call + codes, err := ParseNotebookCodes(notebookCodes) + if err != nil { + config.Debugf("Unable to parse notebook codes: %v", err) + return + } + + codeMap := make(map[string]NotebookCode, 0) + for _, code := range codes { + codeMap[code.InstanceType] = code + } + + config.Debugf("codeMap: %+v", codeMap) + + for _, inst := range instances { + if code, ok := codeMap[inst.InstanceType]; ok { + code.InstanceCount += 1 + } + } + + // Get the instance type from the resource we are checking + var resourceInstanceType string + _, props, _ := s11n.GetMapValue(input.resource, "Properties") + if props == nil { + config.Debugf("expected %s to have Properties", input.logicalId) + return + } + _, instanceTypeNode, _ := s11n.GetMapValue(props, "InstanceType") + if instanceTypeNode == nil { + config.Debugf("expected %s to have InstanceType", input.logicalId) + return + } + resourceInstanceType = instanceTypeNode.Value + config.Debugf("%s InstanceType is %s", input.logicalId, resourceInstanceType) + + code, ok := codeMap[resourceInstanceType] + if !ok { + config.Debugf("InstanceType %s missing from codeMap", resourceInstanceType) + return + } + + quota, err := servicequotas.GetQuota(serviceCode, code.Code) + if err != nil { + config.Debugf("Unable to get quota %s %s: %v", serviceCode, quotaCode, err) + return + } + config.Debugf("Quota for %s (%s) is %v", code.InstanceType, code.Code, quota) + + if code.InstanceCount >= int(math.Round(quota)) { + atLimit = true + } + + } + + forecastCode := F0018 + if !atLimit { + forecast.Add(forecastCode, true, "Quota limit has not been reached") + } else { + forecast.Add(forecastCode, false, "Over quota limit") + } + +} diff --git a/test/templates/forecast/sagemaker-notebook.yaml b/test/templates/forecast/sagemaker-notebook.yaml new file mode 100644 index 00000000..fdc78564 --- /dev/null +++ b/test/templates/forecast/sagemaker-notebook.yaml @@ -0,0 +1,36 @@ +Resources: + + Notebook: + Type: AWS::SageMaker::NotebookInstance + DependsOn: NotebookRolePolicy + Properties: + InstanceType: ml.t2.medium + RoleArn: !GetAtt NotebookRole.Arn + + NotebookRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: sagemaker.amazonaws.com + Action: 'sts:AssumeRole' + + NotebookRolePolicy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyName: SageMakerNotebookInstancePolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - 's3:GetObject' + - 's3:PutObject' + - 's3:ListBucket' + Resource: + - 'arn:aws:s3:::*' + RoleName: !Ref NotebookRole + From 477093bc14a66475f92cabaae484b96b64ee7855 Mon Sep 17 00:00:00 2001 From: Eric Beard Date: Thu, 18 Jul 2024 15:44:37 -0700 Subject: [PATCH 2/2] Fixed instance count in map --- internal/cmd/forecast/sagemaker.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/cmd/forecast/sagemaker.go b/internal/cmd/forecast/sagemaker.go index 94a03fe8..0c85c46b 100644 --- a/internal/cmd/forecast/sagemaker.go +++ b/internal/cmd/forecast/sagemaker.go @@ -80,6 +80,8 @@ func checkNotebookLimit(input *PredictionInput, forecast *Forecast) { return } + config.Debugf("Instances: %+v", instances) + if len(instances) >= int(math.Round(quota)) { atLimit = true } else { @@ -94,9 +96,9 @@ func checkNotebookLimit(input *PredictionInput, forecast *Forecast) { return } - codeMap := make(map[string]NotebookCode, 0) + codeMap := make(map[string]*NotebookCode, 0) for _, code := range codes { - codeMap[code.InstanceType] = code + codeMap[code.InstanceType] = &code } config.Debugf("codeMap: %+v", codeMap) @@ -127,13 +129,15 @@ func checkNotebookLimit(input *PredictionInput, forecast *Forecast) { config.Debugf("InstanceType %s missing from codeMap", resourceInstanceType) return } + config.Debugf("code: %+v", code) quota, err := servicequotas.GetQuota(serviceCode, code.Code) if err != nil { config.Debugf("Unable to get quota %s %s: %v", serviceCode, quotaCode, err) return } - config.Debugf("Quota for %s (%s) is %v", code.InstanceType, code.Code, quota) + config.Debugf("Quota for %s (%s) is %v. Current count is %v", + code.InstanceType, code.Code, quota, code.InstanceCount) if code.InstanceCount >= int(math.Round(quota)) { atLimit = true