EC2 Environment Detection (#361)

Co-authored-by: Jonathan M. Henson <[email protected]>
Co-authored-by: Michael Graeb <[email protected]>

include/aws/s3/private/s3_platform_info.h

@@ -0,0 +1,73 @@
+#ifndef AWS_S3_S3_PLATFORM_INFO_H
+#define AWS_S3_S3_PLATFORM_INFO_H
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#include <aws/s3/s3.h>
+
+struct aws_s3_platform_info_loader;
+
+AWS_EXTERN_C_BEGIN
+
+/**
+ * Initializes and returns a loader for querying the compute platform for information needed for making configuration
+ * decisions.
+ * This will never be NULL.
+ */
+AWS_S3_API
+struct aws_s3_platform_info_loader *aws_s3_platform_info_loader_new(struct aws_allocator *allocator);
+
+AWS_S3_API
+struct aws_s3_platform_info_loader *aws_s3_platform_info_loader_acquire(struct aws_s3_platform_info_loader *loader);
+
+AWS_S3_API
+struct aws_s3_platform_info_loader *aws_s3_platform_info_loader_release(struct aws_s3_platform_info_loader *loader);
+
+/**
+ * Retrieves the pre-configured metadata for a given ec2 instance type. If no such pre-configuration exists, returns
+ * NULL.
+ */
+AWS_S3_API
+const struct aws_s3_platform_info *aws_s3_get_platform_info_for_instance_type(
+    struct aws_s3_platform_info_loader *loader,
+    struct aws_byte_cursor instance_type_name);
+
+/**
+ * Retrieves the  metadata for the current environment. If EC2 instance type is unknown, or it is not an EC2 instance at
+ * all, this value will still include the information about the system that could be determined. This value will never
+ * be NULL.
+ * This API is not thread safe.
+ */
+AWS_S3_API
+const struct aws_s3_platform_info *aws_s3_get_platform_info_for_current_environment(
+    struct aws_s3_platform_info_loader *loader);
+
+/**
+ * Returns true if the current process is running on an Amazon EC2 instance powered by Nitro.
+ */
+AWS_S3_API
+bool aws_s3_is_running_on_ec2_nitro(struct aws_s3_platform_info_loader *loader);
+
+/**
+ * Returns an EC2 instance type assuming this executable is running on Amazon EC2 powered by nitro.
+ *
+ * First this function will check it's running on EC2 via. attempting to read DMI info to avoid making IMDS calls.
+ *
+ * If the function detects it's on EC2, and it was able to detect the instance type without a call to IMDS
+ * it will return it.
+ *
+ * Finally, it will call IMDS and return the instance type from there.
+ *
+ * Note that in the case of the IMDS call, a new client stack is spun up using 1 background thread. The call is made
+ * synchronously with a 1 second timeout: It's not cheap. To make this easier, the underlying result is cached
+ * internally and will be freed when aws_s3_library_clean_up() is called.
+ * @return byte_cursor containing the instance type. If this is empty, the instance type could not be determined.
+ */
+AWS_S3_API
+struct aws_byte_cursor aws_s3_get_ec2_instance_type(struct aws_s3_platform_info_loader *loader);
+
+AWS_EXTERN_C_END
+
+#endif /* AWS_S3_S3_PLATFORM_INFO_H */

include/aws/s3/s3.h

@@ -54,13 +54,16 @@ enum aws_s3_subject {
     AWS_LS_S3_LAST = AWS_LOG_SUBJECT_END_RANGE(AWS_C_S3_PACKAGE_ID)
 };
 
+struct aws_s3_platform_info;
+
 struct aws_s3_cpu_group_info {
     /* group index, this usually refers to a particular numa node */
     uint16_t cpu_group;
     /* array of network devices on this node */
-    const struct aws_byte_cursor *nic_name_array;
+    struct aws_byte_cursor *nic_name_array;
     /* length of network devices array */
     size_t nic_name_array_length;
+    size_t cpus_in_group;
 };
 
 #ifdef _MSC_VER
@@ -69,15 +72,19 @@ struct aws_s3_cpu_group_info {
 #    pragma warning(disable : 5027) /* move assignment operator was implicitly defined as deleted */
 #endif
 
-struct aws_s3_compute_platform_info {
+struct aws_s3_platform_info {
     /* name of the instance-type: example c5n.18xlarge */
-    const struct aws_byte_cursor instance_type;
-    /* max throughput for this instance type */
-    uint16_t max_throughput_gbps;
+    struct aws_byte_cursor instance_type;
+    /* max throughput for this instance type, in gigabits per second */
+    double max_throughput_gbps;
     /* array of cpu group info. This will always have at least one entry. */
-    const struct aws_s3_cpu_group_info *cpu_group_info_array;
+    struct aws_s3_cpu_group_info *cpu_group_info_array;
     /* length of cpu group info array */
     size_t cpu_group_info_array_length;
+
+    /* The current build of this library specifically knows an optimal configuration for this
+     * platform */
+    bool has_recommended_configuration;
 };
 
 #ifdef _MSC_VER
@@ -94,17 +101,17 @@ AWS_S3_API
 void aws_s3_library_init(struct aws_allocator *allocator);
 
 /**
- * Retrieves the pre-configured metadata for an ec2 instance type. If no such pre-configuration exists, returns NULL.
+ * Shuts down the internal datastructures used by aws-c-s3.
  */
 AWS_S3_API
-struct aws_s3_compute_platform_info *aws_s3_get_compute_platform_info_for_instance_type(
-    const struct aws_byte_cursor instance_type_name);
+void aws_s3_library_clean_up(void);
 
-/**
- * Shuts down the internal datastructures used by aws-c-s3.
+/*
+ * Returns the aws_s3_platform_info for current platform
+ * NOTE: THIS API IS EXPERIMENTAL AND UNSTABLE
  */
 AWS_S3_API
-void aws_s3_library_clean_up(void);
+const struct aws_s3_platform_info *aws_s3_get_current_platform_info(void);
 
 AWS_EXTERN_C_END
 AWS_POP_SANE_WARNING_LEVEL

samples/s3/main.c

@@ -19,6 +19,7 @@
 
 int s3_ls_main(int argc, char *const argv[], const char *command_name, void *user_data);
 int s3_cp_main(int argc, char *const argv[], const char *command_name, void *user_data);
+int s3_compute_platform_info_main(int argc, char *const argv[], const char *command_name, void *user_data);
 
 static struct aws_cli_subcommand_dispatch s_dispatch_table[] = {
     {
@@ -29,6 +30,10 @@ static struct aws_cli_subcommand_dispatch s_dispatch_table[] = {
         .command_name = "cp",
         .subcommand_fn = s3_cp_main,
     },
+    {
+        .command_name = "platform-info",
+        .subcommand_fn = s3_compute_platform_info_main,
+    },
 };
 
 static void s_usage(int exit_code) {
@@ -102,11 +107,6 @@ static void s_parse_app_ctx(int argc, char *const argv[], struct app_ctx *app_ct
     }
 
     if (!app_ctx->help_requested) {
-        if (!app_ctx->region) {
-            fprintf(stderr, "region is a required argument\n");
-            s_usage(1);
-        }
-
         if (app_ctx->log_level != AWS_LOG_LEVEL_NONE) {
             s_setup_logger(app_ctx);
         }

samples/s3/s3-cp.c

@@ -122,6 +122,11 @@ int s3_cp_main(int argc, char *argv[], const char *command_name, void *user_data
         s_usage(0);
     }
 
+    if (!app_ctx->region) {
+        fprintf(stderr, "region is a required argument\n");
+        s_usage(1);
+    }
+
     struct cp_app_ctx cp_app_ctx = {
         .app_ctx = app_ctx,
         .mutex = AWS_MUTEX_INIT,

samples/s3/s3-ls.c

@@ -138,6 +138,11 @@ int s3_ls_main(int argc, char *argv[], const char *command_name, void *user_data
         s_usage(0);
     }
 
+    if (!app_ctx->region) {
+        fprintf(stderr, "region is a required argument\n");
+        s_usage(1);
+    }
+
     struct s3_ls_app_data impl_data = {
         .app_ctx = app_ctx,
         .mutex = AWS_MUTEX_INIT,

samples/s3/s3-platform_info.c

@@ -0,0 +1,96 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#include <aws/common/command_line_parser.h>
+
+#include "app_ctx.h"
+
+struct s3_compute_platform_ctx {
+    struct app_ctx *app_ctx;
+    struct aws_byte_cursor instance_type;
+};
+
+static void s_usage(int exit_code) {
+    FILE *output = exit_code == 0 ? stdout : stderr;
+    fprintf(output, "usage: s3 platform-info [options]\n");
+    fprintf(
+        output,
+        "  -instance-type, (optional) Instance type to look up configuration for, if not set it will be the current "
+        "executing environment. \n");
+    fprintf(output, "  -h, --help\n");
+    fprintf(output, "            Display this message and quit.\n");
+    exit(exit_code);
+}
+
+static struct aws_cli_option s_long_options[] = {
+    {"instance-type", AWS_CLI_OPTIONS_REQUIRED_ARGUMENT, NULL, 'i'},
+    /* Per getopt(3) the last element of the array has to be filled with all zeros */
+    {NULL, AWS_CLI_OPTIONS_NO_ARGUMENT, NULL, 0},
+};
+
+static void s_parse_options(int argc, char **argv, struct s3_compute_platform_ctx *ctx) {
+    int option_index = 0;
+
+    int opt_val = 0;
+    do {
+        opt_val = aws_cli_getopt_long(argc, argv, "i:", s_long_options, &option_index);
+        /* START_OF_TEXT means our positional argument */
+        if (opt_val == 'i') {
+            ctx->instance_type = aws_byte_cursor_from_c_str(aws_cli_optarg);
+        }
+    } while (opt_val != -1);
+}
+
+int s3_compute_platform_info_main(int argc, char *argv[], const char *command_name, void *user_data) {
+    (void)command_name;
+
+    struct app_ctx *app_ctx = user_data;
+
+    if (app_ctx->help_requested) {
+        s_usage(0);
+    }
+
+    struct s3_compute_platform_ctx compute_platform_app_ctx = {
+        .app_ctx = app_ctx,
+    };
+    app_ctx->sub_command_data = &compute_platform_app_ctx;
+
+    s_parse_options(argc, argv, &compute_platform_app_ctx);
+
+    const struct aws_s3_platform_info *platform_info = aws_s3_get_current_platform_info();
+
+    printf("{\n");
+    printf("\t'instance_type': '" PRInSTR "',\n", AWS_BYTE_CURSOR_PRI(platform_info->instance_type));
+    printf("\t'max_throughput_gbps': %d,\n", (int)platform_info->max_throughput_gbps);
+    printf("\t'has_recommended_configuration': %s,\n", platform_info->has_recommended_configuration ? "true" : "false");
+
+    printf("\t'cpu_groups': [\n");
+
+    for (size_t i = 0; i < platform_info->cpu_group_info_array_length; ++i) {
+        printf("\t{\n");
+        printf("\t\t'cpu_group_index': %d,\n", (int)platform_info->cpu_group_info_array[i].cpu_group);
+        printf("\t\t'cpus_in_group': %d,\n", (int)platform_info->cpu_group_info_array[i].cpus_in_group);
+        printf("\t\t'usable_network_devices': [\n");
+
+        for (size_t j = 0; j < platform_info->cpu_group_info_array[i].nic_name_array_length; j++) {
+            printf(
+                "\t\t\t'" PRInSTR "'", AWS_BYTE_CURSOR_PRI(platform_info->cpu_group_info_array[i].nic_name_array[j]));
+            if (j < platform_info->cpu_group_info_array[i].nic_name_array_length - 1) {
+                printf(",");
+            }
+            printf("\n");
+        }
+        printf("\t\t]\n");
+        printf("\t}");
+        if (i < platform_info->cpu_group_info_array_length - 1) {
+            printf(",");
+        }
+        printf("\n");
+    }
+    printf("\t]\n");
+    printf("}\n");
+
+    return 0;
+}