From cc5af86774b716f4b17e736153ff5d0f47b4a7e0 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Fri, 10 Nov 2023 14:23:41 -0800 Subject: [PATCH] API to get list of pre optimized systems (#521) Co-authored-by: Michael Graeb --- awscrt/s3.py | 15 +++++++++++++-- crt/aws-c-s3 | 2 +- source/module.c | 1 + source/s3.h | 1 + source/s3_client.c | 33 ++++++++++++++++++++++++++++++++- test/test_s3.py | 6 ++++++ 6 files changed, 54 insertions(+), 4 deletions(-) diff --git a/awscrt/s3.py b/awscrt/s3.py index 6ba298884..f0af97490 100644 --- a/awscrt/s3.py +++ b/awscrt/s3.py @@ -624,6 +624,16 @@ def is_optimized_for_system(): return _awscrt.s3_is_crt_s3_optimized_for_system() +def get_optimized_platforms(): + """ + Returns: + A list[str] of platform identifiers, such as EC2 instance types, for which S3 client is pre-optimized + and have a recommended throughput_target_gbps. You can use `get_recommended_throughput_target_gbps()` + to obtain the recommended throughput_target_gbps for those platforms. + """ + return _awscrt.s3_get_optimized_platforms() + + def get_recommended_throughput_target_gbps() -> Optional[float]: """ Returns: @@ -631,8 +641,9 @@ def get_recommended_throughput_target_gbps() -> Optional[float]: If the best throughput configuration is unknown, returns None. Use this as the S3Client's `throughput_target_gbps`. """ - # Currently the CRT returns 0 if it was unable to make a good guess on configuration. Pre-known configs, have this value set. - # Eventually, the CRT will make a full calculation based on NIC and CPU configuration, but until then handle 0. + # Currently the CRT returns 0 if it was unable to make a good guess on configuration. Pre-known configs, + # have this value set. Eventually, the CRT will make a full calculation based on NIC and CPU configuration, + # but until then handle 0. max_value = _awscrt.s3_get_recommended_throughput_target_gbps() if max_value > 0: return max_value diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index f354c535d..89791b135 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit f354c535db928591d7f44bdc00acccc77650a67c +Subproject commit 89791b135926473c3fdd37099ab69d5161e144c1 diff --git a/source/module.c b/source/module.c index bb423f029..0648957e5 100644 --- a/source/module.c +++ b/source/module.c @@ -802,6 +802,7 @@ static PyMethodDef s_module_methods[] = { AWS_PY_METHOD_DEF(s3_get_ec2_instance_type, METH_NOARGS), AWS_PY_METHOD_DEF(s3_is_crt_s3_optimized_for_system, METH_NOARGS), AWS_PY_METHOD_DEF(s3_get_recommended_throughput_target_gbps, METH_NOARGS), + AWS_PY_METHOD_DEF(s3_get_optimized_platforms, METH_NOARGS), AWS_PY_METHOD_DEF(s3_cross_process_lock_new, METH_VARARGS), AWS_PY_METHOD_DEF(s3_cross_process_lock_acquire, METH_VARARGS), AWS_PY_METHOD_DEF(s3_cross_process_lock_release, METH_VARARGS), diff --git a/source/s3.h b/source/s3.h index 3821acd07..6af8567b8 100644 --- a/source/s3.h +++ b/source/s3.h @@ -10,6 +10,7 @@ PyObject *aws_py_s3_get_ec2_instance_type(PyObject *self, PyObject *args); PyObject *aws_py_s3_is_crt_s3_optimized_for_system(PyObject *self, PyObject *args); PyObject *aws_py_s3_get_recommended_throughput_target_gbps(PyObject *self, PyObject *args); +PyObject *aws_py_s3_get_optimized_platforms(PyObject *self, PyObject *args); PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args); PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args); diff --git a/source/s3_client.c b/source/s3_client.c index 9309fa5de..0a5acb713 100644 --- a/source/s3_client.c +++ b/source/s3_client.c @@ -48,6 +48,38 @@ PyObject *aws_py_s3_get_recommended_throughput_target_gbps(PyObject *self, PyObj return PyFloat_FromDouble(platform_info->max_throughput_gbps); } +PyObject *aws_py_s3_get_optimized_platforms(PyObject *self, PyObject *args) { + (void)self; + (void)args; + + bool success = false; + struct aws_array_list platform_list = aws_s3_get_platforms_with_recommended_config(); + + size_t list_length = aws_array_list_length(&platform_list); + + PyObject *py_list = PyList_New(list_length); + if (!py_list) { + goto clean_up; + } + + for (size_t i = 0; i < list_length; ++i) { + struct aws_byte_cursor cursor; + if (aws_array_list_get_at(&platform_list, &cursor, i) == AWS_OP_SUCCESS) { + PyObject *platform_str = PyUnicode_FromAwsByteCursor(&cursor); + if (!platform_str) { + Py_DECREF(py_list); + goto clean_up; + } + PyList_SetItem(py_list, i, platform_str); /* Steals a Reference */ + } + } + success = true; + +clean_up: + aws_array_list_clean_up(&platform_list); + return success ? py_list : NULL; +} + struct cross_process_lock_binding { struct aws_cross_process_lock *lock; struct aws_string *name; @@ -126,7 +158,6 @@ PyObject *aws_py_s3_cross_process_lock_acquire(PyObject *self, PyObject *args) { PyObject *aws_py_s3_cross_process_lock_release(PyObject *self, PyObject *args) { (void)self; - PyObject *lock_capsule; /* O */ if (!PyArg_ParseTuple(args, "O", &lock_capsule)) { diff --git a/test/test_s3.py b/test/test_s3.py index e0ee0b144..c7e7b3e8c 100644 --- a/test/test_s3.py +++ b/test/test_s3.py @@ -23,6 +23,7 @@ S3ResponseError, CrossProcessLock, create_default_s3_signing_config, + get_optimized_platforms, ) from awscrt.io import ( ClientBootstrap, @@ -216,6 +217,11 @@ def test_wait_shutdown(self): del s3_client self.assertTrue(shutdown_event.wait(self.timeout)) + def test_get_optimized_platforms(self): + platform_list = get_optimized_platforms() + self.assertTrue(len(platform_list) > 0) + self.assertTrue("p4d.24xlarge" in platform_list) + @unittest.skipUnless(os.environ.get('AWS_TEST_S3'), 'set env var to run test: AWS_TEST_S3') class S3RequestTest(NativeResourceTest):