-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
10 changed files
with
5,653 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
#ifndef AWS_S3_ENDPOINT_RESOLVER_PRIVATE_H | ||
#define AWS_S3_ENDPOINT_RESOLVER_PRIVATE_H | ||
|
||
/** | ||
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
* SPDX-License-Identifier: Apache-2.0. | ||
*/ | ||
|
||
extern const struct aws_byte_cursor aws_s3_endpoint_resolver_partitions; | ||
extern const struct aws_byte_cursor aws_s3_endpoint_rule_set; | ||
|
||
#endif /* AWS_S3_ENDPOINT_RESOLVER_PRIVATE_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
#ifndef AWS_S3_ENDPOINT_RESOLVER_H | ||
#define AWS_S3_ENDPOINT_RESOLVER_H | ||
|
||
/** | ||
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
* SPDX-License-Identifier: Apache-2.0. | ||
*/ | ||
|
||
#include <aws/s3/private/s3_endpoint_resolver.h> | ||
#include <aws/s3/s3.h> | ||
AWS_PUSH_SANE_WARNING_LEVEL | ||
|
||
struct aws_endpoints_request_context; | ||
struct aws_endpoints_rule_engine; | ||
AWS_EXTERN_C_BEGIN | ||
|
||
/** | ||
* Creates a new S3 endpoint resolver. | ||
* Warning: Before using this header, you have to enable it by | ||
* setting cmake config AWS_ENABLE_S3_ENDPOINT_RESOLVER=ON | ||
*/ | ||
AWS_S3_API | ||
struct aws_endpoints_rule_engine *aws_s3_endpoint_resolver_new(struct aws_allocator *allocator); | ||
|
||
AWS_EXTERN_C_END | ||
AWS_POP_SANE_WARNING_LEVEL | ||
#endif /* AWS_S3_ENDPOINT_RESOLVER_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,116 @@ | ||
# This script pulls latest 'partitions.json' and 's3-endpoint-rule-set.json' from Git. | ||
# You will need a secret in secrets manager which has the 'ruleset-url' and 'ruleset-token'. | ||
# It uses the latest files to generate 'source/s3_endpoint_resolver/aws_s3_endpoint_rule_set.c' and | ||
# 'source/s3_endpoint_resolver/aws_s3_endpoint_resolver_partition.c' | ||
|
||
import json | ||
import boto3 | ||
import requests | ||
|
||
|
||
def escape_char(c): | ||
escape_dict = { | ||
'\\': '\\\\', | ||
'\'': '\\\'', | ||
'\0': '\\0', | ||
'\a': '\\a', | ||
'\b': '\\b', | ||
'\f': '\\f', | ||
'\n': '\\n', | ||
'\r': '\\r', | ||
'\t': '\\t', | ||
'\v': '\\v' | ||
} | ||
|
||
return escape_dict.get(c, c) | ||
|
||
|
||
def get_header(): | ||
return """\ | ||
/** | ||
* Copyright Amazon.com, Inc. or its affiliates. | ||
* All Rights Reserved. SPDX-License-Identifier: Apache-2.0. | ||
*/ | ||
#include <aws/s3/s3_endpoint_resolver.h> | ||
/** | ||
* This file is generated using scripts/update_s3_endpoint_resolver_artifacts.py. | ||
* Do not modify directly. */ | ||
/* clang-format off */ | ||
""" | ||
|
||
|
||
def generate_c_file_from_json(json_content, c_file_name, c_struct_name): | ||
num_chars_per_line = 20 | ||
|
||
try: | ||
# Compact the json | ||
compact_json_str = json.dumps(json_content, separators=(',', ':')) | ||
compact_c = [] | ||
for i in range(0, len(compact_json_str), num_chars_per_line): | ||
compact_c.append( | ||
', '.join("'{}'".format(escape_char(char)) for char in compact_json_str[i:i + num_chars_per_line])) | ||
|
||
# Write json to a C file | ||
with open(c_file_name, 'w') as f: | ||
f.write(get_header()) | ||
f.write(f"static const char s_generated_array[] = {{\n\t") | ||
f.write(",\n\t".join(compact_c)) | ||
f.write("};\n\n") | ||
|
||
f.write(f"const struct aws_byte_cursor {c_struct_name} = {{\n\t") | ||
f.write(f".len = {len(compact_json_str)},\n\t") | ||
f.write(f".ptr = (uint8_t *) s_generated_array\n}};\n") | ||
|
||
print(f"{c_file_name} has been created successfully.") | ||
|
||
except Exception as e: | ||
print(f"An error occurred: {e}") | ||
|
||
|
||
def get_secret_from_secrets_manager(secret_name, region_name): | ||
session = boto3.session.Session() | ||
client = session.client( | ||
service_name='secretsmanager', | ||
region_name=region_name | ||
) | ||
|
||
try: | ||
get_secret_value_response = client.get_secret_value( | ||
SecretId=secret_name | ||
) | ||
except Exception as e: | ||
raise e | ||
|
||
return json.loads(get_secret_value_response['SecretString']) | ||
|
||
|
||
def download_from_git(url, token=None): | ||
headers = {'Accept': 'application/vnd.github+json'} | ||
if token is not None: | ||
headers['Authorization'] = f"Bearer {token}" | ||
http_response = requests.get(url, headers=headers) | ||
if http_response.status_code != 200: | ||
raise Exception(f"HTTP Status code is {http_response.status_code}") | ||
|
||
return json.loads(http_response.content.decode()) | ||
|
||
|
||
if __name__ == '__main__': | ||
git_secret = get_secret_from_secrets_manager("s3/endpoint/resolver/artifacts/git", "us-east-1") | ||
|
||
rule_set = download_from_git(git_secret['ruleset-url'], git_secret['ruleset-token']) | ||
partition = download_from_git('https://raw.githubusercontent.com/awslabs/smithy/main/smithy-rules-engine/src/main' | ||
'/resources/software/amazon/smithy/rulesengine/language/partitions.json') | ||
|
||
generate_c_file_from_json( | ||
rule_set, | ||
'source/s3_endpoint_resolver/aws_s3_endpoint_rule_set.c', | ||
'aws_s3_endpoint_rule_set') | ||
|
||
generate_c_file_from_json( | ||
partition, | ||
'source/s3_endpoint_resolver/aws_s3_endpoint_resolver_partition.c', | ||
'aws_s3_endpoint_resolver_partitions') |
104 changes: 104 additions & 0 deletions
104
source/s3_endpoint_resolver/aws_s3_endpoint_resolver_partition.c
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
/** | ||
* Copyright Amazon.com, Inc. or its affiliates. | ||
* All Rights Reserved. SPDX-License-Identifier: Apache-2.0. | ||
*/ | ||
|
||
#include <aws/s3/s3_endpoint_resolver.h> | ||
|
||
/** | ||
* This file is generated using scripts/update_s3_endpoint_resolver_artifacts.py. | ||
* Do not modify directly. */ | ||
/* clang-format off */ | ||
|
||
static const char s_generated_array[] = { | ||
'{', '"', 'v', 'e', 'r', 's', 'i', 'o', 'n', '"', ':', '"', '1', '.', '1', '"', ',', '"', 'p', 'a', | ||
'r', 't', 'i', 't', 'i', 'o', 'n', 's', '"', ':', '[', '{', '"', 'i', 'd', '"', ':', '"', 'a', 'w', | ||
's', '"', ',', '"', 'r', 'e', 'g', 'i', 'o', 'n', 'R', 'e', 'g', 'e', 'x', '"', ':', '"', '^', '(', | ||
'u', 's', '|', 'e', 'u', '|', 'a', 'p', '|', 's', 'a', '|', 'c', 'a', '|', 'm', 'e', '|', 'a', 'f', | ||
')', '-', '\\', '\\', 'w', '+', '-', '\\', '\\', 'd', '+', '$', '"', ',', '"', 'r', 'e', 'g', 'i', 'o', | ||
'n', 's', '"', ':', '{', '"', 'a', 'f', '-', 's', 'o', 'u', 't', 'h', '-', '1', '"', ':', '{', '}', | ||
',', '"', 'a', 'p', '-', 'e', 'a', 's', 't', '-', '1', '"', ':', '{', '}', ',', '"', 'a', 'p', '-', | ||
'n', 'o', 'r', 't', 'h', 'e', 'a', 's', 't', '-', '1', '"', ':', '{', '}', ',', '"', 'a', 'p', '-', | ||
'n', 'o', 'r', 't', 'h', 'e', 'a', 's', 't', '-', '2', '"', ':', '{', '}', ',', '"', 'a', 'p', '-', | ||
'n', 'o', 'r', 't', 'h', 'e', 'a', 's', 't', '-', '3', '"', ':', '{', '}', ',', '"', 'a', 'p', '-', | ||
's', 'o', 'u', 't', 'h', '-', '1', '"', ':', '{', '}', ',', '"', 'a', 'p', '-', 's', 'o', 'u', 't', | ||
'h', 'e', 'a', 's', 't', '-', '1', '"', ':', '{', '}', ',', '"', 'a', 'p', '-', 's', 'o', 'u', 't', | ||
'h', 'e', 'a', 's', 't', '-', '2', '"', ':', '{', '}', ',', '"', 'a', 'p', '-', 's', 'o', 'u', 't', | ||
'h', 'e', 'a', 's', 't', '-', '3', '"', ':', '{', '}', ',', '"', 'c', 'a', '-', 'c', 'e', 'n', 't', | ||
'r', 'a', 'l', '-', '1', '"', ':', '{', '}', ',', '"', 'e', 'u', '-', 'c', 'e', 'n', 't', 'r', 'a', | ||
'l', '-', '1', '"', ':', '{', '}', ',', '"', 'e', 'u', '-', 'n', 'o', 'r', 't', 'h', '-', '1', '"', | ||
':', '{', '}', ',', '"', 'e', 'u', '-', 's', 'o', 'u', 't', 'h', '-', '1', '"', ':', '{', '}', ',', | ||
'"', 'e', 'u', '-', 'w', 'e', 's', 't', '-', '1', '"', ':', '{', '}', ',', '"', 'e', 'u', '-', 'w', | ||
'e', 's', 't', '-', '2', '"', ':', '{', '}', ',', '"', 'e', 'u', '-', 'w', 'e', 's', 't', '-', '3', | ||
'"', ':', '{', '}', ',', '"', 'm', 'e', '-', 'c', 'e', 'n', 't', 'r', 'a', 'l', '-', '1', '"', ':', | ||
'{', '}', ',', '"', 'm', 'e', '-', 's', 'o', 'u', 't', 'h', '-', '1', '"', ':', '{', '}', ',', '"', | ||
's', 'a', '-', 'e', 'a', 's', 't', '-', '1', '"', ':', '{', '}', ',', '"', 'u', 's', '-', 'e', 'a', | ||
's', 't', '-', '1', '"', ':', '{', '}', ',', '"', 'u', 's', '-', 'e', 'a', 's', 't', '-', '2', '"', | ||
':', '{', '}', ',', '"', 'u', 's', '-', 'w', 'e', 's', 't', '-', '1', '"', ':', '{', '}', ',', '"', | ||
'u', 's', '-', 'w', 'e', 's', 't', '-', '2', '"', ':', '{', '}', ',', '"', 'a', 'w', 's', '-', 'g', | ||
'l', 'o', 'b', 'a', 'l', '"', ':', '{', '}', '}', ',', '"', 'o', 'u', 't', 'p', 'u', 't', 's', '"', | ||
':', '{', '"', 'n', 'a', 'm', 'e', '"', ':', '"', 'a', 'w', 's', '"', ',', '"', 'd', 'n', 's', 'S', | ||
'u', 'f', 'f', 'i', 'x', '"', ':', '"', 'a', 'm', 'a', 'z', 'o', 'n', 'a', 'w', 's', '.', 'c', 'o', | ||
'm', '"', ',', '"', 'd', 'u', 'a', 'l', 'S', 't', 'a', 'c', 'k', 'D', 'n', 's', 'S', 'u', 'f', 'f', | ||
'i', 'x', '"', ':', '"', 'a', 'p', 'i', '.', 'a', 'w', 's', '"', ',', '"', 's', 'u', 'p', 'p', 'o', | ||
'r', 't', 's', 'F', 'I', 'P', 'S', '"', ':', 't', 'r', 'u', 'e', ',', '"', 's', 'u', 'p', 'p', 'o', | ||
'r', 't', 's', 'D', 'u', 'a', 'l', 'S', 't', 'a', 'c', 'k', '"', ':', 't', 'r', 'u', 'e', '}', '}', | ||
',', '{', '"', 'i', 'd', '"', ':', '"', 'a', 'w', 's', '-', 'u', 's', '-', 'g', 'o', 'v', '"', ',', | ||
'"', 'r', 'e', 'g', 'i', 'o', 'n', 'R', 'e', 'g', 'e', 'x', '"', ':', '"', '^', 'u', 's', '\\', '\\', | ||
'-', 'g', 'o', 'v', '\\', '\\', '-', '\\', '\\', 'w', '+', '\\', '\\', '-', '\\', '\\', 'd', '+', '$', '"', | ||
',', '"', 'r', 'e', 'g', 'i', 'o', 'n', 's', '"', ':', '{', '"', 'u', 's', '-', 'g', 'o', 'v', '-', | ||
'w', 'e', 's', 't', '-', '1', '"', ':', '{', '}', ',', '"', 'u', 's', '-', 'g', 'o', 'v', '-', 'e', | ||
'a', 's', 't', '-', '1', '"', ':', '{', '}', ',', '"', 'a', 'w', 's', '-', 'u', 's', '-', 'g', 'o', | ||
'v', '-', 'g', 'l', 'o', 'b', 'a', 'l', '"', ':', '{', '}', '}', ',', '"', 'o', 'u', 't', 'p', 'u', | ||
't', 's', '"', ':', '{', '"', 'n', 'a', 'm', 'e', '"', ':', '"', 'a', 'w', 's', '-', 'u', 's', '-', | ||
'g', 'o', 'v', '"', ',', '"', 'd', 'n', 's', 'S', 'u', 'f', 'f', 'i', 'x', '"', ':', '"', 'a', 'm', | ||
'a', 'z', 'o', 'n', 'a', 'w', 's', '.', 'c', 'o', 'm', '"', ',', '"', 'd', 'u', 'a', 'l', 'S', 't', | ||
'a', 'c', 'k', 'D', 'n', 's', 'S', 'u', 'f', 'f', 'i', 'x', '"', ':', '"', 'a', 'p', 'i', '.', 'a', | ||
'w', 's', '"', ',', '"', 's', 'u', 'p', 'p', 'o', 'r', 't', 's', 'F', 'I', 'P', 'S', '"', ':', 't', | ||
'r', 'u', 'e', ',', '"', 's', 'u', 'p', 'p', 'o', 'r', 't', 's', 'D', 'u', 'a', 'l', 'S', 't', 'a', | ||
'c', 'k', '"', ':', 't', 'r', 'u', 'e', '}', '}', ',', '{', '"', 'i', 'd', '"', ':', '"', 'a', 'w', | ||
's', '-', 'c', 'n', '"', ',', '"', 'r', 'e', 'g', 'i', 'o', 'n', 'R', 'e', 'g', 'e', 'x', '"', ':', | ||
'"', '^', 'c', 'n', '\\', '\\', '-', '\\', '\\', 'w', '+', '\\', '\\', '-', '\\', '\\', 'd', '+', '$', '"', | ||
',', '"', 'r', 'e', 'g', 'i', 'o', 'n', 's', '"', ':', '{', '"', 'c', 'n', '-', 'n', 'o', 'r', 't', | ||
'h', '-', '1', '"', ':', '{', '}', ',', '"', 'c', 'n', '-', 'n', 'o', 'r', 't', 'h', 'w', 'e', 's', | ||
't', '-', '1', '"', ':', '{', '}', ',', '"', 'a', 'w', 's', '-', 'c', 'n', '-', 'g', 'l', 'o', 'b', | ||
'a', 'l', '"', ':', '{', '}', '}', ',', '"', 'o', 'u', 't', 'p', 'u', 't', 's', '"', ':', '{', '"', | ||
'n', 'a', 'm', 'e', '"', ':', '"', 'a', 'w', 's', '-', 'c', 'n', '"', ',', '"', 'd', 'n', 's', 'S', | ||
'u', 'f', 'f', 'i', 'x', '"', ':', '"', 'a', 'm', 'a', 'z', 'o', 'n', 'a', 'w', 's', '.', 'c', 'o', | ||
'm', '.', 'c', 'n', '"', ',', '"', 'd', 'u', 'a', 'l', 'S', 't', 'a', 'c', 'k', 'D', 'n', 's', 'S', | ||
'u', 'f', 'f', 'i', 'x', '"', ':', '"', 'a', 'p', 'i', '.', 'a', 'm', 'a', 'z', 'o', 'n', 'w', 'e', | ||
'b', 's', 'e', 'r', 'v', 'i', 'c', 'e', 's', '.', 'c', 'o', 'm', '.', 'c', 'n', '"', ',', '"', 's', | ||
'u', 'p', 'p', 'o', 'r', 't', 's', 'F', 'I', 'P', 'S', '"', ':', 't', 'r', 'u', 'e', ',', '"', 's', | ||
'u', 'p', 'p', 'o', 'r', 't', 's', 'D', 'u', 'a', 'l', 'S', 't', 'a', 'c', 'k', '"', ':', 't', 'r', | ||
'u', 'e', '}', '}', ',', '{', '"', 'i', 'd', '"', ':', '"', 'a', 'w', 's', '-', 'i', 's', 'o', '"', | ||
',', '"', 'r', 'e', 'g', 'i', 'o', 'n', 'R', 'e', 'g', 'e', 'x', '"', ':', '"', '^', 'u', 's', '\\', | ||
'\\', '-', 'i', 's', 'o', '\\', '\\', '-', '\\', '\\', 'w', '+', '\\', '\\', '-', '\\', '\\', 'd', '+', '$', | ||
'"', ',', '"', 'o', 'u', 't', 'p', 'u', 't', 's', '"', ':', '{', '"', 'n', 'a', 'm', 'e', '"', ':', | ||
'"', 'a', 'w', 's', '-', 'i', 's', 'o', '"', ',', '"', 'd', 'n', 's', 'S', 'u', 'f', 'f', 'i', 'x', | ||
'"', ':', '"', 'c', '2', 's', '.', 'i', 'c', '.', 'g', 'o', 'v', '"', ',', '"', 's', 'u', 'p', 'p', | ||
'o', 'r', 't', 's', 'F', 'I', 'P', 'S', '"', ':', 't', 'r', 'u', 'e', ',', '"', 's', 'u', 'p', 'p', | ||
'o', 'r', 't', 's', 'D', 'u', 'a', 'l', 'S', 't', 'a', 'c', 'k', '"', ':', 'f', 'a', 'l', 's', 'e', | ||
',', '"', 'd', 'u', 'a', 'l', 'S', 't', 'a', 'c', 'k', 'D', 'n', 's', 'S', 'u', 'f', 'f', 'i', 'x', | ||
'"', ':', '"', 'c', '2', 's', '.', 'i', 'c', '.', 'g', 'o', 'v', '"', '}', ',', '"', 'r', 'e', 'g', | ||
'i', 'o', 'n', 's', '"', ':', '{', '"', 'u', 's', '-', 'i', 's', 'o', '-', 'e', 'a', 's', 't', '-', | ||
'1', '"', ':', '{', '}', ',', '"', 'u', 's', '-', 'i', 's', 'o', '-', 'w', 'e', 's', 't', '-', '1', | ||
'"', ':', '{', '}', ',', '"', 'a', 'w', 's', '-', 'i', 's', 'o', '-', 'g', 'l', 'o', 'b', 'a', 'l', | ||
'"', ':', '{', '}', '}', '}', ',', '{', '"', 'i', 'd', '"', ':', '"', 'a', 'w', 's', '-', 'i', 's', | ||
'o', '-', 'b', '"', ',', '"', 'r', 'e', 'g', 'i', 'o', 'n', 'R', 'e', 'g', 'e', 'x', '"', ':', '"', | ||
'^', 'u', 's', '\\', '\\', '-', 'i', 's', 'o', 'b', '\\', '\\', '-', '\\', '\\', 'w', '+', '\\', '\\', '-', | ||
'\\', '\\', 'd', '+', '$', '"', ',', '"', 'o', 'u', 't', 'p', 'u', 't', 's', '"', ':', '{', '"', 'n', | ||
'a', 'm', 'e', '"', ':', '"', 'a', 'w', 's', '-', 'i', 's', 'o', '-', 'b', '"', ',', '"', 'd', 'n', | ||
's', 'S', 'u', 'f', 'f', 'i', 'x', '"', ':', '"', 's', 'c', '2', 's', '.', 's', 'g', 'o', 'v', '.', | ||
'g', 'o', 'v', '"', ',', '"', 's', 'u', 'p', 'p', 'o', 'r', 't', 's', 'F', 'I', 'P', 'S', '"', ':', | ||
't', 'r', 'u', 'e', ',', '"', 's', 'u', 'p', 'p', 'o', 'r', 't', 's', 'D', 'u', 'a', 'l', 'S', 't', | ||
'a', 'c', 'k', '"', ':', 'f', 'a', 'l', 's', 'e', ',', '"', 'd', 'u', 'a', 'l', 'S', 't', 'a', 'c', | ||
'k', 'D', 'n', 's', 'S', 'u', 'f', 'f', 'i', 'x', '"', ':', '"', 's', 'c', '2', 's', '.', 's', 'g', | ||
'o', 'v', '.', 'g', 'o', 'v', '"', '}', ',', '"', 'r', 'e', 'g', 'i', 'o', 'n', 's', '"', ':', '{', | ||
'"', 'u', 's', '-', 'i', 's', 'o', 'b', '-', 'e', 'a', 's', 't', '-', '1', '"', ':', '{', '}', ',', | ||
'"', 'a', 'w', 's', '-', 'i', 's', 'o', '-', 'b', '-', 'g', 'l', 'o', 'b', 'a', 'l', '"', ':', '{', | ||
'}', '}', '}', ']', '}'}; | ||
|
||
const struct aws_byte_cursor aws_s3_endpoint_resolver_partitions = { | ||
.len = 1705, | ||
.ptr = (uint8_t *) s_generated_array | ||
}; |
Oops, something went wrong.