From 260d71f88b913022b29a50e470429fda897314eb Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Mon, 18 Jul 2022 13:24:12 -0700 Subject: [PATCH] Developer guide for internal devs, and helper scripts (#372) * `docs/dev/README.md`: Getting started guide for contributors to aws-crt-python. I'm going to flesh this out more, or add other docs. But this is a solid start. While writing the guide I realized helper scripts would be useful so: * `scripts/`: new home for helper scripts. I chose to use python instead of bash so that they could be run on Windows machines as well. * `scripts/install-dev.py`: build and install in "develop" mode * `scripts/clean.py`: clean up misc tmp files * `scripts/format-c.py`: format all C code * `scripts/format-python.py`: format all Python code * `scripts/format-all.py`: format all code * `scripts/utils.py`: common helper functions --- .github/workflows/ci.yml | 2 +- .github/workflows/docs.yml | 4 +- .gitignore | 8 +- README.md | 45 +---- docs/README.md | 6 + docs/dev/README.md | 245 +++++++++++++++++++++++++++ docs/dev/vscode/tasks.json | 31 ++++ requirements-dev.txt | 2 + scripts/clean.py | 49 ++++++ scripts/format-all.py | 7 + scripts/format-c.py | 16 ++ scripts/format-python.py | 19 +++ scripts/install-dev.py | 6 + make-docs.sh => scripts/make-docs.sh | 8 +- scripts/utils.py | 36 ++++ setup.py | 3 - 16 files changed, 428 insertions(+), 59 deletions(-) create mode 100644 docs/README.md create mode 100644 docs/dev/README.md create mode 100644 docs/dev/vscode/tasks.json create mode 100644 requirements-dev.txt create mode 100755 scripts/clean.py create mode 100755 scripts/format-all.py create mode 100755 scripts/format-c.py create mode 100755 scripts/format-python.py create mode 100755 scripts/install-dev.py rename make-docs.sh => scripts/make-docs.sh (73%) create mode 100644 scripts/utils.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dbb69357..d42d4237c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: run: | python3 -m pip install sphinx python3 -m pip install --verbose . - ./make-docs.sh + ./scripts/make-docs.sh check-submodules: runs-on: ubuntu-20.04 # latest diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 169627307..0760dd33f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,13 +22,13 @@ jobs: run: | python3 -m pip install sphinx python3 -m pip install --verbose . - ./make-docs.sh + ./scripts/make-docs.sh - name: Commit run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git add --force docs/ + git add --force docs/api git commit --message="update docs" - name: Push to docs branch diff --git a/.gitignore b/.gitignore index bbc5d5bce..857aa76f0 100644 --- a/.gitignore +++ b/.gitignore @@ -129,9 +129,6 @@ instance/ # Scrapy stuff: .scrapy -# Sphinx documentation -docs/_build/ - # PyBuilder target/ @@ -185,7 +182,6 @@ dmypy.json [Ll]ib [Ll]ib64 [Ll]ocal -[Ss]cripts pyvenv.cfg pip-selfcheck.json @@ -533,5 +529,5 @@ ASALocalRun/ # deps from build-deps.sh deps/ -# docs are updated automatically by .github/workflows/docs.yml -docs/ +# API docs are updated automatically by .github/workflows/docs.yml +docs/api diff --git a/README.md b/README.md index 8974274b3..71aef7abd 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Python 3 bindings for the AWS Common Runtime. -API documentation: https://awslabs.github.io/aws-crt-python/ +* [API documentation](https://awslabs.github.io/aws-crt-python/) +* [Development guide](docs/dev/README.md) for contributors to aws-crt-python's source code. ## License @@ -37,45 +38,3 @@ Please note that on Mac, once a private key is used with a certificate, that cer ``` static: certificate has an existing certificate-key pair that was previously imported into the Keychain. Using key from Keychain instead of the one provided. ``` - -## Running tests - -After install, run from project root: -```bash -python3 -m unittest discover --failfast --verbose -``` - -`--failfast` stops after one failed test. -This is useful because a failed test is likely to leak memory, -which will cause the rest of the tests to fail as well. - -`--verbose` makes it easier to see which tests are skipped. - -Many tests require an AWS account. These tests are skipped unless -specific environment variables are set: - -MQTT -* AWS_TEST_IOT_MQTT_ENDPOINT - AWS account-specific endpoint to connect to IoT core by -* AWS_TEST_TLS_CERT_PATH - file path to certificate used to initialize the TLS context of the MQTT connection -* AWS_TEST_TLS_KEY_PATH - file path to the private key used to initialize the TLS context of the MQTT connection -* AWS_TEST_TLS_ROOT_CERT_PATH - file path to the root CA used to initialize the TLS context of the MQTT connection - -PKCS11 -* AWS_TEST_PKCS11_LIB - path to PKCS#11 library -* AWS_TEST_PKCS11_PIN - user PIN for logging into PKCS#11 token -* AWS_TEST_PKCS11_TOKEN_LABEL - label of PKCS#11 token -* AWS_TEST_PKCS11_KEY_LABEL - label of private key on PKCS#11 token, - which must correspond to the cert at AWS_TEST_TLS_CERT_PATH. - -PROXY -* AWS_TEST_HTTP_PROXY_HOST - host address of the proxy to use for tests that make open connections to the proxy -* AWS_TEST_HTTP_PROXY_PORT - port to use for tests that make open connections to the proxy -* AWS_TEST_HTTPS_PROXY_HOST - host address of the proxy to use for tests that make TLS-protected connections to the proxy -* AWS_TEST_HTTPS_PROXY_PORT - port to use for tests that make TLS-protected connections to the proxy -* AWS_TEST_HTTP_PROXY_BASIC_HOST - host address of the proxy to use for tests that make open connections to the proxy with basic authentication -* AWS_TEST_HTTP_PROXY_BASIC_PORT - port to use for tests that make open connections to the proxy with basic authentication -* AWS_TEST_BASIC_AUTH_USERNAME - username to use when using basic authentication to the proxy -* AWS_TEST_BASIC_AUTH_PASSWORD - password to use when using basic authentication to the proxy - -S3 -* AWS_TEST_S3 - set to any value to enable S3 tests diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..3f6bc3489 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,6 @@ +# AWS Common Runtime (CRT) for Python: Documentation +## CRT Usage +- [General information](../README.md) (installation, etc) +- [API documentation](https://awslabs.github.io/aws-crt-python/) +## CRT Development +- [Development guide](dev/README.md) for contributors to aws-crt-python's source code. diff --git a/docs/dev/README.md b/docs/dev/README.md new file mode 100644 index 000000000..67a525436 --- /dev/null +++ b/docs/dev/README.md @@ -0,0 +1,245 @@ +# Development guide for aws-crt-python + +This guide is for contributors to aws-crt-python's source code. +Familiarity (but not necessarily expertise) with Python and C is assumed. + +### Table of Contents + +* [Git](#git) +* [Set up a Virtual Environment](#set-up-a-virtual-environment) +* [Install](#install) +* [Run Tests](#run-tests) + * [Environment Variables for Tests](#environment-variables-for-tests) +* [Using an IDE](#using-an-ide) + * [Visual Studio Code](#using-visual-studio-code-vscode) + * [Debugging Python](#debugging-python-with-vscode) + * [Debugging C](#debugging-c-with-vscode) + +## Git + +Clone to a development folder: +```sh +$ git clone git@github.com:awslabs/aws-crt-python.git +$ cd aws-crt-python +$ git submodule update --init +``` + +Note that you MUST manually update submodules any time you pull latest, or change branches: +```sh +$ git submodule update +``` + +## CMake + +CMake 3 is required to compile the C submodules. To install: + +* On Mac, using homebrew: + ```sh + $ brew install cmake + ``` +* On Linux: use your system's package manager (apt, yum, etc). +* On Windows: [download and install](https://cmake.org/download/) + +## Set up a Virtual Environment + +Set up a [virtual environment](https://docs.python.org/3/library/venv.html) +for development. This guide suggests `aws-crt-python/.venv/` as a default location. +Create a virtual environment like so: +```sh +$ python3 -m venv .venv/ +``` + +To activate the virtual environment in your current terminal: +* On Mac or Linux: + ```sh + $ source .venv/bin/activate + ``` +* In Windows PowerShell: + ```pwsh + > .venv\Scripts\Activate.ps1 + ``` +* In Windows Command Prompt: + ```bat + > .venv\Scripts\Activate.bat + ``` + +Your terminal looks something like this when the virtual environment is active: +``` +(.venv) $ +``` +Now any time you type `python3` or `pip` or even `python`, you are using the +one from your virtual environment. +To stop using the virtual environment, run `deactivate`. + +## Install + +Ensure your tools are up to date: +```sh +(.venv) $ python3 -m pip install --upgrade pip setuptools +``` + +Install dev dependencies: +```sh +(.venv) $ python3 -m pip install --requirement requirements-dev.txt +``` + +Install aws-crt-python (helper script `python3 scripts/install-dev.py` does this): +```sh +(.venv) $ python3 -m pip install --verbose --editable . +``` + +You must re-run this command any time the C source code changes. +But you don't need to re-run it if .py files change +(thanks to the `--editable` aka "develop mode" flag) + +Note that this takes about twice as long on Mac, which compiles C for both `x86_64` and `arm64`. +(TODO: in develop mode, for productivity's sake, only compile C for one architecture, +and ignore resulting warnings from the linker) + +## Run Tests + +To run all tests: +```sh +(.venv) $ python3 -m unittest discover --failfast --verbose +``` + +`discover` automatically finds all tests to run + +`--failfast` stops after one failed test. +This is useful because a failed test is likely to leak memory, +which will cause the rest of the tests to fail as well. + +`--verbose` makes it easier to see which tests are skipped. + +To run specific tests, specify a path. For example: +```sh +(.venv) $ python3 -m unittest --failfast --verbose test.test_http_client.TestClient.test_connect_http +``` + +More path examples: +* `test` - everything under `test/` folder +* `test.test_http_client` - every test in `test_http_client.py` +* `test.test_http_client.TestClient` - every test in `TestClient` class +* `test.test_http_client.TestClient.test_connect_http` - A single test + +When creating new tests, note that the names of test files and test functions must be prefixed with `test_`. + +### Environment Variables for Tests +Many tests require an AWS account. These tests are skipped unless +specific environment variables are set: + +* MQTT Tests + * `AWS_TEST_IOT_MQTT_ENDPOINT` - AWS IoT Core endpoint. This is specific to your account. + * `AWS_TEST_TLS_CERT_PATH` - file path to the certificate used to initialize the TLS context of the MQTT connection + * `AWS_TEST_TLS_KEY_PATH` - file path to the private key used to initialize the TLS context of the MQTT connection + * `AWS_TEST_TLS_ROOT_CERT_PATH` - file path to the root CA used to initialize the TLS context of the MQTT connection +* PKCS#11 Tests + * `AWS_TEST_PKCS11_LIB` - path to PKCS#11 library + * `AWS_TEST_PKCS11_PIN` - user PIN for logging into PKCS#11 token + * `AWS_TEST_PKCS11_TOKEN_LABEL` - label of PKCS#11 token + * `AWS_TEST_PKCS11_KEY_LABEL` - label of private key on PKCS#11 token, which must correspond to the cert at `AWS_TEST_TLS_CERT_PATH`. +* Proxy Tests + * TLS-protected connections to the proxy + * `AWS_TEST_HTTPS_PROXY_HOST` - proxy host address + * `AWS_TEST_HTTPS_PROXY_PORT` - proxy port + * Open connections to the proxy + * `AWS_TEST_HTTP_PROXY_HOST` - proxy host address + * `AWS_TEST_HTTP_PROXY_PORT` - port port + * Open connections to the proxy, with basic authentication + * `AWS_TEST_HTTP_PROXY_BASIC_HOST` - proxy host address + * `AWS_TEST_HTTP_PROXY_BASIC_PORT` - proxy port + * `AWS_TEST_BASIC_AUTH_USERNAME` - username + * `AWS_TEST_BASIC_AUTH_PASSWORD` - password +* S3 Tests + * `AWS_TEST_S3` - set to any value to enable S3 tests. + * **Unfortunately, at this time these tests can only be run by members of the + Common Runtime team, due to hardcoded paths.** + * TODO: alter tests so anyone can run them. + +## Code Formatting + +We use automatic code formatters in this project and pull requests will fail unless +the code is formatted correctly. + +`autopep8` is used for python code. You installed this earlier via `requirements-dev.txt`. + +For C code `clang-format` is used (specifically version 9). +To install this on Mac using homebrew, run: +```sh +(.venv) $ brew install llvm@9 +``` + +Use helper scripts to automatically format your code (or configure your IDE to do it): +```sh +# format all code +(.venv) $ python3 scripts/format-all.py + +# just format Python files +(.venv) $ python3 scripts/format-python.py + +# just format C files +(.venv) $ python3 scripts/format-c.py +``` + +## Using an IDE +### Using Visual Studio Code (VSCode) + +1) Install the following extensions: + * Python (Microsoft) + * C/C++ (Microsoft) + * Code Spell Checker (Street Side Software) - optional + +2) Open the `aws-crt-python/` folder. + +3) Edit workspace settings: `cmd+shift+P -> Preferences: Open Workspace Settings` + * `Python: Default Interpreter Path` - ("python.defaultInterpreterPath" in json view) + * Set the absolute path to Python in your virtual environment. + For example: `/Users/janedoe/dev/aws-crt-python/.venv/bin/python` + * Note that the VSCode terminal ignores this setting and will not use your virtual environment by default. + You must manually run `source .venv/bin/activate` each time you start using the terminal. + Or use the command `cmd+shift+P -> Python: Create Terminal`. + * `C_Cpp > Default: Include Path` - ("C_Cpp.default.includePath" in json view) + * Add item - set path to Python's C headers. + For example: `/Library/Frameworks/Python.framework/Versions/3.10/include/python3.10` + * This is optional, it helps IntelliSense when viewing C files. + * `Files: Insert Final Newline` - ("files.insertFinalNewline" in json view) + * Set to true. It's just good practice. + * `Files: Trim Trailing Whitespace` - ("files.trimTrailingWhitespace" in json view) + * Set to true. It's just good practice. + +4) Add helpful tasks you can run via `cmd+shift+P -> Tasks: Run Task` + * Copy [this file](vscode/tasks.json) to `aws-crt-python/.vscode/tasks.json` for the following tasks: + * `install` - `pip install` in develop mode. `cmd+shift+B` is a special shortcut for this task + * `format` - format all code files + +#### Debugging Python with VSCode +The VSCode `Testing` tab (lab flask/beaker icon) helps run and debug Python tests. +From this tab, click Configure Python Tests: +* Select a test framework/tool to enable - unittest +* Select the directory containing the tests - test +* Select the pattern to identify test files - test_*.py + +Run tests by mousing over the name and clicking the PLAY button, +or use the DEBUG button to hit breakpoints in Python code, and see output in the DEBUG CONSOLE. + +Note that many tests are skipped unless [specific environment variables](#environment-variables-for-tests) are set. +You can set these in a `aws-crt-python/.env` file (don't worry it's ignored by git. For example: +``` +AWS_TEST_IOT_MQTT_ENDPOINT=xxxxxxxxx-ats.iot.xxxxx.amazonaws.com +AWS_TEST_TLS_CERT_PATH=/Users/janedoe/iot/xxxxx-certificate.pem.crt +AWS_TEST_TLS_KEY_PATH=/Users/janedoe/iot/xxxxx-private.pem.key +``` + +#### Debugging C with VSCode +Unfortunately, we haven't figured out how to do interactive debugging of the C code. +Python ultimately builds and links the C module together, and it seems to always strip out the debug info. +Please update this guide if you know how. For now, `printf()` is your best option. + +If you suspect the bug is in the external C code (i.e. [aws-c-http](https://github.com/awslabs/aws-c-http)) +and need to do interactive debugging, your best bet is cloning that external project, +build it in Debug, and step through its tests. + +# TODO +* more about git submodules (latest-submodules.py and working from branches) +* more about logging. consider easy way to turn on logging in tests + diff --git a/docs/dev/vscode/tasks.json b/docs/dev/vscode/tasks.json new file mode 100644 index 000000000..80f69ce6e --- /dev/null +++ b/docs/dev/vscode/tasks.json @@ -0,0 +1,31 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "install", + "type": "shell", + "command": "${config:python.defaultInterpreterPath} scripts/install-dev.py", + "group": { + "kind": "build", + "isDefault": true + }, + "presentation": { + "panel": "shared", + "clear": true + }, + "problemMatcher": [] + }, + { + "label": "format", + "type": "shell", + "command": "${config:python.defaultInterpreterPath} scripts/format-all.py", + "presentation": { + "panel": "shared", + "clear": true + }, + "problemMatcher": [] + } + ] +} diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..3f5acb0f4 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +autopep8 # for code formatting +sphinx # for building docs diff --git a/scripts/clean.py b/scripts/clean.py new file mode 100755 index 000000000..a19a2404c --- /dev/null +++ b/scripts/clean.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +import glob +import os +import shutil +import utils + +# apply these patterns without recursing through subfolders +NONRECURSIVE_PATTERNS = [ + 'build/', + '_awscrt*.*', # compiled _awscrt shared lib + 'awscrt.egg-info/', + 'dist/', + 'wheelhouse/', + 'docsrc/build/', +] + +# recurse through subfolders and apply these patterns +RECURSIVE_PATTERNS = [ + '*.pyc', + '__pycache__/' +] + +# approved list of folders +# because we don't want to clean the virtual environment folder +APPROVED_RECURSIVE_FOLDERS = [ + 'awscrt' + 'scripts' + 'test' + '.builder' +] + +utils.chdir_project_root() + +utils.run('python3 -m pip uninstall -y awscrt') + +paths = [] +for pattern in NONRECURSIVE_PATTERNS: + paths.extend(glob.glob(pattern)) + +for pattern in RECURSIVE_PATTERNS: + for folder in APPROVED_RECURSIVE_FOLDERS: + paths.extend(glob.glob(f'{folder}/**/{pattern}', recursive=True)) + +for path in paths: + print(f'delete: {path}') + if os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) diff --git a/scripts/format-all.py b/scripts/format-all.py new file mode 100755 index 000000000..a185d3669 --- /dev/null +++ b/scripts/format-all.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +import utils + +utils.chdir_project_root() + +utils.run('python3 scripts/format-c.py') +utils.run('python3 scripts/format-python.py') diff --git a/scripts/format-c.py b/scripts/format-c.py new file mode 100755 index 000000000..d52a1c8e6 --- /dev/null +++ b/scripts/format-c.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +import utils +import glob + +FILE_PATTERNS = [ + 'source/**/*.h', + 'source/**/*.c', +] + +utils.chdir_project_root() + +files = [] +for pattern in FILE_PATTERNS: + files.extend(glob.glob(pattern, recursive=True)) + +utils.run(['clang-format', '-i', *files]) diff --git a/scripts/format-python.py b/scripts/format-python.py new file mode 100755 index 000000000..e2572018a --- /dev/null +++ b/scripts/format-python.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import utils + +FILES_AND_FOLDERS_TO_FORMAT = [ + '.builder/', + 'awscrt/', + 'scripts/', + 'test/', + 'setup.py', +] + +utils.chdir_project_root() + +utils.run(['python3', + '-m', 'autopep8', + '--in-place', # edit files in place + '--recursive', + '--jobs', '0', # parallel with all CPUs + *FILES_AND_FOLDERS_TO_FORMAT]) diff --git a/scripts/install-dev.py b/scripts/install-dev.py new file mode 100755 index 000000000..6c158851b --- /dev/null +++ b/scripts/install-dev.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +import utils + +utils.chdir_project_root() + +utils.run('python3 -m pip install --verbose --editable .') diff --git a/make-docs.sh b/scripts/make-docs.sh similarity index 73% rename from make-docs.sh rename to scripts/make-docs.sh index 7c898f1bb..8238e386c 100755 --- a/make-docs.sh +++ b/scripts/make-docs.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -e -pushd `dirname $0` > /dev/null +pushd "$(dirname $0)/.." > /dev/null # clean -rm -rf docs/ +rm -rf docs/api rm -rf docsrc/build/ # build @@ -12,10 +12,10 @@ pushd docsrc > /dev/null make html SPHINXOPTS="-W --keep-going" popd > /dev/null -cp -a docsrc/build/html/. docs +cp -a docsrc/build/html/. docs/api # The existence of this file tells GitHub Pages to just host the HTML as-is # https://github.blog/2009-12-29-bypassing-jekyll-on-github-pages/ -touch docs/.nojekyll +touch docs/api/.nojekyll popd > /dev/null diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 000000000..324bb8147 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,36 @@ +import os +import shlex +import subprocess +import sys + + +def run(args: str | list[str]): + """ + Run a program. + + args may be a string, or list of argument strings. + + If the program is "python3", this is replaced with the + full path to the current python executable. + """ + + # convert string to list + # so that we don't need to pass shell=True to subprocess.run() + # because turning on shell can mess things up (i.e. clang-format hangs forever for some reason) + if isinstance(args, str): + args = shlex.split(args) + + # ensure proper python executable is used + if args[0] == 'python3': + args[0] = sys.executable + + # run + print(f'+ {subprocess.list2cmdline(args)}') + result = subprocess.run(args) + if result.returncode != 0: + sys.exit('FAILED') + + +def chdir_project_root(): + root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + os.chdir(root) diff --git a/setup.py b/setup.py index ab99f8e2b..7c1526343 100644 --- a/setup.py +++ b/setup.py @@ -361,7 +361,4 @@ def _load_version(): ext_modules=[awscrt_ext()], cmdclass={'build_ext': awscrt_build_ext}, test_suite='test', - tests_require=[ - 'boto3' - ] )