New pypi wait strategy to fix release pipeline. (#221)

Our release pipeline needs to wait until a newly uploaded version is available in pypi, before installing the new version for testing. We were using `pip search` to determine that the new version was available, but `pip search` has been disabled at a server level (https://status.python.org/incidents/grk0k7sz6zkp). We've had issues with this approach in the past anyway, where it looked like it was globally available, but then didn't install on some specific platform.

New strategy is to just retry install attempts until they work (or cap out).

codebuild/cd/publish_to_prod_pypi.yml

@@ -22,7 +22,6 @@ phases:
       - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
       - python3 continuous-delivery/pull-pypirc.py prod
       - python3 -m twine upload -r pypi ../dist/*
-      - python3 continuous-delivery/wait-for-pypi.py awscrt $CURRENT_TAG_VERSION --index https://pypi.python.org/pypi
   post_build:
     commands:
       - echo Build completed on `date`

codebuild/cd/publish_to_test_pypi.yml

@@ -22,7 +22,6 @@ phases:
       - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
       - python3 continuous-delivery/pull-pypirc.py alpha
       - python3 -m twine upload -r testpypi ../dist/*
-      - python3 continuous-delivery/wait-for-pypi.py awscrt $CURRENT_TAG_VERSION --index https://test.pypi.org/pypi
   post_build:
     commands:
       - echo Build completed on `date`

codebuild/cd/test_prod_pypi.yml

@@ -14,7 +14,7 @@ phases:
       - echo Build started on `date`
       - cd aws-crt-python
       - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
-      - python3 -m pip install --no-cache-dir --user awscrt==$CURRENT_TAG_VERSION
+      - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user awscrt==$CURRENT_TAG_VERSION
       - python3 continuous-delivery/test-pip-install.py
   post_build:
     commands:

codebuild/cd/test_test_pypi.yml

@@ -14,7 +14,7 @@ phases:
       - echo Build started on `date`
       - cd aws-crt-python
       - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
-      - python3 -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION
+      - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION
       - python3 continuous-delivery/test-pip-install.py
   post_build:
     commands:

continuous-delivery/pip-install-with-retry.py

@@ -0,0 +1,39 @@
+import time
+import sys
+import subprocess
+
+DOCS = """Given cmdline args, executes: python3 -m pip install [args...]
+Keeps retrying until the new version becomes available in pypi (or we time out)"""
+if len(sys.argv) < 2:
+    sys.exit(DOCS)
+
+RETRY_INTERVAL_SECS = 10
+GIVE_UP_AFTER_SECS = 60 * 15
+
+pip_install_args = [sys.executable, '-m', 'pip', 'install'] + sys.argv[1:]
+
+start_time = time.time()
+while True:
+    print(subprocess.list2cmdline(pip_install_args))
+    result = subprocess.run(pip_install_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+    stdout = result.stdout.decode().strip()
+    if stdout:
+        print(stdout)
+
+    if result.returncode == 0:
+        # success
+        sys.exit(0)
+
+    if "could not find a version" in stdout.lower():
+        elapsed_secs = time.time() - start_time
+        if elapsed_secs < GIVE_UP_AFTER_SECS:
+            # try again
+            print("Retrying in", RETRY_INTERVAL_SECS, "secs...")
+            time.sleep(RETRY_INTERVAL_SECS)
+            continue
+        else:
+            print("Giving up on retries after", int(elapsed_secs), "total secs.")
+
+    # fail
+    sys.exit(result.returncode)

continuous-delivery/sanity-check-test-pypi.bat

@@ -1,7 +1,7 @@
 FOR /F "delims=" %%A in ('git describe --tags') do ( set TAG_VERSION=%%A )
 set CURRENT_VERSION=%TAG_VERSION:v=%
 
-"C:\Program Files\Python37\python.exe" -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==%CURRENT_VERSION% || goto error
+"C:\Program Files\Python37\python.exe" continuous-delivery\pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==%CURRENT_VERSION% || goto error
 "C:\Program Files\Python37\python.exe" continuous-delivery\test-pip-install.py || goto error
 
 goto :EOF

continuous-delivery/sanity-check-test-pypi.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 set -ex
 CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
-python3 -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION
+python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION
 python3 continuous-delivery/test-pip-install.py