Merge pull request #650 from asfadmin/rew/pr-3244-s3credentials-readme

PR-3244 Add s3credentialsREADME endpoint

Makefile

@@ -4,7 +4,8 @@ SOURCES := \
 	lambda/tea_bumper.py \
 	lambda/update_lambda.py
 
-RESOURCES := $(wildcard lambda/templates/*)
+HTML_TEMPLATES := $(wildcard lambda/templates/*.html)
+MD_TEMPLATES := $(wildcard lambda/templates/*.md)
 TERRAFORM := $(wildcard terraform/*)
 
 REQUIREMENTS_IN := $(wildcard requirements/*.in)
@@ -15,7 +16,9 @@ DIR := dist
 EMPTY := $(DIR)/empty
 # Temporary artifacts
 DIST_SOURCES := $(SOURCES:lambda/%=$(DIR)/code/%)
-DIST_RESOURCES := $(RESOURCES:lambda/%=$(DIR)/code/%)
+DIST_MD_RESOURCES := $(MD_TEMPLATES:lambda/%.md=$(DIR)/code/%.html)
+DIST_HTML_RESOURCES := $(HTML_TEMPLATES:lambda/%=$(DIR)/code/%)
+DIST_RESOURCES := $(DIST_HTML_RESOURCES) $(DIST_MD_RESOURCES)
 DIST_TERRAFORM := $(TERRAFORM:terraform/%=$(DIR)/terraform/%)
 
 BUCKET_MAP_OBJECT_KEY := DEFAULT
@@ -105,8 +108,13 @@ $(DIR)/thin-egress-app-dependencies.zip: requirements/requirements.txt $(REQUIRE
 	@mkdir -p $(DIR)/python
 	$(DOCKER_LAMBDA_CI) build/dependency_builder.sh "$(DIR)/thin-egress-app-dependencies.zip" "$(DIR)"
 
+.SECONDARY: $(DIST_MD_RESOURCES)
+$(DIST_MD_RESOURCES): $(DIR)/code/%.html: lambda/%.md $(BUILD_VENV)
+	@mkdir -p $(@D)
+	$(BUILD_VENV)/bin/python scripts/render_md.py $< --output $@
+
 .SECONDARY: $(DIST_RESOURCES)
-$(DIST_RESOURCES): $(DIR)/code/%: lambda/%
+$(DIST_HTML_RESOURCES): $(DIR)/code/%: lambda/%
 	@mkdir -p $(@D)
 	cp $< $@
 

cloudformation/thin-egress-app.yaml.j2

@@ -716,6 +716,16 @@ Resources:
       PathPart: 's3credentials'
       RestApiId: !Ref EgressApiGateway
 
+  EgressApiResourceS3CredentialsREADME:
+    Type: AWS::ApiGateway::Resource
+    Condition: S3CredentialsEndpointIsSet
+    DependsOn:
+      - EgressApiGateway
+    Properties:
+      ParentId: !GetAtt EgressApiGateway.RootResourceId
+      PathPart: 's3credentialsREADME'
+      RestApiId: !Ref EgressApiGateway
+
   EgressApiResourceProfile:
     Type: AWS::ApiGateway::Resource
     DependsOn:
@@ -925,6 +935,23 @@ Resources:
       ResourceId: !Ref EgressApiResourceS3Credentials
       RestApiId: !Ref EgressApiGateway
 
+  EgressAPIMethodS3CredentialsREADME:
+    Type: AWS::ApiGateway::Method
+    Condition: S3CredentialsEndpointIsSet
+    Properties:
+      ApiKeyRequired: false
+      AuthorizationType: 'NONE'
+      HttpMethod: 'GET'
+      Integration:
+        IntegrationHttpMethod: 'POST'
+        IntegrationResponses:
+          - StatusCode: 200
+        Type: 'AWS_PROXY'
+        Uri: !Sub "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${EgressLambda.Arn}/invocations"
+      OperationName: 's3credentialsREADME view'
+      ResourceId: !Ref EgressApiResourceS3CredentialsREADME
+      RestApiId: !Ref EgressApiGateway
+
   EgressAPIMethodProfile:
     Type: AWS::ApiGateway::Method
     Properties:

docs/configuration.md

@@ -1,3 +1,5 @@
+# Configuration
+
 ## Bucket Mapping
 
 At the heart of TEA is the concept of the Bucket Map. This YAML file tells TEA how to

docs/deploying.md

@@ -1,3 +1,5 @@
+# Deploying
+
 ## Quickstart
 
 You can either download [`deploy.sh`](https://github.com/asfadmin/thin-egress-app/blob/devel/build/deploy.sh)

docs/index.md

@@ -1,4 +1,5 @@
 # Thin Egress App
+
 ## Purpose of TEA
 
 TEA is a fully Earthdata Cloud (EDC) compliant application to enable

docs/s3access.md

@@ -1,83 +1,130 @@
-## S3 Direct Access
+# S3 Direct Access
+
 *NOTE: Support for S3 direct access is currently experimental*
 
 You can retrieve temporary S3 credentials at the `/s3credentials` endpoint when
-authenticated via earthdata login. These credentials will be valid for 1 hour
+authenticated via Earthdata Login. These credentials will only be valid for
+**1 hour** due to
+[role chaining](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_terms-and-concepts.html)
 and can be used make in-region `s3:ListBucket` and `s3:GetObject` requests.
+Your code must handle expired tokens and request new ones as needed
+for sessions that exceed this 1 hour limit.
 
-### Request
-Credentials are retrieved through a `GET` request to the `/s3credentials`
-endpoint. An optional header `app-name` can be provided to include in the
-generated role session name which will show up in EMS logs.
+## Request
 
-**Params:**
-None.
+Credentials are retrieved through an HTTP `GET` request to the `/s3credentials`
+endpoint. The request must be authenticated with either a JWT token for TEA or
+by using
+[EDL Bearer Tokens](https://urs.earthdata.nasa.gov/documentation/for_users/user_token).
+Unauthenticated requests will be redirected to EDL.
 
 **Headers:**
-  - `app-name`: A string to include in the generated role session name for
-  metric reporting purposes. It is recommended to include this header when
-  making requests on users behalf from another cloud service. The generated
-  role session name is `username@app-name`.
+
+* (optional) `app-name`: An arbitrary string to include in the generated role
+  session name for metric reporting purposes. It can only contain characters
+  that are valid in a `RoleSessionName` see the
+  [AssumeRole documentation](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_RequestParameters).
+  It is recommended to include this header when making requests on users behalf
+  from another cloud service. The generated role session name is
+  `username@app-name`.
 
 **Example:**
 ```python
 import requests
 
-requests.get(
+resp = requests.get(
     "https://your-tea-host/s3credentials",
     headers={"app-name": "my-application"},
     cookies={"asf-urs": "<your jwt token>"}
 )
+print(resp.json())
 ```
 
-### Response
-The response is your temporary credentials as returned by Amazon STS.
-[See the AWS Credentials reference](https://docs.aws.amazon.com/STS/latest/APIReference/API_Credentials.html")
+## Response
+
+The response is your temporary credentials as returned by Amazon STS. See the
+[AWS Credentials reference](https://docs.aws.amazon.com/STS/latest/APIReference/API_Credentials.html) for more details.
 
 **Example:**
 ```json
 {
-  "AccessKeyId": "AKIAIOSFODNN7EXAMPLE",
-  "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
-  "SessionToken": "LONGSTRINGOFCHARACTERS.../HJLgV91QJFCMlmY8slIEOjrOChLQYmzAqrb5U1ekoQAK6f86HKJFTT2dONzPgmJN9ZvW5DBwt6XUxC9HAQ0LDPEYEwbjGVKkzSNQh/",
-  "Expiration": "2021-01-27 00:50:09+00:00"
+    "AccessKeyId": "AKIAIOSFODNN7EXAMPLE",
+    "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+    "SessionToken": "LONGSTRINGOFCHARACTERS.../HJLgV91QJFCMlmY8slIEOjrOChLQYmzAqrb5U1ekoQAK6f86HKJFTT2dONzPgmJN9ZvW5DBwt6XUxC9HAQ0LDPEYEwbjGVKkzSNQh/",
+    "Expiration": "2021-01-27 00:50:09+00:00"
 }
 ```
 
-### Using Temporary Credentials
-To use the credentials you must configure your AWS client with the returned
-access key, secret and token. Note that the credentials will only work in-
-region, so you will get 403 errors if you try to use them with the AWS cli.
+## Using Temporary Credentials
+
+To use the credentials you must configure your AWS SDK library with the
+returned access key, secret and token. Note that the credentials are only valid
+for in-region requests, so using them with your AWS CLI will not work! You must
+make your requests from an AWS service such as Lambda or EC2 in the same region
+as the source bucket you are pulling from. See
+[Using temporary credentials with AWS resources](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html)
+for more information on how to use your temporary credentials.
 
 **Example:**
-```python
-import boto3
-import requests
 
-def get_client():
-  resp = requests.get(
-      "https://your-tea-host/s3credentials",
-      headers={"app-name": "my-application"},
-      cookies={"asf-urs": "<your jwt token>"}
-  )
-  resp.raise_for_status()
-  creds = resp.json()
-
-  return boto3.client(
-      "s3",
-      aws_access_key_id=creds["AccessKeyId"],
-      aws_secret_access_key=creds["SecretAccessKey"],
-      aws_session_token=creds["SessionToken"]
-  )
+This example lambda function uses
+[EDL Bearer Tokens](https://urs.earthdata.nasa.gov/documentation/for_users/user_token)
+to obtain s3 credentials and stream an object from one bucket to another. The
+lambda execution role will need `s3:PutObject` permissions on the destination
+bucket.
 
-```
 
-### Limits
+```python
+import boto3
+import json
+import urllib.request
+
+
+def lambda_handler(event, context):
+    # Get temporary download credentials
+    tea_url = event["CredentialsEndpoint"]
+    bearer_token = event["BearerToken"]
+    req = urllib.request.Request(
+        url=tea_url,
+        headers={"Authorization": f"Bearer {bearer_token}"}
+    )
+    with urllib.request.urlopen(req) as f:
+        creds = json.loads(f.read().decode())
+
+    # Set up separate boto3 clients for download and upload
+    download_client = boto3.client(
+        "s3",
+        aws_access_key_id=creds["AccessKeyId"],
+        aws_secret_access_key=creds["SecretAccessKey"],
+        aws_session_token=creds["SessionToken"]
+    )
+    # Lambda needs to have permission to upload to destination bucket
+    upload_client = boto3.client("s3")
+
+    # Stream from the source bucket to the destination bucket
+    resp = download_client.get_object(
+        Bucket=event["Source"]["Bucket"],
+        Key=event["Source"]["Key"],
+    )
+    upload_client.upload_fileobj(
+        resp["Body"],
+        event["Dest"]["Bucket"],
+        event["Dest"].get("Key") or event["Source"]["Key"],
+    )
+```
 
-The credentials dispensed from the `/s3credentials` endpoint are valid for
-**1 hour**. Your code must handle expired tokens and request new ones as needed
-for sessions that exceed this 1 hour limit. This is an AWS Limit is due to
-[role chaining](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_terms-and-concepts.html)
+The example can be invoked with an event payload as follows:
 
-These credentials will have `s3:GetObject` and `s3:ListBucket` permissions on
-the configured resources.
+```json
+{
+    "CredentialsEndpoint": "https://your-tea-host/s3credentials",
+    "BearerToken": "your bearer token",
+    "Source": {
+        "Bucket": "S3 bucket name from CMR link",
+        "Key": "S3 key from CMR link"
+    },
+    "Dest": {
+        "Bucket": "S3 bucket name to copy to"
+    }
+}
+```

docs/technical.md

@@ -1,3 +1,5 @@
+# Technical
+
 ## System Architecture
 
 TEA is a [Python Chalice](https://github.com/aws/chalice)

docs/troubleshooting.md

@@ -1,3 +1,5 @@
+# Troubleshooting
+
 ## When things go wrong
 
 There is a lot that _can_ go wrong, but we hope that if you followed this

docs/vision.md

@@ -1,3 +1,5 @@
+# Vision
+
 We the **TEA**m strive to create the simplest, most reliable, most feature-rich S3
 distribution app. Our intent is not to provide EVERY feature imaginable, but to maintain the
 **THIN** core of broadly applicable features. We want to continue to engage the user

lambda/app.py

@@ -1032,6 +1032,12 @@ def get_s3_credentials(user_id: str, role_session_name: str, policy: dict):
     return response["Credentials"]
 
 
+@app.route('/s3credentialsREADME', methods=['GET'])
+@with_trace(context={})
+def s3credentials_readme():
+    return make_html_response({}, {}, 200, "s3credentials_readme.html")
+
+
 @app.route('/profile')
 @with_trace(context={})
 def profile():

lambda/templates/base.html

@@ -28,6 +28,7 @@
                 font-size: 85%;
             }
         </style>
+        {% block head %}{% endblock %}
    </head>
 
    <body>

lambda/templates/s3access.md

@@ -0,0 +1 @@
+../../docs/s3access.md