lambda_function_code.py

import pandas as pd
import awswrangler as wr
import os
import urllib.parse

os_input_s3_cleansed_layer = os.environ['s3_cleansed_layer']
os_input_glue_catalog_table_name = os.environ['glue_catalog_table_name']
os_input_glue_catalog_db_name = os.environ['glue_catalog_db_name']
os_input_write_data_operation = os.environ['write_data_operation']

def lambda_handler(event, context):
    
    bucket = event['Records'][0]['s3']['bucket']['name']
    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')
    try:

        # Creating DF 
        df_raw = wr.s3.read_json('s3://{}/{}'.format(bucket, key))

        # Extracting required columns:
        df_step_1 = pd.json_normalize(df_raw['items'])

        # Write to S3
        wr_response = wr.s3.to_parquet(
            df=df_step_1,
            path=os_input_s3_cleansed_layer,
            dataset=True,
            database=os_input_glue_catalog_db_name,
            table=os_input_glue_catalog_table_name,
            mode=os_input_write_data_operation
        )

        return wr_response
    except Exception as e:
        print(e)
        print('We are getting an error in getting object {} from bucket {}. Make sure they exist and the bucket is present in the same region as this function.'.format(key, bucket))
        raise e