diff --git a/Dockerfile b/Dockerfile index d9eea1b..02923a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -111,17 +111,16 @@ RUN pip3 install --upgrade awscli RUN pip3 install click==7.1.2 RUN pip3 install rio-cogeo==1.1.10 --no-binary rasterio --user RUN pip3 install git+https://github.com/NASA-IMPACT/hls-thumbnails@v1.3 - RUN pip3 install git+https://github.com/NASA-IMPACT/hls-metadata@v2.5 - -RUN pip3 install git+https://github.com/NASA-IMPACT/hls-manifest@v2.0 - +RUN pip3 install git+https://github.com/NASA-IMPACT/hls-manifest@v2.1 RUN pip3 install wheel RUN pip3 install git+https://github.com/NASA-IMPACT/hls-browse_imagery@v1.7 RUN pip3 install libxml2-python3 RUN pip3 install git+https://github.com/NASA-IMPACT/hls-hdf_to_cog@v2.1 RUN pip3 install git+https://github.com/NASA-IMPACT/hls-utilities@v1.9 RUN pip3 install git+https://github.com/NASA-IMPACT/hls-cmr_stac@v1.7 +RUN pip3 install git+https://github.com/NASA-IMPACT/hls-vi@v1.15 + COPY ./scripts/* ${PREFIX}/bin/ ENV OMP_NUM_THREADS=4 diff --git a/scripts/sentinel.sh b/scripts/sentinel.sh index 10b5ee7..27cf14d 100755 --- a/scripts/sentinel.sh +++ b/scripts/sentinel.sh @@ -1,6 +1,6 @@ #!/bin/bash # shellcheck disable=SC2153 -# shellcheck disable=SC1091 +# shellcheck disable=SC1091 # Exit on any error set -o errexit @@ -10,6 +10,7 @@ bucket="$OUTPUT_BUCKET" # shellcheck disable=SC2034 inputbucket="$INPUT_BUCKET" workingdir="/var/scratch/${jobid}" +vidir="${workingdir}/vi" bucket_role_arn="$GCC_ROLE_ARN" debug_bucket="$DEBUG_BUCKET" replace_existing="$REPLACE_EXISTING" @@ -46,6 +47,7 @@ set_output_names () { hlsversion="v2.0" day_of_year=$(get_doy "${year}" "${month}" "${day}") outputname="HLS.S30.${granulecomponents[5]}.${year}${day_of_year}${hms}.${hlsversion}" + vi_outputname="HLS-VI.S30.${granulecomponents[5]}.${year}${day_of_year}${hms}.${hlsversion}" output_hdf="${workingdir}/${outputname}.hdf" nbar_name="HLS.S30.${granulecomponents[5]}.${year}${day_of_year}.${hms}.${hlsversion}" nbar_input="${workingdir}/${nbar_name}.hdf" @@ -54,6 +56,7 @@ set_output_names () { output_metadata="${workingdir}/${outputname}.cmr.xml" output_stac_metadata="${workingdir}/${outputname}_stac.json" bucket_key="s3://${bucket}/S30/data/${year}${day_of_year}/${outputname}${twinkey}" + vi_bucket_key="s3://${bucket}/S30_VI/data/${year}${day_of_year}/${vi_outputname}${twinkey}" gibs_dir="${workingdir}/gibs" gibs_bucket_key="s3://${gibs_bucket}/S30/data/${year}${day_of_year}" # We also need to obtain the sensor for the Bandpass parameters file @@ -171,7 +174,7 @@ create_thumbnail -i "$workingdir" -o "$output_thumbnail" -s S30 # Create metadata echo "Creating metadata" -create_metadata "$output_hdf" --save "$output_metadata" +create_metadata "$output_hdf" --save "$output_metadata" # Create STAC metadata cmr_to_stac_item "$output_metadata" "$output_stac_metadata" \ @@ -202,7 +205,7 @@ if [ -z "$debug_bucket" ]; then # Copy manifest to S3 to signal completion. aws s3 cp "$manifest" "${bucket_key}/${manifest_name}" --profile gccprofile else - # Create + # Create # Convert intermediate hdf to COGs hdf_to_cog "$resample30m" --output-dir "$workingdir" --product S30 --debug-mode hdf_to_cog "$nbarIntermediate" --output-dir "$workingdir" --product S30 --debug-mode @@ -227,7 +230,7 @@ for gibs_id_dir in "$gibs_dir"/* ; do subtile_basename=$(basename "$xml" .xml) subtile_manifest_name="${subtile_basename}.json" subtile_manifest="${gibs_id_dir}/${subtile_manifest_name}" - gibs_id_bucket_key="$gibs_bucket_key/${gibsid}" + gibs_id_bucket_key="$gibs_bucket_key/${gibsid}" echo "Gibs id bucket key is ${gibs_id_bucket_key}" create_manifest "$gibs_id_dir" "$subtile_manifest" \ @@ -252,3 +255,29 @@ for gibs_id_dir in "$gibs_dir"/* ; do fi done echo "All GIBS tiles created" + +# Generate VI files +echo "Generating VI files" +vi_generate_indices -i "$workingdir" -o "$vidir" -s "$outputname" +vi_generate_metadata -i "$workingdir" -o "$vidir" + +echo "Generating VI manifest" +vi_manifest_name="${vi_outputname}.json" +vi_manifest="${vidir}/${vi_manifest_name}" +create_manifest "$vidir" "$vi_manifest" "$vi_bucket_key" "HLSS30_VI" \ + "$vi_outputname" "$jobid" false + +if [ -z "$debug_bucket" ]; then + aws s3 cp "$vidir" "$vi_bucket_key" --exclude "*" --include "*.tif" \ + --include "*.xml" --include "*.jpg" --include "*_stac.json" \ + --profile gccprofile --recursive + + # Copy vi manifest to S3 to signal completion. + aws s3 cp "$vi_manifest" "${vi_bucket_key}/${vi_manifest_name}" --profile gccprofile +else + # Copy all vi files to debug bucket. + echo "Copy files to debug bucket" + debug_bucket_key=s3://${debug_bucket}/${outputname} + aws s3 cp "$vidir" "$debug_bucket_key" --recursive --acl public-read +fi +