issues/swodlr-ui-release-1.2.0-sit: large spatial search fix

src/components/map/WorldMap.tsx

@@ -9,7 +9,7 @@ import { EditControl } from 'react-leaflet-draw'
 import { Session } from '../../authentication/session';
 import { lineString } from '@turf/helpers';
 import booleanClockwise from '@turf/boolean-clockwise';
-import { afterCPSL, afterCPSR, beforeCPS, spatialSearchCollectionConceptId, spatialSearchResultLimit } from '../../constants/rasterParameterConstants';
+import { afterCPSL, afterCPSR, beforeCPS, inputBounds, spatialSearchCollectionConceptId, spatialSearchResultsBatchSize } from '../../constants/rasterParameterConstants';
 import { addSpatialSearchResults, setMapFocus, setWaitingForSpatialSearch } from '../sidebar/actions/productSlice';
 import { SpatialSearchResult } from '../../types/constantTypes';
 import { useLocation, useSearchParams } from 'react-router-dom';
@@ -80,6 +80,12 @@ const WorldMap = () => {
   // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
+  /**
+   * Retrieves scenes within the specified coordinates by performing a spatial search query.
+   * 
+   * @param {Array<Array<{lat: number, lng: number}>>} coordinatesToSearch - An array of polygons, where each polygon is an array of coordinates.
+   * @return {Promise<Error | undefined>} A promise that resolves with an error if the query fails, or undefined if the query is successful.
+   */
   const getScenesWithinCoordinates = async (coordinatesToSearch: {lat: number, lng: number}[][]) => {
     try {
       // get session token to use in spatial search query
@@ -114,51 +120,66 @@ const WorldMap = () => {
         return polygonString
       }).join()
 
-      const spatialSearchResponse = await fetch('https://graphql.earthdata.nasa.gov/api', {
-        method: 'POST',
-        headers: {
-          Authorization: `Bearer ${authToken}`,
-          "Content-Type": "application/json"
-        },
-        body: JSON.stringify({ query: getGranules, variables: getSpatialSearchGranuleVariables(polygonUrlString, spatialSearchCollectionConceptId, spatialSearchResultLimit) })
-      }).then(async data => {
-        const responseJson = await data.json()
-        // TODO: make subsequent calls to get granules in spatial search area till everything is found.
-
-        const updatedGranules = responseJson.data.granules.items.map((item: any) => {
-          const itemCopy = structuredClone(item)
-          const cpsString = item.granuleUr.match(`${beforeCPS}([0-9]+(_[0-9]+)+)(${afterCPSR}|${afterCPSL})`)?.[1]
-          itemCopy.cpsString = cpsString
-          return itemCopy
+      let cursor: string | null = 'initialValue'
+      const spatialSearchItems: any[] = []
+      while(cursor !== null) {
+        if(cursor === 'initialValue') cursor = null
+        cursor = await fetch('https://graphql.earthdata.nasa.gov/api', {
+          method: 'POST',
+          headers: {
+            Authorization: `Bearer ${authToken}`,
+            "Content-Type": "application/json"
+          },
+          body: JSON.stringify({ query: getGranules, variables: getSpatialSearchGranuleVariables(polygonUrlString, spatialSearchCollectionConceptId, spatialSearchResultsBatchSize, cursor) })
+        }).then(async data => {
+          const responseJson = await data.json()
+          // check if granules are in valid bounds for cycle pass and scene
+          const filteredResponseItems = responseJson.data.granules.items.filter((item: any) => {
+            const cpsString = item.granuleUr.match(`${beforeCPS}([0-9]+(_[0-9]+)+)(${afterCPSR}|${afterCPSL})`)?.[1]
+            const cyclePassSceneStringArray = cpsString.split('_').map((id: string) => parseInt(id).toString())
+            // cycle in bounds
+            const cycleInBounds: boolean = cyclePassSceneStringArray[0] >= inputBounds.cycle.min && cyclePassSceneStringArray[0] <= inputBounds.cycle.max
+            const passInBounds: boolean = cyclePassSceneStringArray[1] >= inputBounds.pass.min && cyclePassSceneStringArray[1] <= inputBounds.pass.max
+            return cycleInBounds && passInBounds
+          })
+          spatialSearchItems.push(...filteredResponseItems)
+          return spatialSearchItems.length > spatialSearchResultsBatchSize ? null : responseJson.data.granules.cursor
         })
-        const cpsStringTracker: string[] = []
-        const updatedGranulesToUse = updatedGranules.filter((updatedGranuleObject: any) => {
-          // if cpsString not in tracker, it has not been repeated yet. Add to tracker and return
-          const granuleRepeated = cpsStringTracker.includes(updatedGranuleObject.cpsString)
-          if(!granuleRepeated) cpsStringTracker.push(updatedGranuleObject.cpsString)
-          return !granuleRepeated
-          // if cpsString in tracker, it has been repeated. Do not return
-        })
-        const spatialSearchResults = updatedGranulesToUse.map((updatedGranuleObject: any) => {
-          const {producerGranuleId, granuleUr, cpsString, polygons, timeStart, timeEnd} = updatedGranuleObject
-          const cyclePassSceneStringArray = cpsString.split('_').map((id: string) => parseInt(id).toString())
-          const tileValue = parseInt(cyclePassSceneStringArray?.[2] as string)
-          const sceneToUse = String(Math.floor(tileValue))
-          const returnObject: SpatialSearchResult = {
-            cycle: cyclePassSceneStringArray?.[0],
-            pass: cyclePassSceneStringArray?.[1],
-            scene : sceneToUse,
-            producerGranuleId,
-            granuleUr,
-            timeStart,
-            timeEnd,
-            polygons
-          }
-          return returnObject
-        })
-        return spatialSearchResults
+      }
+
+      const updatedGranules = spatialSearchItems.map((item: any) => {
+        const itemCopy = structuredClone(item)
+        const cpsString = item.granuleUr.match(`${beforeCPS}([0-9]+(_[0-9]+)+)(${afterCPSR}|${afterCPSL})`)?.[1]
+        itemCopy.cpsString = cpsString
+        return itemCopy
       })
-      dispatch(addSpatialSearchResults(spatialSearchResponse as SpatialSearchResult[]))
+      const cpsStringTracker: string[] = []
+      const updatedGranulesToUse = updatedGranules.filter((updatedGranuleObject: any) => {
+        // if cpsString not in tracker, it has not been repeated yet. Add to tracker and return
+        const granuleRepeated = cpsStringTracker.includes(updatedGranuleObject.cpsString)
+        if(!granuleRepeated) cpsStringTracker.push(updatedGranuleObject.cpsString)
+        return !granuleRepeated
+        // if cpsString in tracker, it has been repeated. Do not return
+      })
+      const spatialSearchResults = updatedGranulesToUse.map((updatedGranuleObject: any) => {
+        const {producerGranuleId, granuleUr, cpsString, polygons, timeStart, timeEnd} = updatedGranuleObject
+        const cyclePassSceneStringArray = cpsString.split('_').map((id: string) => parseInt(id).toString())
+        const tileValue = parseInt(cyclePassSceneStringArray?.[2] as string)
+        const sceneToUse = String(Math.floor(tileValue))
+        const returnObject: SpatialSearchResult = {
+          cycle: cyclePassSceneStringArray?.[0],
+          pass: cyclePassSceneStringArray?.[1],
+          scene : sceneToUse,
+          producerGranuleId,
+          granuleUr,
+          timeStart,
+          timeEnd,
+          polygons
+        }
+        return returnObject
+      })
+
+      dispatch(addSpatialSearchResults(spatialSearchResults as SpatialSearchResult[]))
     } catch (err) {
       if (err instanceof Error) {
           return err

src/components/sidebar/GranulesTable.tsx

@@ -140,7 +140,6 @@ const GranuleTable = (props: GranuleTableProps) => {
       const fetchData = async () => {
         let scenesFoundArray: string[] = []
         let addedScenes: string[] = []
-        if (spatialSearchResults.length < 1000) {
           for(let i=0; i<spatialSearchResults.length; i++) {
             if ((addedProducts.length + scenesFoundArray.filter(result => result === 'found something').length) >= granuleTableLimit) {
               // don't let more than 10 be added
@@ -165,13 +164,7 @@ const GranuleTable = (props: GranuleTableProps) => {
             // add parameters
             addSearchParamToCurrentUrlState({'cyclePassScene': addedScenes.join('-')})
           }
-        } else {
-          // If too many spatial search results, the search doesn't work because there too many granules and a limit was reached.
-          // In this scenario, make an alert that indicates that the search area was too large.
-          // TODO: remove this alert when there is a fix implemented for cmr spatial search limit.
-          // The valid granules are sometimes not a part of the first 1000 results which is the bug here.
-          scenesFoundArray.push('spatialSearchAreaTooLarge')
-        }
+
         dispatch(setWaitingForSpatialSearch(false))
         return scenesFoundArray
       }
@@ -181,7 +174,6 @@ const GranuleTable = (props: GranuleTableProps) => {
         .then((noScenesFoundResults) => {
           if((noScenesFoundResults.includes('noScenesFound') && !noScenesFoundResults.includes('found something'))) setSaveGranulesAlert('noScenesFound')
           if(noScenesFoundResults.includes('hit granule limit')) setSaveGranulesAlert('granuleLimit')
-          if(noScenesFoundResults.includes('spatialSearchAreaTooLarge')) setSaveGranulesAlert('spatialSearchAreaTooLarge')
         })
         // make sure to catch any error
         .catch(console.error);
@@ -282,9 +274,11 @@ const GranuleTable = (props: GranuleTableProps) => {
       const maxIsValid = checkInBounds(inputType, max)
       validInput = minIsValid && maxIsValid
     } else {
+      console.log('inputValue: ',inputValue)
       const validInBounds = checkInBounds(inputType, inputValue.trim())
       validInput = validInBounds
     }
+    console.log('validInput: ',validInput)
     return validInput
   }
 

src/constants/graphqlQueries.ts

@@ -72,24 +72,11 @@ query($params: GranulesInput) {
           timeEnd
           polygons
         }
+        cursor
     }
 }
 `
 
-// export const getSpatialSearchGranules = `
-// query GetSpatialSearchGranules($params: GranulesInput) {
-//   granules(params: $params) {
-//     items {
-//       producerGranuleId
-//       granuleUr
-//       timeStart
-//       timeEnd
-//       polygons
-//     }
-//   }
-// }
-// `
-
 export const getGranuleVariables = (cycle: number, pass: number, sceneIds: number[]) => {
     const sceneIdsForGranuleName = sceneIds.map(sceneId => `SWOT_L2_HR_Raster_*_${padCPSForCmrQuery(String(sceneId))}F_*`)
     const variables = {
@@ -109,12 +96,13 @@ export const getGranuleVariables = (cycle: number, pass: number, sceneIds: numbe
     return variables
 }
 
-export const getSpatialSearchGranuleVariables = (polygon: string, collectionConceptId: string, limit: number) => {
+export const getSpatialSearchGranuleVariables = (polygon: string, collectionConceptId: string, limit: number, cursor: string | null) => {
     const variables = {
         "params": {
           polygon,
           collectionConceptId,
-          limit
+          limit,
+          cursor
         }
       }
     return variables

src/constants/rasterParameterConstants.ts

@@ -196,10 +196,6 @@ export const granuleAlertMessageConstant: granuleAlertMessageConstantType = {
         message: `Successfully started product generation! Go to the 'My Data' page to track progress.`,
         variant: 'success'
     },
-    spatialSearchAreaTooLarge: {
-        message: `The search area you've selected on the map is too large. Please choose a smaller area to search.`,
-        variant: 'warning'
-    },
     successfullyReGenerated: {
         message: `Successfully re-submitted product generation! Go to the 'My Data' page to track progress.`,
         variant: 'success'
@@ -215,6 +211,7 @@ export const granuleAlertMessageConstant: granuleAlertMessageConstantType = {
   }
 
 export const spatialSearchResultLimit = 2000
+export const spatialSearchResultsBatchSize = 1000
 export const beforeCPS = '_x_x_x_'
 export const afterCPSR = 'F_'
 export const afterCPSL = 'F_'

src/types/constantTypes.ts

@@ -160,7 +160,7 @@ export interface granuleMetadata {
   [key: string]: granuleMetadataInfo
 }
 
-export type alertMessageInput = 'success' | 'alreadyAdded' | 'allScenesNotAvailable' | 'alreadyAddedAndNotFound' | 'noScenesAdded' | 'readyForGeneration' | 'invalidCycle' | 'invalidPass' | 'invalidScene' | 'invalidScene' | 'someScenesNotAvailable' | 'granuleLimit' | 'notInTimeRange' | 'noScenesFound' | 'someSuccess' | 'successfullyGenerated' | 'spatialSearchAreaTooLarge' | 'successfullyReGenerated'
+export type alertMessageInput = 'success' | 'alreadyAdded' | 'allScenesNotAvailable' | 'alreadyAddedAndNotFound' | 'noScenesAdded' | 'readyForGeneration' | 'invalidCycle' | 'invalidPass' | 'invalidScene' | 'invalidScene' | 'someScenesNotAvailable' | 'granuleLimit' | 'notInTimeRange' | 'noScenesFound' | 'someSuccess' | 'successfullyGenerated' | 'successfullyReGenerated'
 
 export interface SpatialSearchResult {
   cycle: string,