Functional updates to R package - improved user options & record nhdplusTools retrieval metadata

pkg/proc.attr.hydfab/DESCRIPTION

@@ -1,6 +1,6 @@
 Package: proc.attr.hydfab
 Title: Grab and process catchment attributes using the hydrofabric
-Version: 0.0.1.0010
+Version: 0.0.1.0013
 Authors@R: 
     c(person("Guy", "Litt", , "[email protected]", role = c("aut", "cre"),
            comment = c(ORCID = "https://orcid.org/0000-0003-1996-7468")),

pkg/proc.attr.hydfab/NAMESPACE

@@ -2,6 +2,7 @@
 
 export(check_attr_selection)
 export(grab_attrs_datasets_fs_wrap)
+export(hfab_config_opt)
 export(proc_attr_exst_wrap)
 export(proc_attr_gageids)
 export(proc_attr_hf)
@@ -11,3 +12,4 @@ export(proc_attr_usgs_nhd)
 export(proc_attr_wrap)
 export(read_loc_data)
 export(retrieve_attr_exst)
+export(write_meta_nldi_feat)

pkg/proc.attr.hydfab/flow/flow.install.proc.attr.hydfab.R

@@ -24,7 +24,7 @@ if ('bolotin' %in% Sys.getenv("HOME")) {
   fs_dir <- file.path(Sys.getenv("HOME"),"git","formulation-selector")
 }
 # Run unit tests?
-RunTest <- FALSE#TRUE Default FALSE prevents s3 data downloading in unit testing (FALSE=fast)
+RunTest <- FALSE #TRUE Default FALSE prevents s3 data downloading in unit testing (FALSE=fast)
 ShowTestCovr <- FALSE # Only possible if RunTest==TRUE. Even slower though.
 # ---------------------------------------------------------------------------- #
 # Enter in all R packages here

pkg/proc.attr.hydfab/flow/fs_attrs_grab.R

@@ -47,6 +47,13 @@ dir_base <- glue::glue(base::unlist(raw_config$file_io)[['dir_base']])#file.path
 dir_std_base <- glue::glue(base::unlist(raw_config$file_io)[['dir_std_base']]) #file.path(dir_base,"input","user_data_std") # The location of standardized data generated by fs_proc python package
 dir_db_hydfab <- glue::glue(base::unlist(raw_config$file_io)[['dir_db_hydfab']]) # file.path(dir_base,'input','hydrofabric') # The local dir where hydrofabric data are stored to limit s3 connections
 dir_db_attrs <- glue::glue(base::unlist(raw_config$file_io)[['dir_db_attrs']])  # file.path(dir_base,'input','attributes') # The parent dir where each comid's attribute parquet file is stored in the subdirectory 'comid/', and each dataset's aggregated parquet attributes are stored in the subdirectory '/{dataset_name}
+ds_type <- try(base::unlist(raw_config$file_io)[['ds_type']])
+if('try-error' %in% base::class(ds_type)){
+  ds_type <- ''
+}
+write_type <- glue::glue(base::unlist(raw_config$file_io)[['write_type']])# file format for writing  writing NLDI feature metadata. Default 'parquet'. May also select 'csv'.
+path_meta <- base::unlist(raw_config$file_io)[['path_meta']]  # Full file path for writing NLDI feature metadata of training data formatted for glue::glue(). Default: "{dir_std_base}/{ds}/nldi_feat_{ds}_{ds_type}.{write_type}"
+
 
 # Read s3 connection details
 s3_base <- base::unlist(raw_config$hydfab_config)[['s3_base']]#s3://lynker-spatial/tabular-resources" # s3 path containing hydrofabric-formatted attribute datasets
@@ -92,12 +99,14 @@ Retr_Params <- base::list(paths = base::list(
                         dir_db_hydfab=dir_db_hydfab,
                         dir_db_attrs=dir_db_attrs,
                         s3_path_hydatl = s3_path_hydatl,
-                        dir_std_base = dir_std_base),
+                        dir_std_base = dir_std_base,
+                        path_meta = path_meta),
                    vars = sub_attr_sel,
-                   datasets = datasets
+                   datasets = datasets,
+                   ds_type = ds_type,
+                   write_type = write_type
        )
 # PROCESS ATTRIBUTES
-
 ls_comids <- proc.attr.hydfab:::grab_attrs_datasets_fs_wrap(Retr_Params,overwrite = TRUE)
 
 # --------------------------- Compile attributes --------------------------- #

pkg/proc.attr.hydfab/inst/extdata/attr_source_types.yml

@@ -0,0 +1,11 @@
+# The name is the expected format for variables specified in proc.attr.hydfab
+hydroatlas_attributes:
+  - 'name': 'ha_vars'
+  - 'internal_dataset_name' : 'hydroatlas__v1'
+camels_attributes:
+  - 'name': 'camels_vars'
+usgs_attributes:
+  - 'name': 'usgs_vars'
+  - 'internal_dataset_name' : 'usgs_nhdplus__v2'
+hydrofabric_attributes:
+  - 'name': 'hfab_vars'

pkg/proc.attr.hydfab/inst/extdata/xssa_attr_config_all_vars_avail.yaml

@@ -19,16 +19,25 @@ file_io: # May define {home_dir} for python's '{home_dir}/string_path'.format(ho
   - 'dir_std_base' : '{dir_base}/user_data_std' # Required. The location of standardized data generated by fs_proc python package
   - 'dir_db_hydfab' : '{dir_base}/hydrofabric' # Required. The local dir where hydrofabric data are stored (limits the total s3 connections)
   - 'dir_db_attrs' : '{dir_base}/attributes' # Required. The parent dir where each comid's attribute parquet file is stored in the subdirectory 'comid/', and each dataset's aggregated parquet attributes are stored in the subdirectory '/{dataset_name}
+  - 'ds_type': 'training' # Required string. Recommended to select 'training' or 'prediction', but any string will work. This string will be used in the filename of the output metadata describing each data point's identifer, COMID, lat/lon, reach name of the location. This string should differ from the string used in the prediction config yaml file. Filename: `"nldi_feat_{dataset}_{ds_type}.csv"` inside `dir_std_base / dataset / `
 formulation_metadata:
   - 'datasets': # Required. Must match directory name inside dir_std_base. May be a list of items, or simply sublist 'all' to select everything inside dir_std_base for attribute grabbing.
     - 'juliemai-xSSA' # Required. In this example case, it's a sublist of just one thing.
   - 'formulation_base': 'Raven_blended' # Informational. Unique name of formulation.
 hydfab_config: # Required section describing hydrofabric connection details and objects of interest
- - 's3_base' : "s3://lynker-spatial/tabular-resources" # Required. s3 path containing hydrofabric-formatted attribute datasets
- - 's3_bucket' : 'lynker-spatial' # Required. s3 bucket containing hydrofabric data
- - 'ext' : 'gpkg' # Required. file extension of the hydrofrabric data. Default 'gpkg'.
- - 'hf_cat_sel': "total" # Required. Options include 'total' or 'all'; total: interested in the single location's aggregated catchment data; all: all subcatchments of interest
-attr_select: # The names of variable sublistings are standardized, e.g. ha_vars, usgs_vars, sc_vars
+ - s3_base: "s3://lynker-spatial/tabular-resources" # Required. s3 path containing hydrofabric-formatted attribute datasets
+ - s3_bucket: 'lynker-spatial' # Required. s3 bucket containing hydrofabric data
+ - hf_cat_sel: "total" # Required. Options include 'total' or 'all'; total: interested in the single location's aggregated catchment data; all: all subcatchments of interest
+ - gpkg:  # Optional. A local gpkg file. Default 'NULL'. See hfsubsetR::get_subset()
+ - hfab_retr: FALSE # Optional, Boolean. Defaults to the hfab_retr argument default in the proc_attr_wrap() function (TRUE). Should the hydrofabric data be downloaded? Hydrofabric data download may not be necessary. Processing is faster if set to FALSE
+ - hf_version: "2.1.1"  # Optional, character string. Defaults to the hf_version argument default in hfsubsetR::get_subset() function. The hydrofabric version.
+ - domain: "conus"  # Optional, character string. Defaults to the hf_version argument default in hfsubsetR::get_subset() function. The hydrofabric domain.
+ - type: "nextgen"  # Optional, character string. Defaults to the hf_version argument default in hfsubsetR::get_subset() function. The hydrofabric type.
+ - lyrs: # Optional, sublist of character strings. Defaults to the hf_version argument default in hfsubsetR::get_subset() function. Hydrofabric layers to extract.
+   - 'divides'
+   - 'network'
+ - source: "s3://lynker-spatial/hydrofabric"
+attr_select: # The names of variable sublistings are standardized with _vars, e.g. ha_vars, usgs_vars, sc_vars
   - 's3_path_hydatl': '{s3_base}/hydroATLAS/hydroatlas_vars.parquet' # path to hydroatlas data formatted for hydrofabric. Required only if hydroatlas variables desired.
   - 'ha_vars':  # hydroatlas variables. Must specify s3_path_hydatl if desired.
     - 'pet_mm_s01'

pkg/proc.attr.hydfab/inst/extdata/xssa_attr_config_missing_vars.yaml

@@ -19,17 +19,26 @@ file_io: # May define {home_dir} for python's '{home_dir}/string_path'.format(ho
   - 'dir_std_base' : '{dir_base}/user_data_std' # Required. The location of standardized data generated by fs_proc python package
   - 'dir_db_hydfab' : '{dir_base}/hydrofabric' # Required. The local dir where hydrofabric data are stored (limits the total s3 connections)
   - 'dir_db_attrs' : '{dir_base}/attributes' # Required. The parent dir where each comid's attribute parquet file is stored in the subdirectory 'comid/', and each dataset's aggregated parquet attributes are stored in the subdirectory '/{dataset_name}
+  - 'ds_type': 'training' # Required string. Recommended to select 'training' or 'prediction', but any string will work. This string will be used in the filename of the output metadata describing each data point's identifer, COMID, lat/lon, reach name of the location. This string should differ from the string used in the prediction config yaml file. Filename: `"nldi_feat_{dataset}_{ds_type}.csv"` inside `dir_std_base / dataset / `
 formulation_metadata:
   - 'datasets': # Required. Must match directory name inside dir_std_base. May be a list of items, or simply sublist 'all' to select everything inside dir_std_base for attribute grabbing.
     - 'juliemai-xSSA' # Required. In this example case, it's a sublist of just one thing.
   - 'formulation_base': 'Raven_blended' # Informational. Unique name of formulation.
 hydfab_config: # Required section describing hydrofabric connection details and objects of interest
- - 's3_base' : "s3://lynker-spatial/tabular-resources" # Required. s3 path containing hydrofabric-formatted attribute datasets
- - 's3_bucket' : 'lynker-spatial' # Required. s3 bucket containing hydrofabric data
- - 'ext' : 'gpkg' # Required. file extension of the hydrofrabric data. Default 'gpkg'.
- - 'hf_cat_sel': "total" # Required. Options include 'total' or 'all'; total: interested in the single location's aggregated catchment data; all: all subcatchments of interest
-attr_select: # The names of variable sublistings are standardized, e.g. ha_vars, usgs_vars, sc_vars
-  - 's3_path_hydatl' : '{s3_base}/hydroATLAS/hydroatlas_vars.parquet' # path to hydroatlas data formatted for hydrofabric. Required only if hydroatlas variables desired.
+ - s3_base: "s3://lynker-spatial/tabular-resources" # Required. s3 path containing hydrofabric-formatted attribute datasets
+ - s3_bucket: 'lynker-spatial' # Required. s3 bucket containing hydrofabric data
+ - hf_cat_sel: "total" # Required. Options include 'total' or 'all'; total: interested in the single location's aggregated catchment data; all: all subcatchments of interest
+ - gpkg:  # Optional. A local gpkg file. Default 'NULL'. See hfsubsetR::get_subset()
+ - hfab_retr: FALSE # Optional, Boolean. Defaults to the hfab_retr argument default in the proc_attr_wrap() function (TRUE). Should the hydrofabric data be downloaded? Hydrofabric data download may not be necessary. Processing is faster if set to FALSE
+ - hf_version: "2.1.1"  # Optional, character string. Defaults to the hf_version argument default in hfsubsetR::get_subset() function. The hydrofabric version.
+ - domain: "conus"  # Optional, character string. Defaults to the hf_version argument default in hfsubsetR::get_subset() function. The hydrofabric domain.
+ - type: "nextgen"  # Optional, character string. Defaults to the hf_version argument default in hfsubsetR::get_subset() function. The hydrofabric type.
+ - lyrs: # Optional, sublist of character strings. Defaults to the hf_version argument default in hfsubsetR::get_subset() function. Hydrofabric layers to extract.
+   - 'divides'
+   - 'network'
+ - source: "s3://lynker-spatial/hydrofabric"
+attr_select: # The names of variable sublistings are standardized with _vars, e.g. ha_vars, usgs_vars, sc_vars
+  - 's3_path_hydatl': '{s3_base}/hydroATLAS/hydroatlas_vars.parquet' # path to hydroatlas data formatted for hydrofabric. Required only if hydroatlas variables desired.
   - 'ha_vars':  # hydroatlas variables. Must specify s3_path_hydatl if desired.
     - 'pet_mm_s01'
     - 'cly_pc_sav'