Merge pull request #43 from danielfromearth/develop

update `main`

.dockerignore

@@ -0,0 +1,103 @@
+TEMPO_*.nc
+tests/
+pytest.ini
+.git
+.gitignore
+.travis.yaml
+.swagger-codegen-ignore
+tox.ini
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+venv/
+.python-version
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+#Ipython Notebook
+.ipynb_checkpoints
+
+# Intellij project settings
+.idea
+
+# VSCode project settings
+.vscode
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# ruff
+.ruff_cache/
+# pytest
+.pytest_cache/
+
+# Pyre type checker
+.pyre/
+
+# Github integration settings
+.github
+
+# .dockerignore and builder script themselves are not needed by
+# docker build.
+.dockerignore
+build-service

.github/dependabot.yml

@@ -3,15 +3,12 @@ updates:
   - package-ecosystem: pip
     directory: "/"
     schedule:
-      interval: daily
+      interval: "weekly"
     # Raise pull requests for version updates
     # to pip against the `develop` branch
     target-branch: "develop"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      # Check for updates to GitHub Actions every weekday
-      interval: "daily"
-    # Raise pull requests for version updates
-    # to pip against the `develop` branch
+      interval: "weekly"
     target-branch: "develop"

.pre-commit-config.yaml

@@ -38,9 +38,3 @@ repos:
       - id: yamllint
         args: ["-d {extends: relaxed, rules: {line-length: {max: 120}}}"]
         stages: [commit, push]
-
-  - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
-    rev: v0.1.0
-    hooks:
-      - id: dockerfilelint
-        stages: [commit, push]

CHANGELOG.md

@@ -8,9 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
   - [PR #1](https://github.com/danielfromearth/stitchee/pull/1): An initial GitHub Actions workflow
+  - [Issue #8](https://github.com/danielfromearth/stitchee/issues/8): Create working Docker image
+  - [Issue #10](https://github.com/danielfromearth/stitchee/issues/10): Add code necessary to communicate with Harmony
+  - [Issue #49](https://github.com/danielfromearth/stitchee/issues/49): More CLI arguments for finer control of concatenation method
 ### Changed
   - [PR #12](https://github.com/danielfromearth/stitchee/pull/12): Changed name to "stitchee"
   - [PR #15](https://github.com/danielfromearth/stitchee/pull/15): Use ruff+black chain for pre-commit lint & format
+  - [Issue #45](https://github.com/danielfromearth/stitchee/issues/45): Rename CLI argument to clarify temporary copying behavior
+  - [Issue #44](https://github.com/danielfromearth/stitchee/issues/44): Concatenation dimension CLI argument is required but isn't listed as such in the help message
 ### Deprecated
 ### Removed
 ### Fixed

Dockerfile

@@ -0,0 +1,47 @@
+
+FROM python:3.10-slim
+
+RUN apt-get update \
+     && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    gcc \
+    libnetcdf-dev \
+    #libhdf5-dev \
+    #hdf5-helpers \
+    && pip3 install --upgrade pip \
+    && pip3 install cython \
+    && pip3 install poetry \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+
+# Create a new user
+RUN adduser --quiet --disabled-password --shell /bin/sh --home /home/dockeruser --gecos "" --uid 1000 dockeruser
+USER dockeruser
+ENV HOME /home/dockeruser
+ENV PYTHONPATH "${PYTHONPATH}:/home/dockeruser/.local/bin"
+ENV PATH="/home/dockeruser/.local/bin:${PATH}"
+
+# The 'SOURCE' argument is what will be used in 'pip install'.
+ARG SOURCE
+
+# Set this argument if running the pip install on a local directory, so
+# the local dist files are copied into the container.
+ARG DIST_PATH
+
+USER root
+RUN mkdir -p /worker && chown dockeruser /worker
+COPY pyproject.toml /worker
+
+WORKDIR /worker
+# ENV PYTHONPATH=${PYTHONPATH}:${PWD}
+COPY --chown=dockeruser $DIST_PATH $DIST_PATH
+#RUN pip3 install --no-cache-dir --force --user --index-url https://pypi.org/simple/ --extra-index-url https://test.pypi.org/simple/ $SOURCE \
+#    && rm -rf $DIST_PATH
+
+#install poetry as root
+RUN poetry config virtualenvs.create false
+RUN poetry install --only main
+
+USER dockeruser
+COPY --chown=dockeruser ./docker-entrypoint.sh docker-entrypoint.sh
+# Run the service
+ENTRYPOINT ["./docker-entrypoint.sh"]

README.md

@@ -25,35 +25,45 @@ poetry run pytest tests/
 
 ```shell
 $ poetry run stitchee --help
-usage: stitchee [-h] -o output_path [--concat_dim concat_dim] [--make_dir_copy] [--keep_tmp_files] [-O] [-v]
+usage: stitchee [-h] -o OUTPUT_PATH [--no_input_file_copies] [--keep_tmp_files] [--concat_method {xarray-concat,xarray-combine}] [--concat_dim CONCAT_DIM]
+                [--xarray_arg_compat XARRAY_ARG_COMPAT] [--xarray_arg_combine_attrs XARRAY_ARG_COMBINE_ATTRS] [--xarray_arg_join XARRAY_ARG_JOIN] [-O]
+                [-v]
                 path/directory or path list [path/directory or path list ...]
 
 Run the along-existing-dimension concatenator.
 
 options:
   -h, --help            show this help message and exit
-  --concat_dim concat_dim
-                        Dimension to concatenate along, if possible.
-  --make_dir_copy       Make a duplicate of the input directory to avoid modification of input files. This is useful for testing, but
-                        uses more disk space.
-  --keep_tmp_files      Prevents removal, after successful execution, of (1) the flattened concatenated file and (2) the input
-                        directory copy if created by '--make_dir_copy'.
+  --no_input_file_copies
+                        By default, input files are copied into a temporary directory to avoid modification of input files. This is useful for testing,
+                        but uses more disk space. By specifying this argument, no copying is performed.
+  --keep_tmp_files      Prevents removal, after successful execution, of (1) the flattened concatenated file and (2) the input directory copy if created
+                        by '--make_dir_copy'.
+  --concat_method {xarray-concat,xarray-combine}
+                        Whether to use the xarray concat method or the combine-by-coords method.
+  --concat_dim CONCAT_DIM
+                        Dimension to concatenate along, if possible. This is required if using the 'xarray-concat' method
+  --xarray_arg_compat XARRAY_ARG_COMPAT
+                        'compat' argument passed to xarray.concat() or xarray.combine_by_coords().
+  --xarray_arg_combine_attrs XARRAY_ARG_COMBINE_ATTRS
+                        'combine_attrs' argument passed to xarray.concat() or xarray.combine_by_coords().
+  --xarray_arg_join XARRAY_ARG_JOIN
+                        'join' argument passed to xarray.concat() or xarray.combine_by_coords().
   -O, --overwrite       Overwrite output file if it already exists.
   -v, --verbose         Enable verbose output to stdout; useful for debugging
 
 Required:
   path/directory or path list
-                        Files to be concatenated, specified via a (1) single directory containing the files to be concatenated, (2)
-                        single text file containing linebreak-separated paths of the files to be concatenated, or (3) multiple
-                        filepaths of the files to be concatenated.
-  -o output_path, --output_path output_path
+                        Files to be concatenated, specified via a (1) single directory containing the files to be concatenated, (2) single text file
+                        containing linebreak-separated paths of the files to be concatenated, or (3) multiple filepaths of the files to be concatenated.
+  -o OUTPUT_PATH, --output_path OUTPUT_PATH
                         The output filename for the merged output.
 ```
 
 For example:
 
 ```shell
-poetry run stitchee /path/to/netcdf/directory/ /path/to/output.nc
+poetry run stitchee /path/to/netcdf/directory/ -o /path/to/output.nc
 ```
 
 ## Roadmap

build-service

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build -t "asdc-trade/stitchee:${VERSION-latest}" .

concatenator/group_handling.py

@@ -174,7 +174,6 @@ def regroup_flattened_dataset(
     output_file : str
         Name of the output file to write the resulting NetCDF file to.
     """
-
     with nc.Dataset(output_file, mode="w", format="NETCDF4") as base_dataset:
         # Copy global attributes
         base_dataset.setncatts(dataset.attrs)

concatenator/harmony/cli.py

@@ -0,0 +1,33 @@
+"""A Harmony CLI wrapper around the concatenate-batcher"""
+from argparse import ArgumentParser
+
+import harmony
+
+from concatenator.harmony.service_adapter import StitcheeAdapter as HarmonyAdapter
+
+
+def main(config: harmony.util.Config = None) -> None:
+    """Parse command line arguments and invoke the service to respond to them.
+
+    Parameters
+    ----------
+    config : harmony.util.Config
+        harmony.util.Config is injectable for tests
+
+    Returns
+    -------
+    None
+    """
+    parser = ArgumentParser(
+        prog="Stitchee", description="Run the STITCH by Extending a dimEnsion service"
+    )
+    harmony.setup_cli(parser)
+    args = parser.parse_args()
+    if harmony.is_harmony_cli(args):
+        harmony.run_cli(parser, args, HarmonyAdapter, cfg=config)
+    else:
+        parser.error("Only --harmony CLIs are supported")
+
+
+if __name__ == "__main__":
+    main()