Merge pull request #300 from GEOS-ESM/develop

Merge develop into main, for release (2.0.0)

Many changes

- Added pyproject.toml to aid with pip installation.
- Engineering -- Formatting with Black -- Linting with Pylint -- Dependency management and packaging with Rye
- Added tests to cover more mepo commands
- Added new command update-state to permanently convert mepo1 style state to mepo2
- Converted mepo to a Python project
  -- Added src/mepo/__init__.py
  -- Renamed mepo.d -> src/mepo
  -- Renamed mepo.d/utest -> tests
  -- Renamed doc --> docs
  -- A mepo config file is now called a mepo registry
  -- More code reorganization
- Helper script mepo, used for development, moved to the bin directory. Not removed yet.
- Added README for docs/make_md_docs.py script
- State: pickle format (mepo1 style) to json format (mepo2 style)
  -- If mepo1 style state is detected, print warning and suggest running mepo update-state

.github/workflows/run-formatter.yaml

@@ -0,0 +1,23 @@
+name: Run formatter
+
+on: [push]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    name: Format code
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements-dev.lock
+      timeout-minutes: 5
+    - name: Run black
+      run: black --check .
+      timeout-minutes: 5

.github/workflows/run-linter.yaml

@@ -0,0 +1,23 @@
+name: Run linter
+
+on: [push]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    name: Lint code
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements-dev.lock
+      timeout-minutes: 5
+    - name: Run pylint
+      run: pylint --exit-zero src/mepo
+      timeout-minutes: 5

.github/workflows/mepo.yaml → .github/workflows/run-tests.yaml

@@ -1,4 +1,4 @@
-name: Unit testing of mepo
+name: Run tests
 
 on: [push]
 
@@ -23,9 +23,11 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        pip install -r requirements.lock
       timeout-minutes: 5
 
-    - name: Run unit tests
-      run: python3 mepo.d/utest/test_mepo_commands.py -v
+    - name: Run tests
+      run: |
+        export PYTHONPATH=$(pwd)/src:$PYTHONPATH
+        python tests/test_mepo_commands.py -v
       timeout-minutes: 5

.gitignore

@@ -1,2 +1,9 @@
 *~
 *.pyc
+*.egg-info
+dist
+venv
+
+# This is generated by docs/make_md_docs.py
+Mepo-Commands.md
+.python-version

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.4.2
+    hooks:
+      - id: black
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        language_version: python3.11

CHANGELOG.md

@@ -13,7 +13,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-### Removed
+## [2.0.0] - 2024-08-09
+
+### Fixed
+
+### Added
+
+- Added `pyproject.toml` to aid with `pip` installation.
+
+- Engineering
+  -- Formatting with Black
+  -- Linting with Pylint
+  -- Dependency management and packaging with Rye
+
+- Added tests to cover more `mepo` commands
+
+- Add new command `update-state` to permanently convert mepo1 style state to mepo2
+
+### Changed
+
+- Converted `mepo` to a Python project via the following renaming
+  -- Added `src/mepo/__init__.py`
+  -- Renamed `mepo.d` -> `src/mepo`
+  -- Renamed `mepo.d/utest` -> `tests`
+  -- Renamed `doc` --> `docs`
+  -- A `mepo` config file is now called a `mepo` registry
+  -- More code reorganization
+
+- Helper script `mepo`, used for development, moved to the `bin` directory.
+- Added README for `docs/make_md_docs.py` script
+
+- State: pickle format (mepo1 style) to json format (mepo2 style)
+  -- If mepo1 style state is detected, print warning and suggest running `mepo update-state`
 
 ## [1.52.0] - 2024-01-10
 

README.md

@@ -1,6 +1,10 @@
-# mepo [![Actions Status](https://github.com/pchakraborty/mepo/workflows/Unit%20testing%20of%20mepo/badge.svg)](https://github.com/pchakraborty/mepo/actions) [![DOI](https://zenodo.org/badge/215067850.svg)](https://zenodo.org/badge/latestdoi/215067850)
+# mepo [![Actions Status](https://github.com/pchakraborty/mepo/workflows/Unit%20testing%20of%20mepo/badge.svg)](https://github.com/pchakraborty/mepo/actions) [![DOI](https://zenodo.org/badge/215067850.svg)](https://zenodo.org/badge/latestdoi/215067850) [![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye-up.com)
 
-`mepo` is a tool, written in Python3 (3.6.0+), to manage (m)ultiple git r(epo)sitories, by attempting to create an illusion of a 'single repository' for multi-repository projects. Please see the [Wiki](../../wiki) for examples of `mepo` workflows.
+`mepo` is a tool, written in Python3 (3.9.0+), to manage (m)ultiple git r(epo)sitories, by attempting to create an illusion of a 'single repository' for multi-repository projects. Please see the [Wiki](../../wiki) for examples of `mepo` workflows.
+
+## Installation
+
+`pip install mepo`
 
 ## Commands
 

bin/mepo

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import traceback
+
+# Version check
+if sys.version_info < (3, 9, 0):
+    sys.exit('ERROR: Python version needs to be >= 3.9.0')
+
+# Add directory containing mepo to path
+SRC_D = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "src")
+sys.path.insert(0, SRC_D)
+
+if __name__ == '__main__':
+    from mepo.__main__ import main
+    main()

doc/make_md_docs.py → docs/make_md_docs.py

@@ -1,61 +1,49 @@
 #!/usr/bin/env python3
 
 import os
-import io
+import glob
 from mdutils.mdutils import MdUtils
 import subprocess as sp
 
-preamble='''
+preamble = """
 mepo provides many different commands for working with a multi-repository fixture.
-'''
+"""
 
 # Assume this script is in mepo/doc. Then we need to get to the mepo/mepo.d/command directory
 doc_dir_path = os.path.dirname(os.path.realpath(__file__))
 # Then we need to get to the mepo/mepo.d/command directory. First the "main" dir
 main_dir_path = os.path.dirname(doc_dir_path)
-# Now add 'mepo.d'
-mepod_dir_path = os.path.join(main_dir_path,'mepo.d')
-# And then 'command'
-command_dir_path = os.path.join(mepod_dir_path,'command')
+# Now add "src/mepo"
+mepod_dir_path = os.path.join(main_dir_path, "src", "mepo")
+# And then "command"
+command_dir_path = os.path.join(mepod_dir_path, "command")
 
-mepo_command_path = os.path.join(main_dir_path,'mepo')
+mepo_command_path = os.path.join(main_dir_path, "bin", "mepo")
 
-def get_command_list(directory):
-    # Walk the tree
-    roots = [x[0] for x in os.walk(directory)]
-
-    # Now remove "." from the list
-    roots = roots[1:]
-
-    # Just get the relative paths
-    rel_roots = [os.path.relpath(x,directory) for x in roots]
 
-    # Now exclude __pycache__
-    command_dirs = [x for x in rel_roots if '__pycache__' not in x]
-
-    # Convert slashes to spaces
-    all_commands = [x.replace('/',' ') for x in command_dirs]
+def get_command_list(directory):
+    # Get all commands
+    all_commands_py = glob.glob(os.path.join(directory, "*.py"))
+    all_commands = [os.path.basename(x).replace(".py", "") for x in all_commands_py]
 
     # Now let's find the commands that have subcommands
-    ## First we get commands with spaces
-    commands_with_spaces = [x for x in all_commands if ' ' in x]
-    ## Now let's just get the first elements
-    temp = [x.split()[0] for x in commands_with_spaces]
-    ## Get the uniques
-    commands_with_subcommands = list(set(temp))
-
-    # Now remove those from our list
-    all_useful_commands = [x for x in all_commands if x not in commands_with_subcommands]
+    ## First we get commands with underscore
+    ## Then replace underscore with a space
+    commands_with_underscore = [x for x in all_commands if "_" in x]
+    commands_with_subcommands = [x.replace("_", " ") for x in commands_with_underscore]
+    all_useful_commands = [x for x in all_commands if x not in commands_with_underscore]
+    all_useful_commands += commands_with_subcommands
 
     return sorted(all_useful_commands)
 
+
 def create_markdown_from_usage(command, mdFile):
-    cmd = [mepo_command_path,command,'--help']
+    cmd = [mepo_command_path, command, "--help"]
 
     # Some commands have spaces, so we need to break it up again
-    cmd = ' '.join(cmd).split()
+    cmd = " ".join(cmd).split()
 
-    result = sp.run(cmd,capture_output=True,universal_newlines=True,env={'COLUMNS':'256'})
+    result = sp.run(cmd, capture_output=True, universal_newlines=True)
     output = result.stdout
 
     output_list = output.split("\n")
@@ -66,25 +54,32 @@ def create_markdown_from_usage(command, mdFile):
 
     # Usage
     usage = output_list[0]
-    usage = usage.replace('usage: ','')
+    usage = usage.replace("usage: ", "")
     mdFile.new_header(level=3, title="Usage")
     mdFile.insert_code(usage)
 
-    positional_arguments = output.partition('positional arguments:\n')[2].partition('\n\n')[0]
+    positional_arguments = output.partition("positional arguments:\n")[2].partition(
+        "\n\n"
+    )[0]
     if positional_arguments:
         mdFile.new_header(level=3, title="Positional Arguments")
         mdFile.insert_code(positional_arguments)
 
-    optional_arguments = output.partition('optional arguments:\n')[2].partition('\n\n')[0]
+    optional_arguments = output.partition("optional arguments:\n")[2].partition("\n\n")[
+        0
+    ]
     # Remove extra blank lines
-    optional_arguments = os.linesep.join([s for s in optional_arguments.splitlines() if s])
+    optional_arguments = os.linesep.join(
+        [s for s in optional_arguments.splitlines() if s]
+    )
     if optional_arguments:
         mdFile.new_header(level=3, title="Optional Arguments")
         mdFile.insert_code(optional_arguments)
 
+
 if __name__ == "__main__":
 
-    doc_file='Mepo-Commands.md'
+    doc_file = "Mepo-Commands.md"
     mdFile = MdUtils(file_name=doc_file)
 
     mdFile.new_header(level=1, title="Overview")
@@ -94,9 +89,9 @@ def create_markdown_from_usage(command, mdFile):
     command_list = get_command_list(command_dir_path)
     for command in command_list:
         mdFile.new_header(level=2, title=command)
-        create_markdown_from_usage(command,mdFile)
+        print(f"mepo command: {command}")
+        create_markdown_from_usage(command, mdFile)
 
-    mdFile.new_table_of_contents(table_title='Table of Contents', depth=2)
+    mdFile.new_table_of_contents(table_title="Table of Contents", depth=2)
     mdFile.create_md_file()
-    print(f'Generated {doc_file}.')
-
+    print(f"Generated {doc_file}.")