Skip to content

Commit

Permalink
Merge pull request #265 from GEOS-ESM/develop
Browse files Browse the repository at this point in the history
GitFlow: Merge develop into main
  • Loading branch information
mathomp4 authored Jan 10, 2024
2 parents a675833 + 64df27f commit e56a187
Show file tree
Hide file tree
Showing 7 changed files with 109 additions and 10 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/mepo.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
python-version: ['3.9', '3.10', '3.11', 'pypy-3.9']
python-version: ['3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10']

name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
Expand Down
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Removed

## [1.52.0] - 2024-01-10

### Added

- Added new `--partial` option to `mepo clone` with two settings: `off`, `blobless`, and `treeless`. If you set, `--partial=blobless` then
the clone will not download blobs by using `--filter=blob:none`. If you set `--partial=treeless` then the clone will not download
trees by using `--filter=tree:0`. The `blobless` option is useful for large repos that have a lot of binary files that you don't
need. The `treeless` option is even more aggressive and *SHOULD NOT* be used unless you know what you are doing. The
`--partial=off` option allows a user to override the default behavior of `--partial` in `.mepoconfig` and turn it off for a
run of `mepo clone`.
- Add a new section for `.mepoconfig` to allow users to set `--partial` as a default for `mepo clone`.

## [1.51.1] - 2023-08-25

### Fixed
Expand Down
43 changes: 42 additions & 1 deletion etc/mepoconfig-example
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
#
# .mepoconfig is a config file a la gitconfig with sections and options.
#
# Currently, .mepoconfig files recognize two sections: [init] and [alias]
# Currently, .mepoconfig files recognize three sections: [init], [alias], and [clone].
#
# =======================================================================
#
Expand All @@ -24,6 +24,10 @@
#
# mepo clone --style postfix
#
# You set these options by running:
#
# mepo config set init.style <value>
#
# =======================================================================
#
# [alias] Section
Expand All @@ -40,3 +44,40 @@
# you can only alias mepo primary commands and not "subcommands" or
# "options". So you can have an alias for "commit" and for "branch",
# but you can't do an option for "commit -m" or "branch create".
#
# You can set an alias by running:
#
# mepo config set alias.<alias> <command>
#
# =======================================================================
#
# [clone] Section
#
# The clone section currently recognizes one option, partial.
# This has two allowed values: blobless and treeless
#
# So if you have:
#
# [clone]
# partial = blobless
#
# This is equivalent to doing:
#
# mepo clone --partial=blobless
#
# which corresponds to the git clone option --filter=blob:none
#
# and similarly for treeless:
#
# [clone]
# partial = treeless
#
# is equivalent to doing:
#
# mepo clone --partial=treeless
#
# which corresponds to the git clone option --filter=tree:0
#
# You set these options by running:
#
# mepo config set clone.partial <value>
7 changes: 7 additions & 0 deletions mepo.d/cmdline/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,13 @@ def __clone(self):
'--allrepos',
action = 'store_true',
help = 'Must be passed with -b/--branch. When set, it not only checkouts out the branch/tag for the fixture, but for all the subrepositories as well.')
clone.add_argument(
'--partial',
metavar = 'partial-type',
nargs = '?',
default = None,
choices = ['off','blobless','treeless'],
help = 'Style of partial clone, default: None, allowed options: %(choices)s. Off means a "normal" full git clone, blobless means cloning with "--filter=blob:none" and treeless means cloning with "--filter=tree:0". NOTE: We do *not* recommend using "treeless" as it is very aggressive and will cause problems with many git commands.')

def __list(self):
listcomps = self.subparsers.add_parser(
Expand Down
42 changes: 37 additions & 5 deletions mepo.d/command/clone/clone.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from state.state import MepoState, StateDoesNotExistError
from repository.git import GitRepository
from command.init import init as mepo_init
from utilities import shellcmd, colors
from utilities import shellcmd, colors, mepoconfig
from urllib.parse import urlparse

import os
Expand All @@ -18,6 +18,23 @@ def run(args):
if args.allrepos and not args.branch:
raise RuntimeError("The allrepos option must be used with a branch/tag.")

# We can get the blobless and treeless options from the config or the args
if args.partial:
# We need to set partial to None if it's off, otherwise we use the
# string. This is safe because argparse only allows for 'off',
# 'blobless', or 'treeless'
partial = None if args.partial == 'off' else args.partial
elif mepoconfig.has_option('clone','partial'):
allowed = ['blobless','treeless']
partial = mepoconfig.get('clone','partial')
if partial not in allowed:
raise Exception(f'Detected partial clone type [{partial}] from .mepoconfig is not an allowed partial clone type: {allowed}')
else:
print(f'Found partial clone type [{partial}] in .mepoconfig')
else:
partial = None


# If you pass in a config, with clone, it could be outside the repo.
# So use the full path
passed_in_config = False
Expand All @@ -34,15 +51,15 @@ def run(args):
last_url_node = p.path.rsplit('/')[-1]
url_suffix = pathlib.Path(last_url_node).suffix
if args.directory:
local_clone(args.repo_url,args.branch,args.directory)
local_clone(args.repo_url,args.branch,args.directory,partial)
os.chdir(args.directory)
else:
if url_suffix == '.git':
git_url_directory = pathlib.Path(last_url_node).stem
else:
git_url_directory = last_url_node

local_clone(args.repo_url,args.branch,git_url_directory)
local_clone(args.repo_url,args.branch,git_url_directory,partial)
os.chdir(git_url_directory)

# Copy the new file into the repo only if we pass it in
Expand All @@ -69,9 +86,16 @@ def run(args):
version = comp.version.name
version = version.replace('origin/','')
recurse = comp.recurse_submodules

# According to Git, treeless clones do not interact well with
# submodules. So we need to see if any comp has the recurse
# option set to True. If so, we need to clone that comp "normally"

_partial = None if partial == 'treeless' and recurse else partial

# We need the type to handle hashes in components.yaml
type = comp.version.type
git.clone(version,recurse,type,comp.name)
git.clone(version,recurse,type,comp.name,_partial)
if comp.sparse:
git.sparsify(comp.sparse)
print_clone_info(comp, max_namelen)
Expand All @@ -89,8 +113,16 @@ def print_clone_info(comp, name_width):
ver_name_type = '({}) {}'.format(comp.version.type, comp.version.name)
print('{:<{width}} | {:<s}'.format(comp.name, ver_name_type, width = name_width))

def local_clone(url,branch=None,directory=None):
def local_clone(url,branch=None,directory=None,partial=None):
cmd1 = 'git clone '

if partial == 'blobless':
cmd1 += '--filter=blob:none '
elif partial == 'treeless':
cmd1 += '--filter=tree:0 '
else:
partial = None

if branch:
cmd1 += '--branch {} '.format(branch)
cmd1 += '--quiet {}'.format(url)
Expand Down
8 changes: 7 additions & 1 deletion mepo.d/repository/git.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,14 @@ def get_full_local_path(self):
def get_remote_url(self):
return self.__remote

def clone(self, version, recurse, type, comp_name):
def clone(self, version, recurse, type, comp_name, partial=None):
cmd1 = 'git clone '

if partial == 'blobless':
cmd1 += '--filter=blob:none '
elif partial == 'treeless':
cmd1 += '--filter=tree:0 '

if recurse:
cmd1 += '--recurse-submodules '

Expand Down
1 change: 1 addition & 0 deletions mepo.d/utest/test_mepo_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def setUpClass(cls):
args.repo_url = None
args.branch = None
args.directory = None
args.partial = 'blobless'
mepo_clone.run(args)
# In order to better test compare, we need to do *something*
args.comp_name = ['env','cmake','fvdycore']
Expand Down

0 comments on commit e56a187

Please sign in to comment.