diff --git a/doc/cache_layout.md b/doc/cache_layout.md index 47c63056c..d782eb5a3 100644 --- a/doc/cache_layout.md +++ b/doc/cache_layout.md @@ -159,13 +159,26 @@ $PUB_CACHE/git/ └── pub-c4e9ddc888c3aa89ef4462f0c4298929191e32b9/ ``` -The `$PUB_CACHE/git/cache/` folder contains a "bare" checkout of each git-url (just the ). The -folders are `$PUB_CACHE/git/cache/$name-$hash/` where `$name` is derived from base-name of the -git url (without `.git`). and `$hash` is the sha1 of the git-url. This makes -them recognizable and unique. +The `$PUB_CACHE/git/cache/` folder contains a "bare" checkout of each git-url +(just the .git folder without a working tree). -The other sub-folders are the actual checkouts. They are clones of respective the `$PUB_CACHE/git/cache/$name-$hash/` -folders checked out at a specific `ref`. The name is `$PUB_CACHE/git/$name-$resolvedRef/` where +The folders are `$PUB_CACHE/git/cache/$name-$hash/` where `$name` is derived +from base-name of the git url (without `.git`). and `$hash` is the sha1 of the +git-url. This makes them recognizable and unique. + +The other sub-folders of `$PUB_CACHE/git` are the actual checkouts. + +Until Dart 3.1 These where clones of the respective +`$PUB_CACHE/git/cache/$name-$hash/` folders checked out at a specific `ref`. + +After Dart 3.1 these are [git worktrees](https://git-scm.com/docs/git-worktree) +instead of clones. This avoids repeating the entire .git folder for each checked +out ref. + +In Dart 3.1 and later a clone will be recognized and replaced by a worktree when +seen. + +The name of the checkout is `$PUB_CACHE/git/$name-$resolvedRef/` where `resolvedRef` is the commit-id that `ref` resolves to. ## Global packages diff --git a/lib/src/source/git.dart b/lib/src/source/git.dart index 54f4dd2e1..a8da9e779 100644 --- a/lib/src/source/git.dart +++ b/lib/src/source/git.dart @@ -392,12 +392,11 @@ class GitSource extends CachedSource { final path = description.path; await _revisionCacheClones.putIfAbsent(revisionCachePath, () async { if (!entryExists(revisionCachePath)) { - await _cloneViaTemp( + await _createWorktree( _repoCachePath(description, cache), revisionCachePath, - cache, + resolvedRef, ); - await _checkOut(revisionCachePath, resolvedRef); _writePackageList(revisionCachePath, [path]); didUpdate = true; } else { @@ -436,7 +435,7 @@ class GitSource extends CachedSource { final result = []; final packages = listDir(rootDir) - .where((entry) => dirExists(p.join(entry, '.git'))) + .where((entry) => entryExists(p.join(entry, '.git'))) .expand((revisionCachePath) { return _readPackageList(revisionCachePath).map((relative) { // If we've already failed to load another package from this @@ -561,7 +560,7 @@ class GitSource extends CachedSource { final path = _repoCachePath(description, cache); assert(!_updatedRepos.contains(path)); try { - await _cloneViaTemp(description.url, path, cache, mirror: true); + await _clone(description.url, path, cache); } catch (_) { await _deleteGitRepoIfInvalid(path); rethrow; @@ -581,7 +580,7 @@ class GitSource extends CachedSource { ) async { final path = _repoCachePath(description, cache); if (_updatedRepos.contains(path)) return false; - await git.run([_gitDirArg(path), 'fetch'], workingDir: path); + await git.run([_gitDirArg(path), 'fetch', 'origin'], workingDir: path); _updatedRepos.add(path); return true; } @@ -613,15 +612,15 @@ class GitSource extends CachedSource { } } - /// Updates the package list file in [revisionCachePath] to include [path], if - /// necessary. + /// Updates the package list file in [revisionCachePath] to include [package], + /// if necessary. /// /// Returns `true` if it had to update anything. - bool _updatePackageList(String revisionCachePath, String path) { + bool _updatePackageList(String revisionCachePath, String package) { final packages = _readPackageList(revisionCachePath); - if (packages.contains(path)) return false; + if (packages.contains(package)) return false; - _writePackageList(revisionCachePath, packages..add(path)); + _writePackageList(revisionCachePath, packages..add(package)); return true; } @@ -645,7 +644,7 @@ class GitSource extends CachedSource { /// The path in a revision cache repository in which we keep a list of the /// packages in the repository. String _packageListPath(String revisionCachePath) => - p.join(revisionCachePath, '.git/pub-packages'); + p.join(revisionCachePath, '.pub-packages'); /// Runs "git rev-list" on [reference] in [path] and returns the first result. /// @@ -669,37 +668,38 @@ class GitSource extends CachedSource { return output; } - /// Clones the repo at the URI [from] to the path [to] on the local - /// filesystem. + /// Makes a working tree of the repo at the path [from] at ref [ref] to the + /// path [to] on the local filesystem. /// - /// If [mirror] is true, creates a bare, mirrored clone. This doesn't check - /// out the working tree, but instead makes the repository a local mirror of - /// the remote repository. See the manpage for `git clone` for more - /// information. - Future _clone( - String from, - String to, { - bool mirror = false, - }) async { + /// Also checks out any submodules. + Future _createWorktree(String from, String to, String ref) async { // Git on Windows does not seem to automatically create the destination // directory. ensureDir(to); - final args = ['clone', if (mirror) '--mirror', from, to]; - - await git.run(args); + await git.run( + [ + _gitDirArg(from), + 'worktree', 'add', + // Checkout even if already checked out in other worktree. + // Should not be necessary, but cannot hurt either. + '--force', + to, + ref, + ], + workingDir: from, + ); } - /// Like [_clone], but clones to a temporary directory (inside the [cache]) - /// and moves - Future _cloneViaTemp( - String from, - String to, - SystemCache cache, { - bool mirror = false, - }) async { + /// Clones git repository at url [from] to the path [to] + /// clones to a temporary directory (inside the [cache]) + /// and moves it to [to] as an atomic operation. + Future _clone(String from, String to, SystemCache cache) async { final tempDir = cache.createTempDir(); try { - await _clone(from, tempDir, mirror: mirror); + // Git on Windows does not seem to automatically create the destination + // directory. + ensureDir(tempDir); + await git.run(['clone', '--mirror', from, tempDir]); } catch (_) { deleteEntry(tempDir); rethrow; @@ -713,12 +713,6 @@ class GitSource extends CachedSource { tryRenameDir(tempDir, to); } - /// Checks out the reference [ref] in [repoPath]. - Future _checkOut(String repoPath, String ref) { - return git - .run(['checkout', ref], workingDir: repoPath).then((result) => null); - } - String _revisionCachePath(PackageId id, SystemCache cache) => p.join( cache.rootDirForSource(this), '${_repoName(id.description.description as GitDescription)}-' diff --git a/test/descriptor.dart b/test/descriptor.dart index c760f983e..833025a14 100644 --- a/test/descriptor.dart +++ b/test/descriptor.dart @@ -310,8 +310,15 @@ Descriptor tokensFile([Map contents = const {}]) { /// Describes the application directory, containing only a pubspec specifying /// the given [dependencies]. -DirectoryDescriptor appDir({Map? dependencies, Map? pubspec}) => - dir(appPath, [appPubspec(dependencies: dependencies, extras: pubspec)]); +DirectoryDescriptor appDir({ + Map? dependencies, + Map? pubspec, + Iterable? contents, +}) => + dir( + appPath, + [appPubspec(dependencies: dependencies, extras: pubspec), ...?contents], + ); /// Describes a `.dart_tools/package_config.json` file. /// diff --git a/test/descriptor/git.dart b/test/descriptor/git.dart index 95ae0b21e..c85260543 100644 --- a/test/descriptor/git.dart +++ b/test/descriptor/git.dart @@ -3,9 +3,11 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:io'; import 'package:path/path.dart' as p; import 'package:pub/src/git.dart' as git; +import 'package:test/test.dart'; import 'package:test_descriptor/test_descriptor.dart'; /// Describes a Git repository and its contents. @@ -80,4 +82,37 @@ class GitRepoDescriptor extends DirectoryDescriptor { await _runGit(command, parent); } } + + /// Serves this git directory on localhost a fresh port + /// returns the port. + Future serve() async { + // Use this to invent a fresh host. + final s = await ServerSocket.bind('localhost', 0); + final port = s.port; + await s.close(); + final process = await Process.start( + 'git', + [ + 'daemon', + '--verbose', + '--export-all', + '--base-path=.git', + '--reuseaddr', + '--strict-paths', + '--port=$port', + '.git/', + ], + workingDirectory: p.join(sandbox, name), + ); + final c = Completer(); + process.stderr.listen((x) { + if (!c.isCompleted) c.complete(); + }); + await c.future; + addTearDown(() async { + process.kill(); + await process.exitCode; + }); + return port; + } } diff --git a/test/get/git/git_lfs_test.dart b/test/get/git/git_lfs_test.dart new file mode 100644 index 000000000..c40d3f06c --- /dev/null +++ b/test/get/git/git_lfs_test.dart @@ -0,0 +1,48 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'package:test/test.dart'; + +import '../../descriptor.dart' as d; +import '../../test_pub.dart'; + +void main() { + test('Can use LFS', () async { + ensureGit(); + + final foo = d.git('foo.git', [d.libPubspec('foo', '1.0.0')]); + await foo.create(); + await foo.runGit(['lfs', 'install']); + + await d.dir('foo.git', [ + d.dir('lib', [d.file('foo.dart', 'main() => print("hi");')]), + ]).create(); + await foo.runGit(['lfs', 'track', 'lib/foo.dart']); + await foo.runGit(['add', '.gitattributes']); + await foo.commit(); + + await d.appDir( + dependencies: { + 'foo': { + 'git': {'url': '../foo.git'}, + }, + }, + contents: [ + d.dir('bin', [d.file('main.dart', 'export "package:foo/foo.dart";')]), + ], + ).create(); + await pubGet(); + + await runPub(args: ['run', 'myapp:main'], output: contains('hi')); + + await d.git( + 'foo.git', + [ + d.dir('lib', [d.file('foo.dart', 'main() => print("bye");')]), + ], + ).commit(); + + await pubUpgrade(); + await runPub(args: ['run', 'myapp:main'], output: contains('bye')); + }); +}