From 977b51b430f220f7d87d017f838bfa2e40d33620 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 28 Feb 2023 12:21:27 +0000 Subject: [PATCH 1/6] Use worktrees for git cache Also call "submodule update --init --recursive" to support submodules. --- test/get/git/git_submodule_test.dart | 56 ++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/get/git/git_submodule_test.dart diff --git a/test/get/git/git_submodule_test.dart b/test/get/git/git_submodule_test.dart new file mode 100644 index 000000000..dae47dba9 --- /dev/null +++ b/test/get/git/git_submodule_test.dart @@ -0,0 +1,56 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'package:test/test.dart'; + +import '../../descriptor.dart' as d; +import '../../test_pub.dart'; + +void main() { + test('Can use and upgrade git dependency with submodule', () async { + ensureGit(); + + final submodule = d.git( + 'lib.git', + [d.file('foo.dart', 'main() => print("hi");')], + ); + await submodule.create(); + final port = await submodule.serve(); + + final foo = d.git('foo.git', [d.libPubspec('foo', '1.0.0')]); + await foo.create(); + await foo.runGit(['submodule', 'add', 'git://localhost:$port/', 'lib']); + await foo.commit(); + + await d.appDir( + dependencies: { + 'foo': { + 'git': {'url': '../foo.git'} + } + }, + contents: [ + d.dir('bin', [d.file('main.dart', 'export "package:foo/foo.dart";')]), + ], + ).create(); + await pubGet(); + + await runPub( + args: ['run', 'myapp:main'], + output: contains('hi'), + ); + + await d.git( + 'lib.git', + [d.file('foo.dart', 'main() => print("bye");')], + ).commit(); + + await foo.runGit(['submodule', 'update', '--remote', '--merge']); + await foo.commit(); + + await pubUpgrade(); + await runPub( + args: ['run', 'myapp:main'], + output: contains('bye'), + ); + }); +} From 5c46570f5b0bd1f1a13fd9ef5823a09f64a191c1 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 28 Feb 2023 13:19:14 +0000 Subject: [PATCH 2/6] Support for lfs and submodules --- lib/src/source/git.dart | 64 ++++++++++++++++------------ test/descriptor.dart | 11 ++++- test/descriptor/git.dart | 32 ++++++++++++++ test/get/git/git_submodule_test.dart | 49 +++++++++++++++++---- 4 files changed, 118 insertions(+), 38 deletions(-) diff --git a/lib/src/source/git.dart b/lib/src/source/git.dart index 36401d68a..52f9b08e2 100644 --- a/lib/src/source/git.dart +++ b/lib/src/source/git.dart @@ -375,8 +375,11 @@ class GitSource extends CachedSource { final path = description.path; await _revisionCacheClones.putIfAbsent(revisionCachePath, () async { if (!entryExists(revisionCachePath)) { - await _clone(_repoCachePath(description, cache), revisionCachePath); - await _checkOut(revisionCachePath, resolvedRef); + await _createWorktree( + _repoCachePath(description, cache), + revisionCachePath, + resolvedRef, + ); _writePackageList(revisionCachePath, [path]); didUpdate = true; } else { @@ -415,7 +418,7 @@ class GitSource extends CachedSource { final result = []; var packages = listDir(rootDir) - .where((entry) => dirExists(p.join(entry, '.git'))) + .where((entry) => entryExists(p.join(entry, '.git'))) .expand((revisionCachePath) { return _readPackageList(revisionCachePath).map((relative) { // If we've already failed to load another package from this @@ -535,7 +538,7 @@ class GitSource extends CachedSource { var path = _repoCachePath(description, cache); assert(!_updatedRepos.contains(path)); try { - await _clone(description.url, path, mirror: true); + await _mirrorClone(description.url, path); } catch (_) { await _deleteGitRepoIfInvalid(path); rethrow; @@ -555,7 +558,7 @@ class GitSource extends CachedSource { ) async { var path = _repoCachePath(description, cache); if (_updatedRepos.contains(path)) return false; - await git.run(['fetch'], workingDir: path); + await git.run(['fetch', 'origin'], workingDir: path); _updatedRepos.add(path); return true; } @@ -619,7 +622,7 @@ class GitSource extends CachedSource { /// The path in a revision cache repository in which we keep a list of the /// packages in the repository. String _packageListPath(String revisionCachePath) => - p.join(revisionCachePath, '.git/pub-packages'); + p.join(revisionCachePath, '.pub-packages'); /// Runs "git rev-list" on [reference] in [path] and returns the first result. /// @@ -640,33 +643,38 @@ class GitSource extends CachedSource { return lines.first; } - /// Clones the repo at the URI [from] to the path [to] on the local - /// filesystem. - /// - /// If [mirror] is true, creates a bare, mirrored clone. This doesn't check - /// out the working tree, but instead makes the repository a local mirror of - /// the remote repository. See the manpage for `git clone` for more - /// information. - /// - /// If [shallow] is true, creates a shallow clone that contains no history - /// for the repository. - Future _clone( - String from, - String to, { - bool mirror = false, - }) async { + /// Creates a bare clone of the repo at the URI [from] to the path [to] on the + /// local filesystem. + Future _mirrorClone(String from, String to) async { // Git on Windows does not seem to automatically create the destination // directory. ensureDir(to); - var args = ['clone', if (mirror) '--mirror', from, to]; - - await git.run(args); + await git.run(['clone', '--mirror', from, to]); } - /// Checks out the reference [ref] in [repoPath]. - Future _checkOut(String repoPath, String ref) { - return git - .run(['checkout', ref], workingDir: repoPath).then((result) => null); + /// Makes a working tree of the repo at the path [from] at ref [ref] to the + /// path [to] on the local filesystem. + /// + /// Also checks out any submodules. + Future _createWorktree(String from, String to, String ref) async { + // Git on Windows does not seem to automatically create the destination + // directory. + ensureDir(to); + await git.run( + [ + 'worktree', 'add', + // Checkout even if already checked out in other worktree. + // Should not be necessary, but cannot hurt either. + '--force', + to, + ref, + ], + workingDir: from, + ); + await git.run( + ['submodule', 'update', '--init', '--recursive'], + workingDir: to, + ); } String _revisionCachePath(PackageId id, SystemCache cache) => p.join( diff --git a/test/descriptor.dart b/test/descriptor.dart index 7f414cc9c..8c918987e 100644 --- a/test/descriptor.dart +++ b/test/descriptor.dart @@ -288,8 +288,15 @@ Descriptor tokensFile([Map contents = const {}]) { /// Describes the application directory, containing only a pubspec specifying /// the given [dependencies]. -DirectoryDescriptor appDir({Map? dependencies, Map? pubspec}) => - dir(appPath, [appPubspec(dependencies: dependencies, extras: pubspec)]); +DirectoryDescriptor appDir({ + Map? dependencies, + Map? pubspec, + Iterable? contents, +}) => + dir( + appPath, + [appPubspec(dependencies: dependencies, extras: pubspec), ...?contents], + ); /// Describes a `.dart_tools/package_config.json` file. /// diff --git a/test/descriptor/git.dart b/test/descriptor/git.dart index d092d9963..731a44bc2 100644 --- a/test/descriptor/git.dart +++ b/test/descriptor/git.dart @@ -3,9 +3,11 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:io'; import 'package:path/path.dart' as path; import 'package:pub/src/git.dart' as git; +import 'package:test/test.dart'; import 'package:test_descriptor/test_descriptor.dart'; /// Describes a Git repository and its contents. @@ -80,4 +82,34 @@ class GitRepoDescriptor extends DirectoryDescriptor { await _runGit(command, parent); } } + + /// Serves this git directory on localhost a fresh port + /// returns the port. + Future serve() async { + // Use this to invent a fresh host. + final s = await ServerSocket.bind('localhost', 0); + int port = s.port; + await s.close(); + final process = await Process.start( + 'git', + [ + 'daemon', + '--verbose', + '--export-all', + '--base-path=.git', + '--reuseaddr', + '--strict-paths', + '--port=$port', + '.git/' + ], + workingDirectory: path.join(sandbox, name), + ); + final c = Completer(); + process.stderr.listen((x) { + if (!c.isCompleted) c.complete(); + }); + await c.future; + addTearDown(process.kill); + return port; + } } diff --git a/test/get/git/git_submodule_test.dart b/test/get/git/git_submodule_test.dart index dae47dba9..2f45162bf 100644 --- a/test/get/git/git_submodule_test.dart +++ b/test/get/git/git_submodule_test.dart @@ -34,10 +34,7 @@ void main() { ).create(); await pubGet(); - await runPub( - args: ['run', 'myapp:main'], - output: contains('hi'), - ); + await runPub(args: ['run', 'myapp:main'], output: contains('hi')); await d.git( 'lib.git', @@ -48,9 +45,45 @@ void main() { await foo.commit(); await pubUpgrade(); - await runPub( - args: ['run', 'myapp:main'], - output: contains('bye'), - ); + await runPub(args: ['run', 'myapp:main'], output: contains('bye')); + }); + + test('Can use LFS', () async { + ensureGit(); + + final foo = d.git('foo.git', [d.libPubspec('foo', '1.0.0')]); + await foo.create(); + await foo.runGit(['lfs', 'install']); + + await d.dir('foo.git', [ + d.dir('lib', [d.file('foo.dart', 'main() => print("hello");')]) + ]).create(); + await foo.runGit(['lfs', 'track', 'lib/foo.dart']); + await foo.runGit(['add', '.gitattributes']); + await foo.commit(); + + await d.appDir( + dependencies: { + 'foo': { + 'git': {'url': '../foo.git'} + } + }, + contents: [ + d.dir('bin', [d.file('main.dart', 'export "package:foo/foo.dart";')]), + ], + ).create(); + await pubGet(); + + await runPub(args: ['run', 'myapp:main'], output: contains('hi')); + + await d.git( + 'foo.git', + [ + d.dir('lib', [d.file('foo.dart', 'main() => print("bye");')]) + ], + ).commit(); + + await pubUpgrade(); + await runPub(args: ['run', 'myapp:main'], output: contains('bye')); }); } From 2b4fcacca31a2a3a6d4131d67c8eafcf8c0ca9c3 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 28 Feb 2023 13:33:11 +0000 Subject: [PATCH 3/6] Undo mistake --- test/get/git/git_submodule_test.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/get/git/git_submodule_test.dart b/test/get/git/git_submodule_test.dart index 2f45162bf..4eea4e58a 100644 --- a/test/get/git/git_submodule_test.dart +++ b/test/get/git/git_submodule_test.dart @@ -56,7 +56,7 @@ void main() { await foo.runGit(['lfs', 'install']); await d.dir('foo.git', [ - d.dir('lib', [d.file('foo.dart', 'main() => print("hello");')]) + d.dir('lib', [d.file('foo.dart', 'main() => print("hi");')]) ]).create(); await foo.runGit(['lfs', 'track', 'lib/foo.dart']); await foo.runGit(['add', '.gitattributes']); From 13b604b09748cbb856c795420485cce3a4d88172 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 28 Feb 2023 14:07:06 +0000 Subject: [PATCH 4/6] Await server shutdown --- test/descriptor/git.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/descriptor/git.dart b/test/descriptor/git.dart index 731a44bc2..ac77adb08 100644 --- a/test/descriptor/git.dart +++ b/test/descriptor/git.dart @@ -109,7 +109,10 @@ class GitRepoDescriptor extends DirectoryDescriptor { if (!c.isCompleted) c.complete(); }); await c.future; - addTearDown(process.kill); + addTearDown(() async { + process.kill(); + await process.exitCode; + }); return port; } } From 75b832ed6aae5bb5d811163fe5868a7676f8d0df Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 6 Jun 2023 12:20:13 +0000 Subject: [PATCH 5/6] Remove the submodule support --- lib/src/source/git.dart | 4 -- ..._submodule_test.dart => git_lfs_test.dart} | 41 ------------------- 2 files changed, 45 deletions(-) rename test/get/git/{git_submodule_test.dart => git_lfs_test.dart} (54%) diff --git a/lib/src/source/git.dart b/lib/src/source/git.dart index d30616c25..85bf1711e 100644 --- a/lib/src/source/git.dart +++ b/lib/src/source/git.dart @@ -675,10 +675,6 @@ class GitSource extends CachedSource { ], workingDir: from, ); - await git.run( - ['submodule', 'update', '--init', '--recursive'], - workingDir: to, - ); } String _revisionCachePath(PackageId id, SystemCache cache) => p.join( diff --git a/test/get/git/git_submodule_test.dart b/test/get/git/git_lfs_test.dart similarity index 54% rename from test/get/git/git_submodule_test.dart rename to test/get/git/git_lfs_test.dart index 4eea4e58a..aed7df155 100644 --- a/test/get/git/git_submodule_test.dart +++ b/test/get/git/git_lfs_test.dart @@ -7,47 +7,6 @@ import '../../descriptor.dart' as d; import '../../test_pub.dart'; void main() { - test('Can use and upgrade git dependency with submodule', () async { - ensureGit(); - - final submodule = d.git( - 'lib.git', - [d.file('foo.dart', 'main() => print("hi");')], - ); - await submodule.create(); - final port = await submodule.serve(); - - final foo = d.git('foo.git', [d.libPubspec('foo', '1.0.0')]); - await foo.create(); - await foo.runGit(['submodule', 'add', 'git://localhost:$port/', 'lib']); - await foo.commit(); - - await d.appDir( - dependencies: { - 'foo': { - 'git': {'url': '../foo.git'} - } - }, - contents: [ - d.dir('bin', [d.file('main.dart', 'export "package:foo/foo.dart";')]), - ], - ).create(); - await pubGet(); - - await runPub(args: ['run', 'myapp:main'], output: contains('hi')); - - await d.git( - 'lib.git', - [d.file('foo.dart', 'main() => print("bye");')], - ).commit(); - - await foo.runGit(['submodule', 'update', '--remote', '--merge']); - await foo.commit(); - - await pubUpgrade(); - await runPub(args: ['run', 'myapp:main'], output: contains('bye')); - }); - test('Can use LFS', () async { ensureGit(); From fa18a3ce07bea4b5b74935376701446f3a09193f Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 26 Nov 2024 13:14:13 +0000 Subject: [PATCH 6/6] Update cache_layout.md --- doc/cache_layout.md | 25 +++++++++++++++++++------ lib/src/source/git.dart | 8 ++++---- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/doc/cache_layout.md b/doc/cache_layout.md index 8cd76a7a4..93e2ff7ab 100644 --- a/doc/cache_layout.md +++ b/doc/cache_layout.md @@ -159,13 +159,26 @@ $PUB_CACHE/git/ └── pub-c4e9ddc888c3aa89ef4462f0c4298929191e32b9/ ``` -The `$PUB_CACHE/git/cache/` folder contains a "bare" checkout of each git-url (just the ). The -folders are `$PUB_CACHE/git/cache/$name-$hash/` where `$name` is derived from base-name of the -git url (without `.git`). and `$hash` is the sha1 of the git-url. This makes -them recognizable and unique. +The `$PUB_CACHE/git/cache/` folder contains a "bare" checkout of each git-url +(just the .git folder without a working tree). -The other sub-folders are the actual checkouts. They are clones of respective the `$PUB_CACHE/git/cache/$name-$hash/` -folders checked out at a specific `ref`. The name is `$PUB_CACHE/git/$name-$resolvedRef/` where +The folders are `$PUB_CACHE/git/cache/$name-$hash/` where `$name` is derived +from base-name of the git url (without `.git`). and `$hash` is the sha1 of the +git-url. This makes them recognizable and unique. + +The other sub-folders of `$PUB_CACHE/git` are the actual checkouts. + +Until Dart 3.1 These where clones of the respective +`$PUB_CACHE/git/cache/$name-$hash/` folders checked out at a specific `ref`. + +After Dart 3.1 these are [git worktrees](https://git-scm.com/docs/git-worktree) +instead of clones. This avoids repeating the entire .git folder for each checked +out ref. + +In Dart 3.1 and later a clone will be recognized and replaced by a worktree when +seen. + +The name of the checkout is `$PUB_CACHE/git/$name-$resolvedRef/` where `resolvedRef` is the commit-id that `ref` resolves to. ## Global packages diff --git a/lib/src/source/git.dart b/lib/src/source/git.dart index 85bf1711e..27983b073 100644 --- a/lib/src/source/git.dart +++ b/lib/src/source/git.dart @@ -594,15 +594,15 @@ class GitSource extends CachedSource { } } - /// Updates the package list file in [revisionCachePath] to include [path], if + /// Updates the package list file in [revisionCachePath] to include [package], if /// necessary. /// /// Returns `true` if it had to update anything. - bool _updatePackageList(String revisionCachePath, String path) { + bool _updatePackageList(String revisionCachePath, String package) { var packages = _readPackageList(revisionCachePath); - if (packages.contains(path)) return false; + if (packages.contains(package)) return false; - _writePackageList(revisionCachePath, packages..add(path)); + _writePackageList(revisionCachePath, packages..add(package)); return true; }