Merge pull request #62 from SwiftPackageIndex/fetch-all-manifests

Fetch all manifests
SwiftPackageIndex · Feb 14, 2024 · 64d2a82 · 64d2a82
2 parents 969af98 + 4d23075
commit 64d2a82
Show file tree

Hide file tree

Showing 11 changed files with 190 additions and 98 deletions.
diff --git a/Sources/ValidatorCore/Commands/CheckDependencies.swift b/Sources/ValidatorCore/Commands/CheckDependencies.swift
@@ -89,8 +89,7 @@ public struct CheckDependencies: AsyncParsableCommand {
 
             do {  // run package dump to validate
                 let repo = try await Current.fetchRepository(client, resolved)
-                let manifest = try await Package.getManifestURL(client: client, repository: repo)
-                _ = try Current.decodeManifest(manifest)
+                _ = try await Current.decodeManifest(client, repo)
             } catch {
                 print("  ... ⛔ \(error)")
                 continue

diff --git a/Sources/ValidatorCore/Environment.swift b/Sources/ValidatorCore/Environment.swift
@@ -19,20 +19,20 @@ import NIO
 
 
 struct Environment {
-    var decodeManifest: (_ url: Package.ManifestURL) throws -> Package
+    var decodeManifest: (_ client: Client, _ repository: Github.Repository) async throws -> Package
     var fileManager: FileManager
-    var fetch: (_ client: HTTPClient, _ url: URL) -> EventLoopFuture<ByteBuffer>
+    var fetch: (_ client: Client, _ url: URL) -> EventLoopFuture<ByteBuffer>
     var fetchDependencies: (_ api: SwiftPackageIndexAPI) async throws -> [SwiftPackageIndexAPI.PackageRecord]
-    var fetchRepository: (_ client: HTTPClient, _ url: PackageURL) async throws -> Github.Repository
+    var fetchRepository: (_ client: Client, _ url: PackageURL) async throws -> Github.Repository
     var githubToken: () -> String?
-    var resolvePackageRedirects: (_ client: HTTPClient, _ url: PackageURL) async throws -> Redirect
+    var resolvePackageRedirects: (_ client: Client, _ url: PackageURL) async throws -> Redirect
     var shell: Shell
 }
 
 
 extension Environment {
     static let live: Self = .init(
-        decodeManifest: { url in try Package.decode(from: url) },
+        decodeManifest: { client, repo in try await Package.decode(client: client, repository: repo) },
         fileManager: .live,
         fetch: Github.fetch(client:url:),
         fetchDependencies: { try await $0.fetchDependencies() },
@@ -43,7 +43,7 @@ extension Environment {
     )
 
     static let mock: Self = .init(
-        decodeManifest: { _ in fatalError("not implemented") },
+        decodeManifest: { _, _ in fatalError("not implemented") },
         fileManager: .mock,
         fetch: { client, _ in client.eventLoopGroup.next().makeFailedFuture(AppError.runtimeError("unimplemented")) },
         fetchDependencies: { _ in [] },
@@ -55,6 +55,19 @@ extension Environment {
 }
 
 
+protocol Client {
+    var eventLoopGroup: EventLoopGroup { get }
+    func execute(request: HTTPClient.Request, deadline: NIODeadline?) -> EventLoopFuture<HTTPClient.Response>
+}
+extension Client {
+    func execute(request: HTTPClient.Request) -> EventLoopFuture<HTTPClient.Response> {
+        execute(request: request, deadline: nil)
+    }
+}
+
+extension HTTPClient: Client { }
+
+
 #if DEBUG
 var Current: Environment = .live
 #else

diff --git a/Sources/ValidatorCore/Github.swift b/Sources/ValidatorCore/Github.swift
@@ -134,12 +134,12 @@ extension Github {
     static var repositoryCache = Cache<Repository>()
 
 
-    static func fetchRepository(client: HTTPClient, url: PackageURL) async throws-> Repository {
+    static func fetchRepository(client: Client, url: PackageURL) async throws-> Repository {
         try await fetchRepository(client: client, url: url, attempt: 0)
     }
 
 
-    static func fetchRepository(client: HTTPClient, url: PackageURL, attempt: Int) async throws-> Repository {
+    static func fetchRepository(client: Client, url: PackageURL, attempt: Int) async throws-> Repository {
         guard attempt < 3 else { throw AppError.retryLimitExceeded }
         let apiURL = URL(string: "https://api.github.com/repos/\(url.owner)/\(url.repository)")!
         let key = Cache<Repository>.Key(string: apiURL.absoluteString)
@@ -158,7 +158,7 @@ extension Github {
     }
 
 
-    static func listRepositoryFilePaths(client: HTTPClient, repository: Repository) async throws -> [String] {
+    static func listRepositoryFilePaths(client: Client, repository: Repository) async throws -> [String] {
         let apiURL = URL( string: "https://api.github.com/repos/\(repository.path)/git/trees/\(repository.defaultBranch)" )!
         struct Response: Decodable {
             var tree: [File]
@@ -180,7 +180,7 @@ extension Github {
     }
 
 
-    static func fetch<T: Decodable>(_ type: T.Type, client: HTTPClient, url: URL) async throws -> T {
+    static func fetch<T: Decodable>(_ type: T.Type, client: Client, url: URL) async throws -> T {
         let body = try await Current.fetch(client, url).get()
         do {
             return try JSONDecoder().decode(type, from: body)
@@ -192,7 +192,7 @@ extension Github {
         }
     }
 
-    static func fetch(client: HTTPClient, url: URL) -> EventLoopFuture<ByteBuffer> {
+    static func fetch(client: Client, url: URL) -> EventLoopFuture<ByteBuffer> {
         let eventLoop = client.eventLoopGroup.next()
         guard let token = Current.githubToken() else {
             return eventLoop.makeFailedFuture(AppError.githubTokenNotSet)

diff --git a/Sources/ValidatorCore/Package.swift b/Sources/ValidatorCore/Package.swift
@@ -86,23 +86,29 @@ extension Package {
     static func loadPackageDumpCache() { packageDumpCache = .load(from: cacheFilename) }
     static func savePackageDumpCache() throws { try packageDumpCache.save(to: cacheFilename) }
 
-    static func decode(from manifestURL: ManifestURL) throws -> Package {
-        if let cached = packageDumpCache[Cache.Key(string: manifestURL.rawValue.absoluteString)] {
+    static func decode(client: Client, repository: Github.Repository) async throws -> Self {
+        let cacheKey = repository.path
+        if let cached = packageDumpCache[Cache.Key(string: cacheKey)] {
             return cached
         }
-        return try withTempDir { tempDir in
-            let fileURL = URL(fileURLWithPath: tempDir).appendingPathComponent("Package.swift")
-            let data = try Data(contentsOf: manifestURL.rawValue)
-            guard Current.fileManager.createFile(fileURL.path, data, nil) else {
-                throw AppError.dumpPackageError("failed to save manifest \(manifestURL.rawValue.absoluteString) to temp directory \(fileURL.absoluteString)")
+        return try await withTempDir { tempDir in
+            for manifestURL in try await Package.getManifestURLs(client: client, repository: repository) {
+                let fileURL = URL(fileURLWithPath: tempDir).appendingPathComponent(manifestURL.lastPathComponent)
+                let buffer = try await Current.fetch(client, manifestURL.rawValue).get()
+                guard let data = buffer.getData(at: 0, length: buffer.readableBytes) else {
+                    throw AppError.dumpPackageError("failed to get data for manifest \(manifestURL.rawValue.absoluteString)")
+                }
+                guard Current.fileManager.createFile(fileURL.path, data, nil) else {
+                    throw AppError.dumpPackageError("failed to save manifest \(manifestURL.rawValue.absoluteString) to temp directory \(fileURL.absoluteString)")
+                }
             }
             do {
                 guard let pkgJSON = try Current.shell.run(command: .packageDump, at: tempDir)
                         .data(using: .utf8) else {
                     throw AppError.dumpPackageError("package dump did not return data")
                 }
                 let pkg = try JSONDecoder().decode(Package.self, from: pkgJSON)
-                packageDumpCache[Cache.Key(string: manifestURL.rawValue.absoluteString)] = pkg
+                packageDumpCache[Cache.Key(string: cacheKey)] = pkg
                 return pkg
             } catch let error as ShellOutError {
                 throw AppError.dumpPackageError("package dump failed: \(error.message)")
@@ -118,16 +124,15 @@ extension Package {
     enum Manifest {}
     typealias ManifestURL = Tagged<Manifest, URL>
 
-    static func getManifestURL(client: HTTPClient, repository: Github.Repository) async throws -> ManifestURL {
+    static func getManifestURLs(client: Client, repository: Github.Repository) async throws -> [ManifestURL] {
         let manifestFiles = try await Github.listRepositoryFilePaths(client: client, repository: repository)
           .filter { $0.hasPrefix("Package") }
           .filter { $0.hasSuffix(".swift") }
           .sorted()
-        guard let manifestFile = manifestFiles.last else {
-            throw AppError.manifestNotFound(owner: repository.owner.login, name: repository.name)
-        }
-        let url = URL(string: "https://raw.githubusercontent.com/\(repository.path)/\(repository.defaultBranch)/\(manifestFile)")!
-        return .init(rawValue: url)
+        guard !manifestFiles.isEmpty else { throw AppError.manifestNotFound(owner: repository.owner.login, name: repository.name) }
+        return manifestFiles
+            .map { URL(string: "https://raw.githubusercontent.com/\(repository.path)/\(repository.defaultBranch)/\($0)")! }
+            .map(ManifestURL.init(rawValue:))
     }
 
 }
diff --git a/Sources/ValidatorCore/RedirectFollower.swift b/Sources/ValidatorCore/RedirectFollower.swift
@@ -46,12 +46,12 @@ enum Redirect: Equatable {
 }
 
 
-private func resolveRedirects(client: HTTPClient, for url: PackageURL) async throws -> Redirect {
+private func resolveRedirects(client: Client, for url: PackageURL) async throws -> Redirect {
     var lastResult = Redirect.initial(url)
     var hopCount = 0
     let maxHops = 10
 
-    func _resolveRedirects(client: HTTPClient, for url: PackageURL) async throws -> Redirect {
+    func _resolveRedirects(client: Client, for url: PackageURL) async throws -> Redirect {
         var request = try HTTPClient.Request(url: url.rawValue, method: .HEAD, headers: .init([
             ("User-Agent", "SPI-Validator")
         ]))
@@ -103,7 +103,7 @@ private func resolveRedirects(client: HTTPClient, for url: PackageURL) async thr
 }
 
 
-func resolvePackageRedirects(client: HTTPClient, for url: PackageURL) async throws -> Redirect {
+func resolvePackageRedirects(client: Client, for url: PackageURL) async throws -> Redirect {
     let res = try await resolveRedirects(client: client, for: url.deletingGitExtension())
     switch res {
         case .initial, .notFound, .error, .unauthorized, .rateLimited:

diff --git a/Sources/ValidatorCore/TempDir.swift b/Sources/ValidatorCore/TempDir.swift
@@ -42,7 +42,7 @@ class TempDir {
 }
 
 
-func withTempDir<T>(body: (String) throws -> T) throws -> T {
+func withTempDir<T>(body: (String) async throws -> T) async throws -> T {
     let tmp = try TempDir()
-    return try body(tmp.path)
+    return try await body(tmp.path)
 }
diff --git a/Tests/ValidatorTests/CheckDependenciesTests.swift b/Tests/ValidatorTests/CheckDependenciesTests.swift
@@ -64,20 +64,9 @@ final class CheckDependenciesTests: XCTestCase {
                 throw Error.unexpectedCall
             }
         }
-        Current.fetch = { client, url in
-            // getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
-            guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
-                return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
-            }
-            return client.eventLoopGroup.next().makeSucceededFuture(
-                ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
-            )
-        }
         var decodeCalled = false
-        Current.decodeManifest = { url in
-            guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
-                throw Error.unexpectedCall
-            }
+        Current.decodeManifest = { _, repo in
+            guard repo.path == "org/3" else { throw Error.unexpectedCall }
             decodeCalled = true
             return .init(name: "3", products: [], dependencies: [])
         }
@@ -122,19 +111,8 @@ final class CheckDependenciesTests: XCTestCase {
                 throw Error.unexpectedCall
             }
         }
-        Current.fetch = { client, url in
-            // getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
-            guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
-                return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
-            }
-            return client.eventLoopGroup.next().makeSucceededFuture(
-                ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
-            )
-        }
-        Current.decodeManifest = { url in
-            guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
-                throw Error.unexpectedCall
-            }
+        Current.decodeManifest = { _, repo in
+            guard repo.path == "org/3" else { throw Error.unexpectedCall }
             return .init(name: "3", products: [], dependencies: [])
         }
         check.packageUrls = [.p1, .p2, .p4]
@@ -170,19 +148,7 @@ final class CheckDependenciesTests: XCTestCase {
                 throw Error.unexpectedCall
             }
         }
-        Current.fetch = { client, url in
-            // getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
-            guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
-                return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
-            }
-            return client.eventLoopGroup.next().makeSucceededFuture(
-                ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
-            )
-        }
-        Current.decodeManifest = { url in
-            guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
-                throw Error.unexpectedCall
-            }
+        Current.decodeManifest = { _, repo in
             // simulate a bad manifest
             throw AppError.dumpPackageError("simulated decoding error")
         }
@@ -258,26 +224,3 @@ private extension SwiftPackageIndexAPI.PackageRecord {
         self.init(id: .init(), url: url, resolvedDependencies: dependencies)
     }
 }
-
-private extension Data {
-    static func listRepositoryFilePaths(for path: String) -> Self {
-        .init("""
-            {
-              "url" : "https://api.github.com/repos/\(path)/git/trees/ea8eea9d89842a29af1b8e6c7677f1c86e72fa42",
-              "tree" : [
-                {
-                  "size" : 1122,
-                  "type" : "blob",
-                  "path" : "Package.swift",
-                  "url" : "https://api.github.com/repos/\(path)/git/blobs/bf4aa0c6a8bd9f749c2f96905c40bf2f70ef97d2",
-                  "mode" : "100644",
-                  "sha" : "bf4aa0c6a8bd9f749c2f96905c40bf2f70ef97d2"
-              }
-              ],
-              "sha" : "ea8eea9d89842a29af1b8e6c7677f1c86e72fa42",
-              "truncated" : false
-            }
-            """.utf8
-        )
-    }
-}
diff --git a/Tests/ValidatorTests/Fixtures/SemanticVersion-Package.swift b/Tests/ValidatorTests/Fixtures/SemanticVersion-Package.swift
@@ -0,0 +1,19 @@
+// swift-tools-version:5.6
+
+import PackageDescription
+
+let package = Package(
+    name: "SemanticVersion",
+    products: [
+        .library(
+            name: "SemanticVersion",
+            targets: ["SemanticVersion"]),
+    ],
+    dependencies: [],
+    targets: [
+        .target(name: "SemanticVersion",
+                dependencies: [],
+                resources: [.process("Documentation.docc")]),
+        .testTarget(name: "SemanticVersionTests", dependencies: ["SemanticVersion"]),
+    ]
+)