Merge pull request #52 from SwiftPackageIndex/run-package-dump-on-new…

…-packages

Run package dump on new packages

Sources/ValidatorCore/AppError.swift

@@ -22,6 +22,7 @@ enum AppError: Error {
     case invalidPackage(url: PackageURL)
     case invalidDenyListUrl(string: String)
     case ioError(String)
+    case manifestNotFound(owner: String, name: String)
     case noData(URL)
     case rateLimited(until: Date)
     case repositoryNotFound(owner: String, name: String)

Sources/ValidatorCore/Commands/CheckDependencies2.swift

@@ -62,20 +62,33 @@ public struct CheckDependencies2: AsyncParsableCommand {
             if idx % 10 == 0 {
                 print("Progress:", idx, "/", missing.count)
             }
+
             // resolve redirects
             print("Processing:", dep.packageURL, "...")
             guard let resolved = await Current.resolvePackageRedirectsAsync(dep.packageURL).url else {
                 // TODO: consider adding retry for some errors
                 print("  ... ⛔ redirect resolution returned nil")
                 continue
             }
+
             if resolved.canonicalPackageURL.canonicalPath != dep.canonicalPath {
                 print("  ... redirected to:", resolved)
             }
+
             if allPackages.contains(resolved.canonicalPackageURL) {
                 print("  ... ⛔ already indexed")
                 continue
             }
+
+            do {  // run package dump to validate
+                let repo = try await Current.fetchRepositoryAsync(client, resolved)
+                let manifest = try await Package.getManifestURL(client: client, repository: repo)
+                _ = try Current.decodeManifest(manifest)
+            } catch {
+                print("  ... ⛔ \(error)")
+                continue
+            }
+
             if newPackages.insert(resolved.appendingGitExtension().canonicalPackageURL).inserted {
                 print("✅ ADD (\(newPackages.count)):", resolved.appendingGitExtension())
             }

Sources/ValidatorCore/Environment.swift

@@ -54,9 +54,10 @@ extension Environment {
         fetch: { client, _ in client.eventLoopGroup.next().makeFailedFuture(AppError.runtimeError("unimplemented")) },
         fetchDependencies: { _ in [] },
         fetchRepository: { client, _, _ in
-            client.eventLoopGroup.next().makeSucceededFuture(
-                Github.Repository(default_branch: "main", fork: false)) },
-        fetchRepositoryAsync: { _, _ in .init(default_branch: "main", fork: false) },
+            client.eventLoopGroup.next()
+                .makeSucceededFuture( Github.Repository(defaultBranch: "main", owner: "foo", name: "bar") )
+        },
+        fetchRepositoryAsync: { _, _ in .init(defaultBranch: "main", owner: "foo", name: "bar") },
         githubToken: { nil },
         resolvePackageRedirects: { eventLoop, url in
             eventLoop.makeSucceededFuture(.initial(url))

Sources/ValidatorCore/Github.swift

@@ -27,8 +27,30 @@ import CDispatch // for NSEC_PER_SEC https://github.com/apple/swift-corelibs-lib
 enum Github {
 
     struct Repository: Codable, Equatable {
-        let default_branch: String
-        let fork: Bool
+        var defaultBranch: String
+        var fork: Bool
+        var name: String
+        var owner: Owner
+
+        struct Owner: Codable, Equatable {
+            var login: String
+        }
+
+        enum CodingKeys: String, CodingKey {
+            case defaultBranch = "default_branch"
+            case fork
+            case name
+            case owner
+        }
+
+        init(defaultBranch: String, fork: Bool = false, owner: String, name: String) {
+            self.defaultBranch = defaultBranch
+            self.fork = fork
+            self.name = name
+            self.owner = .init(login: owner)
+        }
+
+        var path: String { "\(owner.login)/\(name)" }
     }
 
     static func packageList() throws -> [PackageURL] {
@@ -182,6 +204,28 @@ extension Github {
     }
 
 
+    static func listRepositoryFilePaths(client: HTTPClient, repository: Repository) async throws -> [String] {
+        let apiURL = URL( string: "https://api.github.com/repos/\(repository.path)/git/trees/\(repository.defaultBranch)" )!
+        struct Response: Decodable {
+            var tree: [File]
+
+            struct File: Decodable {
+                var type: FileType
+                var path: String
+
+                enum FileType: String, Decodable {
+                    case blob
+                    case tree
+                }
+            }
+        }
+        return try await fetch(Response.self, client: client, url: apiURL)
+            .tree
+            .filter{ $0.type == .blob }
+            .map(\.path)
+    }
+
+
     static func fetch<T: Decodable>(_ type: T.Type, client: HTTPClient, url: URL) async throws -> T {
         let body = try await Current.fetch(client, url).get()
         do {

Sources/ValidatorCore/Package.swift

@@ -118,13 +118,26 @@ extension Package {
     enum Manifest {}
     typealias ManifestURL = Tagged<Manifest, URL>
 
+    @available(*, deprecated)
     static func getManifestURL(client: HTTPClient, packageURL: PackageURL) -> EventLoopFuture<ManifestURL> {
         Current.fetchRepository(client, packageURL.owner, packageURL.repository)
-            .map(\.default_branch)
+            .map(\.defaultBranch)
             .map { defaultBranch in
                 URL(string: "https://raw.githubusercontent.com/\(packageURL.owner)/\(packageURL.repository)/\(defaultBranch)/Package.swift")!
             }
             .map(ManifestURL.init(rawValue:))
     }
 
+    static func getManifestURL(client: HTTPClient, repository: Github.Repository) async throws -> ManifestURL {
+        let manifestFiles = try await Github.listRepositoryFilePaths(client: client, repository: repository)
+          .filter { $0.hasPrefix("Package") }
+          .filter { $0.hasSuffix(".swift") }
+          .sorted()
+        guard let manifestFile = manifestFiles.last else {
+            throw AppError.manifestNotFound(owner: repository.owner.login, name: repository.name)
+        }
+        let url = URL(string: "https://raw.githubusercontent.com/\(repository.path)/\(repository.defaultBranch)/\(manifestFile)")!
+        return .init(rawValue: url)
+    }
+
 }

Tests/ValidatorTests/CheckDependencies2Tests.swift

@@ -17,6 +17,7 @@ import XCTest
 @testable import ValidatorCore
 
 import CanonicalPackageURL
+import NIO
 
 
 final class CheckDependencies2Tests: XCTestCase {
@@ -42,7 +43,8 @@ final class CheckDependencies2Tests: XCTestCase {
             .init(.p2, dependencies: [.p3]),
         ]}
         var saved: [PackageURL]? = nil
-        Current.fileManager.createFile = { _, data, _ in
+        Current.fileManager.createFile = { path, data, _ in
+            guard path.hasSuffix("package.json") else { return false }
             guard let data = data else {
                 XCTFail("data must not be nil")
                 return false
@@ -54,13 +56,39 @@ final class CheckDependencies2Tests: XCTestCase {
             saved = list
             return true
         }
+        Current.fetchRepositoryAsync = { _, url in
+            if url == PackageURL.p3 {
+                return .init(defaultBranch: "main", owner: "org", name: "3")
+            } else {
+                throw Error.unexpectedCall
+            }
+        }
+        Current.fetch = { client, url in
+            // getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
+            guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
+                return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
+            }
+            return client.eventLoopGroup.next().makeSucceededFuture(
+                ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
+            )
+        }
+        var decodeCalled = false
+        Current.decodeManifest = { url in
+            guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
+                throw Error.unexpectedCall
+            }
+            decodeCalled = true
+            return .init(name: "3", products: [], dependencies: [])
+        }
         check.packageUrls = [.p1, .p2]
+        check.output = "package.json"
 
         // MUT
         try await check.run()
 
         // validate
         XCTAssertEqual(saved, [.p1, .p2, .p3])
+        XCTAssertTrue(decodeCalled)
     }
 
     func test_run_list_newer() async throws {
@@ -86,7 +114,30 @@ final class CheckDependencies2Tests: XCTestCase {
             saved = list
             return true
         }
+        Current.fetchRepositoryAsync = { _, url in
+            if url == PackageURL.p3 {
+                return .init(defaultBranch: "main", owner: "org", name: "3")
+            } else {
+                throw Error.unexpectedCall
+            }
+        }
+        Current.fetch = { client, url in
+            // getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
+            guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
+                return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
+            }
+            return client.eventLoopGroup.next().makeSucceededFuture(
+                ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
+            )
+        }
+        Current.decodeManifest = { url in
+            guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
+                throw Error.unexpectedCall
+            }
+            return .init(name: "3", products: [], dependencies: [])
+        }
         check.packageUrls = [.p1, .p2, .p4]
+        check.output = "package.json"
 
         // MUT
         try await check.run()
@@ -95,9 +146,61 @@ final class CheckDependencies2Tests: XCTestCase {
         XCTAssertEqual(saved, [.p1, .p2, .p3, .p4])
     }
 
+    func test_run_manifest_validation() async throws {
+        // Ensure validation via package dump is performed on new packages.
+        // setup
+        Current = .mock
+        Current.fetchDependencies = { _ in [
+            .init(.p1, dependencies: []),
+            .init(.p2, dependencies: [.p3]),
+        ]}
+        var saved: [PackageURL]? = nil
+        Current.fileManager.createFile = { path, data, _ in
+            guard path.hasSuffix("package.json"),
+                  let data = data,
+                  let list = try? JSONDecoder().decode([PackageURL].self, from: data) else { return false }
+            saved = list
+            return true
+        }
+        Current.fetchRepositoryAsync = { _, url in
+            if url == PackageURL.p3 {
+                return .init(defaultBranch: "main", owner: "org", name: "3")
+            } else {
+                throw Error.unexpectedCall
+            }
+        }
+        Current.fetch = { client, url in
+            // getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
+            guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
+                return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
+            }
+            return client.eventLoopGroup.next().makeSucceededFuture(
+                ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
+            )
+        }
+        Current.decodeManifest = { url in
+            guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
+                throw Error.unexpectedCall
+            }
+            // simulate a bad manifest
+            throw AppError.dumpPackageError("simulated decoding error")
+        }
+
+        check.packageUrls = [.p1, .p2]
+        check.output = "package.json"
+
+        // MUT
+        try await check.run()
+
+        // validate
+        XCTAssertEqual(saved, [.p1, .p2])
+    }
+
 }
 
 
+private enum Error: Swift.Error { case unexpectedCall }
+
 private extension PackageURL {
     static let p1 = PackageURL(argument: "https://github.com/org/1.git")!
     static let p2 = PackageURL(argument: "https://github.com/org/2.git")!
@@ -116,3 +219,26 @@ private extension SwiftPackageIndexAPI.PackageRecord {
         self.init(id: .init(), url: url, resolvedDependencies: dependencies)
     }
 }
+
+private extension Data {
+    static func listRepositoryFilePaths(for path: String) -> Self {
+        .init("""
+            {
+              "url" : "https://api.github.com/repos/\(path)/git/trees/ea8eea9d89842a29af1b8e6c7677f1c86e72fa42",
+              "tree" : [
+                {
+                  "size" : 1122,
+                  "type" : "blob",
+                  "path" : "Package.swift",
+                  "url" : "https://api.github.com/repos/\(path)/git/blobs/bf4aa0c6a8bd9f749c2f96905c40bf2f70ef97d2",
+                  "mode" : "100644",
+                  "sha" : "bf4aa0c6a8bd9f749c2f96905c40bf2f70ef97d2"
+              }
+              ],
+              "sha" : "ea8eea9d89842a29af1b8e6c7677f1c86e72fa42",
+              "truncated" : false
+            }
+            """.utf8
+        )
+    }
+}