Skip to content

Commit

Permalink
Fetch all manifests before running package dump
Browse files Browse the repository at this point in the history
  • Loading branch information
@finestructure
finestructure committed Feb 14, 2024
1 parent 969af98 commit cb6c84b
Show file tree
Hide file tree
Showing 6 changed files with 34 additions and 64 deletions.
3 changes: 1 addition & 2 deletions Sources/ValidatorCore/Commands/CheckDependencies.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,7 @@ public struct CheckDependencies: AsyncParsableCommand {

do { // run package dump to validate
let repo = try await Current.fetchRepository(client, resolved)
let manifest = try await Package.getManifestURL(client: client, repository: repo)
_ = try Current.decodeManifest(manifest)
_ = try await Current.decodeManifest(client, repo)
} catch {
print(" ... ⛔ \(error)")
continue
Expand Down
6 changes: 3 additions & 3 deletions Sources/ValidatorCore/Environment.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import NIO


struct Environment {
var decodeManifest: (_ url: Package.ManifestURL) throws -> Package
var decodeManifest: (_ client: HTTPClient, _ repository: Github.Repository) async throws -> Package
var fileManager: FileManager
var fetch: (_ client: HTTPClient, _ url: URL) -> EventLoopFuture<ByteBuffer>
var fetchDependencies: (_ api: SwiftPackageIndexAPI) async throws -> [SwiftPackageIndexAPI.PackageRecord]
Expand All @@ -32,7 +32,7 @@ struct Environment {

extension Environment {
static let live: Self = .init(
decodeManifest: { url in try Package.decode(from: url) },
decodeManifest: { client, repo in try await Package.decode(client: client, repository: repo) },
fileManager: .live,
fetch: Github.fetch(client:url:),
fetchDependencies: { try await $0.fetchDependencies() },
Expand All @@ -43,7 +43,7 @@ extension Environment {
)

static let mock: Self = .init(
decodeManifest: { _ in fatalError("not implemented") },
decodeManifest: { _, _ in fatalError("not implemented") },
fileManager: .mock,
fetch: { client, _ in client.eventLoopGroup.next().makeFailedFuture(AppError.runtimeError("unimplemented")) },
fetchDependencies: { _ in [] },
Expand Down
30 changes: 16 additions & 14 deletions Sources/ValidatorCore/Package.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,23 +86,26 @@ extension Package {
static func loadPackageDumpCache() { packageDumpCache = .load(from: cacheFilename) }
static func savePackageDumpCache() throws { try packageDumpCache.save(to: cacheFilename) }

static func decode(from manifestURL: ManifestURL) throws -> Package {
if let cached = packageDumpCache[Cache.Key(string: manifestURL.rawValue.absoluteString)] {
static func decode(client: HTTPClient, repository: Github.Repository) async throws -> Self {
let cacheKey = repository.path
if let cached = packageDumpCache[Cache.Key(string: cacheKey)] {
return cached
}
return try withTempDir { tempDir in
let fileURL = URL(fileURLWithPath: tempDir).appendingPathComponent("Package.swift")
let data = try Data(contentsOf: manifestURL.rawValue)
guard Current.fileManager.createFile(fileURL.path, data, nil) else {
throw AppError.dumpPackageError("failed to save manifest \(manifestURL.rawValue.absoluteString) to temp directory \(fileURL.absoluteString)")
return try await withTempDir { tempDir in
for manifestURL in try await Package.getManifestURLs(client: client, repository: repository) {
let fileURL = URL(fileURLWithPath: tempDir).appendingPathComponent(manifestURL.lastPathComponent)
let data = try Data(contentsOf: manifestURL.rawValue)
guard Current.fileManager.createFile(fileURL.path, data, nil) else {
throw AppError.dumpPackageError("failed to save manifest \(manifestURL.rawValue.absoluteString) to temp directory \(fileURL.absoluteString)")
}
}
do {
guard let pkgJSON = try Current.shell.run(command: .packageDump, at: tempDir)
.data(using: .utf8) else {
throw AppError.dumpPackageError("package dump did not return data")
}
let pkg = try JSONDecoder().decode(Package.self, from: pkgJSON)
packageDumpCache[Cache.Key(string: manifestURL.rawValue.absoluteString)] = pkg
packageDumpCache[Cache.Key(string: cacheKey)] = pkg
return pkg
} catch let error as ShellOutError {
throw AppError.dumpPackageError("package dump failed: \(error.message)")
Expand All @@ -118,16 +121,15 @@ extension Package {
enum Manifest {}
typealias ManifestURL = Tagged<Manifest, URL>

static func getManifestURL(client: HTTPClient, repository: Github.Repository) async throws -> ManifestURL {
static func getManifestURLs(client: HTTPClient, repository: Github.Repository) async throws -> [ManifestURL] {
let manifestFiles = try await Github.listRepositoryFilePaths(client: client, repository: repository)
.filter { $0.hasPrefix("Package") }
.filter { $0.hasSuffix(".swift") }
.sorted()
guard let manifestFile = manifestFiles.last else {
throw AppError.manifestNotFound(owner: repository.owner.login, name: repository.name)
}
let url = URL(string: "https://raw.githubusercontent.com/\(repository.path)/\(repository.defaultBranch)/\(manifestFile)")!
return .init(rawValue: url)
guard !manifestFiles.isEmpty else { throw AppError.manifestNotFound(owner: repository.owner.login, name: repository.name) }
return manifestFiles
.map { URL(string: "https://raw.githubusercontent.com/\(repository.path)/\(repository.defaultBranch)/\($0)")! }
.map(ManifestURL.init(rawValue:))
}

}
4 changes: 2 additions & 2 deletions Sources/ValidatorCore/TempDir.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ class TempDir {
}


func withTempDir<T>(body: (String) throws -> T) throws -> T {
func withTempDir<T>(body: (String) async throws -> T) async throws -> T {
let tmp = try TempDir()
return try body(tmp.path)
return try await body(tmp.path)
}
44 changes: 5 additions & 39 deletions Tests/ValidatorTests/CheckDependenciesTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,20 +64,9 @@ final class CheckDependenciesTests: XCTestCase {
throw Error.unexpectedCall
}
}
Current.fetch = { client, url in
// getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
}
return client.eventLoopGroup.next().makeSucceededFuture(
ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
)
}
var decodeCalled = false
Current.decodeManifest = { url in
guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
throw Error.unexpectedCall
}
Current.decodeManifest = { _, repo in
guard repo.path == "org/3" else { throw Error.unexpectedCall }
decodeCalled = true
return .init(name: "3", products: [], dependencies: [])
}
Expand Down Expand Up @@ -122,19 +111,8 @@ final class CheckDependenciesTests: XCTestCase {
throw Error.unexpectedCall
}
}
Current.fetch = { client, url in
// getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
}
return client.eventLoopGroup.next().makeSucceededFuture(
ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
)
}
Current.decodeManifest = { url in
guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
throw Error.unexpectedCall
}
Current.decodeManifest = { _, repo in
guard repo.path == "org/3" else { throw Error.unexpectedCall }
return .init(name: "3", products: [], dependencies: [])
}
check.packageUrls = [.p1, .p2, .p4]
Expand Down Expand Up @@ -170,19 +148,7 @@ final class CheckDependenciesTests: XCTestCase {
throw Error.unexpectedCall
}
}
Current.fetch = { client, url in
// getManifestURL -> Github.listRepositoryFilePaths -> Github.fetch
guard url.absoluteString == "https://api.github.com/repos/org/3/git/trees/main" else {
return client.eventLoopGroup.next().makeFailedFuture(Error.unexpectedCall)
}
return client.eventLoopGroup.next().makeSucceededFuture(
ByteBuffer(data: .listRepositoryFilePaths(for: "org/3"))
)
}
Current.decodeManifest = { url in
guard url.absoluteString == "https://raw.githubusercontent.com/org/3/main/Package.swift" else {
throw Error.unexpectedCall
}
Current.decodeManifest = { _, repo in
// simulate a bad manifest
throw AppError.dumpPackageError("simulated decoding error")
}
Expand Down
11 changes: 7 additions & 4 deletions Tests/ValidatorTests/ValidatorTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,11 @@ final class ValidatorTests: XCTestCase {
let repo = Github.Repository(defaultBranch: "main", owner: "SwiftPackageIndex", name: "SemanticVersion")

// MUT
let url = try await Package.getManifestURL(client: client, repository: repo)
let url = try await Package.getManifestURLs(client: client, repository: repo)

// validate
XCTAssertEqual(url,
.init("https://raw.githubusercontent.com/SwiftPackageIndex/SemanticVersion/main/Package.swift"))
[.init("https://raw.githubusercontent.com/SwiftPackageIndex/SemanticVersion/main/Package.swift")])
}

func test_getManifestURL_multiple() async throws {
Expand All @@ -105,11 +105,14 @@ final class ValidatorTests: XCTestCase {
let repo = Github.Repository(defaultBranch: "master", owner: "IBM-Swift", name: "SwiftyJSON")

// MUT
let url = try await Package.getManifestURL(client: client, repository: repo)
let url = try await Package.getManifestURLs(client: client, repository: repo)

// validate
XCTAssertEqual(url,
.init("https://raw.githubusercontent.com/IBM-Swift/SwiftyJSON/master/[email protected]"))
[.init("https://raw.githubusercontent.com/IBM-Swift/SwiftyJSON/master/Package.swift"),
.init("https://raw.githubusercontent.com/IBM-Swift/SwiftyJSON/master/[email protected]"),
.init("https://raw.githubusercontent.com/IBM-Swift/SwiftyJSON/master/[email protected]"),
.init("https://raw.githubusercontent.com/IBM-Swift/SwiftyJSON/master/[email protected]")])
}

func test_ArraySlice_chunk() throws {
Expand Down

0 comments on commit cb6c84b

Please sign in to comment.