Merge pull request #57 from SwiftPackageIndex/async-await-check-redir…

…ects

Move CheckRedirects over to async/await

Sources/ValidatorCore/Commands/CheckDependencies.swift

@@ -55,7 +55,8 @@ public struct CheckDependencies: AsyncParsableCommand {
         let missing = allDependencies.subtracting(allPackages)
         print("Not indexed:", missing.count)
 
-        let client = HTTPClient(eventLoopGroupProvider: .singleton)
+        let client = HTTPClient(eventLoopGroupProvider: .singleton,
+                                configuration: .init(redirectConfiguration: .disallow))
         defer { try? client.syncShutdown() }
 
         var newPackages = UniqueCanonicalPackageURLs()
@@ -69,7 +70,7 @@ public struct CheckDependencies: AsyncParsableCommand {
 
             // resolve redirects
             print("Processing:", dep.packageURL, "...")
-            guard let resolved = try? await Current.resolvePackageRedirects(dep.packageURL).url else {
+            guard let resolved = try? await Current.resolvePackageRedirects(client, dep.packageURL).url else {
                 // TODO: consider adding retry for some errors
                 print("  ... ⛔ redirect resolution returned nil")
                 continue

Sources/ValidatorCore/Commands/CheckRedirects.swift

@@ -19,7 +19,7 @@ import AsyncHTTPClient
 
 
 extension Validator {
-    struct CheckRedirects: ParsableCommand {
+    struct CheckRedirects: AsyncParsableCommand {
         @Option(name: .shortAndLong, help: "read input from file")
         var input: String?
 
@@ -106,7 +106,7 @@ extension Validator {
             }
         }
 
-        mutating func run() throws {
+        func run() async throws {
             let verbose = verbose
             let inputURLs = try inputSource.packageURLs()
             let prefix = limit ?? inputURLs.count
@@ -122,32 +122,34 @@ extension Validator {
             }
 
             var normalized = Set(inputURLs.map { $0.normalized() })
-            let updated = try inputURLs[offset...]
+            var updated = [PackageURL]()
+
+            for (index, packageURL) in inputURLs[offset...]
                 .prefix(prefix)
                 .chunk(index: chunk, of: numberOfChunks)
-                .enumerated()
-                .compactMap { (index, packageURL) -> PackageURL? in
-                    let index = index + offset
-                    let redirect = try resolvePackageRedirects(client: httpClient, for: packageURL)
-                        .wait()
-
-                    if index % 100 == 0, let token = Current.githubToken() {
-                        let rateLimit = try Github.getRateLimit(client: httpClient,
-                                                                token: token).wait()
-                        if rateLimit.remaining < 200 {
-                            print("Rate limit remaining: \(rateLimit.remaining)")
-                            print("Sleeping until reset at \(rateLimit.resetDate) ...")
-                            sleep(UInt32(rateLimit.secondsUntilReset + 0.5))
-                        }
+                .enumerated() {
+                let index = index + offset
+                let redirect = try await resolvePackageRedirects(client: httpClient, for: packageURL)
+
+                if index % 100 == 0, let token = Current.githubToken() {
+                    let rateLimit = try await Github.getRateLimit(client: httpClient, token: token).get()
+                    if rateLimit.remaining < 200 {
+                        print("Rate limit remaining: \(rateLimit.remaining)")
+                        print("Sleeping until reset at \(rateLimit.resetDate) ...")
+                        sleep(UInt32(rateLimit.secondsUntilReset + 0.5))
                     }
+                }
 
-                    return try Self.process(redirect: redirect,
-                                            verbose: verbose,
-                                            index: index,
-                                            packageURL: packageURL,
-                                            normalized: &normalized)
+                if let res = try Self.process(redirect: redirect,
+                                              verbose: verbose,
+                                              index: index,
+                                              packageURL: packageURL,
+                                              normalized: &normalized) {
+                    updated.append(res)
                 }
-                .sorted(by: { $0.lowercased() < $1.lowercased() })
+            }
+
+            updated.sort(by: { $0.lowercased() < $1.lowercased() })
 
             if let path = output {
                 try Current.fileManager.saveList(updated, path: path)

Sources/ValidatorCore/Environment.swift

@@ -25,7 +25,7 @@ struct Environment {
     var fetchDependencies: (_ api: SwiftPackageIndexAPI) async throws -> [SwiftPackageIndexAPI.PackageRecord]
     var fetchRepository: (_ client: HTTPClient, _ url: PackageURL) async throws -> Github.Repository
     var githubToken: () -> String?
-    var resolvePackageRedirects: (PackageURL) async throws -> Redirect
+    var resolvePackageRedirects: (_ client: HTTPClient, _ url: PackageURL) async throws -> Redirect
     var shell: Shell
 }
 
@@ -38,7 +38,7 @@ extension Environment {
         fetchDependencies: { try await $0.fetchDependencies() },
         fetchRepository: Github.fetchRepository(client:url:),
         githubToken: { ProcessInfo.processInfo.environment["GITHUB_TOKEN"] },
-        resolvePackageRedirects: resolveRedirects(for:),
+        resolvePackageRedirects: resolvePackageRedirects(client:for:),
         shell: .live
     )
 
@@ -49,7 +49,7 @@ extension Environment {
         fetchDependencies: { _ in [] },
         fetchRepository: { _, _ in .init(defaultBranch: "main", owner: "foo", name: "bar") },
         githubToken: { nil },
-        resolvePackageRedirects: { .initial($0) },
+        resolvePackageRedirects: { _, url in .initial(url) },
         shell: .mock
     )
 }

Sources/ValidatorCore/RedirectFollower.swift

@@ -46,21 +46,6 @@ enum Redirect: Equatable {
 }
 
 
-
-func resolveRedirects(for url: PackageURL) async throws -> Redirect {
-    let client = HTTPClient(eventLoopGroupProvider: .singleton,
-                            configuration: .init(redirectConfiguration: .disallow))
-    defer { try? client.syncShutdown() }
-    let res = try await resolveRedirects(client: client, for: url.deletingGitExtension())
-    switch res {
-        case .initial, .notFound, .error, .unauthorized, .rateLimited:
-            return res
-        case .redirected(to: let newURL):
-            return .redirected(to: newURL.appendingGitExtension())
-    }
-}
-
-
 private func resolveRedirects(client: HTTPClient, for url: PackageURL) async throws -> Redirect {
     var lastResult = Redirect.initial(url)
     var hopCount = 0
@@ -118,22 +103,12 @@ private func resolveRedirects(client: HTTPClient, for url: PackageURL) async thr
 }
 
 
-/// Resolve redirects for package urls. In particular, this strips the `.git` extension from the test url, because it would always lead to a redirect. It also normalizes the output to always have a `.git` extension.
-/// - Returns: `Redirect`
-@available(*, deprecated)
-func resolvePackageRedirects(client: HTTPClient, for url: PackageURL) -> EventLoopFuture<Redirect> {
-    let promise = client.eventLoopGroup.next().makePromise(of: Redirect.self)
-    promise.completeWithTask {
-        try await resolveRedirects(client: client, for: url.deletingGitExtension())
+func resolvePackageRedirects(client: HTTPClient, for url: PackageURL) async throws -> Redirect {
+    let res = try await resolveRedirects(client: client, for: url.deletingGitExtension())
+    switch res {
+        case .initial, .notFound, .error, .unauthorized, .rateLimited:
+            return res
+        case .redirected(to: let newURL):
+            return .redirected(to: newURL.appendingGitExtension())
     }
-    return promise.futureResult
-        .map {
-            switch $0 {
-                case .initial, .notFound, .error, .unauthorized, .rateLimited:
-                    return $0
-                case .redirected(to: let url):
-                    return .redirected(to: url.appendingGitExtension())
-            }
-        }
 }
-