Merge pull request #65 from SwiftPackageIndex/non-throwing-task-group

Continue on error in the main task group
SwiftPackageIndex · Apr 5, 2024 · d56a0fe · d56a0fe
2 parents 7d48be1 + f0c0a4c
commit d56a0fe
Showing 1 changed file with 24 additions and 19 deletions.
diff --git a/Sources/ValidatorCore/Commands/CheckRedirects.swift b/Sources/ValidatorCore/Commands/CheckRedirects.swift
@@ -139,36 +139,41 @@ extension Validator {
 
             let semaphore = Semaphore(maximum: concurrency ?? 1)
 
-            let updated = try await withThrowingTaskGroup(of: PackageURL?.self) { group in
+            let updated = await withTaskGroup(of: PackageURL?.self) { group in
                 for (index, packageURL) in inputURLs[offset...]
                     .prefix(prefix)
                     .chunk(index: chunk, of: numberOfChunks)
                     .enumerated() {
                     await semaphore.increment()
                     try? await semaphore.waitForAvailability()
                     group.addTask {
-                        let index = index + offset
-                        let redirect = try await resolvePackageRedirects(client: httpClient, for: packageURL)
-
-                        if index % 100 == 0, let token = Current.githubToken() {
-                            let rateLimit = try await Github.getRateLimit(client: httpClient, token: token).get()
-                            if rateLimit.remaining < 200 {
-                                print("Rate limit remaining: \(rateLimit.remaining)")
-                                print("Sleeping until reset at \(rateLimit.resetDate) ...")
-                                sleep(UInt32(rateLimit.secondsUntilReset + 0.5))
+                        do {
+                            let index = index + offset
+                            let redirect = try await resolvePackageRedirects(client: httpClient, for: packageURL)
+
+                            if index % 100 == 0, let token = Current.githubToken() {
+                                let rateLimit = try await Github.getRateLimit(client: httpClient, token: token).get()
+                                if rateLimit.remaining < 200 {
+                                    print("Rate limit remaining: \(rateLimit.remaining)")
+                                    print("Sleeping until reset at \(rateLimit.resetDate) ...")
+                                    sleep(UInt32(rateLimit.secondsUntilReset + 0.5))
+                                }
                             }
+
+                            let res =  try await Self.process(redirect: redirect,
+                                                              verbose: verbose,
+                                                              index: index,
+                                                              packageURL: packageURL)
+
+                            await semaphore.decrement()
+                            return res
+                        } catch {
+                            print("Error in main task group: \(error)")
+                            return nil
                         }
-
-                        let res =  try await Self.process(redirect: redirect,
-                                                          verbose: verbose,
-                                                          index: index,
-                                                          packageURL: packageURL)
-
-                        await semaphore.decrement()
-                        return res
                     }
                 }
-                return try await group
+                return await group
                     .compactMap { $0 }
                     .reduce(into: [], { res, next in res.append(next) })
                     .sorted(by: { $0.lowercased() < $1.lowercased() })