From a91d536e5730c6161aca8a25e03458f505992069 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Tue, 24 Sep 2024 13:54:26 -0700 Subject: [PATCH 01/25] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98ec1e634..be4366616 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# GitHub Enterprise Importer CLI + [![Actions Status: CI](https://github.com/github/gh-gei/workflows/CI/badge.svg)](https://github.com/github/gh-gei/actions?query=workflow%3ACI) From beaf0e9204c26bb1f199558626735e1d17a1ff8b Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Wed, 25 Sep 2024 13:33:04 -0700 Subject: [PATCH 02/25] Retrying GetRepoCommitShas --- src/OctoshiftCLI.IntegrationTests/TestHelper.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs index 4052495b6..040b2ee72 100644 --- a/src/OctoshiftCLI.IntegrationTests/TestHelper.cs +++ b/src/OctoshiftCLI.IntegrationTests/TestHelper.cs @@ -487,8 +487,10 @@ private async Task CreateRepo(string org, string repo, bool isPrivate, bool isIn private async Task> GetRepoCommitShas(string org, string repo) { + var retryPolicy = new RetryPolicy(null); + var url = $"{GithubApiBaseUrl}/repos/{org}/{repo}/commits"; - var commits = await _githubClient.GetAllAsync(url).ToListAsync(); + var commits = await retryPolicy.Retry(async () => await _githubClient.GetAllAsync(url).ToListAsync()); return commits.Select(x => (string)x["sha"]).ToList(); } From bdab8fae7b938de06b5d554d1f79ae82df24d615 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 26 Sep 2024 08:57:22 -0700 Subject: [PATCH 03/25] Create ci-errors.yml --- .github/workflows/ci-errors.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/ci-errors.yml diff --git a/.github/workflows/ci-errors.yml b/.github/workflows/ci-errors.yml new file mode 100644 index 000000000..22606bb6d --- /dev/null +++ b/.github/workflows/ci-errors.yml @@ -0,0 +1,23 @@ +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + test-api-call: + name: Test API Call + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Run API Call + run: | + echo "Running test API call on ${{ matrix.os }}" + curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/e2e-testing-bbs-windows/E2E-WINDOWS-repo-2/commits + shell: bash From 2498132d47a48a217244c924c79378c8e19163d3 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 26 Sep 2024 09:00:18 -0700 Subject: [PATCH 04/25] Update ci-errors.yml --- .github/workflows/ci-errors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-errors.yml b/.github/workflows/ci-errors.yml index 22606bb6d..9c8e80f90 100644 --- a/.github/workflows/ci-errors.yml +++ b/.github/workflows/ci-errors.yml @@ -1,7 +1,7 @@ on: push: branches: - - main + - confirming-ci workflow_dispatch: jobs: From df12e6461acb912c81d5448e9ff369587daf384c Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 26 Sep 2024 11:59:54 -0700 Subject: [PATCH 05/25] testin v 2 orgs --- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index df125fb21..53c14ab22 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -71,7 +71,7 @@ public BbsToGithub(ITestOutputHelper output) public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload) { var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}"; - var githubTargetOrg = $"e2e-testing-bbs-{TestHelper.GetOsName()}"; + var githubTargetOrg = $"e2e-testing-bbs-{TestHelper.GetOsName()}-2"; var repo1 = $"{bbsProjectKey}-repo-1"; var repo2 = $"{bbsProjectKey}-repo-2"; From bb131255611426e20eeba582ccd78d95da305704 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 26 Sep 2024 13:50:52 -0700 Subject: [PATCH 06/25] Delete .github/workflows/ci-errors.yml --- .github/workflows/ci-errors.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .github/workflows/ci-errors.yml diff --git a/.github/workflows/ci-errors.yml b/.github/workflows/ci-errors.yml deleted file mode 100644 index 9c8e80f90..000000000 --- a/.github/workflows/ci-errors.yml +++ /dev/null @@ -1,23 +0,0 @@ -on: - push: - branches: - - confirming-ci - workflow_dispatch: - -jobs: - test-api-call: - name: Test API Call - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Run API Call - run: | - echo "Running test API call on ${{ matrix.os }}" - curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/e2e-testing-bbs-windows/E2E-WINDOWS-repo-2/commits - shell: bash From 2e7e1416077b6ed15bae7fa1690a0d0d7638fa09 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 26 Sep 2024 13:51:10 -0700 Subject: [PATCH 07/25] Updates from editor --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be4366616..98ec1e634 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - +# GitHub Enterprise Importer CLI [![Actions Status: CI](https://github.com/github/gh-gei/workflows/CI/badge.svg)](https://github.com/github/gh-gei/actions?query=workflow%3ACI) From 78fcb2e86912c9f888f7028bd36efc57cdc46a20 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Fri, 27 Sep 2024 13:19:41 -0700 Subject: [PATCH 08/25] Was it a naming issue? --- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index 53c14ab22..8b51c8960 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -87,10 +87,10 @@ await retryPolicy.Retry(async () => await _targetHelper.ResetGithubTestEnvironment(githubTargetOrg); await sourceHelper.CreateBbsProject(bbsProjectKey); - await sourceHelper.CreateBbsRepo(bbsProjectKey, "repo-1"); - await sourceHelper.InitializeBbsRepo(bbsProjectKey, "repo-1"); - await sourceHelper.CreateBbsRepo(bbsProjectKey, "repo-2"); - await sourceHelper.InitializeBbsRepo(bbsProjectKey, "repo-2"); + await sourceHelper.CreateBbsRepo(bbsProjectKey, repo1); + await sourceHelper.InitializeBbsRepo(bbsProjectKey, repo1); + await sourceHelper.CreateBbsRepo(bbsProjectKey, repo2); + await sourceHelper.InitializeBbsRepo(bbsProjectKey, repo2); }); var archiveDownloadOptions = $" --ssh-user octoshift --ssh-private-key {SSH_KEY_FILE}"; From 3443df7e3bbf9631b467d1804e0e639171f29ab9 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Fri, 27 Sep 2024 14:05:44 -0700 Subject: [PATCH 09/25] test --- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index 8b51c8960..9713a71e4 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -74,6 +74,8 @@ public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool us var githubTargetOrg = $"e2e-testing-bbs-{TestHelper.GetOsName()}-2"; var repo1 = $"{bbsProjectKey}-repo-1"; var repo2 = $"{bbsProjectKey}-repo-2"; + var targetRepo1 = $"{bbsProjectKey}-e2e-${TestHelper.GetOsName().ToLower()}-repo-1"; + var targetRepo2 = $"{bbsProjectKey}-e2e-${TestHelper.GetOsName().ToLower()}-repo-2"; var sourceBbsApi = new BbsApi(_sourceBbsClient, bbsServer, _logger); var sourceHelper = new TestHelper(_output, sourceBbsApi, _sourceBbsClient, bbsServer); @@ -123,10 +125,10 @@ await _targetHelper.RunBbsCliMigration( _targetHelper.AssertNoErrorInLogs(_startTime); - await _targetHelper.AssertGithubRepoExists(githubTargetOrg, repo1); - await _targetHelper.AssertGithubRepoExists(githubTargetOrg, repo2); - await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, repo1); - await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, repo2); + await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo1); + await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo2); + await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo1); + await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo2); // TODO: Assert migration logs are downloaded } From fb329cc335b5d9ccc0afed9787c07c825580317e Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Fri, 27 Sep 2024 15:18:54 -0700 Subject: [PATCH 10/25] typo --- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index 9713a71e4..c4d1cf10b 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -74,8 +74,8 @@ public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool us var githubTargetOrg = $"e2e-testing-bbs-{TestHelper.GetOsName()}-2"; var repo1 = $"{bbsProjectKey}-repo-1"; var repo2 = $"{bbsProjectKey}-repo-2"; - var targetRepo1 = $"{bbsProjectKey}-e2e-${TestHelper.GetOsName().ToLower()}-repo-1"; - var targetRepo2 = $"{bbsProjectKey}-e2e-${TestHelper.GetOsName().ToLower()}-repo-2"; + var targetRepo1 = $"{bbsProjectKey}-e2e-{TestHelper.GetOsName().ToLower()}-repo-1"; + var targetRepo2 = $"{bbsProjectKey}-e2e-{TestHelper.GetOsName().ToLower()}-repo-2"; var sourceBbsApi = new BbsApi(_sourceBbsClient, bbsServer, _logger); var sourceHelper = new TestHelper(_output, sourceBbsApi, _sourceBbsClient, bbsServer); From e45313246cb4a5a3030ff5da909e435d6e7a5047 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Mon, 30 Sep 2024 13:19:08 -0700 Subject: [PATCH 11/25] sleep 60 --- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index c4d1cf10b..663b2da9d 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -4,6 +4,7 @@ using System.Net.Http; using System.Text.RegularExpressions; using System.Threading.Tasks; +using System.Threading; using Azure.Storage.Blobs; using OctoshiftCLI.Services; using Xunit; @@ -127,6 +128,7 @@ await _targetHelper.RunBbsCliMigration( await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo1); await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo2); + Thread.Sleep(60000); // 60000 milliseconds = 60 seconds await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo1); await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo2); From 41681610c72604a6005a6921ad19f517ea64f819 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Tue, 1 Oct 2024 08:41:42 -0700 Subject: [PATCH 12/25] Updates from editor --- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index 663b2da9d..1fc30ca6f 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -128,7 +128,6 @@ await _targetHelper.RunBbsCliMigration( await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo1); await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo2); - Thread.Sleep(60000); // 60000 milliseconds = 60 seconds await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo1); await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo2); From aef255b2fc2b451382924ff27701f2665e8f9bfe Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Tue, 1 Oct 2024 11:22:15 -0700 Subject: [PATCH 13/25] Fix formatting issue --- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index 1fc30ca6f..c4d1cf10b 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -4,7 +4,6 @@ using System.Net.Http; using System.Text.RegularExpressions; using System.Threading.Tasks; -using System.Threading; using Azure.Storage.Blobs; using OctoshiftCLI.Services; using Xunit; From f4a92600a5b18e5646e85925ff9ccb25c0748ce1 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Tue, 1 Oct 2024 14:48:43 -0700 Subject: [PATCH 14/25] Pointing to an EMU we control --- src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs index f254e5882..569a7bbb9 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs @@ -15,7 +15,7 @@ public AdoBasicToGithub(ITestOutputHelper output) : base(output) public async Task Basic() { var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}"; - var githubOrg = $"e2e-testing-ado-basic-{TestHelper.GetOsName()}"; + var githubOrg = $"octoshift-e2e-ado-basic-{TestHelper.GetOsName()}"; var teamProject1 = "gei-e2e-1"; var teamProject2 = "gei-e2e-2"; var adoRepo1 = teamProject1; diff --git a/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs index 770dd5e34..d389fec2e 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs @@ -15,7 +15,7 @@ public AdoCsvToGithub(ITestOutputHelper output) : base(output) public async Task With_Inventory_Report_Csv() { var adoOrg = $"gei-e2e-testing-csv-{TestHelper.GetOsName()}"; - var githubOrg = $"e2e-testing-ado-csv-{TestHelper.GetOsName()}"; + var githubOrg = $"octoshift-e2e-ado-csv-{TestHelper.GetOsName()}"; var teamProject1 = "gei-e2e-1"; var teamProject2 = "gei-e2e-2"; var adoRepo1 = teamProject1; diff --git a/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs index d62248e7e..c57689a91 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs @@ -17,7 +17,7 @@ public AdoServerToGithub(ITestOutputHelper output) : base(output, ADO_SERVER_URL public async Task Basic() { var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}"; - var githubOrg = $"e2e-testing-ado-server-{TestHelper.GetOsName()}"; + var githubOrg = $"octoshift-e2e-ado-server-{TestHelper.GetOsName()}"; var teamProject1 = "gei-e2e-1"; var teamProject2 = "gei-e2e-2"; var adoRepo1 = teamProject1; diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index c4d1cf10b..cfdb88f19 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -71,7 +71,7 @@ public BbsToGithub(ITestOutputHelper output) public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload) { var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}"; - var githubTargetOrg = $"e2e-testing-bbs-{TestHelper.GetOsName()}-2"; + var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}"; var repo1 = $"{bbsProjectKey}-repo-1"; var repo2 = $"{bbsProjectKey}-repo-2"; var targetRepo1 = $"{bbsProjectKey}-e2e-{TestHelper.GetOsName().ToLower()}-repo-1"; diff --git a/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs b/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs index ca80223ab..7e0d7905f 100644 --- a/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs @@ -65,7 +65,7 @@ public GhesToGithub(ITestOutputHelper output) public async Task Basic() { var githubSourceOrg = $"e2e-testing-{TestHelper.GetOsName()}"; - var githubTargetOrg = $"e2e-testing-ghes-{TestHelper.GetOsName()}"; + var githubTargetOrg = $"octoshift-e2e-ghes-{TestHelper.GetOsName()}"; const string repo1 = "repo-1"; const string repo2 = "repo-2"; diff --git a/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs b/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs index 0269e9bfb..089cada6a 100644 --- a/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs @@ -44,7 +44,7 @@ public GithubToGithub(ITestOutputHelper output) public async Task Basic() { var githubSourceOrg = $"e2e-testing-source-{TestHelper.GetOsName()}"; - var githubTargetOrg = $"e2e-testing-ghec-{TestHelper.GetOsName()}"; + var githubTargetOrg = $"octoshift-e2e-ghec-{TestHelper.GetOsName()}"; var repo1 = "repo-1"; var repo2 = "repo-2"; From 951b0c7cf01341943960f1cc728016c85fe066ae Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Wed, 2 Oct 2024 11:08:27 -0700 Subject: [PATCH 15/25] Add pagination --- src/Octoshift/Services/GithubApi.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index baca7a80f..8ff5d58e4 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -574,11 +574,10 @@ public virtual async Task GetIdpGroupId(string org, string groupName) { var url = $"{_apiUrl}/orgs/{org.EscapeDataString()}/external-groups"; - // TODO: Need to implement paging - var response = await _client.GetAsync(url); - var data = JObject.Parse(response); + var response = await _client.GetAllAsync(url) + .SingleAsync(x => ((string)x["group_name"]).ToUpper() == groupName.ToUpper()); - return (int)data["groups"].Children().Single(x => ((string)x["group_name"]).ToUpper() == groupName.ToUpper())["group_id"]; + return (int)response["group_id"]; } public virtual async Task GetTeamSlug(string org, string teamName) From 2d85d53d340c7d8fc399407d6dc5302636c213a4 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Wed, 2 Oct 2024 12:55:13 -0700 Subject: [PATCH 16/25] Update logic --- src/Octoshift/Services/GithubApi.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index 8ff5d58e4..0e44431af 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -575,7 +575,7 @@ public virtual async Task GetIdpGroupId(string org, string groupName) var url = $"{_apiUrl}/orgs/{org.EscapeDataString()}/external-groups"; var response = await _client.GetAllAsync(url) - .SingleAsync(x => ((string)x["group_name"]).ToUpper() == groupName.ToUpper()); + .SingleAsync(x => GroupNames(x["group"], groupName)); return (int)response["group_id"]; } @@ -1177,4 +1177,8 @@ private static CodeScanningAlertInstance BuildCodeScanningAlertInstance(JToken s StartColumn = (int)scanningAlertInstance["location"]["start_column"], EndColumn = (int)scanningAlertInstance["location"]["end_column"] }; + private static bool GroupNames(dynamic x, string groupName) + { + return ((string)x["group_name"]).ToUpper() == groupName.ToUpper(); + } } From d4d22e8677e117875f6bd0c09057a41d79184a9c Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Wed, 2 Oct 2024 13:33:05 -0700 Subject: [PATCH 17/25] update --- src/Octoshift/Services/GithubApi.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index 0e44431af..2bab8bc53 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -575,7 +575,7 @@ public virtual async Task GetIdpGroupId(string org, string groupName) var url = $"{_apiUrl}/orgs/{org.EscapeDataString()}/external-groups"; var response = await _client.GetAllAsync(url) - .SingleAsync(x => GroupNames(x["group"], groupName)); + .SingleAsync(x => GroupNames(x["groups"], groupName)); return (int)response["group_id"]; } From 12361dfc7738ed4b90e71404291f2ffaecc83742 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Wed, 2 Oct 2024 14:55:25 -0700 Subject: [PATCH 18/25] Parse tokens correctly --- src/Octoshift/Services/GithubApi.cs | 5 +++-- src/Octoshift/Services/GithubClient.cs | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index 2bab8bc53..5def9ec5b 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -574,8 +574,9 @@ public virtual async Task GetIdpGroupId(string org, string groupName) { var url = $"{_apiUrl}/orgs/{org.EscapeDataString()}/external-groups"; - var response = await _client.GetAllAsync(url) - .SingleAsync(x => GroupNames(x["groups"], groupName)); + + var response = await _client.GetAllAsyncGroupId(url) + .SingleAsync(x => ((string)x["group_name"]).ToUpper() == groupName.ToUpper()); return (int)response["group_id"]; } diff --git a/src/Octoshift/Services/GithubClient.cs b/src/Octoshift/Services/GithubClient.cs index 2cae6402d..ed976836e 100644 --- a/src/Octoshift/Services/GithubClient.cs +++ b/src/Octoshift/Services/GithubClient.cs @@ -62,6 +62,23 @@ public virtual async IAsyncEnumerable GetAllAsync(string url, Dictionary } while (nextUrl != null); } + public virtual async IAsyncEnumerable GetAllAsyncGroupId(string url, Dictionary customHeaders = null) + { + var nextUrl = url; + do + { + var (content, headers) = await GetWithRetry(nextUrl, customHeaders: customHeaders); + var jResponse = JObject.Parse(content)["groups"].Children(); + + foreach (var jToken in jResponse) + { + yield return jToken; + } + + nextUrl = GetNextUrl(headers); + } while (nextUrl != null); + } + public virtual async Task PostAsync(string url, object body, Dictionary customHeaders = null) => (await SendAsync(HttpMethod.Post, url, body, customHeaders: customHeaders)).Content; From 9e8d9e4f8014585bff1d777b1785c8ede11e0e50 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 3 Oct 2024 08:27:45 -0700 Subject: [PATCH 19/25] Fix source repo name --- src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs b/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs index 089cada6a..405588351 100644 --- a/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs @@ -43,7 +43,7 @@ public GithubToGithub(ITestOutputHelper output) [Fact] public async Task Basic() { - var githubSourceOrg = $"e2e-testing-source-{TestHelper.GetOsName()}"; + var githubSourceOrg = $"octoshift-e2e-source-{TestHelper.GetOsName()}"; var githubTargetOrg = $"octoshift-e2e-ghec-{TestHelper.GetOsName()}"; var repo1 = "repo-1"; var repo2 = "repo-2"; From 00d7a71a4f565675d5eeff43a3cd3dd0d9448de7 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 3 Oct 2024 13:15:52 -0700 Subject: [PATCH 20/25] Update target ADO names --- src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs | 2 +- src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs index 569a7bbb9..aef851c4d 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs @@ -15,7 +15,7 @@ public AdoBasicToGithub(ITestOutputHelper output) : base(output) public async Task Basic() { var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}"; - var githubOrg = $"octoshift-e2e-ado-basic-{TestHelper.GetOsName()}"; + var githubOrg = $"octoshift-e2e-ado-basic-{TestHelper.GetOsName()}-2"; var teamProject1 = "gei-e2e-1"; var teamProject2 = "gei-e2e-2"; var adoRepo1 = teamProject1; diff --git a/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs index d389fec2e..df2040064 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs @@ -15,7 +15,7 @@ public AdoCsvToGithub(ITestOutputHelper output) : base(output) public async Task With_Inventory_Report_Csv() { var adoOrg = $"gei-e2e-testing-csv-{TestHelper.GetOsName()}"; - var githubOrg = $"octoshift-e2e-ado-csv-{TestHelper.GetOsName()}"; + var githubOrg = $"octoshift-e2e-ado-csv-{TestHelper.GetOsName()}-2"; var teamProject1 = "gei-e2e-1"; var teamProject2 = "gei-e2e-2"; var adoRepo1 = teamProject1; diff --git a/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs index c57689a91..65871fe9c 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs @@ -17,7 +17,7 @@ public AdoServerToGithub(ITestOutputHelper output) : base(output, ADO_SERVER_URL public async Task Basic() { var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}"; - var githubOrg = $"octoshift-e2e-ado-server-{TestHelper.GetOsName()}"; + var githubOrg = $"octoshift-e2e-ado-server-{TestHelper.GetOsName()}-2"; var teamProject1 = "gei-e2e-1"; var teamProject2 = "gei-e2e-2"; var adoRepo1 = teamProject1; From 130e83e32dc7762abccdd8ebed156205029e4624 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 3 Oct 2024 13:44:52 -0700 Subject: [PATCH 21/25] revert --- src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs index df2040064..d389fec2e 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs @@ -15,7 +15,7 @@ public AdoCsvToGithub(ITestOutputHelper output) : base(output) public async Task With_Inventory_Report_Csv() { var adoOrg = $"gei-e2e-testing-csv-{TestHelper.GetOsName()}"; - var githubOrg = $"octoshift-e2e-ado-csv-{TestHelper.GetOsName()}-2"; + var githubOrg = $"octoshift-e2e-ado-csv-{TestHelper.GetOsName()}"; var teamProject1 = "gei-e2e-1"; var teamProject2 = "gei-e2e-2"; var adoRepo1 = teamProject1; From 5ebcc023fd29f7b96bba2b3187ba018f7f9b38a1 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Thu, 3 Oct 2024 14:45:42 -0700 Subject: [PATCH 22/25] Update --- src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs b/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs index 65871fe9c..c57689a91 100644 --- a/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs @@ -17,7 +17,7 @@ public AdoServerToGithub(ITestOutputHelper output) : base(output, ADO_SERVER_URL public async Task Basic() { var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}"; - var githubOrg = $"octoshift-e2e-ado-server-{TestHelper.GetOsName()}-2"; + var githubOrg = $"octoshift-e2e-ado-server-{TestHelper.GetOsName()}"; var teamProject1 = "gei-e2e-1"; var teamProject2 = "gei-e2e-2"; var adoRepo1 = teamProject1; From c86c7c6a01aff92d92f33606d6cc655d3dcdaa85 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Fri, 4 Oct 2024 06:52:08 -0700 Subject: [PATCH 23/25] Add unit tests --- src/Octoshift/Services/GithubApi.cs | 5 - src/Octoshift/Services/GithubClient.cs | 11 ++- .../Octoshift/Services/GithubClientTests.cs | 94 +++++++++++++++++++ 3 files changed, 104 insertions(+), 6 deletions(-) diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index 5def9ec5b..bc4602bfa 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -574,7 +574,6 @@ public virtual async Task GetIdpGroupId(string org, string groupName) { var url = $"{_apiUrl}/orgs/{org.EscapeDataString()}/external-groups"; - var response = await _client.GetAllAsyncGroupId(url) .SingleAsync(x => ((string)x["group_name"]).ToUpper() == groupName.ToUpper()); @@ -1178,8 +1177,4 @@ private static CodeScanningAlertInstance BuildCodeScanningAlertInstance(JToken s StartColumn = (int)scanningAlertInstance["location"]["start_column"], EndColumn = (int)scanningAlertInstance["location"]["end_column"] }; - private static bool GroupNames(dynamic x, string groupName) - { - return ((string)x["group_name"]).ToUpper() == groupName.ToUpper(); - } } diff --git a/src/Octoshift/Services/GithubClient.cs b/src/Octoshift/Services/GithubClient.cs index ed976836e..c43fe4004 100644 --- a/src/Octoshift/Services/GithubClient.cs +++ b/src/Octoshift/Services/GithubClient.cs @@ -68,13 +68,22 @@ public virtual async IAsyncEnumerable GetAllAsyncGroupId(string url, Dic do { var (content, headers) = await GetWithRetry(nextUrl, customHeaders: customHeaders); - var jResponse = JObject.Parse(content)["groups"].Children(); + // Parse the response and check if "groups" exists + var jObject = JObject.Parse(content); + var jResponse = jObject["groups"]; + if (jResponse == null || !jResponse.HasValues) + { + yield break; // No groups found, exit the iterator + } + + // Yield each group foreach (var jToken in jResponse) { yield return jToken; } + // Check for the next page URL nextUrl = GetNextUrl(headers); } while (nextUrl != null); } diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs index fec7f8bd6..07313d1f4 100644 --- a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs @@ -1351,6 +1351,100 @@ public async Task GetAllAsync_Retries_On_Non_Success() results.Should().BeEquivalentTo(firstItem, secondItem); } + [Fact] + public async Task GetAllAsyncGroupId_Should_Get_All_Pages() + { + // Arrange + const string url = "https://api.github.com/orgs/my-org/external-groups"; + + const string firstGroup = "first-group"; + const string secondGroup = "second-group"; + const string thirdGroup = "third-group"; + + var handlerMock = new Mock(); + + // first request + MockHttpHandler( + req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, + CreateHttpResponseFactory( + content: $"{{ \"groups\": [{{ \"group_name\": \"{firstGroup}\" }}, {{ \"group_name\": \"{secondGroup}\" }}] }}", + headers: new[] + { + ("Link", $"<{url}&page=2>; rel=\"next\", " + + $"<{url}&page=2>; rel=\"last\"") + }), + handlerMock); + + // second request + MockHttpHandler( + req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == $"{url}&page=2", + CreateHttpResponseFactory( + content: $"{{ \"groups\": [{{ \"group_name\": \"{thirdGroup}\" }}] }}"), + handlerMock); + + using var httpClient = new HttpClient(handlerMock.Object); + var githubClient = new GithubClient(_mockOctoLogger.Object, httpClient, null, _retryPolicy, _dateTimeProvider.Object, PERSONAL_ACCESS_TOKEN); + + // Act + var results = await githubClient.GetAllAsyncGroupId(url).ToListAsync(); + + // Assert + results.Should().HaveCount(3); + results[0]["group_name"].Value().Should().Be(firstGroup); + results[1]["group_name"].Value().Should().Be(secondGroup); + results[2]["group_name"].Value().Should().Be(thirdGroup); + } + + [Fact] + public async Task GetAllAsyncGroupId_Should_Handle_Empty_Groups_Response() + { + // Arrange + const string url = "https://api.github.com/orgs/my-org/external-groups"; + + var handlerMock = new Mock(); + + // first request + MockHttpHandler( + req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, + CreateHttpResponseFactory( + content: "{ \"groups\": [] }"), // Empty groups array + handlerMock); + + using var httpClient = new HttpClient(handlerMock.Object); + var githubClient = new GithubClient(_mockOctoLogger.Object, httpClient, null, _retryPolicy, _dateTimeProvider.Object, PERSONAL_ACCESS_TOKEN); + + // Act + var results = await githubClient.GetAllAsyncGroupId(url).ToListAsync(); + + // Assert + results.Should().BeEmpty(); // No groups should be returned + } + + [Fact] + public async Task GetAllAsyncGroupId_Throws_HttpRequestException_On_Non_Success_Response() + { + // Arrange + const string url = "https://api.github.com/orgs/my-org/external-groups"; + + var handlerMock = new Mock(); + + // first request + MockHttpHandler( + req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, + CreateHttpResponseFactory(statusCode: HttpStatusCode.InternalServerError), + handlerMock); + + using var httpClient = new HttpClient(handlerMock.Object); + var githubClient = new GithubClient(_mockOctoLogger.Object, httpClient, null, _retryPolicy, _dateTimeProvider.Object, PERSONAL_ACCESS_TOKEN); + + // Act, Assert + await FluentActions + .Invoking(async () => await githubClient.GetAllAsyncGroupId(url).ToListAsync()) + .Should() + .ThrowExactlyAsync(); + } + + [Fact] public async Task PostGraphQLWithPaginationAsync_Should_Return_All_Pages() { From 8fd1b584204753ad13e57a40b819db54adc2e463 Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Fri, 4 Oct 2024 14:08:21 -0700 Subject: [PATCH 24/25] Clean up --- src/Octoshift/Services/GithubApi.cs | 6 +- src/Octoshift/Services/GithubClient.cs | 16 ++- .../Octoshift/Services/GithubApiTests.cs | 36 +++--- .../Octoshift/Services/GithubClientTests.cs | 116 +++++------------- 4 files changed, 68 insertions(+), 106 deletions(-) diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index bc4602bfa..e523998ed 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -574,10 +574,10 @@ public virtual async Task GetIdpGroupId(string org, string groupName) { var url = $"{_apiUrl}/orgs/{org.EscapeDataString()}/external-groups"; - var response = await _client.GetAllAsyncGroupId(url) - .SingleAsync(x => ((string)x["group_name"]).ToUpper() == groupName.ToUpper()); + var group = await _client.GetAllAsync(url, data => (JArray)data["groups"]) + .SingleAsync(x => string.Equals((string)x["group_name"], groupName, StringComparison.OrdinalIgnoreCase)); - return (int)response["group_id"]; + return (int)group["group_id"]; } public virtual async Task GetTeamSlug(string org, string teamName) diff --git a/src/Octoshift/Services/GithubClient.cs b/src/Octoshift/Services/GithubClient.cs index c43fe4004..57b1fdb92 100644 --- a/src/Octoshift/Services/GithubClient.cs +++ b/src/Octoshift/Services/GithubClient.cs @@ -47,13 +47,25 @@ public GithubClient(OctoLogger log, HttpClient httpClient, IVersionProvider vers public virtual async Task GetAsync(string url, Dictionary customHeaders = null) => (await GetWithRetry(url, customHeaders)).Content; - public virtual async IAsyncEnumerable GetAllAsync(string url, Dictionary customHeaders = null) + public virtual IAsyncEnumerable GetAllAsync(string url, Dictionary customHeaders = null) => + GetAllAsync(url, jToken => (JArray)jToken, customHeaders); + + public virtual async IAsyncEnumerable GetAllAsync( + string url, + Func resultCollectionSelector, + Dictionary customHeaders = null) { + if (resultCollectionSelector is null) + { + throw new ArgumentNullException(nameof(resultCollectionSelector)); + } + var nextUrl = url; do { var (content, headers) = await GetWithRetry(nextUrl, customHeaders: customHeaders); - foreach (var jToken in JArray.Parse(content)) + var jContent = JToken.Parse(content); + foreach (var jToken in resultCollectionSelector(jContent)) { yield return jToken; } diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs index 962922ded..826305389 100644 --- a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs @@ -1606,25 +1606,27 @@ public async Task GetIdpGroupId_Returns_The_Idp_Group_Id() var url = $"https://api.github.com/orgs/{GITHUB_ORG}/external-groups"; const int expectedGroupId = 123; - var response = $@" - {{ - ""groups"": [ - {{ - ""group_id"": ""{expectedGroupId}"", - ""group_name"": ""{groupName}"", - ""updated_at"": ""2021-01-24T11:31:04-06:00"" - }}, - {{ - ""group_id"": ""456"", - ""group_name"": ""Octocat admins"", - ""updated_at"": ""2021-03-24T11:31:04-06:00"" - }}, - ] - }}"; + + var group1 = new + { + group_id = expectedGroupId, + group_name = groupName, + updated_at = DateTime.Parse("2021-01-24T11:31:04-06:00") + }; + var group2 = new + { + group_id = "456", + group_name = "Octocat admins", + updated_at = DateTime.Parse("2021-03-24T11:31:04-06:00") + }; _githubClientMock - .Setup(m => m.GetAsync(url, null)) - .ReturnsAsync(response); + .Setup(m => m.GetAllAsync(url, It.IsAny>(), null)) + .Returns(new[] + { + JToken.FromObject(group1), + JToken.FromObject(group2) + }.ToAsyncEnumerable()); // Act var actualGroupId = await _githubApi.GetIdpGroupId(GITHUB_ORG, groupName); diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs index 07313d1f4..9a2c97965 100644 --- a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs @@ -1352,99 +1352,47 @@ public async Task GetAllAsync_Retries_On_Non_Success() } [Fact] - public async Task GetAllAsyncGroupId_Should_Get_All_Pages() - { - // Arrange - const string url = "https://api.github.com/orgs/my-org/external-groups"; - - const string firstGroup = "first-group"; - const string secondGroup = "second-group"; - const string thirdGroup = "third-group"; - - var handlerMock = new Mock(); - - // first request - MockHttpHandler( - req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, - CreateHttpResponseFactory( - content: $"{{ \"groups\": [{{ \"group_name\": \"{firstGroup}\" }}, {{ \"group_name\": \"{secondGroup}\" }}] }}", - headers: new[] - { - ("Link", $"<{url}&page=2>; rel=\"next\", " + - $"<{url}&page=2>; rel=\"last\"") - }), - handlerMock); - - // second request - MockHttpHandler( - req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == $"{url}&page=2", - CreateHttpResponseFactory( - content: $"{{ \"groups\": [{{ \"group_name\": \"{thirdGroup}\" }}] }}"), - handlerMock); - - using var httpClient = new HttpClient(handlerMock.Object); - var githubClient = new GithubClient(_mockOctoLogger.Object, httpClient, null, _retryPolicy, _dateTimeProvider.Object, PERSONAL_ACCESS_TOKEN); - - // Act - var results = await githubClient.GetAllAsyncGroupId(url).ToListAsync(); - - // Assert - results.Should().HaveCount(3); - results[0]["group_name"].Value().Should().Be(firstGroup); - results[1]["group_name"].Value().Should().Be(secondGroup); - results[2]["group_name"].Value().Should().Be(thirdGroup); - } - - [Fact] - public async Task GetAllAsyncGroupId_Should_Handle_Empty_Groups_Response() - { - // Arrange - const string url = "https://api.github.com/orgs/my-org/external-groups"; - - var handlerMock = new Mock(); - - // first request - MockHttpHandler( - req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, - CreateHttpResponseFactory( - content: "{ \"groups\": [] }"), // Empty groups array - handlerMock); + public async Task GetAllAsync_Should_Use_Result_Collection_Selector() + { + // Arrange + const string url = "https://api.github.com/orgs/foo/external-groups"; + + const string firstGroupId = "123"; + const string firstGroupName = "Octocat readers"; + const string secondGroupId = "456"; + const string secondGroupName = "Octocat admins"; + const string response = $@" + {{ + ""groups"": [ + {{ + ""group_id"": ""{firstGroupId}"", + ""group_name"": ""{firstGroupName}"", + ""updated_at"": ""2021-01-24T11:31:04-06:00"" + }}, + {{ + ""group_id"": ""{secondGroupId}"", + ""group_name"": ""{secondGroupName}"", + ""updated_at"": ""2021-03-24T11:31:04-06:00"" + }}, + ] + }}"; + + var handlerMock = MockHttpHandler(req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, CreateHttpResponseFactory(content: response)); using var httpClient = new HttpClient(handlerMock.Object); var githubClient = new GithubClient(_mockOctoLogger.Object, httpClient, null, _retryPolicy, _dateTimeProvider.Object, PERSONAL_ACCESS_TOKEN); // Act - var results = await githubClient.GetAllAsyncGroupId(url).ToListAsync(); + var results = await githubClient.GetAllAsync(url, data => (JArray)data["groups"]).ToListAsync(); // Assert - results.Should().BeEmpty(); // No groups should be returned - } - - [Fact] - public async Task GetAllAsyncGroupId_Throws_HttpRequestException_On_Non_Success_Response() - { - // Arrange - const string url = "https://api.github.com/orgs/my-org/external-groups"; - - var handlerMock = new Mock(); - - // first request - MockHttpHandler( - req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, - CreateHttpResponseFactory(statusCode: HttpStatusCode.InternalServerError), - handlerMock); - - using var httpClient = new HttpClient(handlerMock.Object); - var githubClient = new GithubClient(_mockOctoLogger.Object, httpClient, null, _retryPolicy, _dateTimeProvider.Object, PERSONAL_ACCESS_TOKEN); - - // Act, Assert - await FluentActions - .Invoking(async () => await githubClient.GetAllAsyncGroupId(url).ToListAsync()) - .Should() - .ThrowExactlyAsync(); + results.Should().HaveCount(2); + results[0]["group_id"].Value().Should().Be(firstGroupId); + results[0]["group_name"].Value().Should().Be(firstGroupName); + results[1]["group_id"].Value().Should().Be(secondGroupId); + results[1]["group_name"].Value().Should().Be(secondGroupName); } - [Fact] public async Task PostGraphQLWithPaginationAsync_Should_Return_All_Pages() { From b77dbf239ac3cc78a8acc15e119b1375127f1b5b Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Fri, 4 Oct 2024 14:09:59 -0700 Subject: [PATCH 25/25] merge with Arins branch --- src/Octoshift/Services/GithubClient.cs | 27 -------------------------- 1 file changed, 27 deletions(-) diff --git a/src/Octoshift/Services/GithubClient.cs b/src/Octoshift/Services/GithubClient.cs index 57b1fdb92..10c52390e 100644 --- a/src/Octoshift/Services/GithubClient.cs +++ b/src/Octoshift/Services/GithubClient.cs @@ -73,33 +73,6 @@ public virtual async IAsyncEnumerable GetAllAsync( nextUrl = GetNextUrl(headers); } while (nextUrl != null); } - - public virtual async IAsyncEnumerable GetAllAsyncGroupId(string url, Dictionary customHeaders = null) - { - var nextUrl = url; - do - { - var (content, headers) = await GetWithRetry(nextUrl, customHeaders: customHeaders); - // Parse the response and check if "groups" exists - var jObject = JObject.Parse(content); - var jResponse = jObject["groups"]; - - if (jResponse == null || !jResponse.HasValues) - { - yield break; // No groups found, exit the iterator - } - - // Yield each group - foreach (var jToken in jResponse) - { - yield return jToken; - } - - // Check for the next page URL - nextUrl = GetNextUrl(headers); - } while (nextUrl != null); - } - public virtual async Task PostAsync(string url, object body, Dictionary customHeaders = null) => (await SendAsync(HttpMethod.Post, url, body, customHeaders: customHeaders)).Content;