Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Getting CLI working again #1267

Merged
merged 25 commits into from
Oct 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/Octoshift/Services/GithubApi.cs
Original file line number Diff line number Diff line change
Expand Up @@ -574,11 +574,10 @@ public virtual async Task<int> GetIdpGroupId(string org, string groupName)
{
var url = $"{_apiUrl}/orgs/{org.EscapeDataString()}/external-groups";

// TODO: Need to implement paging
var response = await _client.GetAsync(url);
var data = JObject.Parse(response);
var group = await _client.GetAllAsync(url, data => (JArray)data["groups"])
.SingleAsync(x => string.Equals((string)x["group_name"], groupName, StringComparison.OrdinalIgnoreCase));

return (int)data["groups"].Children().Single(x => ((string)x["group_name"]).ToUpper() == groupName.ToUpper())["group_id"];
return (int)group["group_id"];
}

public virtual async Task<string> GetTeamSlug(string org, string teamName)
Expand Down
17 changes: 14 additions & 3 deletions src/Octoshift/Services/GithubClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,21 +47,32 @@ public GithubClient(OctoLogger log, HttpClient httpClient, IVersionProvider vers

public virtual async Task<string> GetAsync(string url, Dictionary<string, string> customHeaders = null) => (await GetWithRetry(url, customHeaders)).Content;

public virtual async IAsyncEnumerable<JToken> GetAllAsync(string url, Dictionary<string, string> customHeaders = null)
public virtual IAsyncEnumerable<JToken> GetAllAsync(string url, Dictionary<string, string> customHeaders = null) =>
GetAllAsync(url, jToken => (JArray)jToken, customHeaders);

public virtual async IAsyncEnumerable<JToken> GetAllAsync(
string url,
Func<JToken, JArray> resultCollectionSelector,
Dictionary<string, string> customHeaders = null)
{
if (resultCollectionSelector is null)
{
throw new ArgumentNullException(nameof(resultCollectionSelector));
}

var nextUrl = url;
do
{
var (content, headers) = await GetWithRetry(nextUrl, customHeaders: customHeaders);
foreach (var jToken in JArray.Parse(content))
var jContent = JToken.Parse(content);
foreach (var jToken in resultCollectionSelector(jContent))
{
yield return jToken;
}

nextUrl = GetNextUrl(headers);
} while (nextUrl != null);
}

public virtual async Task<string> PostAsync(string url, object body, Dictionary<string, string> customHeaders = null) =>
(await SendAsync(HttpMethod.Post, url, body, customHeaders: customHeaders)).Content;

Expand Down
2 changes: 1 addition & 1 deletion src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public AdoBasicToGithub(ITestOutputHelper output) : base(output)
public async Task Basic()
{
var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}";
var githubOrg = $"e2e-testing-ado-basic-{TestHelper.GetOsName()}";
var githubOrg = $"octoshift-e2e-ado-basic-{TestHelper.GetOsName()}-2";
var teamProject1 = "gei-e2e-1";
var teamProject2 = "gei-e2e-2";
var adoRepo1 = teamProject1;
Expand Down
2 changes: 1 addition & 1 deletion src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public AdoCsvToGithub(ITestOutputHelper output) : base(output)
public async Task With_Inventory_Report_Csv()
{
var adoOrg = $"gei-e2e-testing-csv-{TestHelper.GetOsName()}";
var githubOrg = $"e2e-testing-ado-csv-{TestHelper.GetOsName()}";
var githubOrg = $"octoshift-e2e-ado-csv-{TestHelper.GetOsName()}";
var teamProject1 = "gei-e2e-1";
var teamProject2 = "gei-e2e-2";
var adoRepo1 = teamProject1;
Expand Down
2 changes: 1 addition & 1 deletion src/OctoshiftCLI.IntegrationTests/AdoServerToGithub.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public AdoServerToGithub(ITestOutputHelper output) : base(output, ADO_SERVER_URL
public async Task Basic()
{
var adoOrg = $"gei-e2e-testing-basic-{TestHelper.GetOsName()}";
var githubOrg = $"e2e-testing-ado-server-{TestHelper.GetOsName()}";
var githubOrg = $"octoshift-e2e-ado-server-{TestHelper.GetOsName()}";
var teamProject1 = "gei-e2e-1";
var teamProject2 = "gei-e2e-2";
var adoRepo1 = teamProject1;
Expand Down
20 changes: 11 additions & 9 deletions src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,11 @@ public BbsToGithub(ITestOutputHelper output)
public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload)
{
var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}";
var githubTargetOrg = $"e2e-testing-bbs-{TestHelper.GetOsName()}";
var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}";
var repo1 = $"{bbsProjectKey}-repo-1";
var repo2 = $"{bbsProjectKey}-repo-2";
var targetRepo1 = $"{bbsProjectKey}-e2e-{TestHelper.GetOsName().ToLower()}-repo-1";
var targetRepo2 = $"{bbsProjectKey}-e2e-{TestHelper.GetOsName().ToLower()}-repo-2";

var sourceBbsApi = new BbsApi(_sourceBbsClient, bbsServer, _logger);
var sourceHelper = new TestHelper(_output, sourceBbsApi, _sourceBbsClient, bbsServer);
Expand All @@ -87,10 +89,10 @@ await retryPolicy.Retry(async () =>
await _targetHelper.ResetGithubTestEnvironment(githubTargetOrg);

await sourceHelper.CreateBbsProject(bbsProjectKey);
await sourceHelper.CreateBbsRepo(bbsProjectKey, "repo-1");
await sourceHelper.InitializeBbsRepo(bbsProjectKey, "repo-1");
await sourceHelper.CreateBbsRepo(bbsProjectKey, "repo-2");
await sourceHelper.InitializeBbsRepo(bbsProjectKey, "repo-2");
await sourceHelper.CreateBbsRepo(bbsProjectKey, repo1);
await sourceHelper.InitializeBbsRepo(bbsProjectKey, repo1);
await sourceHelper.CreateBbsRepo(bbsProjectKey, repo2);
await sourceHelper.InitializeBbsRepo(bbsProjectKey, repo2);
ArinGhazarian marked this conversation as resolved.
Show resolved Hide resolved
});

var archiveDownloadOptions = $" --ssh-user octoshift --ssh-private-key {SSH_KEY_FILE}";
Expand Down Expand Up @@ -123,10 +125,10 @@ await _targetHelper.RunBbsCliMigration(

_targetHelper.AssertNoErrorInLogs(_startTime);

await _targetHelper.AssertGithubRepoExists(githubTargetOrg, repo1);
await _targetHelper.AssertGithubRepoExists(githubTargetOrg, repo2);
await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, repo1);
await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, repo2);
await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo1);
await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo2);
await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo1);
await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo2);

// TODO: Assert migration logs are downloaded
}
Expand Down
2 changes: 1 addition & 1 deletion src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public GhesToGithub(ITestOutputHelper output)
public async Task Basic()
{
var githubSourceOrg = $"e2e-testing-{TestHelper.GetOsName()}";
var githubTargetOrg = $"e2e-testing-ghes-{TestHelper.GetOsName()}";
var githubTargetOrg = $"octoshift-e2e-ghes-{TestHelper.GetOsName()}";
const string repo1 = "repo-1";
const string repo2 = "repo-2";

Expand Down
4 changes: 2 additions & 2 deletions src/OctoshiftCLI.IntegrationTests/GithubToGithub.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ public GithubToGithub(ITestOutputHelper output)
[Fact]
public async Task Basic()
{
var githubSourceOrg = $"e2e-testing-source-{TestHelper.GetOsName()}";
var githubTargetOrg = $"e2e-testing-ghec-{TestHelper.GetOsName()}";
var githubSourceOrg = $"octoshift-e2e-source-{TestHelper.GetOsName()}";
var githubTargetOrg = $"octoshift-e2e-ghec-{TestHelper.GetOsName()}";
var repo1 = "repo-1";
var repo2 = "repo-2";

Expand Down
4 changes: 3 additions & 1 deletion src/OctoshiftCLI.IntegrationTests/TestHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -487,8 +487,10 @@ private async Task CreateRepo(string org, string repo, bool isPrivate, bool isIn

private async Task<IEnumerable<string>> GetRepoCommitShas(string org, string repo)
{
var retryPolicy = new RetryPolicy(null);

var url = $"{GithubApiBaseUrl}/repos/{org}/{repo}/commits";
var commits = await _githubClient.GetAllAsync(url).ToListAsync();
var commits = await retryPolicy.Retry(async () => await _githubClient.GetAllAsync(url).ToListAsync());
return commits.Select(x => (string)x["sha"]).ToList();
}

Expand Down
36 changes: 19 additions & 17 deletions src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1606,25 +1606,27 @@ public async Task GetIdpGroupId_Returns_The_Idp_Group_Id()

var url = $"https://api.github.com/orgs/{GITHUB_ORG}/external-groups";
const int expectedGroupId = 123;
var response = $@"
{{
""groups"": [
{{
""group_id"": ""{expectedGroupId}"",
""group_name"": ""{groupName}"",
""updated_at"": ""2021-01-24T11:31:04-06:00""
}},
{{
""group_id"": ""456"",
""group_name"": ""Octocat admins"",
""updated_at"": ""2021-03-24T11:31:04-06:00""
}},
]
}}";

var group1 = new
{
group_id = expectedGroupId,
group_name = groupName,
updated_at = DateTime.Parse("2021-01-24T11:31:04-06:00")
};
var group2 = new
{
group_id = "456",
group_name = "Octocat admins",
updated_at = DateTime.Parse("2021-03-24T11:31:04-06:00")
};

_githubClientMock
.Setup(m => m.GetAsync(url, null))
.ReturnsAsync(response);
.Setup(m => m.GetAllAsync(url, It.IsAny<Func<JToken, JArray>>(), null))
.Returns(new[]
{
JToken.FromObject(group1),
JToken.FromObject(group2)
}.ToAsyncEnumerable());

// Act
var actualGroupId = await _githubApi.GetIdpGroupId(GITHUB_ORG, groupName);
Expand Down
42 changes: 42 additions & 0 deletions src/OctoshiftCLI.Tests/Octoshift/Services/GithubClientTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1351,6 +1351,48 @@ public async Task GetAllAsync_Retries_On_Non_Success()
results.Should().BeEquivalentTo(firstItem, secondItem);
}

[Fact]
public async Task GetAllAsync_Should_Use_Result_Collection_Selector()
{
// Arrange
const string url = "https://api.github.com/orgs/foo/external-groups";

const string firstGroupId = "123";
const string firstGroupName = "Octocat readers";
const string secondGroupId = "456";
const string secondGroupName = "Octocat admins";
const string response = $@"
{{
""groups"": [
{{
""group_id"": ""{firstGroupId}"",
""group_name"": ""{firstGroupName}"",
""updated_at"": ""2021-01-24T11:31:04-06:00""
}},
{{
""group_id"": ""{secondGroupId}"",
""group_name"": ""{secondGroupName}"",
""updated_at"": ""2021-03-24T11:31:04-06:00""
}},
]
}}";

var handlerMock = MockHttpHandler(req => req.Method == HttpMethod.Get && req.RequestUri.ToString() == url, CreateHttpResponseFactory(content: response));

using var httpClient = new HttpClient(handlerMock.Object);
var githubClient = new GithubClient(_mockOctoLogger.Object, httpClient, null, _retryPolicy, _dateTimeProvider.Object, PERSONAL_ACCESS_TOKEN);

// Act
var results = await githubClient.GetAllAsync(url, data => (JArray)data["groups"]).ToListAsync();

// Assert
results.Should().HaveCount(2);
results[0]["group_id"].Value<string>().Should().Be(firstGroupId);
results[0]["group_name"].Value<string>().Should().Be(firstGroupName);
results[1]["group_id"].Value<string>().Should().Be(secondGroupId);
results[1]["group_name"].Value<string>().Should().Be(secondGroupName);
}

[Fact]
public async Task PostGraphQLWithPaginationAsync_Should_Return_All_Pages()
{
Expand Down
Loading