From 8376fd490b3e60317f7ace8d4387aa21f0f846c2 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Mon, 4 Nov 2024 10:34:16 -0800 Subject: [PATCH 01/17] Add BBS to GenerateScript command --- .../BbsToGithub.cs | 20 ++++++++++++------- .../GenerateScript/GenerateScriptCommand.cs | 7 +++++++ .../GenerateScriptCommandArgs.cs | 1 + .../GenerateScriptCommandHandler.cs | 3 ++- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index de22e99a8..aa561db1e 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -14,7 +14,6 @@ namespace OctoshiftCLI.IntegrationTests; [Collection("Integration Tests")] public sealed class BbsToGithub : IDisposable { - private const string SSH_KEY_FILE = "ssh_key.pem"; private const string AWS_REGION = "us-east-1"; @@ -33,6 +32,8 @@ public sealed class BbsToGithub : IDisposable private readonly DateTime _startTime; private readonly string _azureStorageConnectionString; + public enum ArchiveUploadOption { AzureStorage, AwsS3, GithubStorage } + public BbsToGithub(ITestOutputHelper output) { _startTime = DateTime.Now; @@ -67,10 +68,11 @@ public BbsToGithub(ITestOutputHelper output) } [Theory] - [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, true)] - [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, true)] - [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, false)] - public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload) + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AzureStorage)] + [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, ArchiveUploadOption.AzureStorage)] + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)] + [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)] + public async Task Basic(string bbsServer, bool useSshForArchiveDownload, ArchiveUploadOption uploadOption) { var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}"; var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}"; @@ -110,17 +112,21 @@ await retryPolicy.Retry(async () => } var archiveUploadOptions = ""; - if (useAzureForArchiveUpload) + if (uploadOption == ArchiveUploadOption.AzureStorage) { _tokens.Add("AZURE_STORAGE_CONNECTION_STRING", _azureStorageConnectionString); } - else + else if (uploadOption == ArchiveUploadOption.AwsS3) { _tokens.Add("AWS_ACCESS_KEY_ID", Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID")); _tokens.Add("AWS_SECRET_ACCESS_KEY", Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY")); var awsBucketName = Environment.GetEnvironmentVariable("AWS_BUCKET_NAME"); archiveUploadOptions = $" --aws-bucket-name {awsBucketName} --aws-region {AWS_REGION}"; } + else if (uploadOption == ArchiveUploadOption.GithubStorage) + { + archiveUploadOptions = " --use-github-storage true"; + } await _targetHelper.RunBbsCliMigration( $"generate-script --github-org {githubTargetOrg} --bbs-server-url {bbsServer} --bbs-project {bbsProjectKey}{archiveDownloadOptions}{archiveUploadOptions}", _tokens); diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs index b5016f6af..5a09fcc84 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -35,6 +35,7 @@ public GenerateScriptCommand() : base( AddOption(AwsRegion); AddOption(KeepArchive); AddOption(NoSslVerify); + AddOption(UseGithubStorage); } public Option BbsServerUrl { get; } = new( @@ -124,6 +125,12 @@ public GenerateScriptCommand() : base( Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" }; + public Option UseGithubStorage { get; } = new("--use-github-storage") + { + IsHidden = true, + Description = "Enables multipart uploads to a GitHub owned storage for use during migration", + }; + public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandArgs args, IServiceProvider sp) { if (args is null) diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 2e2c6c93f..c6b653d35 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -27,6 +27,7 @@ public class GenerateScriptCommandArgs : CommandArgs public bool KeepArchive { get; set; } public bool NoSslVerify { get; set; } public string TargetApiUrl { get; set; } + public bool UseGithubStorage { get; set; } public override void Validate(OctoLogger log) { diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index 68be3956b..f7e330d91 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -135,9 +135,10 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bb var noSslVerify = args.NoSslVerify ? " --no-ssl-verify" : ""; var targetRepoVisibility = " --target-repo-visibility private"; var targetApiUrlOption = args.TargetApiUrl.HasValue() ? $" --target-api-url \"{args.TargetApiUrl}\"" : ""; + var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : ""; return $"gh bbs2gh migrate-repo{targetApiUrlOption}{bbsServerUrlOption}{bbsUsernameOption}{bbsSharedHomeOption}{bbsProjectOption}{bbsRepoOption}{sshArchiveDownloadOptions}" + - $"{smbArchiveDownloadOptions}{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}"; + $"{smbArchiveDownloadOptions}{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}{githubStorageOption}"; } private string Exec(string script) => Wrap(script, "Exec"); From 8d411d900df5af5a9d80b34910646a6490d4263c Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Mon, 4 Nov 2024 18:34:06 -0800 Subject: [PATCH 02/17] Add generate script test --- .../GenerateScriptCommandHandlerTests.cs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs index 211306a5b..f311e7d0f 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -701,6 +701,45 @@ public async Task One_Repo_With_Aws_Bucket_Name_And_Region() _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); } + [Fact] + public async Task BBS_Single_Repo_With_UseGithubStorage() + { + // Arrange + var TARGET_API_URL = "https://foo.com/api/v3"; + const string BBS_SERVER_URL = "http://bbs-server-url"; + const string BBS_PROJECT_KEY = "BBS-PROJECT"; + const string BBS_REPO_SLUG = "repo-slug"; + const string GITHUB_ORG = "GITHUB-ORG"; + + _mockBbsApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_PROJECT_KEY, Name: "BBS Project Name"), + }); + _mockBbsApi.Setup(m => m.GetRepos(BBS_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_REPO_SLUG, Name: "RepoName"), + }); + + var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-project \"{BBS_PROJECT_KEY}\" --bbs-repo \"{BBS_REPO_SLUG}\" --github-org \"{GITHUB_ORG}\" --target-api-url \"{TARGET_API_URL}\" --github-repo \"{BBS_PROJECT_KEY}-{BBS_REPO_SLUG}\" --use-github-storage --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + BbsServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo("unit-test-output"), + UseGithubStorage = true, + TargetApiUrl = TARGET_API_URL, + BbsProject = BBS_PROJECT_KEY, + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + } + + + private string TrimNonExecutableLines(string script, int skipFirst = 9, int skipLast = 0) { var lines = script.Split(new[] { Environment.NewLine, "\n" }, StringSplitOptions.RemoveEmptyEntries).AsEnumerable(); From cfdc6471fe3092ccac1de68e483be13825687c3e Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Mon, 4 Nov 2024 19:01:30 -0800 Subject: [PATCH 03/17] Add unit test for Command tests --- .../Commands/GenerateScript/GenerateScriptCommandTests.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs index db247883e..48732cbb5 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -58,6 +58,7 @@ public void Should_Have_Options() TestHelpers.VerifyCommandOption(_command.Options, "keep-archive", false); TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(_command.Options, "use-github-storage", false, true); } [Fact] @@ -125,4 +126,8 @@ public void It_Gets_A_No_Ssl_Verify_HttpClient_When_No_Ssl_Verify_Is_Set() _mockBbsApiFactory.Verify(m => m.Create(BBS_SERVER_URL, bbsTestUser, bbsTestPassword, true)); } + + [Fact] + + } From cc6d92ae18c19231cde3976a5639cdb0e6f0d861 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 07:43:09 -0800 Subject: [PATCH 04/17] fix test --- .../Commands/GenerateScript/GenerateScriptCommandTests.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs index 48732cbb5..335ce5d4c 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -126,8 +126,4 @@ public void It_Gets_A_No_Ssl_Verify_HttpClient_When_No_Ssl_Verify_Is_Set() _mockBbsApiFactory.Verify(m => m.Create(BBS_SERVER_URL, bbsTestUser, bbsTestPassword, true)); } - - [Fact] - - } From 9537674e957d4e8dc05093dcf7a3979f1a0cd435 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 08:04:08 -0800 Subject: [PATCH 05/17] Fix unit tests --- .../GenerateScript/GenerateScriptCommandHandlerTests.cs | 8 +++++++- .../Commands/GenerateScript/GenerateScriptCommandTests.cs | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs index f311e7d0f..b699dad12 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -735,7 +735,13 @@ public async Task BBS_Single_Repo_With_UseGithubStorage() await _handler.Handle(args); // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => + script.Contains("--bbs-server-url \"http://bbs-server-url\"") && + script.Contains("--bbs-project \"BBS-PROJECT\"") && + script.Contains("--github-org \"GITHUB-ORG\"") && + script.Contains("--use-github-storage") +))); + } diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs index 335ce5d4c..284822db0 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -36,7 +36,7 @@ public void Should_Have_Options() { _command.Should().NotBeNull(); _command.Name.Should().Be("generate-script"); - _command.Options.Count.Should().Be(20); + _command.Options.Count.Should().Be(21); TestHelpers.VerifyCommandOption(_command.Options, "bbs-server-url", true); TestHelpers.VerifyCommandOption(_command.Options, "github-org", true); From fbf1c8d62dfe1baf9a358bfc3a455a568774dd24 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 08:13:12 -0800 Subject: [PATCH 06/17] Update integration test --- src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs index aa561db1e..dc54136d0 100644 --- a/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs +++ b/src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs @@ -125,7 +125,7 @@ await retryPolicy.Retry(async () => } else if (uploadOption == ArchiveUploadOption.GithubStorage) { - archiveUploadOptions = " --use-github-storage true"; + archiveUploadOptions = " --use-github-storage"; } await _targetHelper.RunBbsCliMigration( From e673cede51f2fe9230383748c755fb590f306456 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 08:46:26 -0800 Subject: [PATCH 07/17] Fix unit test --- .../GenerateScript/GenerateScriptCommandHandlerTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs index b699dad12..472e1656e 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -706,10 +706,8 @@ public async Task BBS_Single_Repo_With_UseGithubStorage() { // Arrange var TARGET_API_URL = "https://foo.com/api/v3"; - const string BBS_SERVER_URL = "http://bbs-server-url"; const string BBS_PROJECT_KEY = "BBS-PROJECT"; const string BBS_REPO_SLUG = "repo-slug"; - const string GITHUB_ORG = "GITHUB-ORG"; _mockBbsApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] { @@ -720,7 +718,6 @@ public async Task BBS_Single_Repo_With_UseGithubStorage() (Id: 1, Slug: BBS_REPO_SLUG, Name: "RepoName"), }); - var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-project \"{BBS_PROJECT_KEY}\" --bbs-repo \"{BBS_REPO_SLUG}\" --github-org \"{GITHUB_ORG}\" --target-api-url \"{TARGET_API_URL}\" --github-repo \"{BBS_PROJECT_KEY}-{BBS_REPO_SLUG}\" --use-github-storage --target-repo-visibility private }}"; // Act var args = new GenerateScriptCommandArgs From 30350b1f0d4f6819915e351b8f339eeacdf0102e Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 10:28:11 -0800 Subject: [PATCH 08/17] Add check with AWS storgae --- .../GenerateScriptCommandArgsTests.cs | 15 +++++++++++++++ .../GenerateScript/GenerateScriptCommandArgs.cs | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs index 2064072d0..e45ba5a54 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs @@ -34,4 +34,19 @@ public void Invoke_With_Ssh_Port_Set_To_7999_Logs_Warning() _mockOctoLogger.Verify(x => x.LogWarning(It.Is(x => x.ToLower().Contains("--ssh-port is set to 7999")))); } + + [Fact] + public void It_Throws_If_Both_AwsBucketName_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsBucketName = "my-bucket"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + } diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index c6b653d35..07415d9de 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -36,6 +36,11 @@ public override void Validate(OctoLogger log) throw new OctoshiftCliException("--no-ssl-verify can only be provided with --bbs-server-url."); } + if (!string.IsNullOrWhiteSpace(AwsBucketName) && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + if (SshPort == 7999) { log?.LogWarning("--ssh-port is set to 7999, which is the default port that Bitbucket Server and Bitbucket Data Center use for Git operations over SSH. This is probably the wrong value, because --ssh-port should be configured with the SSH port used to manage the server where Bitbucket Server/Bitbucket Data Center is running, not the port used for Git operations over SSH."); From a3bce61ac606e3b3f5e381619c42095fe1fe1411 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 11:19:05 -0800 Subject: [PATCH 09/17] Update release notes --- releasenotes/v1.8.1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/releasenotes/v1.8.1.md b/releasenotes/v1.8.1.md index 3485c7b1c..b31d9e2d8 100644 --- a/releasenotes/v1.8.1.md +++ b/releasenotes/v1.8.1.md @@ -1,2 +1,2 @@ -Add `--use-github-storage` to `gh [gei|bbs2gh] migrate-repo` command to support uploading to a GitHub owned storage -Add `--use-github-storage` to `gh gei generate-script` command to support uploading to a GitHub owned storage \ No newline at end of file +Add `--use-github-storage` to gh [gei|bbs2gh] migrate-repo command to support uploading to a GitHub owned storage +Add `--use-github-storage` to gh [gei|bbs2gh] generate-script command to support uploading to a GitHub owned storage From 791cb33a9ad63d778c06e8e6ef6f5268dbc0fbe5 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 11:22:40 -0800 Subject: [PATCH 10/17] Update release notes --- releasenotes/v1.8.1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/releasenotes/v1.8.1.md b/releasenotes/v1.8.1.md index b31d9e2d8..834f3442b 100644 --- a/releasenotes/v1.8.1.md +++ b/releasenotes/v1.8.1.md @@ -1,2 +1,2 @@ -Add `--use-github-storage` to gh [gei|bbs2gh] migrate-repo command to support uploading to a GitHub owned storage -Add `--use-github-storage` to gh [gei|bbs2gh] generate-script command to support uploading to a GitHub owned storage +Add `--use-github-storage` to `gh [gei|bbs2gh] migrate-repo` command to support uploading to a GitHub owned storage +Add `--use-github-storage` to `gh [gei|bbs2gh] generate-script` command to support uploading to a GitHub owned storage From 80a7cf716011866768f94e78590f17ce2342b90a Mon Sep 17 00:00:00 2001 From: Begona Guereca Date: Wed, 6 Nov 2024 11:31:26 -0800 Subject: [PATCH 11/17] Update src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs Co-authored-by: Arin Ghazarian --- src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 07415d9de..71c0e5eb5 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -36,7 +36,7 @@ public override void Validate(OctoLogger log) throw new OctoshiftCliException("--no-ssl-verify can only be provided with --bbs-server-url."); } - if (!string.IsNullOrWhiteSpace(AwsBucketName) && UseGithubStorage) + if (AwsBucketName.HasValue() && UseGithubStorage) { throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); } From dd78abb9ab41f87aae359e4bc0ae76c828a74e06 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 16:32:29 -0600 Subject: [PATCH 12/17] test --- .vscode/launch.json | 22 +++++++++++++++++-- .../GenerateScriptCommandArgs.cs | 8 +++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ae9629318..21f9b19f0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -48,11 +48,29 @@ "preLaunchTask": "build", // If you have changed target frameworks, make sure to update the program path. "program": "${workspaceFolder}/src/bbs2gh/bin/Debug/net6.0/bbs2gh.dll", - "args": [], + "env": { + "BBS_PASSWORD": "8ecTR38ng6doP/g", + "BBS_USERNAME": "bbsadmin" + }, + "args": [ + "generate-script", + "--bbs-project", + "IM", + "--ssh-user", + "bbs", + "--ssh-private-key", + "/Users/begonaguereca/.ssh/id_rsa_bbs_octoshift", + "--use-github-storage", + "--bbs-server-url", + "https://test-bbs-o.githubapp.com", + "--github-org", + "octoshift-staging", + "--verbose" + ], "cwd": "${workspaceFolder}/src/bbs2gh", // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console "console": "internalConsole", "stopAtEntry": false } ] -} +} \ No newline at end of file diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 71c0e5eb5..7a46a4524 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -36,10 +36,10 @@ public override void Validate(OctoLogger log) throw new OctoshiftCliException("--no-ssl-verify can only be provided with --bbs-server-url."); } - if (AwsBucketName.HasValue() && UseGithubStorage) - { - throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); - } + // if (!string.IsNullOrWhiteSpace(AwsBucketName) && UseGithubStorage) + // { + // throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + // } if (SshPort == 7999) { From 75564e7aa426d7b72e271c368f07cb551148508b Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 17:00:54 -0600 Subject: [PATCH 13/17] launch --- .vscode/launch.json | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 21f9b19f0..c3f958891 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -48,25 +48,7 @@ "preLaunchTask": "build", // If you have changed target frameworks, make sure to update the program path. "program": "${workspaceFolder}/src/bbs2gh/bin/Debug/net6.0/bbs2gh.dll", - "env": { - "BBS_PASSWORD": "8ecTR38ng6doP/g", - "BBS_USERNAME": "bbsadmin" - }, - "args": [ - "generate-script", - "--bbs-project", - "IM", - "--ssh-user", - "bbs", - "--ssh-private-key", - "/Users/begonaguereca/.ssh/id_rsa_bbs_octoshift", - "--use-github-storage", - "--bbs-server-url", - "https://test-bbs-o.githubapp.com", - "--github-org", - "octoshift-staging", - "--verbose" - ], + "args": [], "cwd": "${workspaceFolder}/src/bbs2gh", // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console "console": "internalConsole", From f099c19d40d7f8247c061f9701910f787ef469f2 Mon Sep 17 00:00:00 2001 From: bbsadmin Date: Wed, 6 Nov 2024 17:03:31 -0600 Subject: [PATCH 14/17] revert --- .../Commands/GenerateScript/GenerateScriptCommandArgs.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 7a46a4524..71c0e5eb5 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -36,10 +36,10 @@ public override void Validate(OctoLogger log) throw new OctoshiftCliException("--no-ssl-verify can only be provided with --bbs-server-url."); } - // if (!string.IsNullOrWhiteSpace(AwsBucketName) && UseGithubStorage) - // { - // throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); - // } + if (AwsBucketName.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } if (SshPort == 7999) { From 3f9d76a2a2bff223f55a410202e8d159e1d7ff76 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Wed, 6 Nov 2024 17:35:22 -0800 Subject: [PATCH 15/17] Don't add AWS and Azure validations when --use-github-storage is provided --- .../GenerateScriptCommandHandlerTests.cs | 46 ++++++++++++++++++- .../GenerateScriptCommandHandler.cs | 2 +- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs index 472e1656e..6394f51c7 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -2,7 +2,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using FluentAssertions; using Moq; using OctoshiftCLI.BbsToGithub.Commands.GenerateScript; using OctoshiftCLI.Contracts; @@ -261,6 +260,51 @@ exit 1 _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); } + [Fact] + public async Task Validates_Env_Vars_AZURE_STORAGE_CONNECTION_STRING_And_AWS_Not_Validated_When_UseGithubStorage() + { + // Arrange + _mockBbsApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + BbsServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT), + UseGithubStorage = true + }; + await _handler.Handle(args); + + var expectedAws = @" +if (-not $env:AWS_ACCESS_KEY_ID) { + Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" +} +if (-not $env:AWS_SECRET_ACCESS_KEY) { + Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; + + var expectedAzure = @" +if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { + Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" + exit 1 +} else { + Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAws, 0, 0))))); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAzure, 0, 0))))); + } + [Fact] public async Task Validates_Env_Vars_SMB_PASSWORD() { diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index f7e330d91..0656cc13f 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -71,7 +71,7 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) content.AppendLine(VALIDATE_AWS_ACCESS_KEY_ID); content.AppendLine(VALIDATE_AWS_SECRET_ACCESS_KEY); } - else + else if (!args.UseGithubStorage) { content.AppendLine(VALIDATE_AZURE_STORAGE_CONNECTION_STRING); } From 48e64698a7c646ba87770be687ed3fe0d4600daf Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Wed, 6 Nov 2024 17:36:44 -0800 Subject: [PATCH 16/17] Add missing new line --- .vscode/launch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index c3f958891..ae9629318 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -55,4 +55,4 @@ "stopAtEntry": false } ] -} \ No newline at end of file +} From 13c453e3030f64180769dbcfe7de7010622f4650 Mon Sep 17 00:00:00 2001 From: Arin Ghazarian Date: Wed, 6 Nov 2024 17:43:49 -0800 Subject: [PATCH 17/17] Add an extra validation for when both AWS region and use github storage are provided --- .../GenerateScriptCommandArgsTests.cs | 13 +++++++++++++ .../GenerateScript/GenerateScriptCommandArgs.cs | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs index e45ba5a54..91d402928 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs @@ -49,4 +49,17 @@ public void It_Throws_If_Both_AwsBucketName_And_UseGithubStorage_Are_Provided() .WithMessage("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); } + [Fact] + public void It_Throws_If_Both_AwsRegion_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsRegion = "aws-region"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } } diff --git a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 71c0e5eb5..d54077566 100644 --- a/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -41,6 +41,11 @@ public override void Validate(OctoLogger log) throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); } + if (AwsRegion.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } + if (SshPort == 7999) { log?.LogWarning("--ssh-port is set to 7999, which is the default port that Bitbucket Server and Bitbucket Data Center use for Git operations over SSH. This is probably the wrong value, because --ssh-port should be configured with the SSH port used to manage the server where Bitbucket Server/Bitbucket Data Center is running, not the port used for Git operations over SSH.");