Merge pull request #1292 from github/bbs-git-storage-option-generate-…

…script

Update BBS generate script to accept `--use-github-storage` option

releasenotes/v1.8.1.md

@@ -1,2 +1,2 @@
 Add `--use-github-storage` to `gh [gei|bbs2gh] migrate-repo` command to support uploading to a GitHub owned storage
-Add `--use-github-storage` to `gh gei generate-script` command to support uploading to a GitHub owned storage
+Add `--use-github-storage` to `gh [gei|bbs2gh] generate-script` command to support uploading to a GitHub owned storage

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

@@ -14,7 +14,6 @@ namespace OctoshiftCLI.IntegrationTests;
 [Collection("Integration Tests")]
 public sealed class BbsToGithub : IDisposable
 {
-
     private const string SSH_KEY_FILE = "ssh_key.pem";
     private const string AWS_REGION = "us-east-1";
 
@@ -33,6 +32,8 @@ public sealed class BbsToGithub : IDisposable
     private readonly DateTime _startTime;
     private readonly string _azureStorageConnectionString;
 
+    public enum ArchiveUploadOption { AzureStorage, AwsS3, GithubStorage }
+
     public BbsToGithub(ITestOutputHelper output)
     {
         _startTime = DateTime.Now;
@@ -67,10 +68,11 @@ public BbsToGithub(ITestOutputHelper output)
     }
 
     [Theory]
-    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, true)]
-    [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, true)]
-    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, false)]
-    public async Task Basic(string bbsServer, bool useSshForArchiveDownload, bool useAzureForArchiveUpload)
+    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AzureStorage)]
+    [InlineData("http://e2e-bbs-7-21-9-win-2019.eastus.cloudapp.azure.com:7990", false, ArchiveUploadOption.AzureStorage)]
+    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.AwsS3)]
+    [InlineData("http://e2e-bbs-8-5-0-linux-2204.eastus.cloudapp.azure.com:7990", true, ArchiveUploadOption.GithubStorage)]
+    public async Task Basic(string bbsServer, bool useSshForArchiveDownload, ArchiveUploadOption uploadOption)
     {
         var bbsProjectKey = $"E2E-{TestHelper.GetOsName().ToUpper()}";
         var githubTargetOrg = $"octoshift-e2e-bbs-{TestHelper.GetOsName()}";
@@ -110,17 +112,21 @@ await retryPolicy.Retry(async () =>
         }
 
         var archiveUploadOptions = "";
-        if (useAzureForArchiveUpload)
+        if (uploadOption == ArchiveUploadOption.AzureStorage)
         {
             _tokens.Add("AZURE_STORAGE_CONNECTION_STRING", _azureStorageConnectionString);
         }
-        else
+        else if (uploadOption == ArchiveUploadOption.AwsS3)
         {
             _tokens.Add("AWS_ACCESS_KEY_ID", Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID"));
             _tokens.Add("AWS_SECRET_ACCESS_KEY", Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY"));
             var awsBucketName = Environment.GetEnvironmentVariable("AWS_BUCKET_NAME");
             archiveUploadOptions = $" --aws-bucket-name {awsBucketName} --aws-region {AWS_REGION}";
         }
+        else if (uploadOption == ArchiveUploadOption.GithubStorage)
+        {
+            archiveUploadOptions = " --use-github-storage";
+        }
 
         await _targetHelper.RunBbsCliMigration(
             $"generate-script --github-org {githubTargetOrg} --bbs-server-url {bbsServer} --bbs-project {bbsProjectKey}{archiveDownloadOptions}{archiveUploadOptions}", _tokens);

src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs

@@ -34,4 +34,32 @@ public void Invoke_With_Ssh_Port_Set_To_7999_Logs_Warning()
 
         _mockOctoLogger.Verify(x => x.LogWarning(It.Is<string>(x => x.ToLower().Contains("--ssh-port is set to 7999"))));
     }
+
+    [Fact]
+    public void It_Throws_If_Both_AwsBucketName_And_UseGithubStorage_Are_Provided()
+    {
+        // Arrange
+        _args.AwsBucketName = "my-bucket";
+        _args.UseGithubStorage = true;
+
+        // Act & Assert
+        _args.Invoking(x => x.Validate(_mockOctoLogger.Object))
+            .Should()
+            .ThrowExactly<OctoshiftCliException>()
+            .WithMessage("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations.");
+    }
+
+    [Fact]
+    public void It_Throws_If_Both_AwsRegion_And_UseGithubStorage_Are_Provided()
+    {
+        // Arrange
+        _args.AwsRegion = "aws-region";
+        _args.UseGithubStorage = true;
+
+        // Act & Assert
+        _args.Invoking(x => x.Validate(_mockOctoLogger.Object))
+            .Should()
+            .ThrowExactly<OctoshiftCliException>()
+            .WithMessage("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations.");
+    }
 }

src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs

@@ -2,7 +2,6 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using FluentAssertions;
 using Moq;
 using OctoshiftCLI.BbsToGithub.Commands.GenerateScript;
 using OctoshiftCLI.Contracts;
@@ -261,6 +260,51 @@ exit 1
         _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny<string>(), It.Is<string>(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0)))));
     }
 
+    [Fact]
+    public async Task Validates_Env_Vars_AZURE_STORAGE_CONNECTION_STRING_And_AWS_Not_Validated_When_UseGithubStorage()
+    {
+        // Arrange
+        _mockBbsApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>());
+
+        // Act
+        var args = new GenerateScriptCommandArgs()
+        {
+            BbsServerUrl = BBS_SERVER_URL,
+            GithubOrg = GITHUB_ORG,
+            SshUser = SSH_USER,
+            SshPrivateKey = SSH_PRIVATE_KEY,
+            Output = new FileInfo(OUTPUT),
+            UseGithubStorage = true
+        };
+        await _handler.Handle(args);
+
+        var expectedAws = @"
+if (-not $env:AWS_ACCESS_KEY_ID) {
+    Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3.""
+    exit 1
+} else {
+    Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3.""
+}
+if (-not $env:AWS_SECRET_ACCESS_KEY) {
+    Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3.""
+    exit 1
+} else {
+    Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3.""
+}";
+
+        var expectedAzure = @"
+if (-not $env:AZURE_STORAGE_CONNECTION_STRING) {
+    Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage.""
+    exit 1
+} else {
+    Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage.""
+}";
+
+        // Assert
+        _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny<string>(), It.Is<string>(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAws, 0, 0)))));
+        _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny<string>(), It.Is<string>(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAzure, 0, 0)))));
+    }
+
     [Fact]
     public async Task Validates_Env_Vars_SMB_PASSWORD()
     {
@@ -701,6 +745,48 @@ public async Task One_Repo_With_Aws_Bucket_Name_And_Region()
         _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny<string>(), It.Is<string>(script => script.Contains(migrateRepoCommand))));
     }
 
+    [Fact]
+    public async Task BBS_Single_Repo_With_UseGithubStorage()
+    {
+        // Arrange
+        var TARGET_API_URL = "https://foo.com/api/v3";
+        const string BBS_PROJECT_KEY = "BBS-PROJECT";
+        const string BBS_REPO_SLUG = "repo-slug";
+
+        _mockBbsApi.Setup(m => m.GetProjects()).ReturnsAsync(new[]
+        {
+            (Id: 1, Key: BBS_PROJECT_KEY, Name: "BBS Project Name"),
+        });
+        _mockBbsApi.Setup(m => m.GetRepos(BBS_PROJECT_KEY)).ReturnsAsync(new[]
+        {
+            (Id: 1, Slug: BBS_REPO_SLUG, Name: "RepoName"),
+         });
+
+
+        // Act
+        var args = new GenerateScriptCommandArgs
+        {
+            BbsServerUrl = BBS_SERVER_URL,
+            GithubOrg = GITHUB_ORG,
+            Output = new FileInfo("unit-test-output"),
+            UseGithubStorage = true,
+            TargetApiUrl = TARGET_API_URL,
+            BbsProject = BBS_PROJECT_KEY,
+        };
+        await _handler.Handle(args);
+
+        // Assert
+        _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny<string>(), It.Is<string>(script =>
+            script.Contains("--bbs-server-url \"http://bbs-server-url\"") &&
+            script.Contains("--bbs-project \"BBS-PROJECT\"") &&
+            script.Contains("--github-org \"GITHUB-ORG\"") &&
+            script.Contains("--use-github-storage")
+)));
+
+    }
+
+
+
     private string TrimNonExecutableLines(string script, int skipFirst = 9, int skipLast = 0)
     {
         var lines = script.Split(new[] { Environment.NewLine, "\n" }, StringSplitOptions.RemoveEmptyEntries).AsEnumerable();

src/OctoshiftCLI.Tests/bbs2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs

@@ -36,7 +36,7 @@ public void Should_Have_Options()
     {
         _command.Should().NotBeNull();
         _command.Name.Should().Be("generate-script");
-        _command.Options.Count.Should().Be(20);
+        _command.Options.Count.Should().Be(21);
 
         TestHelpers.VerifyCommandOption(_command.Options, "bbs-server-url", true);
         TestHelpers.VerifyCommandOption(_command.Options, "github-org", true);
@@ -58,6 +58,7 @@ public void Should_Have_Options()
         TestHelpers.VerifyCommandOption(_command.Options, "keep-archive", false);
         TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false);
         TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "use-github-storage", false, true);
     }
 
     [Fact]

src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs

@@ -35,6 +35,7 @@ public GenerateScriptCommand() : base(
         AddOption(AwsRegion);
         AddOption(KeepArchive);
         AddOption(NoSslVerify);
+        AddOption(UseGithubStorage);
     }
 
     public Option<string> BbsServerUrl { get; } = new(
@@ -124,6 +125,12 @@ public GenerateScriptCommand() : base(
         Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com"
     };
 
+    public Option<bool> UseGithubStorage { get; } = new("--use-github-storage")
+    {
+        IsHidden = true,
+        Description = "Enables multipart uploads to a GitHub owned storage for use during migration",
+    };
+
     public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandArgs args, IServiceProvider sp)
     {
         if (args is null)

src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs

@@ -27,6 +27,7 @@ public class GenerateScriptCommandArgs : CommandArgs
     public bool KeepArchive { get; set; }
     public bool NoSslVerify { get; set; }
     public string TargetApiUrl { get; set; }
+    public bool UseGithubStorage { get; set; }
 
     public override void Validate(OctoLogger log)
     {
@@ -35,6 +36,16 @@ public override void Validate(OctoLogger log)
             throw new OctoshiftCliException("--no-ssl-verify can only be provided with --bbs-server-url.");
         }
 
+        if (AwsBucketName.HasValue() && UseGithubStorage)
+        {
+            throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations.");
+        }
+
+        if (AwsRegion.HasValue() && UseGithubStorage)
+        {
+            throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations.");
+        }
+
         if (SshPort == 7999)
         {
             log?.LogWarning("--ssh-port is set to 7999, which is the default port that Bitbucket Server and Bitbucket Data Center use for Git operations over SSH. This is probably the wrong value, because --ssh-port should be configured with the SSH port used to manage the server where Bitbucket Server/Bitbucket Data Center is running, not the port used for Git operations over SSH.");

src/bbs2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs

@@ -71,7 +71,7 @@ private async Task<string> GenerateScript(GenerateScriptCommandArgs args)
             content.AppendLine(VALIDATE_AWS_ACCESS_KEY_ID);
             content.AppendLine(VALIDATE_AWS_SECRET_ACCESS_KEY);
         }
-        else
+        else if (!args.UseGithubStorage)
         {
             content.AppendLine(VALIDATE_AZURE_STORAGE_CONNECTION_STRING);
         }
@@ -135,9 +135,10 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bb
         var noSslVerify = args.NoSslVerify ? " --no-ssl-verify" : "";
         var targetRepoVisibility = " --target-repo-visibility private";
         var targetApiUrlOption = args.TargetApiUrl.HasValue() ? $" --target-api-url \"{args.TargetApiUrl}\"" : "";
+        var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : "";
 
         return $"gh bbs2gh migrate-repo{targetApiUrlOption}{bbsServerUrlOption}{bbsUsernameOption}{bbsSharedHomeOption}{bbsProjectOption}{bbsRepoOption}{sshArchiveDownloadOptions}" +
-               $"{smbArchiveDownloadOptions}{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}";
+               $"{smbArchiveDownloadOptions}{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}{githubStorageOption}";
     }
 
     private string Exec(string script) => Wrap(script, "Exec");