-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Flattening work by @neildsouth to add Azure Blob Storage support
Signed-off-by: Alex Woodhead <[email protected]>
- Loading branch information
1 parent
1757165
commit b1b2a76
Showing
23 changed files
with
1,434 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/AssemblyInfo.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| /* | ||
| * Copyright 2022 MONAI Consortium | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| using Xunit; | ||
| //Optional | ||
| [assembly: CollectionBehavior(DisableTestParallelization = true)] | ||
| //Optional | ||
| [assembly: TestCaseOrderer("Xunit.Extensions.Ordering.TestCaseOrderer", "Xunit.Extensions.Ordering")] | ||
| //Optional | ||
| [assembly: TestCollectionOrderer("Xunit.Extensions.Ordering.CollectionOrderer", "Xunit.Extensions.Ordering")] |
7 changes: 7 additions & 0 deletions
7
src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Class1.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| namespace Monai.Deploy.Storage.AzureBlob.Tests | ||
| { | ||
| public class Class1 | ||
| { | ||
|
|
||
| } | ||
| } |
200 changes: 200 additions & 0 deletions
200
...ugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureBlobServiceTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,200 @@ | ||
| using FluentAssertions; | ||
| using Microsoft.Extensions.Logging; | ||
| using Moq; | ||
| using Xunit; | ||
| using Xunit.Extensions.Ordering; | ||
|
|
||
| namespace Monai.Deploy.Storage.AzureBlob.Tests.Integration | ||
| { | ||
| [Order(0), Collection("AzureBlobStorage")] | ||
| public class AzureBlobServiceTests | ||
| { | ||
| private readonly Mock<ILogger<AzureBlobStorageService>> _logger; | ||
| private readonly AzureBlobStorageFixture _fixture; | ||
| private readonly AzureBlobStorageService _azureBlobService; | ||
| private readonly string _testFileName; | ||
| private readonly string _testFileNameCopy; | ||
|
|
||
| public AzureBlobServiceTests(AzureBlobStorageFixture fixture) | ||
| { | ||
| _logger = new Mock<ILogger<AzureBlobStorageService>>(); | ||
| _fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); | ||
| _azureBlobService = new AzureBlobStorageService(_fixture.ClientFactory, _fixture.Configurations, _logger.Object); | ||
| _testFileName = $"Tao-Te-Ching/Laozi/chapter-one.zip"; | ||
| _testFileNameCopy = $"Tao-Te-Ching/Laozi/chapter-one=backup.zip"; | ||
| } | ||
|
|
||
|
|
||
| [Fact, Order(1)] | ||
| public async Task S01_GivenABucketToAzureBlob() | ||
| { | ||
| var exception = await Record.ExceptionAsync(async () => | ||
| { | ||
| //await _azureBlobService.CreateFolderAsync(_fixture.ContainerName, ""); | ||
| var containerClient = _fixture.ClientFactory.GetBlobContainerClient(_fixture.ContainerName); | ||
| await containerClient.CreateIfNotExistsAsync().ConfigureAwait(false); | ||
| }).ConfigureAwait(false); | ||
|
|
||
| Assert.Null(exception); | ||
| } | ||
|
|
||
| [Fact, Order(2)] | ||
| public async Task S02_GivenASetOfDataAvailableToAzureBlob() | ||
| { | ||
| var exception = await Record.ExceptionAsync(async () => | ||
| { | ||
| await _fixture.GenerateAndUploadData().ConfigureAwait(false); | ||
| }).ConfigureAwait(false); | ||
|
|
||
| Assert.Null(exception); | ||
| } | ||
|
|
||
| [Theory, Order(3)] | ||
| [InlineData(null, 4)] | ||
| [InlineData("dir-1/", 1)] | ||
| [InlineData("dir-2/", 2)] | ||
| public async Task S03_WhenListObjectsAsyncIsCalled_ExpectItToListObjectsBasedOnParameters(string? prefix, int count) | ||
| { | ||
| var actual = await _azureBlobService.ListObjectsAsync(_fixture.ContainerName, prefix, true).ConfigureAwait(false); | ||
|
|
||
| actual.Should().NotBeEmpty() | ||
| .And.HaveCount(count); | ||
|
|
||
| var expected = _fixture.Files.ToList(); | ||
| if (prefix is not null) | ||
| { | ||
| expected = expected.Where(p => p.StartsWith(prefix)).ToList(); | ||
| } | ||
| actual.Select(p => p.FilePath).Should().BeEquivalentTo(expected); | ||
| } | ||
|
|
||
| [Fact, Order(4)] | ||
| public async Task S04_WhenVerifyObjectsExistAsyncIsCalled_ExpectToReturnAll() | ||
| { | ||
| var actual = await _azureBlobService.VerifyObjectsExistAsync(_fixture.ContainerName, _fixture.Files).ConfigureAwait(false); | ||
|
|
||
| actual.Should().NotBeEmpty() | ||
| .And.HaveCount(_fixture.Files.Count); | ||
|
|
||
| actual.Should().ContainValues(true); | ||
| } | ||
|
|
||
| [Fact, Order(5)] | ||
| public async Task S05_GivenAFileUploadedToAzureBlob() | ||
| { | ||
| var data = _fixture.GetRandomBytes(); | ||
| var stream = new MemoryStream(data); | ||
| await _azureBlobService.PutObjectAsync(_fixture.ContainerName, _testFileName, stream, data.Length, "application/binary", null).ConfigureAwait(false); | ||
|
|
||
| var callback = (Stream stream) => | ||
| { | ||
| var actual = new MemoryStream(); | ||
| stream.CopyTo(actual); | ||
| actual.ToArray().Should().Equal(data); | ||
| }; | ||
| var client = _fixture.ClientFactory.GetBlobClient(_fixture.ContainerName, _testFileName); | ||
|
|
||
| var fileContentsStream = new MemoryStream(); | ||
| await client.DownloadToAsync(fileContentsStream).ConfigureAwait(false); | ||
| fileContentsStream.ToArray().Should().Equal(data); | ||
|
|
||
| var prop = await client.GetPropertiesAsync().ConfigureAwait(false); | ||
| prop.Value.ContentLength.Should().Be(data.Length); | ||
| } | ||
|
|
||
| [Fact, Order(6)] | ||
| public async Task S06_ExpectTheFileToBeBeDownloadable() | ||
| { | ||
| var stream = await _azureBlobService.GetObjectAsync(_fixture.ContainerName, _testFileName).ConfigureAwait(false); | ||
| Assert.NotNull(stream); | ||
| var ms = new MemoryStream(); | ||
| stream.CopyTo(ms); | ||
| var data = ms.ToArray(); | ||
|
|
||
| var original = await DownloadData(_testFileName).ConfigureAwait(false); | ||
|
|
||
| Assert.NotNull(original); | ||
|
|
||
| data.Should().Equal(original); | ||
| } | ||
|
|
||
| [Fact, Order(7)] | ||
| public async Task S07_GivenACopyOfTheFile() | ||
| { | ||
| await _azureBlobService.CopyObjectAsync(_fixture.ContainerName, _testFileName, _fixture.ContainerName, _testFileNameCopy).ConfigureAwait(false); | ||
|
|
||
| var original = await DownloadData(_testFileName).ConfigureAwait(false); | ||
| var copy = await DownloadData(_testFileNameCopy).ConfigureAwait(false); | ||
|
|
||
| Assert.NotNull(original); | ||
| Assert.NotNull(copy); | ||
|
|
||
| copy.Should().Equal(original); | ||
| } | ||
|
|
||
| [Fact, Order(8)] | ||
| public async Task S08_ExpectedBothOriginalAndCopiedToExist() | ||
| { | ||
| var files = new List<string>() { _testFileName, _testFileNameCopy, "file-does-not-exist" }; | ||
| var expectedResults = new List<bool>() { true, true, false }; | ||
| var results = await _azureBlobService.VerifyObjectsExistAsync(_fixture.ContainerName, files).ConfigureAwait(false); | ||
|
|
||
| Assert.NotNull(results); | ||
|
|
||
| results.Should().ContainKeys(files); | ||
| results.Should().ContainValues(expectedResults); | ||
|
|
||
| for (var i = 0; i < files.Count; i++) | ||
| { | ||
| var file = files[i]; | ||
| var result = await _azureBlobService.VerifyObjectExistsAsync(_fixture.ContainerName, file).ConfigureAwait(false); | ||
| Assert.Equal(expectedResults[i], result); | ||
| } | ||
| } | ||
|
|
||
| [Fact, Order(9)] | ||
| public async Task S09_GivenADirectoryCreatedToAzureBlob() | ||
| { | ||
| var folderName = "my-folder"; | ||
| await _azureBlobService.CreateFolderAsync(_fixture.ContainerName, folderName).ConfigureAwait(false); | ||
| var result = await _azureBlobService.VerifyObjectExistsAsync(_fixture.ContainerName, $"{folderName}/stubFile.txt").ConfigureAwait(false); | ||
|
|
||
| Assert.True(result); | ||
| } | ||
|
|
||
| [Fact, Order(10)] | ||
| public async Task S10_ExpectTheDirectoryToBeRemovable() | ||
| { | ||
| var folderName = "my - folder / stubFile.txt"; | ||
| await _azureBlobService.RemoveObjectAsync(_fixture.ContainerName, folderName).ConfigureAwait(false); | ||
| var result = await _azureBlobService.VerifyObjectExistsAsync(_fixture.ContainerName, $"{folderName}/stubFile.txt").ConfigureAwait(false); | ||
| Assert.False(result); | ||
|
|
||
| var files = new List<string>() { _testFileName, _testFileNameCopy, "file-does-not-exist" }; | ||
| await _azureBlobService.RemoveObjectsAsync(_fixture.ContainerName, files).ConfigureAwait(false); | ||
| } | ||
|
|
||
| [Fact, Order(11)] | ||
| public async Task S11_ExpectTheFilesToBeRemovable() | ||
| { | ||
| var files = new List<string>() { _testFileName, _testFileNameCopy, "file-does-not-exist" }; | ||
| await _azureBlobService.RemoveObjectsAsync(_fixture.ContainerName, files).ConfigureAwait(false); | ||
|
|
||
| for (var i = 0; i < files.Count; i++) | ||
| { | ||
| var file = files[i]; | ||
| var result = await _azureBlobService.VerifyObjectExistsAsync(_fixture.ContainerName, file).ConfigureAwait(false); | ||
| Assert.False(result); | ||
| } | ||
| } | ||
|
|
||
| private async Task<byte[]> DownloadData(string filename) | ||
| { | ||
| var copiedStream = new MemoryStream(); | ||
|
|
||
| var client = _fixture.ClientFactory.GetBlobClient(_fixture.ContainerName, filename); | ||
| await client.DownloadToAsync(copiedStream).ConfigureAwait(false); | ||
| return copiedStream.ToArray(); | ||
| } | ||
| } | ||
| } |
107 changes: 107 additions & 0 deletions
107
...ins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureBlobStorageFixture.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| /* | ||
| * Copyright 2022 MONAI Consortium | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| using Microsoft.Extensions.Logging; | ||
| using Microsoft.Extensions.Options; | ||
| using Monai.Deploy.Storage.Configuration; | ||
| using Moq; | ||
| using Xunit; | ||
|
|
||
| namespace Monai.Deploy.Storage.AzureBlob.Tests.Integration | ||
| { | ||
| [CollectionDefinition("AzureBlobStorage")] | ||
| public class AzureBlobStorageCollection : ICollectionFixture<AzureBlobStorageFixture> | ||
| { | ||
| // This class has no code, and is never created. Its purpose is simply | ||
| // to be the place to apply [CollectionDefinition] and all the | ||
| // ICollectionFixture<> interfaces. | ||
| } | ||
|
|
||
| public class AzureBlobStorageFixture : IAsyncDisposable | ||
| { | ||
| const int MaxFileSize = 104857600; | ||
| private readonly Random _random; | ||
| private readonly List<string> _files; | ||
|
|
||
| public IOptions<StorageServiceConfiguration> Configurations { get; } | ||
| public AzureBlobClientFactory ClientFactory { get; } | ||
| public IReadOnlyList<string> Files { get => _files; } | ||
| public string ContainerName { get; } | ||
|
|
||
| public AzureBlobStorageFixture() | ||
| { | ||
| _random = new Random(); | ||
| _files = new List<string>(); | ||
|
|
||
| Configurations = Options.Create(new StorageServiceConfiguration()); | ||
| Configurations.Value.Settings.Add(ConfigurationKeys.ConnectionString, "UseDevelopmentStorage=true"); | ||
|
|
||
| ClientFactory = new AzureBlobClientFactory(Configurations, new Mock<ILogger<AzureBlobClientFactory>>().Object); | ||
| ContainerName = $"md-test-{_random.Next(1000)}"; | ||
| } | ||
|
|
||
| internal async Task GenerateAndUploadData() | ||
| { | ||
| await GenerateAndUploadFile($"{Guid.NewGuid()}").ConfigureAwait(false); | ||
| await GenerateAndUploadFile($"dir-1/{Guid.NewGuid()}").ConfigureAwait(false); | ||
| await GenerateAndUploadFile($"dir-2/{Guid.NewGuid()}").ConfigureAwait(false); | ||
| await GenerateAndUploadFile($"dir-2/a/b/{Guid.NewGuid()}").ConfigureAwait(false); | ||
| } | ||
|
|
||
| private async Task GenerateAndUploadFile(string filePath) | ||
| { | ||
| var data = GetRandomBytes(); | ||
| var stream = new MemoryStream(data); | ||
| var client = ClientFactory.GetBlobClient(ContainerName, filePath); | ||
| await client.UploadAsync(stream).ConfigureAwait(false); | ||
| _files.Add(filePath); | ||
| } | ||
|
|
||
| public byte[] GetRandomBytes() | ||
| { | ||
| return new byte[_random.Next(1, MaxFileSize)]; | ||
| } | ||
|
|
||
| public async ValueTask DisposeAsync() | ||
| { | ||
| await RemoveData().ConfigureAwait(false); | ||
| await RemoveBucket().ConfigureAwait(false); | ||
| } | ||
|
|
||
| private async Task RemoveBucket() | ||
| { | ||
| var client = ClientFactory.GetBlobContainerClient(ContainerName); | ||
| var exists = await client.ExistsAsync().ConfigureAwait(false); | ||
| if (exists) | ||
| { | ||
| var resultSegment = client.GetBlobsAsync(prefix: ContainerName).AsPages(default, 100); | ||
|
|
||
| await foreach (var blobPage in resultSegment) | ||
| { | ||
| foreach (var blobItem in blobPage.Values) | ||
| { | ||
| await ClientFactory.GetBlobClient(ContainerName, blobItem.Name).DeleteAsync().ConfigureAwait(false); | ||
| }; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private async Task RemoveData() | ||
| { | ||
| await RemoveBucket().ConfigureAwait(false); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.