From 57fecb1494584a2a542403e5d272b31b94b7bd66 Mon Sep 17 00:00:00 2001 From: Gabriel Roldan Date: Sun, 11 Aug 2024 00:06:43 -0300 Subject: [PATCH] Run Azure blob store integration tests with Azure and Azurite Azurite is run as a testcontainer, and the github CI build runs all integration tests for Java 11, 17, and 21. In order to save on Azure resources, online integration tests are run only if the Azurite based tests succeeded, using github secrets to set up the account, key, and container; and for Java 11 only. --- .github/workflows/azure-integration.yml | 82 +++++ geowebcache/azureblob/REAME.md | 34 ++ geowebcache/azureblob/pom.xml | 71 ++++ .../geowebcache/azure/AzureBlobStoreData.java | 4 +- .../org/geowebcache/azure/AzureClient.java | 2 +- .../azure/AzureBlobStoreConformanceTest.java | 18 +- ...ava => AzureBlobStoreIntegrationTest.java} | 11 +- .../azure/AzureBlobStoreSuitabilityTest.java | 54 +-- .../AzuriteAzureBlobStoreConformanceIT.java | 51 +++ .../AzuriteAzureBlobStoreIntegrationIT.java | 51 +++ .../AzuriteAzureBlobStoreSuitabilityIT.java | 67 ++++ .../OnlineAzureBlobStoreConformanceIT.java | 34 ++ .../OnlineAzureBlobStoreIntegrationIT.java} | 8 +- .../OnlineAzureBlobStoreSuitabilityIT.java | 70 ++++ .../{ => tests/online}/PropertiesLoader.java | 2 +- .../online}/TemporaryAzureFolder.java | 4 +- .../azure/AzuriteContainer.java | 262 ++++++++++++++ .../azure/AzuriteContainerLegacyProxy.java | 331 ++++++++++++++++++ geowebcache/core/pom.xml | 5 + .../storage/AbstractBlobStoreTest.java | 36 +- geowebcache/pom.xml | 56 ++- geowebcache/s3storage/pom.xml | 6 +- geowebcache/sqlite/pom.xml | 5 + 23 files changed, 1172 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/azure-integration.yml create mode 100644 geowebcache/azureblob/REAME.md rename geowebcache/azureblob/src/test/java/org/geowebcache/azure/{AbstractAzureBlobStoreIntegrationTest.java => AzureBlobStoreIntegrationTest.java} (98%) create mode 100644 geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreConformanceIT.java create mode 100644 geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreIntegrationIT.java create mode 100644 geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreSuitabilityIT.java create mode 100644 geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreConformanceIT.java rename geowebcache/azureblob/src/test/java/org/geowebcache/azure/{OnlineAzureBlobStoreIntegrationTest.java => tests/online/OnlineAzureBlobStoreIntegrationIT.java} (85%) create mode 100644 geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreSuitabilityIT.java rename geowebcache/azureblob/src/test/java/org/geowebcache/azure/{ => tests/online}/PropertiesLoader.java (98%) rename geowebcache/azureblob/src/test/java/org/geowebcache/azure/{ => tests/online}/TemporaryAzureFolder.java (97%) create mode 100644 geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainer.java create mode 100644 geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainerLegacyProxy.java diff --git a/.github/workflows/azure-integration.yml b/.github/workflows/azure-integration.yml new file mode 100644 index 000000000..58e98d44a --- /dev/null +++ b/.github/workflows/azure-integration.yml @@ -0,0 +1,82 @@ +name: Azure BlobStore Integration + +on: + push: + tags: + - "**" + pull_request: + paths: + - ".github/workflows/azure-integration.yml" + - "pom.xml" + - "geowebcache/pom.xml" + - "geowebcache/core/**" + - "geowebcache/azureblob/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + azurite: + name: Azurite container + runs-on: ubuntu-latest + strategy: + matrix: + java-version: [ 11, 17, 21 ] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java-version }} + cache: 'maven' + + - name: Tests against Azurite TestContainers + #-PexcludeOnlineTests includes Azurite container tests and excludes Azure online tests + run: | + mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \ + -Ponline,excludeAzureOnlineTests \ + -DskipTests=true \ + -DskipITs=false -B -ntp + + - name: Remove SNAPSHOT jars from repository + run: | + find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} + + azure: + name: Azure online + #if: github.repository == 'geowebcache/geowebcache' + runs-on: ubuntu-latest + needs: azurite + if: | + always() && + !contains(needs.*.result, 'cancelled') && + !contains(needs.*.result, 'failure') + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 11 + cache: 'maven' + + - name: Tests against Azure + env: + azure_account: ${{ secrets.AZURE_ACCOUNT }} + azure_account_key: ${{ secrets.AZURE_ACCOUNT_KEY }} + azure_container: ${{ secrets.AZURE_CONTAINER }} + if: ${{ env.azure_account != null }} && ${{ env.azure_account_key != null }} + run: | #-PexcludeDockerTests includes Azure online tests and excludes Azurite container tests + echo "accountName=$azure_account" > $HOME/.gwc_azure_tests.properties + echo "accountKey=$azure_account_key" >> $HOME/.gwc_azure_tests.properties + echo "container=$azure_container" >> $HOME/.gwc_azure_tests.properties + echo 'maxConnections=8' >> $HOME/.gwc_azure_tests.properties + echo 'useHTTPS=true' >> $HOME/.gwc_azure_tests.properties + mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \ + -Ponline,excludeDockerTests \ + -DskipTests=true \ + -DskipITs=false -B -ntp + + - name: Remove SNAPSHOT jars from repository + run: | + find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} diff --git a/geowebcache/azureblob/REAME.md b/geowebcache/azureblob/REAME.md new file mode 100644 index 000000000..821f33df2 --- /dev/null +++ b/geowebcache/azureblob/REAME.md @@ -0,0 +1,34 @@ +# Azure BlobStore + +GeoWebCache `BlobStore` implementation to store tiles on [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs) + + +## Building + +`mvn install|test|verify` will run only unit tests. + +In order to run the integration tests, build with the `-Ponline` maven profile. For example: + +``` +mvn verify -Ponline +``` + +There are two sets of integration tests: + +- `org.geowebcache.azure.tests.container.*IT.java`: run integration tests using [Testcontainers](https://testcontainers.com/), with a Microsoft [Azurite](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite) Docker image. +- `org.geowebcache.azure.tests.online.*IT.java`: run integration tests against a real Azure account, only if there's a configuration file in `$HOME/.gwc_azure_tests.properties`, which must have the following contents: + +``` +accountName= +accountKey= +container= +useHTTPS=true +maxConnections= +``` + +## Continuous integration + +There's a Github Actions CI job defined at `/.github/workflows/azure-integration.yml` +that will run the Docker-based integration tests first, and then the online ones against +a real Azure account, using Github repository secrets to define the values for the +`$HOME/.gwc_azure_tests.properties` file's `accountName`, `accountSecret`, and `container` keys. diff --git a/geowebcache/azureblob/pom.xml b/geowebcache/azureblob/pom.xml index 7da2dcfdf..88538c828 100644 --- a/geowebcache/azureblob/pom.xml +++ b/geowebcache/azureblob/pom.xml @@ -51,5 +51,76 @@ javax.servlet-api provided + + org.testcontainers + testcontainers + test + + + org.awaitility + awaitility + test + + + + online + + false + + + + + maven-failsafe-plugin + + 1 + false + + + + + + + + excludeAzureOnlineTests + + false + + + + + maven-failsafe-plugin + + 1 + false + + org.geowebcache.azure.tests.online.*IT + + + + + + + + + excludeDockerTests + + false + + + + + maven-failsafe-plugin + + 1 + false + + org.geowebcache.azure.tests.container.*IT + + + + + + + diff --git a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java index 0b7809e3c..0bd12e95a 100644 --- a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java +++ b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java @@ -22,7 +22,7 @@ * Azure Blobstore type-resolved data from a {@link AzureBlobStoreInfo} using enviroment variables * if enabled. */ -class AzureBlobStoreData { +public class AzureBlobStoreData { private String container; private String prefix; @@ -36,7 +36,7 @@ class AzureBlobStoreData { private String proxyPassword; private String serviceURL; - AzureBlobStoreData() {} + public AzureBlobStoreData() {} public AzureBlobStoreData( final AzureBlobStoreInfo storeInfo, final GeoWebCacheEnvironment environment) { diff --git a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java index b71421c14..c56deab32 100644 --- a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java +++ b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java @@ -50,7 +50,7 @@ import org.geowebcache.util.URLs; import org.springframework.http.HttpStatus; -class AzureClient implements Closeable { +public class AzureClient implements Closeable { private final NettyClient.Factory factory; private AzureBlobStoreData configuration; diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java index cc6516fad..e38cc0ad9 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java @@ -25,25 +25,25 @@ import java.util.stream.Stream; import org.easymock.EasyMock; import org.geowebcache.GeoWebCacheException; +import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT; +import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreConformanceIT; import org.geowebcache.layer.TileLayer; import org.geowebcache.layer.TileLayerDispatcher; import org.geowebcache.locks.LockProvider; import org.geowebcache.locks.NoOpLockProvider; import org.geowebcache.storage.AbstractBlobStoreTest; -import org.junit.Assume; -import org.junit.Rule; -public class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest { - public PropertiesLoader testConfigLoader = new PropertiesLoader(); +/** + * @see OnlineAzureBlobStoreConformanceIT + * @see AzuriteAzureBlobStoreConformanceIT + */ +public abstract class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest { - @Rule - public TemporaryAzureFolder tempFolder = - new TemporaryAzureFolder(testConfigLoader.getProperties()); + protected abstract AzureBlobStoreData getConfiguration(); @Override public void createTestUnit() throws Exception { - Assume.assumeTrue(tempFolder.isConfigured()); - AzureBlobStoreData config = tempFolder.getConfig(); + AzureBlobStoreData config = getConfiguration(); TileLayerDispatcher layers = createMock(TileLayerDispatcher.class); LockProvider lockProvider = new NoOpLockProvider(); diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AbstractAzureBlobStoreIntegrationTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreIntegrationTest.java similarity index 98% rename from geowebcache/azureblob/src/test/java/org/geowebcache/azure/AbstractAzureBlobStoreIntegrationTest.java rename to geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreIntegrationTest.java index 9ddb3aded..2c4ba5f34 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AbstractAzureBlobStoreIntegrationTest.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreIntegrationTest.java @@ -42,6 +42,8 @@ import java.util.Map; import java.util.logging.Logger; import org.geotools.util.logging.Logging; +import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreIntegrationIT; +import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreIntegrationIT; import org.geowebcache.config.DefaultGridsets; import org.geowebcache.grid.GridSet; import org.geowebcache.grid.GridSetBroker; @@ -69,10 +71,13 @@ * Integration tests for {@link AzureBlobStore}. * *

This is an abstract class for both online and offline integration tests. + * + * @see OnlineAzureBlobStoreIntegrationIT + * @see AzuriteAzureBlobStoreIntegrationIT */ -public abstract class AbstractAzureBlobStoreIntegrationTest { +public abstract class AzureBlobStoreIntegrationTest { - private static Logger log = Logging.getLogger(PropertiesLoader.class.getName()); + private static Logger log = Logging.getLogger(AzureBlobStoreIntegrationTest.class.getName()); private static final String DEFAULT_FORMAT = "png"; @@ -80,8 +85,6 @@ public abstract class AbstractAzureBlobStoreIntegrationTest { private static final String DEFAULT_LAYER = "topp:world"; - public PropertiesLoader testConfigLoader = new PropertiesLoader(); - private AzureBlobStore blobStore; protected abstract AzureBlobStoreData getConfiguration(); diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java index 4612b9474..22e6688b9 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java @@ -17,11 +17,12 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItemInArray; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeFalse; import io.reactivex.Flowable; import java.nio.ByteBuffer; import org.easymock.EasyMock; +import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreSuitabilityIT; +import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreSuitabilityIT; import org.geowebcache.layer.TileLayerDispatcher; import org.geowebcache.locks.LockProvider; import org.geowebcache.locks.NoOpLockProvider; @@ -30,23 +31,14 @@ import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.Before; -import org.junit.Rule; import org.junit.experimental.theories.DataPoints; -import org.junit.experimental.theories.Theories; -import org.junit.runner.RunWith; -import org.junit.runners.model.FrameworkMethod; -import org.junit.runners.model.InitializationError; -import org.junit.runners.model.Statement; import org.springframework.http.HttpStatus; -@RunWith(AzureBlobStoreSuitabilityTest.MyTheories.class) -public class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest { - - public PropertiesLoader testConfigLoader = new PropertiesLoader(); - - @Rule - public TemporaryAzureFolder tempFolder = - new TemporaryAzureFolder(testConfigLoader.getProperties()); +/** + * @see OnlineAzureBlobStoreSuitabilityIT + * @see AzuriteAzureBlobStoreSuitabilityIT + */ +public abstract class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest { @DataPoints public static String[][] persistenceLocations = { @@ -67,6 +59,10 @@ public void setup() throws Exception { EasyMock.replay(tld); } + protected abstract AzureBlobStoreData getConfiguration(); + + protected abstract AzureClient getClient(); + @SuppressWarnings("unchecked") @Override protected Matcher existing() { @@ -81,13 +77,12 @@ protected Matcher empty() { @Override public BlobStore create(Object dir) throws Exception { - AzureBlobStoreData info = tempFolder.getConfig(); + AzureBlobStoreData info = getConfiguration(); for (String path : (String[]) dir) { String fullPath = info.getPrefix() + "/" + path; ByteBuffer byteBuffer = ByteBuffer.wrap("testAbc".getBytes()); int statusCode = - tempFolder - .getClient() + getClient() .getBlockBlobURL(fullPath) .upload(Flowable.just(byteBuffer), byteBuffer.limit()) .blockingGet() @@ -96,27 +91,4 @@ public BlobStore create(Object dir) throws Exception { } return new AzureBlobStore(info, tld, locks); } - - // Sorry, this bit of evil makes the Theories runner gracefully ignore the - // tests if Azure is unavailable. There's probably a better way to do this. - public static class MyTheories extends Theories { - - public MyTheories(Class klass) throws InitializationError { - super(klass); - } - - @Override - public Statement methodBlock(FrameworkMethod method) { - if (new PropertiesLoader().getProperties().containsKey("container")) { - return super.methodBlock(method); - } else { - return new Statement() { - @Override - public void evaluate() { - assumeFalse("Azure unavailable", true); - } - }; - } - } - } } diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreConformanceIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreConformanceIT.java new file mode 100644 index 000000000..58c0abfab --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreConformanceIT.java @@ -0,0 +1,51 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.azure.tests.container; + +import org.geowebcache.azure.AzureBlobStoreConformanceTest; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.testcontainers.azure.AzuriteContainer; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TestName; + +/** + * Runs {@link AzureBlobStoreConformanceTest} tests against a local ephemeral Docker container using + * {@link AzuriteContainer}. + * + *

If there's no Docker environment, the test is {@link AzuriteContainer#disabledWithoutDocker() + * ignored} + */ +public class AzuriteAzureBlobStoreConformanceIT extends AzureBlobStoreConformanceTest { + + /** + * Use "legacy" container to work with {@literal + * com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as + * AzuriteContainer.legacy().debugLegacy() to print out request/response information for + * debugging purposes + */ + @ClassRule + public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker(); + + /** Used to get a per-test case Azure container */ + @Rule public TestName testName = new TestName(); + + @Override + protected AzureBlobStoreData getConfiguration() { + // container must be lower case or we get a 400 bad request + String container = testName.getMethodName().toLowerCase(); + return azurite.getConfiguration(container); + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreIntegrationIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreIntegrationIT.java new file mode 100644 index 000000000..9f4fba858 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreIntegrationIT.java @@ -0,0 +1,51 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.azure.tests.container; + +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureBlobStoreIntegrationTest; +import org.geowebcache.testcontainers.azure.AzuriteContainer; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TestName; + +/** + * Runs {@link AzureBlobStoreIntegrationTest} tests against a local ephemeral Docker container using + * {@link AzuriteContainer}. + * + *

If there's no Docker environment, the test is {@link AzuriteContainer#disabledWithoutDocker() + * ignored} + */ +public class AzuriteAzureBlobStoreIntegrationIT extends AzureBlobStoreIntegrationTest { + + /** + * Use "legacy" container to work with {@literal + * com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as + * AzuriteContainer.legacy().debugLegacy() to print out request/response information for + * debugging purposes + */ + @ClassRule + public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker(); + + /** Used to get a per-test case Azure container */ + @Rule public TestName testName = new TestName(); + + @Override + protected AzureBlobStoreData getConfiguration() { + // container must be lower case or we get a 400 bad request + String container = testName.getMethodName().toLowerCase(); + return azurite.getConfiguration(container); + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreSuitabilityIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreSuitabilityIT.java new file mode 100644 index 000000000..3cdc8c164 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreSuitabilityIT.java @@ -0,0 +1,67 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.azure.tests.container; + +import java.io.UncheckedIOException; +import org.geowebcache.azure.AzureBlobStoreConformanceTest; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureBlobStoreSuitabilityTest; +import org.geowebcache.azure.AzureClient; +import org.geowebcache.storage.StorageException; +import org.geowebcache.testcontainers.azure.AzuriteContainer; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TestName; + +/** + * Runs {@link AzureBlobStoreConformanceTest} tests against a local ephemeral Docker container using + * {@link AzuriteContainer}. + * + *

If there's no Docker environment, the test is {@link AzuriteContainer#disabledWithoutDocker() + * ignored} + */ +public class AzuriteAzureBlobStoreSuitabilityIT extends AzureBlobStoreSuitabilityTest { + + /** + * Use "legacy" container to work with {@literal + * com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as + * AzuriteContainer.legacy().debugLegacy() to print out request/response information for + * debugging purposes + */ + @ClassRule + public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker(); + + /** Used to get a per-test case Azure container */ + @Rule public TestName testName = new TestName(); + + @Override + protected AzureBlobStoreData getConfiguration() { + // container must be lower case or we get a 400 bad request + String container = testName.getMethodName().toLowerCase(); + AzureBlobStoreData configuration = azurite.getConfiguration(container); + // AzureBlobStoreSuitabilityTest requires a prefix to be set + configuration.setPrefix("test-prefix"); + return configuration; + } + + @Override + protected AzureClient getClient() { + try { + return new AzureClient(getConfiguration()); + } catch (StorageException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreConformanceIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreConformanceIT.java new file mode 100644 index 000000000..ab89916d5 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreConformanceIT.java @@ -0,0 +1,34 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + * @author Kevin Smith, Boundless, 2017 + */ +package org.geowebcache.azure.tests.online; + +import org.geowebcache.azure.AzureBlobStoreConformanceTest; +import org.geowebcache.azure.AzureBlobStoreData; +import org.junit.Assume; +import org.junit.Rule; + +public class OnlineAzureBlobStoreConformanceIT extends AzureBlobStoreConformanceTest { + public PropertiesLoader testConfigLoader = new PropertiesLoader(); + + @Rule + public TemporaryAzureFolder tempFolder = + new TemporaryAzureFolder(testConfigLoader.getProperties()); + + @Override + protected AzureBlobStoreData getConfiguration() { + Assume.assumeTrue(tempFolder.isConfigured()); + return tempFolder.getConfig(); + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/OnlineAzureBlobStoreIntegrationTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreIntegrationIT.java similarity index 85% rename from geowebcache/azureblob/src/test/java/org/geowebcache/azure/OnlineAzureBlobStoreIntegrationTest.java rename to geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreIntegrationIT.java index cc002a7e8..25a07b89d 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/OnlineAzureBlobStoreIntegrationTest.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreIntegrationIT.java @@ -12,16 +12,20 @@ * * @author Andrea Aime, GeoSolutions, Copyright 2019 */ -package org.geowebcache.azure; +package org.geowebcache.azure.tests.online; import static org.junit.Assert.assertTrue; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureBlobStoreIntegrationTest; import org.junit.Assume; import org.junit.Rule; import org.junit.Test; import org.springframework.http.HttpStatus; -public class OnlineAzureBlobStoreIntegrationTest extends AbstractAzureBlobStoreIntegrationTest { +public class OnlineAzureBlobStoreIntegrationIT extends AzureBlobStoreIntegrationTest { + + private PropertiesLoader testConfigLoader = new PropertiesLoader(); @Rule public TemporaryAzureFolder tempFolder = diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreSuitabilityIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreSuitabilityIT.java new file mode 100644 index 000000000..c996d0752 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreSuitabilityIT.java @@ -0,0 +1,70 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + * @author Kevin Smith, Boundless, 2018 + */ +package org.geowebcache.azure.tests.online; + +import static org.junit.Assume.assumeFalse; + +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureBlobStoreSuitabilityTest; +import org.geowebcache.azure.AzureClient; +import org.junit.Rule; +import org.junit.experimental.theories.Theories; +import org.junit.runner.RunWith; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.Statement; + +@RunWith(OnlineAzureBlobStoreSuitabilityIT.MyTheories.class) +public class OnlineAzureBlobStoreSuitabilityIT extends AzureBlobStoreSuitabilityTest { + + public PropertiesLoader testConfigLoader = new PropertiesLoader(); + + @Rule + public TemporaryAzureFolder tempFolder = + new TemporaryAzureFolder(testConfigLoader.getProperties()); + + @Override + protected AzureBlobStoreData getConfiguration() { + return tempFolder.getConfig(); + } + + @Override + protected AzureClient getClient() { + return tempFolder.getClient(); + } + + // Sorry, this bit of evil makes the Theories runner gracefully ignore the + // tests if Azure is unavailable. There's probably a better way to do this. + public static class MyTheories extends Theories { + + public MyTheories(Class klass) throws InitializationError { + super(klass); + } + + @Override + public Statement methodBlock(FrameworkMethod method) { + if (new PropertiesLoader().getProperties().containsKey("container")) { + return super.methodBlock(method); + } else { + return new Statement() { + @Override + public void evaluate() { + assumeFalse("Azure unavailable", true); + } + }; + } + } + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/PropertiesLoader.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/PropertiesLoader.java similarity index 98% rename from geowebcache/azureblob/src/test/java/org/geowebcache/azure/PropertiesLoader.java rename to geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/PropertiesLoader.java index ca3e9c878..80c9887ce 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/PropertiesLoader.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/PropertiesLoader.java @@ -12,7 +12,7 @@ * * @author Andrea Aime, GeoSolutions, Copyright 2019 */ -package org.geowebcache.azure; +package org.geowebcache.azure.tests.online; import static com.google.common.base.Preconditions.checkArgument; diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/TemporaryAzureFolder.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/TemporaryAzureFolder.java similarity index 97% rename from geowebcache/azureblob/src/test/java/org/geowebcache/azure/TemporaryAzureFolder.java rename to geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/TemporaryAzureFolder.java index 71c5a2c9b..89774b3db 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/TemporaryAzureFolder.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/TemporaryAzureFolder.java @@ -12,7 +12,7 @@ * * @author Andrea Aime, GeoSolutions, Copyright 2019 */ -package org.geowebcache.azure; +package org.geowebcache.azure.tests.online; import static com.google.common.base.Preconditions.checkState; import static org.junit.Assert.assertTrue; @@ -22,6 +22,8 @@ import java.util.List; import java.util.Properties; import java.util.UUID; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureClient; import org.junit.rules.ExternalResource; import org.springframework.http.HttpStatus; diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainer.java b/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainer.java new file mode 100644 index 000000000..fb8d06b60 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainer.java @@ -0,0 +1,262 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.testcontainers.azure; + +import static java.lang.String.format; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import java.io.IOException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.UncheckedException; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT; +import org.junit.Assume; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Testcontainers container for AWS Azurite + * blobstore test environment. + * + *

Runs the Azurite + * Docker image for local Azure Storage development with testcontainers. + * + *

Azurite accepts the same well-known account and key used by the legacy Azure Storage Emulator: + * + *

    + *
  • Account name: {@code devstoreaccount1} + *
  • Account key: {@code + * Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==} + *
+ * + *

Usage: For Junit 4, use it as a {@code @Rule} or {@code @ClassRule}: + * + * 

+ * 
+ *   @Rule public AzuriteContainer azurite = AzuriteContainer.legacy();
+ * 
+ *
+ * + * works with the old {@code com.microsoft.azure:azure-storage-blob:jar:11.0.0} as a dependency. + * + * 
+ * 
+ *   @Rule public AzuriteContainer azurite = AzuriteContainer.legacy();
+ * 
+ *
+ * + * works with the latest {@code com.azure:azure-storage-blob:jar:12.27.0} as a dependency. + * + *

Sample test: + * + * 

+ * 
+ *   @ClassRule public static AzuriteContainer azurite = AzuriteContainer.legacy();
+ *
+ *   @Test
+ *   public void azureBlobStoreSmokeTest(){
+ *      String container = "testcontainer";//ought to be lower case
+ *      AzureBlobStoreData config = azurite.getConfiguration(container);
+ *      AzureBlobStore store = new AzureBlobStore(config, tileLayerDispatcher, lockProvider);
+ *      assertFalse(store.layerExists("layer1");
+ *   }
+ * 
+ *
+ */ +public class AzuriteContainer extends GenericContainer { + + private static final DockerImageName LATEST_IMAGE = + DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:latest"); + + private static final DockerImageName LEGACY_IMAGE = + DockerImageName.parse("arafato/azurite:2.6.5"); + + private final String accountName = "devstoreaccount1"; + private final String accountKey = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + + private final int blobsPort = 10_000; + + private AzuriteContainerLegacyProxy proxy; + + private final boolean doProxy; + + /** Whether to print request/response debug information when in {@link #legacy} mode */ + private boolean debugRequests; + + /** flag for {@link #disabledWithoutDocker()} */ + private boolean disabledWithoutDocker; + + private AzuriteContainer(DockerImageName imageName, boolean doProxy) { + super(imageName); + this.doProxy = doProxy; + super.setWaitStrategy(Wait.forListeningPort()); + super.addExposedPort(blobsPort); + } + + /** + * @return a container running {@code arafato/azurite:2.6.5} and {@link #getBlobsPort() proxied} + * to fix protocol discrepancies so it works correctly with older {@code + * com.microsoft.azure:azure-storage-blob} dependencies + */ + public static AzuriteContainer legacy() { + return new AzuriteContainer(LEGACY_IMAGE, true); + } + + /** @return a container running {@code mcr.microsoft.com/azure-storage/azurite:latest} */ + public static AzuriteContainer latest() { + return new AzuriteContainer(LATEST_IMAGE, false); + } + + /** + * Enables request/response debugging when in legacy mode + * + *

Sample output: + * + * 

+     * 
+     * routing GET http://localhost:44445/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png to GET http://localhost:33319/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png
+     * 	applied request header Authorization: SharedKey devstoreaccount1:6UeSk1Qf8XRibLI1sE3tasmDxOtVxGUSMDQqRUDIW9Y=
+     * 	applied request header x-ms-version: 2018-11-09
+     * 	applied request header x-ms-date: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied request header host: localhost
+     * 	applied request header x-ms-client-request-id: 526b726a-13af-49a3-b277-fdf645d77903
+     * 	applied request header User-Agent: Azure-Storage/11.0.0 (JavaJRE 11.0.23; Linux 6.8.0-39-generic)
+     * 	response: 200 OK
+     * 	applied response header X-Powered-By: Express
+     * 	applied response header ETag: "jzUOHaHcch36ue3TFspQaLiWSvo"
+     * 	applied response header Last-Modified: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied response header x-ms-version: 2016-05-31
+     * 	applied response header date: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied response header x-ms-request-id: 05130dd1-5672-11ef-a96b-c7f08f042b95
+     * 	applied response header accept-ranges: bytes
+     * 	applied response header x-ms-blob-type: BlockBlob
+     * 	applied response header x-ms-request-server-encrypted: false
+     * 	applied response header Content-Type: image/png
+     * 	Content-Type: image/png
+     * 
+     *
+ */ + public AzuriteContainer debugLegacy() { + this.debugRequests = true; + return this; + } + + /** + * Disables the tests using this testcontainer if there's no Docker environment available. + * + *

Same effect as JUnit 5's {@code + * org.testcontainers.junit.jupiter.@Testcontainers(disabledWithoutDocker = true)} + */ + public AzuriteContainer disabledWithoutDocker() { + this.disabledWithoutDocker = true; + return this; + } + + /** + * Overrides to apply the {@link Assume assumption} checking the Docker environment is available + * if {@link #disabledWithoutDocker() enabled}, so this test container can be used as a {@code + * ClassRule @ClassRule} and hence avoid running a container for each test case. + */ + @Override + @SuppressWarnings("deprecation") + public Statement apply(Statement base, Description description) { + if (disabledWithoutDocker) { + assumeTrue( + "Docker environment unavailable, ignoring test " + + AzuriteAzureBlobStoreConformanceIT.class.getSimpleName(), + DockerClientFactory.instance().isDockerAvailable()); + } + return super.apply(base, description); + } + + @Override + public void start() { + super.start(); + if (doProxy && proxy == null) { + int targetPort = getRealBlobsPort(); + proxy = new AzuriteContainerLegacyProxy(targetPort).debugRequests(debugRequests); + try { + proxy.start(); + } catch (IOException e) { + throw new UncheckedException(e); + } + } + } + + @Override + public void stop() { + super.stop(); + if (doProxy && null != proxy) { + try { + proxy.stop(); + } finally { + proxy = null; + } + } + } + + public String getAccountName() { + return accountName; + } + + public String getAccountKey() { + return accountKey; + } + + /** + * Returns the localhost port where the azurite blob storage service is running. + * + *

when in {@link #legacy() legacy} mode, a small http proxy is run and the proxy port is + * returned. The proxy fixes some protocol issues. For instance, re-writes the returned response + * headers {@code etag}, {@code last-modified}, and {@code content-type}, as {@code Etag}, + * {@code Last-Modified}, and {@code Content-Type}, respectively, as expected by the Netty + * version the legacy {@code com.microsoft.azure:azure-storage-blob} dependency transitively + * carries over. + */ + public int getBlobsPort() { + if (doProxy) { + if (proxy == null) throw new IllegalStateException(""); + return proxy.getLocalPort(); + } + return getRealBlobsPort(); + } + + int getRealBlobsPort() { + return super.getMappedPort(blobsPort); + } + + public String getBlobServiceUrl() { + return format("http://localhost:%d/%s", getBlobsPort(), getAccountName()); + } + + public AzureBlobStoreData getConfiguration(String container) { + assertTrue("Container must be lower case", StringUtils.isAllLowerCase(container)); + AzureBlobStoreData config = new AzureBlobStoreData(); + config.setServiceURL(getBlobServiceUrl()); + config.setAccountName(getAccountName()); + config.setAccountKey(getAccountKey()); + config.setMaxConnections(10); + config.setContainer(container); + return config; + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainerLegacyProxy.java b/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainerLegacyProxy.java new file mode 100644 index 000000000..6874f31a1 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainerLegacyProxy.java @@ -0,0 +1,331 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.testcontainers.azure; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.ServerSocket; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Stream; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.RequestLine; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.DefaultBHttpServerConnectionFactory; +import org.apache.http.impl.bootstrap.HttpServer; +import org.apache.http.impl.bootstrap.ServerBootstrap; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpProcessor; +import org.apache.http.protocol.HttpProcessorBuilder; +import org.apache.http.protocol.HttpRequestHandler; +import org.apache.http.protocol.ResponseConnControl; +import org.apache.http.protocol.ResponseContent; +import org.apache.http.util.EntityUtils; +import org.geotools.util.logging.Logging; + +/** + * A simple HTTP proxy to adapt some Azure Blob storage protocol issues to the netty version used by + * older {@code com.microsoft.azure:azure-storage-blob} dependencies. + * + *

For instance, re-writes the returned response headers {@code etag}, {@code last-modified}, and + * {@code content-type}, as {@code Etag}, {@code Last-Modified}, and {@code Content-Type}, + * respectively, as expected by the Netty version the legacy {@code + * com.microsoft.azure:azure-storage-blob} dependency transitively carries over. + * + *

Even though HTTP request and response headers should be case-insensitive, this older netty + * version ({@code 4.1.28}, and even newer ones) fail to parse the lower-case names returned by + * Azurite. + */ +class AzuriteContainerLegacyProxy { + public static Logger LOGGER = Logging.getLogger(AzuriteContainerLegacyProxy.class.getName()); + + private int localPort; + private HttpServer proxyServer; + + private int targetPort; + + private final AtomicBoolean started = new AtomicBoolean(); + + private boolean debug; + + AzuriteContainerLegacyProxy(int targetPort) { + this.targetPort = targetPort; + } + + /** + * @return the random port where the proxy server is running + * @throws IllegalStateException if the proxy is not {@link #start() running} + */ + public int getLocalPort() { + if (!started.get()) { + throw new IllegalStateException( + "Proxy not running, local port is allocated at start()"); + } + return localPort; + } + + /** + * Whether to print request/response debugging information to stderr. + * + *

Sample output: + * + * 

+     * 
+     * routing GET http://localhost:44445/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png to GET http://localhost:33319/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png
+     * 	applied request header Authorization: SharedKey devstoreaccount1:6UeSk1Qf8XRibLI1sE3tasmDxOtVxGUSMDQqRUDIW9Y=
+     * 	applied request header x-ms-version: 2018-11-09
+     * 	applied request header x-ms-date: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied request header host: localhost
+     * 	applied request header x-ms-client-request-id: 526b726a-13af-49a3-b277-fdf645d77903
+     * 	applied request header User-Agent: Azure-Storage/11.0.0 (JavaJRE 11.0.23; Linux 6.8.0-39-generic)
+     * 	response: 200 OK
+     * 	applied response header X-Powered-By: Express
+     * 	applied response header ETag: "jzUOHaHcch36ue3TFspQaLiWSvo"
+     * 	applied response header Last-Modified: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied response header x-ms-version: 2016-05-31
+     * 	applied response header date: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied response header x-ms-request-id: 05130dd1-5672-11ef-a96b-c7f08f042b95
+     * 	applied response header accept-ranges: bytes
+     * 	applied response header x-ms-blob-type: BlockBlob
+     * 	applied response header x-ms-request-server-encrypted: false
+     * 	applied response header Content-Type: image/png
+     * 	Content-Type: image/png
+     * 
+     *
+ */ + public AzuriteContainerLegacyProxy debugRequests(boolean debug) { + this.debug = debug; + return this; + } + + /** Allocates a free port and runs the proxy server on it. This method is idempotent. */ + public void start() throws IOException { + if (started.compareAndSet(false, true)) { + this.localPort = findFreePort(); + + // this is the request handler that performs the proxying and fixes the response headers + HttpRequestHandler proxyHandler = new ProxyHandler(localPort, targetPort, debug); + + HttpProcessor httpproc = + HttpProcessorBuilder.create() + // handles Transfer-Encoding and Content-Length + .add(new ResponseContent(true)) + // handles connection keep-alive + .add(new ResponseConnControl()) + .build(); + + proxyServer = + ServerBootstrap.bootstrap() + .setConnectionFactory(DefaultBHttpServerConnectionFactory.INSTANCE) + .setHttpProcessor(httpproc) + .setListenerPort(localPort) + .registerHandler("*", proxyHandler) + .create(); + proxyServer.start(); + } + } + + /** Stops the proxy server. This method is idempotent. */ + public void stop() { + if (started.compareAndSet(true, false)) { + proxyServer.stop(); + } + } + + private int findFreePort() { + try (ServerSocket s = new ServerSocket(0)) { + return s.getLocalPort(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static class ProxyHandler implements HttpRequestHandler { + private final int sourcePort; + private final int targetPort; + private boolean debug; + + final CloseableHttpClient client; + Function responseHeaderNameTransform = Function.identity(); + + ProxyHandler(int sourcePort, int targetPort, boolean debug) { + this.sourcePort = sourcePort; + this.targetPort = targetPort; + this.debug = debug; + + @SuppressWarnings("PMD.CloseResource") + PoolingHttpClientConnectionManager connManager = + new PoolingHttpClientConnectionManager(); + client = + HttpClients.custom() + .setConnectionManager(connManager) + .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) + .build(); + } + + @Override + public void handle(HttpRequest request, HttpResponse response, HttpContext context) + throws HttpException, IOException { + HttpUriRequest proxyRequest = proxify(request); + logRequest(request, proxyRequest); + + try (CloseableHttpResponse proxyResponse = client.execute(proxyRequest)) { + response.setStatusLine(proxyResponse.getStatusLine()); // status and reason phrase + logResponseStatus(response); + + Header[] headers = proxyResponse.getAllHeaders(); + applyResponseHeaders(response, headers); + transferResponseEntity(response, proxyResponse); + } + } + + private void transferResponseEntity(HttpResponse localResponse, HttpResponse remoteResponse) + throws IOException { + final HttpEntity remoteResponseEntity = remoteResponse.getEntity(); + HttpEntity entity; + if (null == remoteResponseEntity) { + entity = emptyBodyEntity(remoteResponse); + } else { + entity = extractResponseBody(remoteResponseEntity); + } + EntityUtils.updateEntity(localResponse, entity); + } + + private HttpEntity extractResponseBody(final HttpEntity remoteResponseEntity) + throws IOException { + ContentType contentType = ContentType.get(remoteResponseEntity); + byte[] rawContent = EntityUtils.toByteArray(remoteResponseEntity); + logResponseBody(contentType, rawContent); + return new ByteArrayEntity(rawContent, 0, rawContent.length, contentType); + } + + private HttpEntity emptyBodyEntity(HttpResponse remoteResponse) { + BasicHttpEntity entity = new BasicHttpEntity(); + Optional.ofNullable(remoteResponse.getFirstHeader("Content-Length")) + .map(Header::getValue) + .map(Long::parseLong) + .ifPresent(cl -> entity.setContentLength(cl)); + Header contentType = remoteResponse.getFirstHeader("Content-Type"); + entity.setContentType(contentType); + return entity; + } + + private void logResponseStatus(HttpResponse response) { + StatusLine statusLine = response.getStatusLine(); + info("\tresponse: %d %s", statusLine.getStatusCode(), statusLine.getReasonPhrase()); + } + + private void logResponseBody(ContentType contentType, byte[] rawContent) { + if (null != contentType) { + info("\tContent-Type: %s", contentType); + if (contentType.getMimeType().startsWith("application/xml") + || contentType.getMimeType().contains("json")) { + info("\tcontent:\t%s", new String(rawContent)); + } + } + } + + private void logRequest(HttpRequest request, HttpUriRequest proxyRequest) { + info( + "routing %s %s to %s %s", + request.getRequestLine().getMethod(), + request.getRequestLine().getUri(), + proxyRequest.getRequestLine().getMethod(), + proxyRequest.getRequestLine().getUri()); + + Stream.of(proxyRequest.getAllHeaders()) + .forEach( + header -> + info( + "\tapplied request header %s: %s", + header.getName(), header.getValue())); + } + + private void applyResponseHeaders(HttpResponse response, Header[] headers) { + if (null == headers || headers.length == 0) return; + + Stream.of(headers) + .forEach( + header -> { + String name = header.getName(); + String value = header.getValue(); + name = responseHeaderNameTransform.apply(name); + if ("Connection".equalsIgnoreCase(name) + || "Transfer-Encoding".equalsIgnoreCase(name) + || "Content-Length".equalsIgnoreCase(name)) { + // these will produce a 'Connection reset by peer', let the + // proxy handle them + return; + } + // Fix the problematic response header names + if ("etag".equalsIgnoreCase(name)) { + name = "ETag"; + } else if ("last-modified".equalsIgnoreCase(name)) { + name = "Last-Modified"; + } else if ("content-type".equalsIgnoreCase(name)) { + name = "Content-Type"; + } + response.addHeader(name, value); + info("\tapplied response header %s: %s", name, value); + }); + } + + private HttpUriRequest proxify(HttpRequest request) { + + RequestLine requestLine = request.getRequestLine(); + + String uri = + requestLine + .getUri() + .replace( + "http://localhost:" + sourcePort, + "http://localhost:" + targetPort); + + HttpUriRequest proxyRequest = + RequestBuilder.copy(request) + .setUri(uri) + // these will produce a 'Connection reset by peer', let the + // proxy handle them + .removeHeaders("Connection") + .removeHeaders("Transfer-Encoding") + .removeHeaders("Content-Length") + .build(); + return proxyRequest; + } + + @SuppressWarnings("PMD.SystemPrintln") + private void info(String msg, Object... params) { + if (debug) { + System.err.printf(msg + "%n", params); + } + } + } +} diff --git a/geowebcache/core/pom.xml b/geowebcache/core/pom.xml index 64d8f36d8..6c1b96d5b 100644 --- a/geowebcache/core/pom.xml +++ b/geowebcache/core/pom.xml @@ -200,6 +200,11 @@ xmlunit-legacy test + + org.awaitility + awaitility + test + diff --git a/geowebcache/core/src/test/java/org/geowebcache/storage/AbstractBlobStoreTest.java b/geowebcache/core/src/test/java/org/geowebcache/storage/AbstractBlobStoreTest.java index 5f298eb06..dcbeeaebe 100644 --- a/geowebcache/core/src/test/java/org/geowebcache/storage/AbstractBlobStoreTest.java +++ b/geowebcache/core/src/test/java/org/geowebcache/storage/AbstractBlobStoreTest.java @@ -14,6 +14,8 @@ */ package org.geowebcache.storage; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.captureLong; @@ -512,8 +514,7 @@ public void testDeleteGridset() throws Exception { EasyMock.replay(listener); assertThat(store.deleteByGridsetId("testLayer", "testGridSet1"), is(true)); EasyMock.verify(listener); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + assertNoTile(fromCache1_2); } @Test @@ -575,8 +576,9 @@ public void testDeleteGridsetDoesntDeleteOthers() throws Exception { new ByteArrayResource( "7,8,9,10 test".getBytes(StandardCharsets.UTF_8))))); store.deleteByGridsetId("testLayer", "testGridSet1"); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + + assertNoTile(fromCache1_2); + assertThat(store.get(fromCache2_3), is(true)); assertThat(fromCache2_3, hasProperty("blobSize", is((int) size2))); assertThat( @@ -707,8 +709,9 @@ public void testParameters() throws Exception { EasyMock.replay(listener); store.delete(remove); EasyMock.verify(listener); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + + assertNoTile(fromCache1_2); + assertThat(store.get(fromCache2_3), is(true)); assertThat(fromCache2_3, hasProperty("blobSize", is((int) size2))); assertThat( @@ -925,8 +928,8 @@ public void testDeleteByParametersId() throws Exception { EasyMock.replay(listener); store.deleteByParametersId("testLayer", paramID1); EasyMock.verify(listener); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + + assertNoTile(fromCache1_2); } @Test @@ -959,7 +962,9 @@ public void testDeleteByParametersIdDoesNotDeleteOthers() throws Exception { store.put(toCache1); store.put(toCache2); store.deleteByParametersId("testLayer", paramID1); - assertThat(store.get(fromCache2_3), is(true)); + + await().atMost(30, SECONDS) // give stores with async deletes a chance to complete + .untilAsserted(() -> assertThat(store.get(fromCache2_3), is(true))); assertThat(fromCache2_3, hasProperty("blobSize", is((int) size2))); assertThat( fromCache2_3, @@ -1071,8 +1076,7 @@ public void testPurgeOrphans() throws Exception { EasyMock.replay(listener); store.purgeOrphans(layer); EasyMock.verify(listener); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + assertNoTile(fromCache1_2); } protected void cacheTile( @@ -1129,7 +1133,15 @@ protected void assertNoTile( TileObject to = TileObject.createQueryTileObject( layerName, new long[] {x, y, z}, gridSetId, format, parameters); - assertThat(store.get(to), describedAs("don't get a tile", is(false))); + assertNoTile(to); + } + + private void assertNoTile(TileObject to) { + await().atMost(30, SECONDS) // give stores with async deletes a chance to complete + .untilAsserted( + () -> + assertThat( + store.get(to), describedAs("don't get a tile", is(false)))); assertThat(to, hasProperty("blob", nullValue())); assertThat(to, hasProperty("blobSize", is(0))); } diff --git a/geowebcache/pom.xml b/geowebcache/pom.xml index fd290a1e9..1d1783ece 100644 --- a/geowebcache/pom.xml +++ b/geowebcache/pom.xml @@ -97,6 +97,7 @@ 2.3.0 5.12.0 UTF-8 + ${skipTests} @@ -109,6 +110,14 @@ test + + org.testcontainers + testcontainers-bom + 1.20.1 + pom + import + + org.locationtech.jts jts-core @@ -299,7 +308,12 @@ ${hamcrest.version} test - + + org.awaitility + awaitility + 4.2.2 + test + org.easymock easymock @@ -548,6 +562,29 @@ cobertura-maven-plugin 2.0 + + maven-failsafe-plugin + 3.3.1 + + + + integration-test + verify + + + + true + + + src/test/resources/logging.properties + + true + + ${maven.test.jvmargs} -XX:+IgnoreUnrecognizedVMOptions --illegal-access=warn --add-exports=java.desktop/sun.awt.image=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.desktop/java.awt.font=ALL-UNNAMED + + ${skipITs} + + @@ -643,23 +680,6 @@ - - maven-failsafe-plugin - 2.19.1 - - - - integration-test - verify - - - - - -Djava.awt.headless=true - - - - pl.project13.maven diff --git a/geowebcache/s3storage/pom.xml b/geowebcache/s3storage/pom.xml index 0d4fc25e6..4db2f9023 100644 --- a/geowebcache/s3storage/pom.xml +++ b/geowebcache/s3storage/pom.xml @@ -59,7 +59,11 @@ log4j-slf4j-impl test - + + org.awaitility + awaitility + test + diff --git a/geowebcache/sqlite/pom.xml b/geowebcache/sqlite/pom.xml index 8eb9f72f8..50cdeda13 100644 --- a/geowebcache/sqlite/pom.xml +++ b/geowebcache/sqlite/pom.xml @@ -63,5 +63,10 @@ easymock test + + org.awaitility + awaitility + test +