diff --git a/.github/workflows/azure-integration.yml b/.github/workflows/azure-integration.yml new file mode 100644 index 000000000..fdb94e8b2 --- /dev/null +++ b/.github/workflows/azure-integration.yml @@ -0,0 +1,88 @@ +name: Azure BlobStore Integration + +on: + push: + branches: + - "**" + paths: + - ".github/workflows/azure-integration.yml" + - "pom.xml" + - "geowebcache/pom.xml" + - "geowebcache/core/**" + - "geowebcache/azureblob/**" + pull_request: + branches: + - "main" + paths: + - ".github/workflows/azure-integration.yml" + - "pom.xml" + - "geowebcache/pom.xml" + - "geowebcache/core/**" + - "geowebcache/azureblob/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + azurite: + name: Azurite container + runs-on: ubuntu-latest + strategy: + matrix: + java-version: [ 11, 17, 21 ] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java-version }} + cache: 'maven' + + - name: Tests against Azurite TestContainers + #-PexcludeOnlineTests includes Azurite container tests and excludes Azure online tests + run: | + mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \ + -DskipTests=true -DskipITs=false -B -ntp \ + -PexcludeOnlineTests + + - name: Remove SNAPSHOT jars from repository + run: | + find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} + + azure: + name: Azure online + #if: github.repository == 'geowebcache/geowebcache' + runs-on: ubuntu-latest + needs: azurite + if: | + always() && + !contains(needs.*.result, 'cancelled') && + !contains(needs.*.result, 'failure') + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 11 + cache: 'maven' + + - name: Tests against Azure + env: + azure_account: ${{ secrets.AZURE_ACCOUNT }} + azure_account_key: ${{ secrets.AZURE_ACCOUNT_KEY }} + azure_container: ${{ secrets.AZURE_CONTAINER }} + if: ${{ env.azure_account != null }} && ${{ env.azure_account_key != null }} + run: | #-PexcludeDockerTests includes Azure online tests and excludes Azurite container tests + echo "accountName=$azure_account" > $HOME/.gwc_azure_tests.properties + echo "accountKey=$azure_account_key" >> $HOME/.gwc_azure_tests.properties + echo "container=$azure_container" >> $HOME/.gwc_azure_tests.properties + echo 'maxConnections=8' >> $HOME/.gwc_azure_tests.properties + echo 'useHTTPS=true' >> $HOME/.gwc_azure_tests.properties + mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \ + -DskipTests=true -DskipITs=false -B -ntp \ + -PexcludeDockerTests + + - name: Remove SNAPSHOT jars from repository + run: | + find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 82dfed58e..8deac2617 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -27,7 +27,8 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Build with Maven - run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml + # -DskipITs skips failsafe integration tests but runs unit tests with surefire. Integration tests have their own jobs + run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml -DskipITs - name: Remove SNAPSHOT jars from repository run: | find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} @@ -49,7 +50,8 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Build with Maven - run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml + # -DskipITs skips failsafe integration tests but runs unit tests with surefire. Integration tests have their own jobs + run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml -DskipITs - name: Remove SNAPSHOT jars from repository run: | find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} @@ -71,7 +73,8 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Build with Maven - run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml + # -DskipITs skips failsafe integration tests but runs unit tests with surefire. Integration tests have their own jobs + run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml -DskipITs - name: Remove SNAPSHOT jars from repository run: | find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} diff --git a/geowebcache/azureblob/pom.xml b/geowebcache/azureblob/pom.xml index 2084bdeb0..64788d369 100644 --- a/geowebcache/azureblob/pom.xml +++ b/geowebcache/azureblob/pom.xml @@ -61,5 +61,70 @@ javax.servlet-api provided + + org.testcontainers + testcontainers + test + + + org.awaitility + awaitility + test + + + + + maven-failsafe-plugin + + 1 + false + + + + + + + + excludeOnlineTests + + false + + + + + maven-failsafe-plugin + + 1 + false + + org.geowebcache.azure.tests.online.*IT + + + + + + + + + excludeDockerTests + + false + + + + + maven-failsafe-plugin + + 1 + false + + org.geowebcache.azure.tests.container.*IT + + + + + + + diff --git a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java index 0b7809e3c..0bd12e95a 100644 --- a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java +++ b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java @@ -22,7 +22,7 @@ * Azure Blobstore type-resolved data from a {@link AzureBlobStoreInfo} using enviroment variables * if enabled. */ -class AzureBlobStoreData { +public class AzureBlobStoreData { private String container; private String prefix; @@ -36,7 +36,7 @@ class AzureBlobStoreData { private String proxyPassword; private String serviceURL; - AzureBlobStoreData() {} + public AzureBlobStoreData() {} public AzureBlobStoreData( final AzureBlobStoreInfo storeInfo, final GeoWebCacheEnvironment environment) { diff --git a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java index feaa447c4..c56deab32 100644 --- a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java +++ b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java @@ -50,7 +50,7 @@ import org.geowebcache.util.URLs; import org.springframework.http.HttpStatus; -class AzureClient implements Closeable { +public class AzureClient implements Closeable { private final NettyClient.Factory factory; private AzureBlobStoreData configuration; @@ -93,8 +93,13 @@ public AzureClient(AzureBlobStoreData configuration) throws StorageException { this.container = serviceURL.createContainerURL(containerName); // no way to see if the containerURL already exists, try to create and see if // we get a 409 CONFLICT + int status; + try { + status = this.container.getProperties().blockingGet().statusCode(); + } catch (com.microsoft.azure.storage.blob.StorageException se) { + status = se.statusCode(); + } try { - int status = this.container.getProperties().blockingGet().statusCode(); if (status == HttpStatus.NOT_FOUND.value()) { status = this.container.create(null, null, null).blockingGet().statusCode(); if (!HttpStatus.valueOf(status).is2xxSuccessful() diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java index cc6516fad..e38cc0ad9 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java @@ -25,25 +25,25 @@ import java.util.stream.Stream; import org.easymock.EasyMock; import org.geowebcache.GeoWebCacheException; +import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT; +import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreConformanceIT; import org.geowebcache.layer.TileLayer; import org.geowebcache.layer.TileLayerDispatcher; import org.geowebcache.locks.LockProvider; import org.geowebcache.locks.NoOpLockProvider; import org.geowebcache.storage.AbstractBlobStoreTest; -import org.junit.Assume; -import org.junit.Rule; -public class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest { - public PropertiesLoader testConfigLoader = new PropertiesLoader(); +/** + * @see OnlineAzureBlobStoreConformanceIT + * @see AzuriteAzureBlobStoreConformanceIT + */ +public abstract class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest { - @Rule - public TemporaryAzureFolder tempFolder = - new TemporaryAzureFolder(testConfigLoader.getProperties()); + protected abstract AzureBlobStoreData getConfiguration(); @Override public void createTestUnit() throws Exception { - Assume.assumeTrue(tempFolder.isConfigured()); - AzureBlobStoreData config = tempFolder.getConfig(); + AzureBlobStoreData config = getConfiguration(); TileLayerDispatcher layers = createMock(TileLayerDispatcher.class); LockProvider lockProvider = new NoOpLockProvider(); diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AbstractAzureBlobStoreIntegrationTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreIntegrationTest.java similarity index 98% rename from geowebcache/azureblob/src/test/java/org/geowebcache/azure/AbstractAzureBlobStoreIntegrationTest.java rename to geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreIntegrationTest.java index 9ddb3aded..2c4ba5f34 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AbstractAzureBlobStoreIntegrationTest.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreIntegrationTest.java @@ -42,6 +42,8 @@ import java.util.Map; import java.util.logging.Logger; import org.geotools.util.logging.Logging; +import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreIntegrationIT; +import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreIntegrationIT; import org.geowebcache.config.DefaultGridsets; import org.geowebcache.grid.GridSet; import org.geowebcache.grid.GridSetBroker; @@ -69,10 +71,13 @@ * Integration tests for {@link AzureBlobStore}. * *

This is an abstract class for both online and offline integration tests. + * + * @see OnlineAzureBlobStoreIntegrationIT + * @see AzuriteAzureBlobStoreIntegrationIT */ -public abstract class AbstractAzureBlobStoreIntegrationTest { +public abstract class AzureBlobStoreIntegrationTest { - private static Logger log = Logging.getLogger(PropertiesLoader.class.getName()); + private static Logger log = Logging.getLogger(AzureBlobStoreIntegrationTest.class.getName()); private static final String DEFAULT_FORMAT = "png"; @@ -80,8 +85,6 @@ public abstract class AbstractAzureBlobStoreIntegrationTest { private static final String DEFAULT_LAYER = "topp:world"; - public PropertiesLoader testConfigLoader = new PropertiesLoader(); - private AzureBlobStore blobStore; protected abstract AzureBlobStoreData getConfiguration(); diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java index 4612b9474..22e6688b9 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java @@ -17,11 +17,12 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItemInArray; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeFalse; import io.reactivex.Flowable; import java.nio.ByteBuffer; import org.easymock.EasyMock; +import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreSuitabilityIT; +import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreSuitabilityIT; import org.geowebcache.layer.TileLayerDispatcher; import org.geowebcache.locks.LockProvider; import org.geowebcache.locks.NoOpLockProvider; @@ -30,23 +31,14 @@ import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.Before; -import org.junit.Rule; import org.junit.experimental.theories.DataPoints; -import org.junit.experimental.theories.Theories; -import org.junit.runner.RunWith; -import org.junit.runners.model.FrameworkMethod; -import org.junit.runners.model.InitializationError; -import org.junit.runners.model.Statement; import org.springframework.http.HttpStatus; -@RunWith(AzureBlobStoreSuitabilityTest.MyTheories.class) -public class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest { - - public PropertiesLoader testConfigLoader = new PropertiesLoader(); - - @Rule - public TemporaryAzureFolder tempFolder = - new TemporaryAzureFolder(testConfigLoader.getProperties()); +/** + * @see OnlineAzureBlobStoreSuitabilityIT + * @see AzuriteAzureBlobStoreSuitabilityIT + */ +public abstract class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest { @DataPoints public static String[][] persistenceLocations = { @@ -67,6 +59,10 @@ public void setup() throws Exception { EasyMock.replay(tld); } + protected abstract AzureBlobStoreData getConfiguration(); + + protected abstract AzureClient getClient(); + @SuppressWarnings("unchecked") @Override protected Matcher existing() { @@ -81,13 +77,12 @@ protected Matcher empty() { @Override public BlobStore create(Object dir) throws Exception { - AzureBlobStoreData info = tempFolder.getConfig(); + AzureBlobStoreData info = getConfiguration(); for (String path : (String[]) dir) { String fullPath = info.getPrefix() + "/" + path; ByteBuffer byteBuffer = ByteBuffer.wrap("testAbc".getBytes()); int statusCode = - tempFolder - .getClient() + getClient() .getBlockBlobURL(fullPath) .upload(Flowable.just(byteBuffer), byteBuffer.limit()) .blockingGet() @@ -96,27 +91,4 @@ public BlobStore create(Object dir) throws Exception { } return new AzureBlobStore(info, tld, locks); } - - // Sorry, this bit of evil makes the Theories runner gracefully ignore the - // tests if Azure is unavailable. There's probably a better way to do this. - public static class MyTheories extends Theories { - - public MyTheories(Class klass) throws InitializationError { - super(klass); - } - - @Override - public Statement methodBlock(FrameworkMethod method) { - if (new PropertiesLoader().getProperties().containsKey("container")) { - return super.methodBlock(method); - } else { - return new Statement() { - @Override - public void evaluate() { - assumeFalse("Azure unavailable", true); - } - }; - } - } - } } diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreConformanceIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreConformanceIT.java new file mode 100644 index 000000000..58c0abfab --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreConformanceIT.java @@ -0,0 +1,51 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.azure.tests.container; + +import org.geowebcache.azure.AzureBlobStoreConformanceTest; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.testcontainers.azure.AzuriteContainer; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TestName; + +/** + * Runs {@link AzureBlobStoreConformanceTest} tests against a local ephemeral Docker container using + * {@link AzuriteContainer}. + * + *

If there's no Docker environment, the test is {@link AzuriteContainer#disabledWithoutDocker() + * ignored} + */ +public class AzuriteAzureBlobStoreConformanceIT extends AzureBlobStoreConformanceTest { + + /** + * Use "legacy" container to work with {@literal + * com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as + * AzuriteContainer.legacy().debugLegacy() to print out request/response information for + * debugging purposes + */ + @ClassRule + public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker(); + + /** Used to get a per-test case Azure container */ + @Rule public TestName testName = new TestName(); + + @Override + protected AzureBlobStoreData getConfiguration() { + // container must be lower case or we get a 400 bad request + String container = testName.getMethodName().toLowerCase(); + return azurite.getConfiguration(container); + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreIntegrationIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreIntegrationIT.java new file mode 100644 index 000000000..9f4fba858 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreIntegrationIT.java @@ -0,0 +1,51 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.azure.tests.container; + +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureBlobStoreIntegrationTest; +import org.geowebcache.testcontainers.azure.AzuriteContainer; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TestName; + +/** + * Runs {@link AzureBlobStoreIntegrationTest} tests against a local ephemeral Docker container using + * {@link AzuriteContainer}. + * + *

If there's no Docker environment, the test is {@link AzuriteContainer#disabledWithoutDocker() + * ignored} + */ +public class AzuriteAzureBlobStoreIntegrationIT extends AzureBlobStoreIntegrationTest { + + /** + * Use "legacy" container to work with {@literal + * com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as + * AzuriteContainer.legacy().debugLegacy() to print out request/response information for + * debugging purposes + */ + @ClassRule + public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker(); + + /** Used to get a per-test case Azure container */ + @Rule public TestName testName = new TestName(); + + @Override + protected AzureBlobStoreData getConfiguration() { + // container must be lower case or we get a 400 bad request + String container = testName.getMethodName().toLowerCase(); + return azurite.getConfiguration(container); + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreSuitabilityIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreSuitabilityIT.java new file mode 100644 index 000000000..3cdc8c164 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/container/AzuriteAzureBlobStoreSuitabilityIT.java @@ -0,0 +1,67 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.azure.tests.container; + +import java.io.UncheckedIOException; +import org.geowebcache.azure.AzureBlobStoreConformanceTest; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureBlobStoreSuitabilityTest; +import org.geowebcache.azure.AzureClient; +import org.geowebcache.storage.StorageException; +import org.geowebcache.testcontainers.azure.AzuriteContainer; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TestName; + +/** + * Runs {@link AzureBlobStoreConformanceTest} tests against a local ephemeral Docker container using + * {@link AzuriteContainer}. + * + *

If there's no Docker environment, the test is {@link AzuriteContainer#disabledWithoutDocker() + * ignored} + */ +public class AzuriteAzureBlobStoreSuitabilityIT extends AzureBlobStoreSuitabilityTest { + + /** + * Use "legacy" container to work with {@literal + * com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as + * AzuriteContainer.legacy().debugLegacy() to print out request/response information for + * debugging purposes + */ + @ClassRule + public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker(); + + /** Used to get a per-test case Azure container */ + @Rule public TestName testName = new TestName(); + + @Override + protected AzureBlobStoreData getConfiguration() { + // container must be lower case or we get a 400 bad request + String container = testName.getMethodName().toLowerCase(); + AzureBlobStoreData configuration = azurite.getConfiguration(container); + // AzureBlobStoreSuitabilityTest requires a prefix to be set + configuration.setPrefix("test-prefix"); + return configuration; + } + + @Override + protected AzureClient getClient() { + try { + return new AzureClient(getConfiguration()); + } catch (StorageException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreConformanceIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreConformanceIT.java new file mode 100644 index 000000000..ab89916d5 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreConformanceIT.java @@ -0,0 +1,34 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + * @author Kevin Smith, Boundless, 2017 + */ +package org.geowebcache.azure.tests.online; + +import org.geowebcache.azure.AzureBlobStoreConformanceTest; +import org.geowebcache.azure.AzureBlobStoreData; +import org.junit.Assume; +import org.junit.Rule; + +public class OnlineAzureBlobStoreConformanceIT extends AzureBlobStoreConformanceTest { + public PropertiesLoader testConfigLoader = new PropertiesLoader(); + + @Rule + public TemporaryAzureFolder tempFolder = + new TemporaryAzureFolder(testConfigLoader.getProperties()); + + @Override + protected AzureBlobStoreData getConfiguration() { + Assume.assumeTrue(tempFolder.isConfigured()); + return tempFolder.getConfig(); + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/OnlineAzureBlobStoreIntegrationTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreIntegrationIT.java similarity index 85% rename from geowebcache/azureblob/src/test/java/org/geowebcache/azure/OnlineAzureBlobStoreIntegrationTest.java rename to geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreIntegrationIT.java index cc002a7e8..25a07b89d 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/OnlineAzureBlobStoreIntegrationTest.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreIntegrationIT.java @@ -12,16 +12,20 @@ * * @author Andrea Aime, GeoSolutions, Copyright 2019 */ -package org.geowebcache.azure; +package org.geowebcache.azure.tests.online; import static org.junit.Assert.assertTrue; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureBlobStoreIntegrationTest; import org.junit.Assume; import org.junit.Rule; import org.junit.Test; import org.springframework.http.HttpStatus; -public class OnlineAzureBlobStoreIntegrationTest extends AbstractAzureBlobStoreIntegrationTest { +public class OnlineAzureBlobStoreIntegrationIT extends AzureBlobStoreIntegrationTest { + + private PropertiesLoader testConfigLoader = new PropertiesLoader(); @Rule public TemporaryAzureFolder tempFolder = diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreSuitabilityIT.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreSuitabilityIT.java new file mode 100644 index 000000000..c996d0752 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/OnlineAzureBlobStoreSuitabilityIT.java @@ -0,0 +1,70 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + * @author Kevin Smith, Boundless, 2018 + */ +package org.geowebcache.azure.tests.online; + +import static org.junit.Assume.assumeFalse; + +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureBlobStoreSuitabilityTest; +import org.geowebcache.azure.AzureClient; +import org.junit.Rule; +import org.junit.experimental.theories.Theories; +import org.junit.runner.RunWith; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.Statement; + +@RunWith(OnlineAzureBlobStoreSuitabilityIT.MyTheories.class) +public class OnlineAzureBlobStoreSuitabilityIT extends AzureBlobStoreSuitabilityTest { + + public PropertiesLoader testConfigLoader = new PropertiesLoader(); + + @Rule + public TemporaryAzureFolder tempFolder = + new TemporaryAzureFolder(testConfigLoader.getProperties()); + + @Override + protected AzureBlobStoreData getConfiguration() { + return tempFolder.getConfig(); + } + + @Override + protected AzureClient getClient() { + return tempFolder.getClient(); + } + + // Sorry, this bit of evil makes the Theories runner gracefully ignore the + // tests if Azure is unavailable. There's probably a better way to do this. + public static class MyTheories extends Theories { + + public MyTheories(Class klass) throws InitializationError { + super(klass); + } + + @Override + public Statement methodBlock(FrameworkMethod method) { + if (new PropertiesLoader().getProperties().containsKey("container")) { + return super.methodBlock(method); + } else { + return new Statement() { + @Override + public void evaluate() { + assumeFalse("Azure unavailable", true); + } + }; + } + } + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/PropertiesLoader.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/PropertiesLoader.java similarity index 98% rename from geowebcache/azureblob/src/test/java/org/geowebcache/azure/PropertiesLoader.java rename to geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/PropertiesLoader.java index ca3e9c878..80c9887ce 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/PropertiesLoader.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/PropertiesLoader.java @@ -12,7 +12,7 @@ * * @author Andrea Aime, GeoSolutions, Copyright 2019 */ -package org.geowebcache.azure; +package org.geowebcache.azure.tests.online; import static com.google.common.base.Preconditions.checkArgument; diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/TemporaryAzureFolder.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/TemporaryAzureFolder.java similarity index 97% rename from geowebcache/azureblob/src/test/java/org/geowebcache/azure/TemporaryAzureFolder.java rename to geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/TemporaryAzureFolder.java index 71c5a2c9b..89774b3db 100644 --- a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/TemporaryAzureFolder.java +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/tests/online/TemporaryAzureFolder.java @@ -12,7 +12,7 @@ * * @author Andrea Aime, GeoSolutions, Copyright 2019 */ -package org.geowebcache.azure; +package org.geowebcache.azure.tests.online; import static com.google.common.base.Preconditions.checkState; import static org.junit.Assert.assertTrue; @@ -22,6 +22,8 @@ import java.util.List; import java.util.Properties; import java.util.UUID; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.AzureClient; import org.junit.rules.ExternalResource; import org.springframework.http.HttpStatus; diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainer.java b/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainer.java new file mode 100644 index 000000000..fb8d06b60 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainer.java @@ -0,0 +1,262 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.testcontainers.azure; + +import static java.lang.String.format; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import java.io.IOException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.UncheckedException; +import org.geowebcache.azure.AzureBlobStoreData; +import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT; +import org.junit.Assume; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Testcontainers container for AWS Azurite + * blobstore test environment. + * + *

Runs the Azurite + * Docker image for local Azure Storage development with testcontainers. + * + *

Azurite accepts the same well-known account and key used by the legacy Azure Storage Emulator: + * + *

    + *
  • Account name: {@code devstoreaccount1} + *
  • Account key: {@code + * Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==} + *
+ * + *

Usage: For Junit 4, use it as a {@code @Rule} or {@code @ClassRule}: + * + * 

+ * 
+ *   @Rule public AzuriteContainer azurite = AzuriteContainer.legacy();
+ * 
+ *
+ * + * works with the old {@code com.microsoft.azure:azure-storage-blob:jar:11.0.0} as a dependency. + * + * 
+ * 
+ *   @Rule public AzuriteContainer azurite = AzuriteContainer.legacy();
+ * 
+ *
+ * + * works with the latest {@code com.azure:azure-storage-blob:jar:12.27.0} as a dependency. + * + *

Sample test: + * + * 

+ * 
+ *   @ClassRule public static AzuriteContainer azurite = AzuriteContainer.legacy();
+ *
+ *   @Test
+ *   public void azureBlobStoreSmokeTest(){
+ *      String container = "testcontainer";//ought to be lower case
+ *      AzureBlobStoreData config = azurite.getConfiguration(container);
+ *      AzureBlobStore store = new AzureBlobStore(config, tileLayerDispatcher, lockProvider);
+ *      assertFalse(store.layerExists("layer1");
+ *   }
+ * 
+ *
+ */ +public class AzuriteContainer extends GenericContainer { + + private static final DockerImageName LATEST_IMAGE = + DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:latest"); + + private static final DockerImageName LEGACY_IMAGE = + DockerImageName.parse("arafato/azurite:2.6.5"); + + private final String accountName = "devstoreaccount1"; + private final String accountKey = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + + private final int blobsPort = 10_000; + + private AzuriteContainerLegacyProxy proxy; + + private final boolean doProxy; + + /** Whether to print request/response debug information when in {@link #legacy} mode */ + private boolean debugRequests; + + /** flag for {@link #disabledWithoutDocker()} */ + private boolean disabledWithoutDocker; + + private AzuriteContainer(DockerImageName imageName, boolean doProxy) { + super(imageName); + this.doProxy = doProxy; + super.setWaitStrategy(Wait.forListeningPort()); + super.addExposedPort(blobsPort); + } + + /** + * @return a container running {@code arafato/azurite:2.6.5} and {@link #getBlobsPort() proxied} + * to fix protocol discrepancies so it works correctly with older {@code + * com.microsoft.azure:azure-storage-blob} dependencies + */ + public static AzuriteContainer legacy() { + return new AzuriteContainer(LEGACY_IMAGE, true); + } + + /** @return a container running {@code mcr.microsoft.com/azure-storage/azurite:latest} */ + public static AzuriteContainer latest() { + return new AzuriteContainer(LATEST_IMAGE, false); + } + + /** + * Enables request/response debugging when in legacy mode + * + *

Sample output: + * + * 

+     * 
+     * routing GET http://localhost:44445/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png to GET http://localhost:33319/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png
+     * 	applied request header Authorization: SharedKey devstoreaccount1:6UeSk1Qf8XRibLI1sE3tasmDxOtVxGUSMDQqRUDIW9Y=
+     * 	applied request header x-ms-version: 2018-11-09
+     * 	applied request header x-ms-date: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied request header host: localhost
+     * 	applied request header x-ms-client-request-id: 526b726a-13af-49a3-b277-fdf645d77903
+     * 	applied request header User-Agent: Azure-Storage/11.0.0 (JavaJRE 11.0.23; Linux 6.8.0-39-generic)
+     * 	response: 200 OK
+     * 	applied response header X-Powered-By: Express
+     * 	applied response header ETag: "jzUOHaHcch36ue3TFspQaLiWSvo"
+     * 	applied response header Last-Modified: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied response header x-ms-version: 2016-05-31
+     * 	applied response header date: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied response header x-ms-request-id: 05130dd1-5672-11ef-a96b-c7f08f042b95
+     * 	applied response header accept-ranges: bytes
+     * 	applied response header x-ms-blob-type: BlockBlob
+     * 	applied response header x-ms-request-server-encrypted: false
+     * 	applied response header Content-Type: image/png
+     * 	Content-Type: image/png
+     * 
+     *
+ */ + public AzuriteContainer debugLegacy() { + this.debugRequests = true; + return this; + } + + /** + * Disables the tests using this testcontainer if there's no Docker environment available. + * + *

Same effect as JUnit 5's {@code + * org.testcontainers.junit.jupiter.@Testcontainers(disabledWithoutDocker = true)} + */ + public AzuriteContainer disabledWithoutDocker() { + this.disabledWithoutDocker = true; + return this; + } + + /** + * Overrides to apply the {@link Assume assumption} checking the Docker environment is available + * if {@link #disabledWithoutDocker() enabled}, so this test container can be used as a {@code + * ClassRule @ClassRule} and hence avoid running a container for each test case. + */ + @Override + @SuppressWarnings("deprecation") + public Statement apply(Statement base, Description description) { + if (disabledWithoutDocker) { + assumeTrue( + "Docker environment unavailable, ignoring test " + + AzuriteAzureBlobStoreConformanceIT.class.getSimpleName(), + DockerClientFactory.instance().isDockerAvailable()); + } + return super.apply(base, description); + } + + @Override + public void start() { + super.start(); + if (doProxy && proxy == null) { + int targetPort = getRealBlobsPort(); + proxy = new AzuriteContainerLegacyProxy(targetPort).debugRequests(debugRequests); + try { + proxy.start(); + } catch (IOException e) { + throw new UncheckedException(e); + } + } + } + + @Override + public void stop() { + super.stop(); + if (doProxy && null != proxy) { + try { + proxy.stop(); + } finally { + proxy = null; + } + } + } + + public String getAccountName() { + return accountName; + } + + public String getAccountKey() { + return accountKey; + } + + /** + * Returns the localhost port where the azurite blob storage service is running. + * + *

when in {@link #legacy() legacy} mode, a small http proxy is run and the proxy port is + * returned. The proxy fixes some protocol issues. For instance, re-writes the returned response + * headers {@code etag}, {@code last-modified}, and {@code content-type}, as {@code Etag}, + * {@code Last-Modified}, and {@code Content-Type}, respectively, as expected by the Netty + * version the legacy {@code com.microsoft.azure:azure-storage-blob} dependency transitively + * carries over. + */ + public int getBlobsPort() { + if (doProxy) { + if (proxy == null) throw new IllegalStateException(""); + return proxy.getLocalPort(); + } + return getRealBlobsPort(); + } + + int getRealBlobsPort() { + return super.getMappedPort(blobsPort); + } + + public String getBlobServiceUrl() { + return format("http://localhost:%d/%s", getBlobsPort(), getAccountName()); + } + + public AzureBlobStoreData getConfiguration(String container) { + assertTrue("Container must be lower case", StringUtils.isAllLowerCase(container)); + AzureBlobStoreData config = new AzureBlobStoreData(); + config.setServiceURL(getBlobServiceUrl()); + config.setAccountName(getAccountName()); + config.setAccountKey(getAccountKey()); + config.setMaxConnections(10); + config.setContainer(container); + return config; + } +} diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainerLegacyProxy.java b/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainerLegacyProxy.java new file mode 100644 index 000000000..6874f31a1 --- /dev/null +++ b/geowebcache/azureblob/src/test/java/org/geowebcache/testcontainers/azure/AzuriteContainerLegacyProxy.java @@ -0,0 +1,331 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the + * GNU Lesser General Public License as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this + * program. If not, see . + * + *

Copyright 2024 + */ +package org.geowebcache.testcontainers.azure; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.ServerSocket; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Stream; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.RequestLine; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.DefaultBHttpServerConnectionFactory; +import org.apache.http.impl.bootstrap.HttpServer; +import org.apache.http.impl.bootstrap.ServerBootstrap; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpProcessor; +import org.apache.http.protocol.HttpProcessorBuilder; +import org.apache.http.protocol.HttpRequestHandler; +import org.apache.http.protocol.ResponseConnControl; +import org.apache.http.protocol.ResponseContent; +import org.apache.http.util.EntityUtils; +import org.geotools.util.logging.Logging; + +/** + * A simple HTTP proxy to adapt some Azure Blob storage protocol issues to the netty version used by + * older {@code com.microsoft.azure:azure-storage-blob} dependencies. + * + *

For instance, re-writes the returned response headers {@code etag}, {@code last-modified}, and + * {@code content-type}, as {@code Etag}, {@code Last-Modified}, and {@code Content-Type}, + * respectively, as expected by the Netty version the legacy {@code + * com.microsoft.azure:azure-storage-blob} dependency transitively carries over. + * + *

Even though HTTP request and response headers should be case-insensitive, this older netty + * version ({@code 4.1.28}, and even newer ones) fail to parse the lower-case names returned by + * Azurite. + */ +class AzuriteContainerLegacyProxy { + public static Logger LOGGER = Logging.getLogger(AzuriteContainerLegacyProxy.class.getName()); + + private int localPort; + private HttpServer proxyServer; + + private int targetPort; + + private final AtomicBoolean started = new AtomicBoolean(); + + private boolean debug; + + AzuriteContainerLegacyProxy(int targetPort) { + this.targetPort = targetPort; + } + + /** + * @return the random port where the proxy server is running + * @throws IllegalStateException if the proxy is not {@link #start() running} + */ + public int getLocalPort() { + if (!started.get()) { + throw new IllegalStateException( + "Proxy not running, local port is allocated at start()"); + } + return localPort; + } + + /** + * Whether to print request/response debugging information to stderr. + * + *

Sample output: + * + * 

+     * 
+     * routing GET http://localhost:44445/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png to GET http://localhost:33319/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png
+     * 	applied request header Authorization: SharedKey devstoreaccount1:6UeSk1Qf8XRibLI1sE3tasmDxOtVxGUSMDQqRUDIW9Y=
+     * 	applied request header x-ms-version: 2018-11-09
+     * 	applied request header x-ms-date: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied request header host: localhost
+     * 	applied request header x-ms-client-request-id: 526b726a-13af-49a3-b277-fdf645d77903
+     * 	applied request header User-Agent: Azure-Storage/11.0.0 (JavaJRE 11.0.23; Linux 6.8.0-39-generic)
+     * 	response: 200 OK
+     * 	applied response header X-Powered-By: Express
+     * 	applied response header ETag: "jzUOHaHcch36ue3TFspQaLiWSvo"
+     * 	applied response header Last-Modified: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied response header x-ms-version: 2016-05-31
+     * 	applied response header date: Fri, 09 Aug 2024 17:08:38 GMT
+     * 	applied response header x-ms-request-id: 05130dd1-5672-11ef-a96b-c7f08f042b95
+     * 	applied response header accept-ranges: bytes
+     * 	applied response header x-ms-blob-type: BlockBlob
+     * 	applied response header x-ms-request-server-encrypted: false
+     * 	applied response header Content-Type: image/png
+     * 	Content-Type: image/png
+     * 
+     *
+ */ + public AzuriteContainerLegacyProxy debugRequests(boolean debug) { + this.debug = debug; + return this; + } + + /** Allocates a free port and runs the proxy server on it. This method is idempotent. */ + public void start() throws IOException { + if (started.compareAndSet(false, true)) { + this.localPort = findFreePort(); + + // this is the request handler that performs the proxying and fixes the response headers + HttpRequestHandler proxyHandler = new ProxyHandler(localPort, targetPort, debug); + + HttpProcessor httpproc = + HttpProcessorBuilder.create() + // handles Transfer-Encoding and Content-Length + .add(new ResponseContent(true)) + // handles connection keep-alive + .add(new ResponseConnControl()) + .build(); + + proxyServer = + ServerBootstrap.bootstrap() + .setConnectionFactory(DefaultBHttpServerConnectionFactory.INSTANCE) + .setHttpProcessor(httpproc) + .setListenerPort(localPort) + .registerHandler("*", proxyHandler) + .create(); + proxyServer.start(); + } + } + + /** Stops the proxy server. This method is idempotent. */ + public void stop() { + if (started.compareAndSet(true, false)) { + proxyServer.stop(); + } + } + + private int findFreePort() { + try (ServerSocket s = new ServerSocket(0)) { + return s.getLocalPort(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static class ProxyHandler implements HttpRequestHandler { + private final int sourcePort; + private final int targetPort; + private boolean debug; + + final CloseableHttpClient client; + Function responseHeaderNameTransform = Function.identity(); + + ProxyHandler(int sourcePort, int targetPort, boolean debug) { + this.sourcePort = sourcePort; + this.targetPort = targetPort; + this.debug = debug; + + @SuppressWarnings("PMD.CloseResource") + PoolingHttpClientConnectionManager connManager = + new PoolingHttpClientConnectionManager(); + client = + HttpClients.custom() + .setConnectionManager(connManager) + .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) + .build(); + } + + @Override + public void handle(HttpRequest request, HttpResponse response, HttpContext context) + throws HttpException, IOException { + HttpUriRequest proxyRequest = proxify(request); + logRequest(request, proxyRequest); + + try (CloseableHttpResponse proxyResponse = client.execute(proxyRequest)) { + response.setStatusLine(proxyResponse.getStatusLine()); // status and reason phrase + logResponseStatus(response); + + Header[] headers = proxyResponse.getAllHeaders(); + applyResponseHeaders(response, headers); + transferResponseEntity(response, proxyResponse); + } + } + + private void transferResponseEntity(HttpResponse localResponse, HttpResponse remoteResponse) + throws IOException { + final HttpEntity remoteResponseEntity = remoteResponse.getEntity(); + HttpEntity entity; + if (null == remoteResponseEntity) { + entity = emptyBodyEntity(remoteResponse); + } else { + entity = extractResponseBody(remoteResponseEntity); + } + EntityUtils.updateEntity(localResponse, entity); + } + + private HttpEntity extractResponseBody(final HttpEntity remoteResponseEntity) + throws IOException { + ContentType contentType = ContentType.get(remoteResponseEntity); + byte[] rawContent = EntityUtils.toByteArray(remoteResponseEntity); + logResponseBody(contentType, rawContent); + return new ByteArrayEntity(rawContent, 0, rawContent.length, contentType); + } + + private HttpEntity emptyBodyEntity(HttpResponse remoteResponse) { + BasicHttpEntity entity = new BasicHttpEntity(); + Optional.ofNullable(remoteResponse.getFirstHeader("Content-Length")) + .map(Header::getValue) + .map(Long::parseLong) + .ifPresent(cl -> entity.setContentLength(cl)); + Header contentType = remoteResponse.getFirstHeader("Content-Type"); + entity.setContentType(contentType); + return entity; + } + + private void logResponseStatus(HttpResponse response) { + StatusLine statusLine = response.getStatusLine(); + info("\tresponse: %d %s", statusLine.getStatusCode(), statusLine.getReasonPhrase()); + } + + private void logResponseBody(ContentType contentType, byte[] rawContent) { + if (null != contentType) { + info("\tContent-Type: %s", contentType); + if (contentType.getMimeType().startsWith("application/xml") + || contentType.getMimeType().contains("json")) { + info("\tcontent:\t%s", new String(rawContent)); + } + } + } + + private void logRequest(HttpRequest request, HttpUriRequest proxyRequest) { + info( + "routing %s %s to %s %s", + request.getRequestLine().getMethod(), + request.getRequestLine().getUri(), + proxyRequest.getRequestLine().getMethod(), + proxyRequest.getRequestLine().getUri()); + + Stream.of(proxyRequest.getAllHeaders()) + .forEach( + header -> + info( + "\tapplied request header %s: %s", + header.getName(), header.getValue())); + } + + private void applyResponseHeaders(HttpResponse response, Header[] headers) { + if (null == headers || headers.length == 0) return; + + Stream.of(headers) + .forEach( + header -> { + String name = header.getName(); + String value = header.getValue(); + name = responseHeaderNameTransform.apply(name); + if ("Connection".equalsIgnoreCase(name) + || "Transfer-Encoding".equalsIgnoreCase(name) + || "Content-Length".equalsIgnoreCase(name)) { + // these will produce a 'Connection reset by peer', let the + // proxy handle them + return; + } + // Fix the problematic response header names + if ("etag".equalsIgnoreCase(name)) { + name = "ETag"; + } else if ("last-modified".equalsIgnoreCase(name)) { + name = "Last-Modified"; + } else if ("content-type".equalsIgnoreCase(name)) { + name = "Content-Type"; + } + response.addHeader(name, value); + info("\tapplied response header %s: %s", name, value); + }); + } + + private HttpUriRequest proxify(HttpRequest request) { + + RequestLine requestLine = request.getRequestLine(); + + String uri = + requestLine + .getUri() + .replace( + "http://localhost:" + sourcePort, + "http://localhost:" + targetPort); + + HttpUriRequest proxyRequest = + RequestBuilder.copy(request) + .setUri(uri) + // these will produce a 'Connection reset by peer', let the + // proxy handle them + .removeHeaders("Connection") + .removeHeaders("Transfer-Encoding") + .removeHeaders("Content-Length") + .build(); + return proxyRequest; + } + + @SuppressWarnings("PMD.SystemPrintln") + private void info(String msg, Object... params) { + if (debug) { + System.err.printf(msg + "%n", params); + } + } + } +} diff --git a/geowebcache/core/pom.xml b/geowebcache/core/pom.xml index b7b0305d5..9deb0b4a6 100644 --- a/geowebcache/core/pom.xml +++ b/geowebcache/core/pom.xml @@ -215,6 +215,11 @@ xmlunit-legacy test + + org.awaitility + awaitility + test + diff --git a/geowebcache/core/src/test/java/org/geowebcache/storage/AbstractBlobStoreTest.java b/geowebcache/core/src/test/java/org/geowebcache/storage/AbstractBlobStoreTest.java index 5f298eb06..672c32ba9 100644 --- a/geowebcache/core/src/test/java/org/geowebcache/storage/AbstractBlobStoreTest.java +++ b/geowebcache/core/src/test/java/org/geowebcache/storage/AbstractBlobStoreTest.java @@ -14,6 +14,8 @@ */ package org.geowebcache.storage; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.captureLong; @@ -512,8 +514,7 @@ public void testDeleteGridset() throws Exception { EasyMock.replay(listener); assertThat(store.deleteByGridsetId("testLayer", "testGridSet1"), is(true)); EasyMock.verify(listener); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + assertNoTile(fromCache1_2); } @Test @@ -575,8 +576,9 @@ public void testDeleteGridsetDoesntDeleteOthers() throws Exception { new ByteArrayResource( "7,8,9,10 test".getBytes(StandardCharsets.UTF_8))))); store.deleteByGridsetId("testLayer", "testGridSet1"); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + + assertNoTile(fromCache1_2); + assertThat(store.get(fromCache2_3), is(true)); assertThat(fromCache2_3, hasProperty("blobSize", is((int) size2))); assertThat( @@ -707,8 +709,9 @@ public void testParameters() throws Exception { EasyMock.replay(listener); store.delete(remove); EasyMock.verify(listener); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + + assertNoTile(fromCache1_2); + assertThat(store.get(fromCache2_3), is(true)); assertThat(fromCache2_3, hasProperty("blobSize", is((int) size2))); assertThat( @@ -925,8 +928,8 @@ public void testDeleteByParametersId() throws Exception { EasyMock.replay(listener); store.deleteByParametersId("testLayer", paramID1); EasyMock.verify(listener); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + + assertNoTile(fromCache1_2); } @Test @@ -959,7 +962,9 @@ public void testDeleteByParametersIdDoesNotDeleteOthers() throws Exception { store.put(toCache1); store.put(toCache2); store.deleteByParametersId("testLayer", paramID1); - assertThat(store.get(fromCache2_3), is(true)); + + await().atMost(5, SECONDS) // give stores with async deletes a chance to complete + .untilAsserted(() -> assertThat(store.get(fromCache2_3), is(true))); assertThat(fromCache2_3, hasProperty("blobSize", is((int) size2))); assertThat( fromCache2_3, @@ -1071,8 +1076,7 @@ public void testPurgeOrphans() throws Exception { EasyMock.replay(listener); store.purgeOrphans(layer); EasyMock.verify(listener); - assertThat(store.get(fromCache1_2), is(false)); - assertThat(fromCache1_2, hasProperty("blobSize", is(0))); + assertNoTile(fromCache1_2); } protected void cacheTile( @@ -1129,7 +1133,15 @@ protected void assertNoTile( TileObject to = TileObject.createQueryTileObject( layerName, new long[] {x, y, z}, gridSetId, format, parameters); - assertThat(store.get(to), describedAs("don't get a tile", is(false))); + assertNoTile(to); + } + + private void assertNoTile(TileObject to) { + await().atMost(5, SECONDS) // give stores with async deletes a chance to complete + .untilAsserted( + () -> + assertThat( + store.get(to), describedAs("don't get a tile", is(false)))); assertThat(to, hasProperty("blob", nullValue())); assertThat(to, hasProperty("blobSize", is(0))); } diff --git a/geowebcache/pom.xml b/geowebcache/pom.xml index 83bd5ce44..3afcab37b 100644 --- a/geowebcache/pom.xml +++ b/geowebcache/pom.xml @@ -108,6 +108,14 @@ test + + org.testcontainers + testcontainers-bom + 1.20.1 + pom + import + + org.locationtech.jts jts-core @@ -286,7 +294,12 @@ 1.3 test - + + org.awaitility + awaitility + 4.2.2 + test + org.easymock easymock @@ -521,6 +534,28 @@ cobertura-maven-plugin 2.0 + + maven-failsafe-plugin + 3.3.1 + + + + integration-test + verify + + + + true + + + src/test/resources/logging.properties + + true + + ${maven.test.jvmargs} -XX:+IgnoreUnrecognizedVMOptions --illegal-access=warn --add-exports=java.desktop/sun.awt.image=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.desktop/java.awt.font=ALL-UNNAMED + ${skipITs} + + @@ -618,19 +653,6 @@ maven-failsafe-plugin - 2.19.1 - - - - integration-test - verify - - - - - -Djava.awt.headless=true - - diff --git a/geowebcache/s3storage/pom.xml b/geowebcache/s3storage/pom.xml index 3f05769f5..c74da3195 100644 --- a/geowebcache/s3storage/pom.xml +++ b/geowebcache/s3storage/pom.xml @@ -69,7 +69,11 @@ log4j-slf4j-impl test - + + org.awaitility + awaitility + test + diff --git a/geowebcache/sqlite/pom.xml b/geowebcache/sqlite/pom.xml index 6bd247850..595a88d89 100644 --- a/geowebcache/sqlite/pom.xml +++ b/geowebcache/sqlite/pom.xml @@ -73,5 +73,10 @@ easymock test + + org.awaitility + awaitility + test +