Run Azure blob store integration tests with Azure and Azurite

Azurite is run as a testcontainer, and the github CI build runs all integration tests for Java 11, 17, and 21. In order to save on Azure resources, online integration tests are run only if the Azurite based tests succeeded, using github secrets to set up the account, key, and container; and for Java 11 only.
GeoWebCache · Aug 29, 2024 · 8d1146e · 8d1146e
1 parent d04281e
commit 8d1146e
Show file tree

Hide file tree

Showing 23 changed files with 1,172 additions and 92 deletions.
diff --git a/.github/workflows/azure-integration.yml b/.github/workflows/azure-integration.yml
@@ -0,0 +1,82 @@
+name: Azure BlobStore Integration
+
+on:
+  push:
+    tags:
+      - "**"
+  pull_request:
+    paths:
+      - ".github/workflows/azure-integration.yml"
+      - "pom.xml"
+      - "geowebcache/pom.xml"
+      - "geowebcache/core/**"
+      - "geowebcache/azureblob/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  azurite:
+    name: Azurite container
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ 11, 17, 21 ]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java-version }}
+        cache: 'maven'
+
+    - name: Tests against Azurite TestContainers
+      #-PexcludeOnlineTests includes Azurite container tests and excludes Azure online tests
+      run: |
+         mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \
+         -Ponline,excludeAzureOnlineTests \
+         -DskipTests=true \
+         -DskipITs=false -B -ntp
+
+    - name: Remove SNAPSHOT jars from repository
+      run: |
+        find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
+
+  azure:
+    name: Azure online
+    #if: github.repository == 'geowebcache/geowebcache'
+    runs-on: ubuntu-latest
+    needs: azurite
+    if: |
+      always() &&
+      !contains(needs.*.result, 'cancelled') &&
+      !contains(needs.*.result, 'failure')
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: 11
+        cache: 'maven'
+
+    - name: Tests against Azure
+      env:
+        azure_account: ${{ secrets.AZURE_ACCOUNT }}
+        azure_account_key: ${{ secrets.AZURE_ACCOUNT_KEY }}
+        azure_container: ${{ secrets.AZURE_CONTAINER }}
+      if: ${{ env.azure_account != null }} && ${{ env.azure_account_key != null }}
+      run: | #-PexcludeDockerTests includes Azure online tests and excludes Azurite container tests
+         echo "accountName=$azure_account"     > $HOME/.gwc_azure_tests.properties
+         echo "accountKey=$azure_account_key" >> $HOME/.gwc_azure_tests.properties
+         echo "container=$azure_container"    >> $HOME/.gwc_azure_tests.properties
+         echo 'maxConnections=8'              >> $HOME/.gwc_azure_tests.properties
+         echo 'useHTTPS=true'                 >> $HOME/.gwc_azure_tests.properties
+         mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \
+         -Ponline,excludeDockerTests \
+         -DskipTests=true \
+         -DskipITs=false -B -ntp
+
+    - name: Remove SNAPSHOT jars from repository
+      run: |
+        find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
diff --git a/geowebcache/azureblob/REAME.md b/geowebcache/azureblob/REAME.md
@@ -0,0 +1,34 @@
+# Azure BlobStore
+
+GeoWebCache `BlobStore` implementation to store tiles on [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs)
+
+
+## Building
+
+`mvn install|test|verify` will run only unit tests.
+
+In order to run the integration tests, build with the `-Ponline` maven profile. For example:
+
+```
+mvn verify -Ponline
+```
+
+There are two sets of integration tests:
+
+- `org.geowebcache.azure.tests.container.*IT.java`: run integration tests using [Testcontainers](https://testcontainers.com/), with a Microsoft [Azurite](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite) Docker image.
+- `org.geowebcache.azure.tests.online.*IT.java`: run integration tests against a real Azure account, only if there's a configuration file in `$HOME/.gwc_azure_tests.properties`, which must have the following contents:
+
+```
+accountName=<Azure account name>
+accountKey=<Azure account key>
+container=<Azure blob storage container name>
+useHTTPS=true
+maxConnections=<optional, max number of concurrent connections>
+```
+
+## Continuous integration
+
+There's a Github Actions CI job defined at `<gwc git root>/.github/workflows/azure-integration.yml`
+that will run the Docker-based integration tests first, and then the online ones against
+a real Azure account, using Github repository secrets to define the values for the
+`$HOME/.gwc_azure_tests.properties` file's `accountName`, `accountSecret`, and `container` keys.
diff --git a/geowebcache/azureblob/pom.xml b/geowebcache/azureblob/pom.xml
@@ -51,5 +51,76 @@
       <artifactId>javax.servlet-api</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>testcontainers</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.awaitility</groupId>
+      <artifactId>awaitility</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
+  <profiles>
+    <profile>
+      <id>online</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <configuration>
+              <forkCount>1</forkCount>
+              <reuseForks>false</reuseForks>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <!-- skips the integration tests against a real Azure account, used to split up the CI builds-->
+      <id>excludeAzureOnlineTests</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <configuration>
+              <forkCount>1</forkCount>
+              <reuseForks>false</reuseForks>
+              <excludes>
+                <exclude>org.geowebcache.azure.tests.online.*IT</exclude>
+              </excludes>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <!-- skips the integration tests against the Azurite test container, used to split up the CI builds-->
+      <id>excludeDockerTests</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <configuration>
+              <forkCount>1</forkCount>
+              <reuseForks>false</reuseForks>
+              <excludes>
+                <exclude>org.geowebcache.azure.tests.container.*IT</exclude>
+              </excludes>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>
diff --git a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java
@@ -22,7 +22,7 @@
  * Azure Blobstore type-resolved data from a {@link AzureBlobStoreInfo} using enviroment variables
  * if enabled.
  */
-class AzureBlobStoreData {
+public class AzureBlobStoreData {
 
     private String container;
     private String prefix;
@@ -36,7 +36,7 @@ class AzureBlobStoreData {
     private String proxyPassword;
     private String serviceURL;
 
-    AzureBlobStoreData() {}
+    public AzureBlobStoreData() {}
 
     public AzureBlobStoreData(
             final AzureBlobStoreInfo storeInfo, final GeoWebCacheEnvironment environment) {

diff --git a/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java b/geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java
@@ -50,7 +50,7 @@
 import org.geowebcache.util.URLs;
 import org.springframework.http.HttpStatus;
 
-class AzureClient implements Closeable {
+public class AzureClient implements Closeable {
 
     private final NettyClient.Factory factory;
     private AzureBlobStoreData configuration;

diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java
@@ -25,25 +25,25 @@
 import java.util.stream.Stream;
 import org.easymock.EasyMock;
 import org.geowebcache.GeoWebCacheException;
+import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT;
+import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreConformanceIT;
 import org.geowebcache.layer.TileLayer;
 import org.geowebcache.layer.TileLayerDispatcher;
 import org.geowebcache.locks.LockProvider;
 import org.geowebcache.locks.NoOpLockProvider;
 import org.geowebcache.storage.AbstractBlobStoreTest;
-import org.junit.Assume;
-import org.junit.Rule;
 
-public class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest<AzureBlobStore> {
-    public PropertiesLoader testConfigLoader = new PropertiesLoader();
+/**
+ * @see OnlineAzureBlobStoreConformanceIT
+ * @see AzuriteAzureBlobStoreConformanceIT
+ */
+public abstract class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest<AzureBlobStore> {
 
-    @Rule
-    public TemporaryAzureFolder tempFolder =
-            new TemporaryAzureFolder(testConfigLoader.getProperties());
+    protected abstract AzureBlobStoreData getConfiguration();
 
     @Override
     public void createTestUnit() throws Exception {
-        Assume.assumeTrue(tempFolder.isConfigured());
-        AzureBlobStoreData config = tempFolder.getConfig();
+        AzureBlobStoreData config = getConfiguration();
 
         TileLayerDispatcher layers = createMock(TileLayerDispatcher.class);
         LockProvider lockProvider = new NoOpLockProvider();

diff --git a/...bstractAzureBlobStoreIntegrationTest.java → .../azure/AzureBlobStoreIntegrationTest.java b/...bstractAzureBlobStoreIntegrationTest.java → .../azure/AzureBlobStoreIntegrationTest.java
@@ -42,6 +42,8 @@
 import java.util.Map;
 import java.util.logging.Logger;
 import org.geotools.util.logging.Logging;
+import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreIntegrationIT;
+import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreIntegrationIT;
 import org.geowebcache.config.DefaultGridsets;
 import org.geowebcache.grid.GridSet;
 import org.geowebcache.grid.GridSetBroker;
@@ -69,19 +71,20 @@
  * Integration tests for {@link AzureBlobStore}.
  *
  * <p>This is an abstract class for both online and offline integration tests.
+ *
+ * @see OnlineAzureBlobStoreIntegrationIT
+ * @see AzuriteAzureBlobStoreIntegrationIT
  */
-public abstract class AbstractAzureBlobStoreIntegrationTest {
+public abstract class AzureBlobStoreIntegrationTest {
 
-    private static Logger log = Logging.getLogger(PropertiesLoader.class.getName());
+    private static Logger log = Logging.getLogger(AzureBlobStoreIntegrationTest.class.getName());
 
     private static final String DEFAULT_FORMAT = "png";
 
     private static final String DEFAULT_GRIDSET = "EPSG:4326";
 
     private static final String DEFAULT_LAYER = "topp:world";
 
-    public PropertiesLoader testConfigLoader = new PropertiesLoader();
-
     private AzureBlobStore blobStore;
 
     protected abstract AzureBlobStoreData getConfiguration();

diff --git a/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java b/geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java
@@ -17,11 +17,12 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItemInArray;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeFalse;
 
 import io.reactivex.Flowable;
 import java.nio.ByteBuffer;
 import org.easymock.EasyMock;
+import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreSuitabilityIT;
+import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreSuitabilityIT;
 import org.geowebcache.layer.TileLayerDispatcher;
 import org.geowebcache.locks.LockProvider;
 import org.geowebcache.locks.NoOpLockProvider;
@@ -30,23 +31,14 @@
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.experimental.theories.DataPoints;
-import org.junit.experimental.theories.Theories;
-import org.junit.runner.RunWith;
-import org.junit.runners.model.FrameworkMethod;
-import org.junit.runners.model.InitializationError;
-import org.junit.runners.model.Statement;
 import org.springframework.http.HttpStatus;
 
-@RunWith(AzureBlobStoreSuitabilityTest.MyTheories.class)
-public class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest {
-
-    public PropertiesLoader testConfigLoader = new PropertiesLoader();
-
-    @Rule
-    public TemporaryAzureFolder tempFolder =
-            new TemporaryAzureFolder(testConfigLoader.getProperties());
+/**
+ * @see OnlineAzureBlobStoreSuitabilityIT
+ * @see AzuriteAzureBlobStoreSuitabilityIT
+ */
+public abstract class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest {
 
     @DataPoints
     public static String[][] persistenceLocations = {
@@ -67,6 +59,10 @@ public void setup() throws Exception {
         EasyMock.replay(tld);
     }
 
+    protected abstract AzureBlobStoreData getConfiguration();
+
+    protected abstract AzureClient getClient();
+
     @SuppressWarnings("unchecked")
     @Override
     protected Matcher<Object> existing() {
@@ -81,13 +77,12 @@ protected Matcher<Object> empty() {
 
     @Override
     public BlobStore create(Object dir) throws Exception {
-        AzureBlobStoreData info = tempFolder.getConfig();
+        AzureBlobStoreData info = getConfiguration();
         for (String path : (String[]) dir) {
             String fullPath = info.getPrefix() + "/" + path;
             ByteBuffer byteBuffer = ByteBuffer.wrap("testAbc".getBytes());
             int statusCode =
-                    tempFolder
-                            .getClient()
+                    getClient()
                             .getBlockBlobURL(fullPath)
                             .upload(Flowable.just(byteBuffer), byteBuffer.limit())
                             .blockingGet()
@@ -96,27 +91,4 @@ public BlobStore create(Object dir) throws Exception {
         }
         return new AzureBlobStore(info, tld, locks);
     }
-
-    // Sorry, this bit of evil makes the Theories runner gracefully ignore the
-    // tests if Azure is unavailable.  There's probably a better way to do this.
-    public static class MyTheories extends Theories {
-
-        public MyTheories(Class<?> klass) throws InitializationError {
-            super(klass);
-        }
-
-        @Override
-        public Statement methodBlock(FrameworkMethod method) {
-            if (new PropertiesLoader().getProperties().containsKey("container")) {
-                return super.methodBlock(method);
-            } else {
-                return new Statement() {
-                    @Override
-                    public void evaluate() {
-                        assumeFalse("Azure unavailable", true);
-                    }
-                };
-            }
-        }
-    }
 }