Skip to content

Commit

Permalink
Run Azure blob store integration tests with Azure and Azurite
Browse files Browse the repository at this point in the history
Azurite is run as a testcontainer, and the github CI build runs all
integration tests for Java 11, 17, and 21.

In order to save on Azure resources, online integration tests are
run only if the Azurite based tests succeeded, using github secrets
to set up the account, key, and container; and for Java 11 only.
  • Loading branch information
groldan committed Aug 27, 2024
1 parent 0d2fc0b commit 57fecb1
Show file tree
Hide file tree
Showing 23 changed files with 1,172 additions and 92 deletions.
82 changes: 82 additions & 0 deletions .github/workflows/azure-integration.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
name: Azure BlobStore Integration

on:
push:
tags:
- "**"
pull_request:
paths:
- ".github/workflows/azure-integration.yml"
- "pom.xml"
- "geowebcache/pom.xml"
- "geowebcache/core/**"
- "geowebcache/azureblob/**"

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
azurite:
name: Azurite container
runs-on: ubuntu-latest
strategy:
matrix:
java-version: [ 11, 17, 21 ]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java-version }}
cache: 'maven'

- name: Tests against Azurite TestContainers
#-PexcludeOnlineTests includes Azurite container tests and excludes Azure online tests
run: |
mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \
-Ponline,excludeAzureOnlineTests \
-DskipTests=true \
-DskipITs=false -B -ntp
- name: Remove SNAPSHOT jars from repository
run: |
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
azure:
name: Azure online
#if: github.repository == 'geowebcache/geowebcache'
runs-on: ubuntu-latest
needs: azurite
if: |
always() &&
!contains(needs.*.result, 'cancelled') &&
!contains(needs.*.result, 'failure')
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 11
cache: 'maven'

- name: Tests against Azure
env:
azure_account: ${{ secrets.AZURE_ACCOUNT }}
azure_account_key: ${{ secrets.AZURE_ACCOUNT_KEY }}
azure_container: ${{ secrets.AZURE_CONTAINER }}
if: ${{ env.azure_account != null }} && ${{ env.azure_account_key != null }}
run: | #-PexcludeDockerTests includes Azure online tests and excludes Azurite container tests
echo "accountName=$azure_account" > $HOME/.gwc_azure_tests.properties
echo "accountKey=$azure_account_key" >> $HOME/.gwc_azure_tests.properties
echo "container=$azure_container" >> $HOME/.gwc_azure_tests.properties
echo 'maxConnections=8' >> $HOME/.gwc_azure_tests.properties
echo 'useHTTPS=true' >> $HOME/.gwc_azure_tests.properties
mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \
-Ponline,excludeDockerTests \
-DskipTests=true \
-DskipITs=false -B -ntp
- name: Remove SNAPSHOT jars from repository
run: |
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
34 changes: 34 additions & 0 deletions geowebcache/azureblob/REAME.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Azure BlobStore

GeoWebCache `BlobStore` implementation to store tiles on [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs)


## Building

`mvn install|test|verify` will run only unit tests.

In order to run the integration tests, build with the `-Ponline` maven profile. For example:

```
mvn verify -Ponline
```

There are two sets of integration tests:

- `org.geowebcache.azure.tests.container.*IT.java`: run integration tests using [Testcontainers](https://testcontainers.com/), with a Microsoft [Azurite](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite) Docker image.
- `org.geowebcache.azure.tests.online.*IT.java`: run integration tests against a real Azure account, only if there's a configuration file in `$HOME/.gwc_azure_tests.properties`, which must have the following contents:

```
accountName=<Azure account name>
accountKey=<Azure account key>
container=<Azure blob storage container name>
useHTTPS=true
maxConnections=<optional, max number of concurrent connections>
```

## Continuous integration

There's a Github Actions CI job defined at `<gwc git root>/.github/workflows/azure-integration.yml`
that will run the Docker-based integration tests first, and then the online ones against
a real Azure account, using Github repository secrets to define the values for the
`$HOME/.gwc_azure_tests.properties` file's `accountName`, `accountSecret`, and `container` keys.
71 changes: 71 additions & 0 deletions geowebcache/azureblob/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,76 @@
<artifactId>javax.servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
<profile>
<id>online</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<!-- skips the integration tests against a real Azure account, used to split up the CI builds-->
<id>excludeAzureOnlineTests</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<excludes>
<exclude>org.geowebcache.azure.tests.online.*IT</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<!-- skips the integration tests against the Azurite test container, used to split up the CI builds-->
<id>excludeDockerTests</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<excludes>
<exclude>org.geowebcache.azure.tests.container.*IT</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
* Azure Blobstore type-resolved data from a {@link AzureBlobStoreInfo} using enviroment variables
* if enabled.
*/
class AzureBlobStoreData {
public class AzureBlobStoreData {

private String container;
private String prefix;
Expand All @@ -36,7 +36,7 @@ class AzureBlobStoreData {
private String proxyPassword;
private String serviceURL;

AzureBlobStoreData() {}
public AzureBlobStoreData() {}

public AzureBlobStoreData(
final AzureBlobStoreInfo storeInfo, final GeoWebCacheEnvironment environment) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
import org.geowebcache.util.URLs;
import org.springframework.http.HttpStatus;

class AzureClient implements Closeable {
public class AzureClient implements Closeable {

private final NettyClient.Factory factory;
private AzureBlobStoreData configuration;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,25 @@
import java.util.stream.Stream;
import org.easymock.EasyMock;
import org.geowebcache.GeoWebCacheException;
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT;
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreConformanceIT;
import org.geowebcache.layer.TileLayer;
import org.geowebcache.layer.TileLayerDispatcher;
import org.geowebcache.locks.LockProvider;
import org.geowebcache.locks.NoOpLockProvider;
import org.geowebcache.storage.AbstractBlobStoreTest;
import org.junit.Assume;
import org.junit.Rule;

public class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest<AzureBlobStore> {
public PropertiesLoader testConfigLoader = new PropertiesLoader();
/**
* @see OnlineAzureBlobStoreConformanceIT
* @see AzuriteAzureBlobStoreConformanceIT
*/
public abstract class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest<AzureBlobStore> {

@Rule
public TemporaryAzureFolder tempFolder =
new TemporaryAzureFolder(testConfigLoader.getProperties());
protected abstract AzureBlobStoreData getConfiguration();

@Override
public void createTestUnit() throws Exception {
Assume.assumeTrue(tempFolder.isConfigured());
AzureBlobStoreData config = tempFolder.getConfig();
AzureBlobStoreData config = getConfiguration();

TileLayerDispatcher layers = createMock(TileLayerDispatcher.class);
LockProvider lockProvider = new NoOpLockProvider();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
import java.util.Map;
import java.util.logging.Logger;
import org.geotools.util.logging.Logging;
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreIntegrationIT;
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreIntegrationIT;
import org.geowebcache.config.DefaultGridsets;
import org.geowebcache.grid.GridSet;
import org.geowebcache.grid.GridSetBroker;
Expand Down Expand Up @@ -69,19 +71,20 @@
* Integration tests for {@link AzureBlobStore}.
*
* <p>This is an abstract class for both online and offline integration tests.
*
* @see OnlineAzureBlobStoreIntegrationIT
* @see AzuriteAzureBlobStoreIntegrationIT
*/
public abstract class AbstractAzureBlobStoreIntegrationTest {
public abstract class AzureBlobStoreIntegrationTest {

private static Logger log = Logging.getLogger(PropertiesLoader.class.getName());
private static Logger log = Logging.getLogger(AzureBlobStoreIntegrationTest.class.getName());

private static final String DEFAULT_FORMAT = "png";

private static final String DEFAULT_GRIDSET = "EPSG:4326";

private static final String DEFAULT_LAYER = "topp:world";

public PropertiesLoader testConfigLoader = new PropertiesLoader();

private AzureBlobStore blobStore;

protected abstract AzureBlobStoreData getConfiguration();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItemInArray;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeFalse;

import io.reactivex.Flowable;
import java.nio.ByteBuffer;
import org.easymock.EasyMock;
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreSuitabilityIT;
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreSuitabilityIT;
import org.geowebcache.layer.TileLayerDispatcher;
import org.geowebcache.locks.LockProvider;
import org.geowebcache.locks.NoOpLockProvider;
Expand All @@ -30,23 +31,14 @@
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.Rule;
import org.junit.experimental.theories.DataPoints;
import org.junit.experimental.theories.Theories;
import org.junit.runner.RunWith;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.InitializationError;
import org.junit.runners.model.Statement;
import org.springframework.http.HttpStatus;

@RunWith(AzureBlobStoreSuitabilityTest.MyTheories.class)
public class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest {

public PropertiesLoader testConfigLoader = new PropertiesLoader();

@Rule
public TemporaryAzureFolder tempFolder =
new TemporaryAzureFolder(testConfigLoader.getProperties());
/**
* @see OnlineAzureBlobStoreSuitabilityIT
* @see AzuriteAzureBlobStoreSuitabilityIT
*/
public abstract class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest {

@DataPoints
public static String[][] persistenceLocations = {
Expand All @@ -67,6 +59,10 @@ public void setup() throws Exception {
EasyMock.replay(tld);
}

protected abstract AzureBlobStoreData getConfiguration();

protected abstract AzureClient getClient();

@SuppressWarnings("unchecked")
@Override
protected Matcher<Object> existing() {
Expand All @@ -81,13 +77,12 @@ protected Matcher<Object> empty() {

@Override
public BlobStore create(Object dir) throws Exception {
AzureBlobStoreData info = tempFolder.getConfig();
AzureBlobStoreData info = getConfiguration();
for (String path : (String[]) dir) {
String fullPath = info.getPrefix() + "/" + path;
ByteBuffer byteBuffer = ByteBuffer.wrap("testAbc".getBytes());
int statusCode =
tempFolder
.getClient()
getClient()
.getBlockBlobURL(fullPath)
.upload(Flowable.just(byteBuffer), byteBuffer.limit())
.blockingGet()
Expand All @@ -96,27 +91,4 @@ public BlobStore create(Object dir) throws Exception {
}
return new AzureBlobStore(info, tld, locks);
}

// Sorry, this bit of evil makes the Theories runner gracefully ignore the
// tests if Azure is unavailable. There's probably a better way to do this.
public static class MyTheories extends Theories {

public MyTheories(Class<?> klass) throws InitializationError {
super(klass);
}

@Override
public Statement methodBlock(FrameworkMethod method) {
if (new PropertiesLoader().getProperties().containsKey("container")) {
return super.methodBlock(method);
} else {
return new Statement() {
@Override
public void evaluate() {
assumeFalse("Azure unavailable", true);
}
};
}
}
}
}
Loading

0 comments on commit 57fecb1

Please sign in to comment.