diff --git a/README.md b/README.md index 2e573850..d609a185 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ repositories { } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons', version: '0.13.1' + implementation group: 'org.radarbase', name: 'radar-commons', version: '0.13.2' } ``` @@ -69,7 +69,7 @@ repositories { } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons-server', version: '0.13.1' + implementation group: 'org.radarbase', name: 'radar-commons-server', version: '0.13.2' } ``` @@ -82,7 +82,7 @@ repositories { } dependencies { - testImplementation group: 'org.radarbase', name: 'radar-commons-testing', version: '0.13.1' + testImplementation group: 'org.radarbase', name: 'radar-commons-testing', version: '0.13.2' } ``` @@ -94,7 +94,7 @@ repositories { } dependencies { - runtimeOnly group: 'org.radarbase', name: 'radar-commons-unsafe', version: '0.13.1' + runtimeOnly group: 'org.radarbase', name: 'radar-commons-unsafe', version: '0.13.2' } ``` @@ -119,7 +119,7 @@ configurations.all { } dependencies { - compile group: 'org.radarbase', name: 'radar-commons', version: '0.13.2-SNAPSHOT' + compile group: 'org.radarbase', name: 'radar-commons', version: '0.13.3-SNAPSHOT' } ``` diff --git a/build.gradle b/build.gradle index 24fe6990..ce9139a6 100644 --- a/build.gradle +++ b/build.gradle @@ -15,6 +15,8 @@ */ plugins { id 'com.commercehub.gradle.plugin.avro' version '0.19.1' + id("io.github.gradle-nexus.publish-plugin") version "1.0.0" + id("com.github.ben-manes.versions") version "0.38.0" } subprojects { @@ -22,28 +24,29 @@ subprojects { apply plugin: 'java' apply plugin: 'java-library' apply plugin: 'idea' + apply plugin: 'com.github.ben-manes.versions' //---------------------------------------------------------------------------// // Configuration // //---------------------------------------------------------------------------// - version = '0.13.1' + version = '0.13.2' group = 'org.radarbase' ext.githubRepoName = 'RADAR-base/radar-commons' ext.slf4jVersion = '1.7.30' - ext.kafkaVersion = '2.7.0' + ext.kafkaVersion = '6.1.1-ce' ext.avroVersion = '1.9.2' - ext.confluentVersion = '5.5.3' - ext.jacksonVersion = '2.12.1' - ext.jacksonYamlVersion = '2.12.1' + ext.confluentVersion = '6.1.1' + ext.jacksonVersion = '2.12.3' + ext.jacksonYamlVersion = '2.12.3' ext.okhttpVersion = '4.9.1' ext.junitVersion = '4.13.2' - ext.mockitoVersion = '3.7.7' + ext.mockitoVersion = '3.9.0' ext.hamcrestVersion = '1.3' - ext.codacyVersion = '7.9.0' - ext.radarSchemasVersion = '0.5.15' - ext.orgJsonVersion = '20201115' + ext.codacyVersion = '11.15.0' + ext.radarSchemasVersion = '0.6.0' + ext.orgJsonVersion = '20210307' ext.githubUrl = "https://github.com/$githubRepoName" ext.issueUrl = "https://github.com/$githubRepoName/issues" @@ -53,9 +56,8 @@ subprojects { // Dependencies // //---------------------------------------------------------------------------// repositories { - jcenter() + mavenCentral() maven { url 'https://packages.confluent.io/maven/' } - maven { url 'https://dl.bintray.com/typesafe/maven-releases' } flatDir { dirs "${project.rootDir}/libs" } @@ -108,6 +110,15 @@ subprojects { } } +nexusPublishing { + repositories { + sonatype { + username = project.hasProperty("ossrh.user") ? project.property("ossrh.user") : System.getenv("OSSRH_USER") + password = project.hasProperty("ossrh.password") ? project.property("ossrh.password") : System.getenv("OSSRH_PASSWORD") + } + } +} + wrapper { - gradleVersion '6.8.3' + gradleVersion '7.0' } diff --git a/gradle/publishing.gradle b/gradle/publishing.gradle index 65846e2e..7076577b 100644 --- a/gradle/publishing.gradle +++ b/gradle/publishing.gradle @@ -72,20 +72,6 @@ publishing { } } } - - repositories { - maven { - name = "OSSRH" - credentials { - username = project.hasProperty("ossrh.user") ? project.property("ossrh.user") : System.getenv("OSSRH_USER") - password = project.hasProperty("ossrh.password") ? project.property("ossrh.password") : System.getenv("OSSRH_PASSWORD") - } - - def releasesRepoUrl = uri("https://oss.sonatype.org/service/local/staging/deploy/maven2/") - def snapshotsRepoUrl = uri("https://oss.sonatype.org/content/repositories/snapshots/") - url = version.toString().endsWith("SNAPSHOT") ? snapshotsRepoUrl : releasesRepoUrl - } - } } signing { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 442d9132..f371643e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/radar-commons-server/build.gradle b/radar-commons-server/build.gradle index 8cb9307f..cdba36cc 100644 --- a/radar-commons-server/build.gradle +++ b/radar-commons-server/build.gradle @@ -40,7 +40,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion // Direct producer uses KafkaAvroSerializer if initialized testImplementation group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion - testImplementation group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion + testImplementation group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion // Direct producer uses KafkaAvroSerializer if initialized testImplementation group: 'junit', name: 'junit', version: junitVersion testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion diff --git a/radar-commons-testing/build.gradle b/radar-commons-testing/build.gradle index e11e26a3..5e139e49 100644 --- a/radar-commons-testing/build.gradle +++ b/radar-commons-testing/build.gradle @@ -44,9 +44,9 @@ dependencies { api project(':radar-commons') api project(':radar-commons-server') api group: 'org.apache.avro', name: 'avro', version: avroVersion - api group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion + api group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion - implementation group: 'com.opencsv', name: 'opencsv', version: '5.3' + implementation group: 'com.opencsv', name: 'opencsv', version: '5.4' implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion implementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java index ff8e23a6..a7d4b133 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java @@ -22,9 +22,12 @@ import java.io.Closeable; import java.io.IOException; import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,7 +62,6 @@ public class MockCsvParser implements Closeable { */ public MockCsvParser(MockDataConfig config, Path root) throws IOException, CsvValidationException { - //noinspection unchecked topic = config.parseAvroTopic(); bufferedReader = Files.newBufferedReader(config.getDataFile(root)); @@ -72,7 +74,7 @@ public MockCsvParser(MockDataConfig config, Path root) currentLine = csvReader.readNext(); } - public AvroTopic getTopic() { + public AvroTopic getTopic() { return topic; } @@ -111,7 +113,13 @@ private V parseRecord(String[] rawValues, V record = (V) SpecificData.newInstance(recordClass, schema); for (Field field : schema.getFields()) { - String fieldString = rawValues[header.get(field.name())]; + Integer fieldHeader = header.get(field.name()); + if (fieldHeader == null) { + throw new IllegalArgumentException( + "Cannot map record field " + field.name() + + ": no corresponding header in " + header.keySet()); + } + String fieldString = rawValues[fieldHeader]; Object fieldValue = parseValue(field.schema(), fieldString); record.put(field.pos(), fieldValue); } @@ -119,7 +127,7 @@ private V parseRecord(String[] rawValues, return record; } - private static Object parseValue(Schema schema, String fieldString) { + public static Object parseValue(Schema schema, String fieldString) { switch (schema.getType()) { case INT: return Integer.parseInt(fieldString); @@ -139,12 +147,20 @@ private static Object parseValue(Schema schema, String fieldString) { return parseUnion(schema, fieldString); case ENUM: return parseEnum(schema, fieldString); + case BYTES: + return parseBytes(fieldString); default: throw new IllegalArgumentException("Cannot handle schemas of type " + schema.getType()); } } + private static ByteBuffer parseBytes(String fieldString) { + byte[] result = Base64.getDecoder() + .decode(fieldString.getBytes(StandardCharsets.UTF_8)); + return ByteBuffer.wrap(result); + } + private static Object parseUnion(Schema schema, String fieldString) { if (schema.getTypes().size() != 2) { throw new IllegalArgumentException( diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockCsvParserTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockCsvParserTest.java new file mode 100644 index 00000000..287d2db3 --- /dev/null +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockCsvParserTest.java @@ -0,0 +1,16 @@ +package org.radarbase.mock.data; + +import static org.junit.Assert.assertArrayEquals; + +import java.nio.ByteBuffer; +import org.apache.avro.Schema; +import org.junit.Test; + +public class MockCsvParserTest { + @Test + public void parseValue() { + Schema schema = new Schema.Parser().parse("{\"type\":\"bytes\"}"); + ByteBuffer buffer = (ByteBuffer) MockCsvParser.parseValue(schema, "ab"); + assertArrayEquals(new byte[] { 105 }, buffer.array()); + } +} diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java index 60e14339..f38b33d0 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java @@ -152,7 +152,7 @@ public void validateMissingKeyField() throws Exception { writer.append("test,a,1,2,1\n"); } - assertValidateThrows(NullPointerException.class, config); + assertValidateThrows(IllegalArgumentException.class, config); } @Test @@ -165,7 +165,7 @@ public void validateMissingValueField() throws Exception { writer.append("test,a,b,1,2\n"); } - assertValidateThrows(NullPointerException.class, config); + assertValidateThrows(IllegalArgumentException.class, config); } @Test diff --git a/radar-commons-unsafe/build.gradle b/radar-commons-unsafe/build.gradle index 1d148256..1b6fe4b4 100644 --- a/radar-commons-unsafe/build.gradle +++ b/radar-commons-unsafe/build.gradle @@ -7,14 +7,13 @@ sourceCompatibility = '1.8' // Sources and classpath configurations // //---------------------------------------------------------------------------// -configurations.compile { +configurations.implementation { resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' } // In this section you declare where to find the dependencies of your project repositories { maven { url 'https://jitpack.io' } - maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } } // In this section you declare the dependencies for your production and test code diff --git a/radar-commons/build.gradle b/radar-commons/build.gradle index 57880421..a208d566 100644 --- a/radar-commons/build.gradle +++ b/radar-commons/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation group: 'org.slf4j', name:'slf4j-api', version: slf4jVersion testImplementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion - testImplementation group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion + testImplementation group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion testImplementation group: 'junit', name: 'junit', version: junitVersion testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: okhttpVersion