diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..5afb0577 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,43 @@ +# Continuous integration, including test and integration test +name: CI + +# Run in master and dev branches and in all pull requests to those branches +on: + push: + branches: [ master, dev ] + pull_request: {} + +jobs: + # Build and test the code + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - uses: actions/setup-java@v1 + with: + java-version: 11 + + - name: Cache + uses: actions/cache@v2 + with: + # Cache gradle directories + path: | + ~/.gradle/caches + ~/.gradle/wrapper + # Key for restoring and saving the cache + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle', 'gradle.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + # Compile the code + - name: Compile code + run: ./gradlew assemble + + # Gradle check + - name: Check + run: ./gradlew check diff --git a/.github/workflows/publish_snapshots.yml b/.github/workflows/publish_snapshots.yml new file mode 100644 index 00000000..a89a77bf --- /dev/null +++ b/.github/workflows/publish_snapshots.yml @@ -0,0 +1,49 @@ +# Continuous integration, including test and integration test +name: Publish snapshots + +# Run in master and dev branches and in all pull requests to those branches +on: + push: + branches: [ dev ] + +jobs: + # Build and test the code + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Has SNAPSHOT version + id: is-snapshot + run: grep "version = '.*-SNAPSHOT'" build.gradle + + - uses: actions/setup-java@v1 + with: + java-version: 11 + + - name: Cache + uses: actions/cache@v2 + with: + # Cache gradle directories + path: | + ~/.gradle/caches + ~/.gradle/wrapper + # Key for restoring and saving the cache + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle', 'gradle.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Install gpg secret key + run: | + cat <(echo -e "${{ secrets.OSSRH_GPG_SECRET_KEY }}") | gpg --batch --import + gpg --list-secret-keys --keyid-format LONG + + - name: Publish + env: + OSSRH_USER: ${{ secrets.OSSRH_USER }} + OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + run: ./gradlew -Psigning.gnupg.keyName=CBEF2CF0 -Psigning.gnupg.executable=gpg -Psigning.gnupg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} publish diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..9feed302 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,53 @@ +# Create release files +name: Release + +on: + release: + types: [published] + +jobs: + upload: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 11 + + - name: Cache + uses: actions/cache@v2 + with: + # Cache gradle directories + path: | + ~/.gradle/caches + ~/.gradle/wrapper + # Key for restoring and saving the cache + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle', 'gradle.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + # Compile code + - name: Compile code + run: ./gradlew assemble + + # Upload it to GitHub + - name: Upload to GitHub + uses: AButler/upload-release-assets@v2.0 + with: + files: 'radar-jersey/build/libs/*;radar-jersey-hibernate/build/libs/*' + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install gpg secret key + run: | + cat <(echo -e "${{ secrets.OSSRH_GPG_SECRET_KEY }}") | gpg --batch --import + gpg --list-secret-keys --keyid-format LONG + + - name: Publish + env: + OSSRH_USER: ${{ secrets.OSSRH_USER }} + OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + run: ./gradlew -Psigning.gnupg.keyName=CBEF2CF0 -Psigning.gnupg.executable=gpg -Psigning.gnupg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} publish closeAndReleaseSonatypeStagingRepository diff --git a/.github/workflows/scheduled_snyk.yaml b/.github/workflows/scheduled_snyk.yaml new file mode 100644 index 00000000..46074dfb --- /dev/null +++ b/.github/workflows/scheduled_snyk.yaml @@ -0,0 +1,25 @@ +name: Snyk scheduled test +on: + schedule: + - cron: '0 2 * * 1' +jobs: + security: + runs-on: ubuntu-latest + env: + REPORT_FILE: test.json + steps: + - uses: actions/checkout@master + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/gradle-jdk11@master + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + command: test + args: --json-file-output=${{ env.REPORT_FILE }} + - name: Report new vulnerabilities + uses: thehyve/report-vulnerability@master + with: + report-file: ${{ env.REPORT_FILE }} + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: ${{ failure() }} diff --git a/.github/workflows/snyk.yaml b/.github/workflows/snyk.yaml new file mode 100644 index 00000000..890b84f6 --- /dev/null +++ b/.github/workflows/snyk.yaml @@ -0,0 +1,16 @@ +name: Snyk test +on: + pull_request: + branches: + - master +jobs: + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/gradle-jdk11@master + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + args: --severity-threshold=high diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9e4cf878..00000000 --- a/.travis.yml +++ /dev/null @@ -1,26 +0,0 @@ -language: java -jdk: - - oraclejdk11 -sudo: false - -cache: - directories: - - $HOME/.gradle/caches/jars-1 - - $HOME/.gradle/caches/jars-2 - - $HOME/.gradle/caches/jars-3 - - $HOME/.gradle/caches/modules-2/files-2.1/ - - $HOME/.gradle/native - - $HOME/.gradle/wrapper - -after_script: - - ./gradlew sendCoverageToCodacy - -deploy: - - provider: releases - api_key: ${GH_TOKEN} - file_glob: true - file: - - "*/build/libs/*.jar" - skip_cleanup: true - on: - tags: true diff --git a/README.md b/README.md index d609a185..8c41caf7 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ repositories { } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons', version: '0.13.2' + implementation group: 'org.radarbase', name: 'radar-commons', version: '0.14.0' } ``` @@ -69,7 +69,7 @@ repositories { } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons-server', version: '0.13.2' + implementation group: 'org.radarbase', name: 'radar-commons-server', version: '0.14.0' } ``` @@ -82,19 +82,7 @@ repositories { } dependencies { - testImplementation group: 'org.radarbase', name: 'radar-commons-testing', version: '0.13.2' -} -``` - -Finally, if the schema registry is losing old schemas and your code is not recovering, include `radar-commons-unsafe`. Ensure that it comes in the classpath before any Confluent code. This will override the Confluent Avro deserializer to recover from failure when a message with unknown schema ID is passed. -```gradle -repositories { - mavenCentral() - maven { url 'https://packages.confluent.io/maven/' } -} - -dependencies { - runtimeOnly group: 'org.radarbase', name: 'radar-commons-unsafe', version: '0.13.2' + testImplementation group: 'org.radarbase', name: 'radar-commons-testing', version: '0.14.0' } ``` @@ -119,7 +107,7 @@ configurations.all { } dependencies { - compile group: 'org.radarbase', name: 'radar-commons', version: '0.13.3-SNAPSHOT' + compile group: 'org.radarbase', name: 'radar-commons', version: '0.14.1-SNAPSHOT' } ``` diff --git a/build.gradle b/build.gradle index 4246723e..d17019e8 100644 --- a/build.gradle +++ b/build.gradle @@ -14,13 +14,13 @@ * limitations under the License. */ plugins { - id 'com.commercehub.gradle.plugin.avro' version '0.19.1' - id("io.github.gradle-nexus.publish-plugin") version "1.0.0" - id("com.github.ben-manes.versions") version "0.38.0" + id 'com.github.davidmc24.gradle.plugin.avro' version '1.3.0' + id("io.github.gradle-nexus.publish-plugin") version "1.1.0" + id("com.github.ben-manes.versions") version "0.39.0" } allprojects { - version = '0.13.2' + version = '0.14.0' group = 'org.radarbase' } @@ -29,27 +29,14 @@ subprojects { apply plugin: 'java' apply plugin: 'java-library' apply plugin: 'idea' - apply plugin: 'com.github.ben-manes.versions' + + targetCompatibility = '11' + sourceCompatibility = '11' //---------------------------------------------------------------------------// // Configuration // //---------------------------------------------------------------------------// ext.githubRepoName = 'RADAR-base/radar-commons' - - ext.slf4jVersion = '1.7.30' - ext.kafkaVersion = '6.1.1-ce' - ext.avroVersion = '1.9.2' - ext.confluentVersion = '6.1.1' - ext.jacksonVersion = '2.12.3' - ext.jacksonYamlVersion = '2.12.3' - ext.okhttpVersion = '4.9.1' - ext.junitVersion = '4.13.2' - ext.mockitoVersion = '3.9.0' - ext.hamcrestVersion = '1.3' - ext.codacyVersion = '11.15.0' - ext.radarSchemasVersion = '0.6.0' - ext.orgJsonVersion = '20210307' - ext.githubUrl = "https://github.com/$githubRepoName" ext.issueUrl = "https://github.com/$githubRepoName/issues" ext.website = 'https://radar-base.org' @@ -122,5 +109,5 @@ nexusPublishing { } wrapper { - gradleVersion '7.0' + gradleVersion '7.3.3' } diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml index b9f3a458..a252c58c 100644 --- a/config/checkstyle/checkstyle.xml +++ b/config/checkstyle/checkstyle.xml @@ -1,7 +1,7 @@ + "-//Puppy Crawl//DTD Check Configuration 1.3//EN" + "http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd"> - + - + @@ -27,6 +27,11 @@ + + + + + @@ -39,10 +44,6 @@ - - - - @@ -51,10 +52,7 @@ - - - - + @@ -87,6 +85,23 @@ + + + + + + + + + + + + + + + + + - - + - + + + + + - - + diff --git a/config/pmd/ruleset.xml b/config/pmd/ruleset.xml index ca3f5871..1157371b 100644 --- a/config/pmd/ruleset.xml +++ b/config/pmd/ruleset.xml @@ -23,6 +23,11 @@ + + + + + @@ -37,6 +42,7 @@ + @@ -52,8 +58,13 @@ - - + + + + + + + @@ -62,13 +73,17 @@ - + + + + + diff --git a/gradle.properties b/gradle.properties index 09426f9c..d415f4dc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,12 @@ -#bintrayUser=username -#bintrayApiKey=apikey +slf4jVersion=1.7.32 +confluentVersion=7.0.1 +kafkaVersion=7.0.1-ce +avroVersion=1.11.0 +jacksonVersion=2.12.6 +jacksonYamlVersion=2.12.6 +okhttpVersion=4.9.3 +junitVersion=4.13.2 +mockitoVersion=4.2.0 +hamcrestVersion=1.3 +radarSchemasVersion=0.7.5 +orgJsonVersion=20211205 diff --git a/gradle/codestyle.gradle b/gradle/codestyle.gradle index f2bd9bd1..9a24dbb0 100644 --- a/gradle/codestyle.gradle +++ b/gradle/codestyle.gradle @@ -1,55 +1,24 @@ apply plugin: 'checkstyle' apply plugin: 'pmd' -apply plugin: 'jacoco' - -configurations { - codacy -} - -dependencies { - codacy group: 'com.github.codacy', name: 'codacy-coverage-reporter', version: codacyVersion -} checkstyle { - // codacy version - toolVersion '6.16' - ignoreFailures false + toolVersion = '9.2' + ignoreFailures = false configFile = rootProject.file('config/checkstyle/checkstyle.xml') + // ignore tests + sourceSets = [sourceSets.main] } pmd { - // codacy version - toolVersion = '6.12.0' + toolVersion = '6.41.0' ignoreFailures = false consoleOutput = true ruleSets = [] - ruleSetFiles = files(rootProject.file("config/pmd/ruleset.xml")) -} - -checkstyle { + ruleSetFiles = rootProject.files("config/pmd/ruleset.xml") // ignore tests sourceSets = [sourceSets.main] } -pmd { - // ignore tests - sourceSets = [sourceSets.main] -} - -jacocoTestReport { - reports { - xml.enabled = true - csv.enabled = false - html.enabled = false - } -} - -task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { - main = 'com.codacy.CodacyCoverageReporter' - classpath = configurations.codacy - args = ['report', '-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] -} - diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c0..7454180f 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f371643e..2e6e5897 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 4f906e0c..1b6c7873 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,101 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/radar-commons-server/build.gradle b/radar-commons-server/build.gradle index cdba36cc..8ed0c63d 100644 --- a/radar-commons-server/build.gradle +++ b/radar-commons-server/build.gradle @@ -14,17 +14,10 @@ * limitations under the License. */ -apply plugin: 'com.commercehub.gradle.plugin.avro' +apply plugin: 'com.github.davidmc24.gradle.plugin.avro' description = 'RADAR Common server library utilities.' -targetCompatibility = '11.0' -sourceCompatibility = '11.0' - -repositories { - maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } -} - dependencies { api project(':radar-commons') diff --git a/radar-commons-server/src/main/java/org/radarbase/config/YamlConfigLoader.java b/radar-commons-server/src/main/java/org/radarbase/config/YamlConfigLoader.java index 1b8ae639..2ffe713c 100644 --- a/radar-commons-server/src/main/java/org/radarbase/config/YamlConfigLoader.java +++ b/radar-commons-server/src/main/java/org/radarbase/config/YamlConfigLoader.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.IOException; @@ -38,7 +38,7 @@ public class YamlConfigLoader { private static final ObjectMapper YAML_MAPPER = new ObjectMapper(YAML_FACTORY); static { - YAML_MAPPER.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); + YAML_MAPPER.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); // only serialize fields, not getters, etc. YAML_MAPPER.setVisibility(YAML_MAPPER.getSerializationConfig().getDefaultVisibilityChecker() .withFieldVisibility(JsonAutoDetect.Visibility.ANY) diff --git a/radar-commons-testing/build.gradle b/radar-commons-testing/build.gradle index 5e139e49..d00ca98c 100644 --- a/radar-commons-testing/build.gradle +++ b/radar-commons-testing/build.gradle @@ -33,20 +33,13 @@ run { description = 'RADAR Common testing library mocking code and utilities.' -targetCompatibility = '11.0' -sourceCompatibility = '11.0' - -repositories { - maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } -} - dependencies { api project(':radar-commons') api project(':radar-commons-server') api group: 'org.apache.avro', name: 'avro', version: avroVersion api group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion - implementation group: 'com.opencsv', name: 'opencsv', version: '5.4' + implementation group: 'com.opencsv', name: 'opencsv', version: '5.5.2' implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion implementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java index 16e547d5..f2592fb2 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java @@ -63,6 +63,7 @@ public MockDevice(KafkaSender sender, K key, List> generators } @Override + @SuppressWarnings("PMD.CloseResource") public void run() { List> topicSenders = new ArrayList<>(generators.size()); diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java index a7d4b133..8c0056c8 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java @@ -40,9 +40,10 @@ import org.radarbase.topic.AvroTopic; /** - * Parse mock data from a CSV file + * Parse mock data from a CSV file. * @param key type. */ +@SuppressWarnings("PMD.GodClass") public class MockCsvParser implements Closeable { private static final char ARRAY_SEPARATOR = ';'; private static final char ARRAY_START = '['; @@ -127,6 +128,7 @@ private V parseRecord(String[] rawValues, return record; } + /** Parse value from Schema. */ public static Object parseValue(Schema schema, String fieldString) { switch (schema.getType()) { case INT: diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java index 88b9dcbf..dfcc0b30 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java @@ -145,6 +145,7 @@ public Iterator> iterateValues(final K key, final long } /** + * Get a random double. * @return random {@code Double} using {@code ThreadLocalRandom}. **/ private double getRandomDouble() { diff --git a/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java b/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java index 75a2e4db..3941f7de 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java @@ -38,7 +38,7 @@ public void beat() throws Exception { // time of one beat is about 1/128 seconds = 7.8125 milliseconds long beatDiff = System.currentTimeMillis() - time; assertThat(beatDiff, greaterThanOrEqualTo(7L)); - assertThat(beatDiff, lessThanOrEqualTo(13L)); + assertThat(beatDiff, lessThanOrEqualTo(14L)); } } while (!oscilloscope.willRestart()); diff --git a/radar-commons-unsafe/build.gradle b/radar-commons-unsafe/build.gradle deleted file mode 100644 index 86411178..00000000 --- a/radar-commons-unsafe/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -description = 'RADAR Common unsafe libraries overriding stock Kafka and Confluent code.' - -targetCompatibility = '1.8' -sourceCompatibility = '1.8' - -//---------------------------------------------------------------------------// -// Sources and classpath configurations // -//---------------------------------------------------------------------------// - -configurations.implementation { - resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' -} - -// In this section you declare where to find the dependencies of your project -repositories { - maven { url 'https://jitpack.io' } -} - -// In this section you declare the dependencies for your production and test code -dependencies { - compileOnly group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion - compileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion - compileOnly group: 'org.apache.kafka', name: 'kafka_2.12', version: kafkaVersion -} - -apply from: '../gradle/publishing.gradle' diff --git a/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java b/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java deleted file mode 100644 index 4a74187c..00000000 --- a/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java +++ /dev/null @@ -1,367 +0,0 @@ -/* - * Copyright 2018 Confluent Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.confluent.kafka.serializers; - -import io.confluent.kafka.schemaregistry.avro.AvroSchema; -import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider; -import io.confluent.kafka.schemaregistry.avro.AvroSchemaUtils; -import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import kafka.utils.VerifiableProperties; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericContainer; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificRecord; -import org.apache.kafka.common.errors.SerializationException; - -public abstract class AbstractKafkaAvroDeserializer extends AbstractKafkaSchemaSerDe { - - private final ConcurrentMap> oldToNewIdMap = new ConcurrentHashMap<>(); - - private final DecoderFactory decoderFactory = DecoderFactory.get(); - protected boolean useSpecificAvroReader = false; - private final Map readerSchemaCache = new ConcurrentHashMap<>(); - - /** - * Sets properties for this deserializer without overriding the schema registry client itself. - * Useful for testing, where a mock client is injected. - */ - protected void configure(KafkaAvroDeserializerConfig config) { - configureClientProperties(config, new AvroSchemaProvider()); - useSpecificAvroReader = config - .getBoolean(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG); - } - - protected KafkaAvroDeserializerConfig deserializerConfig(Map props) { - return new KafkaAvroDeserializerConfig(props); - } - - protected KafkaAvroDeserializerConfig deserializerConfig(VerifiableProperties props) { - return new KafkaAvroDeserializerConfig(props.props()); - } - - /** - * Deserializes the payload without including schema information for primitive types, maps, and - * arrays. Just the resulting deserialized object is returned. - * - *

This behavior is the norm for Decoders/Deserializers. - * - * @param payload serialized data - * @return the deserialized object - */ - protected Object deserialize(byte[] payload) throws SerializationException { - return deserialize(null, null, payload, null); - } - - /** - * Just like single-parameter version but accepts an Avro schema to use for reading - * - * @param payload serialized data - * @param readerSchema schema to use for Avro read (optional, enables Avro projection) - * @return the deserialized object - */ - protected Object deserialize(byte[] payload, Schema readerSchema) throws SerializationException { - return deserialize(null, null, payload, readerSchema); - } - - protected Object deserialize(String topic, Boolean isKey, byte[] payload, Schema readerSchema) - throws SerializationException { - if (payload == null) { - return null; - } - - DeserializationContext context = new DeserializationContext(topic, isKey, payload); - return context.read(context.schemaFromRegistry().rawSchema(), readerSchema); - } - - private Integer schemaVersion(String topic, - Boolean isKey, - int id, - String subject, - AvroSchema schema, - Object result) throws IOException, RestClientException { - Integer version; - if (isDeprecatedSubjectNameStrategy(isKey)) { - subject = getSubjectName(topic, isKey, result, schema); - AvroSchema subjectSchema = (AvroSchema) schemaRegistry.getSchemaBySubjectAndId(subject, id); - version = schemaRegistry.getVersion(subject, subjectSchema); - } else { - //we already got the subject name - version = schemaRegistry.getVersion(subject, schema); - } - return version; - } - - private String subjectName(String topic, Boolean isKey, AvroSchema schemaFromRegistry) { - return isDeprecatedSubjectNameStrategy(isKey) - ? null - : getSubjectName(topic, isKey, null, schemaFromRegistry); - } - - /** - * Deserializes the payload and includes schema information, with version information from the - * schema registry embedded in the schema. - * - * @param payload the serialized data - * @return a GenericContainer with the schema and data, either as a {@link NonRecordContainer}, - * {@link org.apache.avro.generic.GenericRecord}, or {@link SpecificRecord} - */ - protected GenericContainerWithVersion deserializeWithSchemaAndVersion( - String topic, boolean isKey, byte[] payload) - throws SerializationException { - // Even if the caller requests schema & version, if the payload is null we cannot include it. - // The caller must handle this case. - if (payload == null) { - return null; - } - - // Annotate the schema with the version. Note that we only do this if the schema + - // version are requested, i.e. in Kafka Connect converters. This is critical because that - // code *will not* rely on exact schema equality. Regular deserializers *must not* include - // this information because it would return schemas which are not equivalent. - // - // Note, however, that we also do not fill in the connect.version field. This allows the - // Converter to let a version provided by a Kafka Connect source take priority over the - // schema registry's ordering (which is implicit by auto-registration time rather than - // explicit from the Connector). - DeserializationContext context = new DeserializationContext(topic, isKey, payload); - AvroSchema schema = context.schemaForDeserialize(); - Object result = context.read(schema.rawSchema(), null); - - try { - Integer version = schemaVersion(topic, isKey, context.getSchemaId(), - context.getSubject(), schema, result); - if (schema.rawSchema().getType().equals(Schema.Type.RECORD)) { - return new GenericContainerWithVersion((GenericContainer) result, version); - } else { - return new GenericContainerWithVersion(new NonRecordContainer(schema.rawSchema(), result), - version); - } - } catch (RestClientException | IOException e) { - throw new SerializationException("Error retrieving Avro " - + getSchemaType(isKey) - + " schema version for id " - + context.getSchemaId(), e); - } - } - - protected DatumReader getDatumReader(Schema writerSchema, Schema readerSchema) { - // normalize reader schema - readerSchema = getReaderSchema(writerSchema, readerSchema); - boolean writerSchemaIsPrimitive = - AvroSchemaUtils.getPrimitiveSchemas().containsValue(writerSchema); - if (writerSchemaIsPrimitive) { - return new GenericDatumReader<>(writerSchema, readerSchema); - } else if (useSchemaReflection) { - return new ReflectDatumReader<>(writerSchema, readerSchema); - } else if (useSpecificAvroReader) { - return new SpecificDatumReader<>(writerSchema, readerSchema); - } else { - return new GenericDatumReader<>(writerSchema, readerSchema); - } - } - - /** - * Normalizes the reader schema, puts the resolved schema into the cache. - *

  • - *
      if the reader schema is provided, use the provided one
    - *
      if the reader schema is cached for the writer schema full name, use the cached value
    - *
      if the writer schema is primitive, use the writer one
    - *
      if schema reflection is used, generate one from the class referred by writer schema
    - *
      if generated classes are used, query the class referred by writer schema
    - *
      otherwise use the writer schema
    - *
  • - */ - private Schema getReaderSchema(Schema writerSchema, Schema readerSchema) { - if (readerSchema != null) { - return readerSchema; - } - readerSchema = readerSchemaCache.get(writerSchema.getFullName()); - if (readerSchema != null) { - return readerSchema; - } - boolean writerSchemaIsPrimitive = - AvroSchemaUtils.getPrimitiveSchemas().values().contains(writerSchema); - if (writerSchemaIsPrimitive) { - readerSchema = writerSchema; - } else if (useSchemaReflection) { - readerSchema = getReflectionReaderSchema(writerSchema); - readerSchemaCache.put(writerSchema.getFullName(), readerSchema); - } else if (useSpecificAvroReader) { - readerSchema = getSpecificReaderSchema(writerSchema); - readerSchemaCache.put(writerSchema.getFullName(), readerSchema); - } else { - readerSchema = writerSchema; - } - return readerSchema; - } - - @SuppressWarnings("unchecked") - private Schema getSpecificReaderSchema(Schema writerSchema) { - Class readerClass = SpecificData.get().getClass(writerSchema); - if (readerClass == null) { - throw new SerializationException("Could not find class " - + writerSchema.getFullName() - + " specified in writer's schema whilst finding reader's " - + "schema for a SpecificRecord."); - } - try { - return readerClass.getConstructor().newInstance().getSchema(); - } catch (InstantiationException | NoSuchMethodException | InvocationTargetException e) { - throw new SerializationException(writerSchema.getFullName() - + " specified by the " - + "writers schema could not be instantiated to " - + "find the readers schema."); - } catch (IllegalAccessException e) { - throw new SerializationException(writerSchema.getFullName() - + " specified by the " - + "writers schema is not allowed to be instantiated " - + "to find the readers schema."); - } - } - - private Schema getReflectionReaderSchema(Schema writerSchema) { - // shall we use ReflectData.AllowNull.get() instead? - Class readerClass = ReflectData.get().getClass(writerSchema); - if (readerClass == null) { - throw new SerializationException("Could not find class " - + writerSchema.getFullName() - + " specified in writer's schema whilst finding reader's " - + "schema for a reflected class."); - } - return ReflectData.get().getSchema(readerClass); - } - - class DeserializationContext { - private final String topic; - private final Boolean isKey; - private final ByteBuffer buffer; - private final int schemaId; - - DeserializationContext(final String topic, final Boolean isKey, final byte[] payload) { - this.topic = topic; - this.isKey = isKey; - this.buffer = getByteBuffer(payload); - this.schemaId = buffer.getInt(); - } - - AvroSchema schemaFromRegistry() { - String subject = getSubject(); - ConcurrentMap subjectIdMap = oldToNewIdMap.computeIfAbsent(subject, sub -> new ConcurrentHashMap<>()); - int id = subjectIdMap.getOrDefault(schemaId, schemaId); - try { - return (AvroSchema) schemaRegistry.getSchemaBySubjectAndId(subject, id); - } catch (RestClientException e) { - if (e.getErrorCode() == 40403) { - try { - List versions = schemaRegistry.getAllVersions(subject); - int latestId = versions.get(versions.size() - 1); - subjectIdMap.put(schemaId, latestId); - - return (AvroSchema) schemaRegistry.getSchemaBySubjectAndId(subject, latestId); - } catch (RestClientException | IOException ex) { - throw new SerializationException("Error retrieving Avro " - + getSchemaType(isKey) - + " schema for id " - + schemaId, e); - } - } - throw new SerializationException("Error retrieving Avro " - + getSchemaType(isKey) - + " schema for id " - + schemaId, e); - } catch (IOException e) { - throw new SerializationException("Error retrieving Avro " - + getSchemaType(isKey) - + " schema for id " - + schemaId, e); - } - } - - AvroSchema schemaForDeserialize() { - return schemaFromRegistry(); - } - - String getSubject() { - return subjectName(topic, isKey, schemaFromRegistry()); - } - - String getTopic() { - return topic; - } - - boolean isKey() { - return isKey; - } - - - int getSchemaId() { - return schemaId; - } - - Object read(Schema writerSchema) { - return read(writerSchema, null); - } - - Object read(Schema writerSchema, Schema readerSchema) { - DatumReader reader = getDatumReader(writerSchema, readerSchema); - int length = buffer.limit() - 1 - idSize; - if (writerSchema.getType().equals(Schema.Type.BYTES)) { - byte[] bytes = new byte[length]; - buffer.get(bytes, 0, length); - return bytes; - } else { - int start = buffer.position() + buffer.arrayOffset(); - try { - Object result = reader.read(null, decoderFactory.binaryDecoder(buffer.array(), - start, length, null)); - if (writerSchema.getType().equals(Schema.Type.STRING)) { - return result.toString(); - } else { - return result; - } - } catch (IOException | RuntimeException e) { - // avro deserialization may throw AvroRuntimeException, NullPointerException, etc - throw new SerializationException("Error deserializing Avro message for id " - + schemaId, e); - } - } - } - } - - private static String getSchemaType(Boolean isKey) { - if (isKey == null) { - return "unknown"; - } else if (isKey) { - return "key"; - } else { - return "value"; - } - } -} diff --git a/radar-commons/build.gradle b/radar-commons/build.gradle index a208d566..ad6bcec4 100644 --- a/radar-commons/build.gradle +++ b/radar-commons/build.gradle @@ -1,8 +1,5 @@ description = 'RADAR Common utilities library.' -targetCompatibility = '1.8' -sourceCompatibility = '1.8' - //---------------------------------------------------------------------------// // Sources and classpath configurations // //---------------------------------------------------------------------------// @@ -16,17 +13,11 @@ configurations { // In this section you declare where to find the dependencies of your project repositories { maven { url 'https://jitpack.io' } - maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } } // In this section you declare the dependencies for your production and test code dependencies { - api (group: 'org.apache.avro', name: 'avro', version: avroVersion) { - exclude group: 'org.xerial.snappy', module: 'snappy-java' - exclude group: 'com.thoughtworks.paranamer', module: 'paranamer' - exclude group: 'org.apache.commons', module: 'commons-compress' - exclude group: 'org.tukaani', module: 'xz' - } + api (group: 'org.apache.avro', name: 'avro', version: avroVersion) // to implement producers and consumers api group: 'com.squareup.okhttp3', name: 'okhttp', version: okhttpVersion diff --git a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java index 5ad6b177..90a5d401 100644 --- a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java +++ b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java @@ -28,6 +28,7 @@ /** * POJO representing a ServerConfig configuration. */ +@SuppressWarnings("PMD.GodClass") public class ServerConfig { private String host; private int port = -1; @@ -57,21 +58,7 @@ public ServerConfig(String urlString) throws MalformedURLException { /** Get the path of the server as a string. This does not include proxyHost information. */ public String getUrlString() { - return addUrlString(new StringBuilder(40)).toString(); - } - - private StringBuilder addUrlString(StringBuilder builder) { - if (protocol != null) { - builder.append(protocol).append("://"); - } - builder.append(host); - if (port != -1) { - builder.append(':').append(port); - } - if (path != null) { - builder.append(path); - } - return builder; + return getUrl().toString(); } /** Get the paths of a list of servers, concatenated with commas. */ @@ -84,7 +71,7 @@ public static String getPaths(List configList) { } else { builder.append(','); } - server.addUrlString(builder); + builder.append(server.getUrl()); } return builder.toString(); } @@ -109,22 +96,11 @@ public URL getUrl() { * @throws IllegalStateException if the URL is invalid */ public HttpUrl getHttpUrl() { - HttpUrl.Builder urlBuilder = new HttpUrl.Builder() - .scheme(protocol) - .host(host); - - if (port != -1) { - urlBuilder.port(port); - } - if (path != null) { - urlBuilder.encodedPath(path); - } - - return urlBuilder.build(); + return HttpUrl.get(getUrl()); } /** - * Get the HTTP proxyHost associated to given server + * Get the HTTP proxyHost associated to given server. * @return http proxyHost if specified, or null if none is specified. * @throws IllegalStateException if proxyHost is set but proxyPort is not or if the server * protocol is not HTTP(s) @@ -206,23 +182,29 @@ public String getPath() { * @throws IllegalArgumentException if the path contains a question mark. */ public final void setPath(String path) { - if (path == null) { - this.path = "/"; - } else if (path.contains("?")) { + this.path = cleanPath(path); + } + + @SuppressWarnings("PMD.UseStringBufferForStringAppends") + private static String cleanPath(String path) { + String newPath = path; + if (newPath == null) { + newPath = "/"; + } + if (newPath.contains("?")) { throw new IllegalArgumentException("Cannot set server path with query string"); - } else { - this.path = path.trim(); - if (this.path.isEmpty()) { - this.path = "/"; - } else { - if (this.path.charAt(0) != '/') { - this.path = '/' + this.path; - } - if (this.path.charAt(this.path.length() - 1) != '/') { - this.path += '/'; - } - } } + newPath = newPath.trim(); + if (newPath.isEmpty()) { + newPath = "/"; + } + if (newPath.charAt(0) != '/') { + newPath = '/' + newPath; + } + if (newPath.charAt(newPath.length() - 1) != '/') { + newPath += '/'; + } + return newPath; } @Override diff --git a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java index c8f40ba0..bbd65b48 100644 --- a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java @@ -41,11 +41,12 @@ private class SchemaEncoderWriter implements AvroWriter { this.schema = schema; GenericData genericData; + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (SpecificRecord.class.isAssignableFrom(clazz)) { - genericData = new SpecificData(RemoteSchemaEncoder.class.getClassLoader()); + genericData = new SpecificData(classLoader); isGeneric = false; } else { - genericData = new GenericData(RemoteSchemaEncoder.class.getClassLoader()); + genericData = new GenericData(classLoader); isGeneric = true; } recordEncoder = new AvroDatumEncoder(genericData, binary); diff --git a/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java b/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java index 6e2fedce..8bc7f22c 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java +++ b/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java @@ -152,6 +152,7 @@ public void flush() throws IOException { } @Override + @SuppressWarnings("PMD.UseTryWithResources") public void close() throws IOException { try { flush(); diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java index 51b68772..7f70ce66 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java @@ -368,10 +368,10 @@ private AvroDataMapper mapBytes(Schema from, final Schema to, final Object defau private AvroDataMapper mapRecord(Schema from, Schema to) - throws SchemaValidationException { + throws SchemaValidationException { if (to.getType() != Schema.Type.RECORD) { throw new SchemaValidationException(to, from, - new IllegalArgumentException("From and to schemas must be records.")); + new IllegalArgumentException("From and to schemas must be records.")); } List fromFields = from.getFields(); Schema.Field[] toFields = new Schema.Field[fromFields.size()]; @@ -398,8 +398,8 @@ private AvroDataMapper mapRecord(Schema from, Schema to) for (int i = 0; i < filledPositions.length; i++) { if (!filledPositions[i] && to.getFields().get(i).defaultVal() == null) { throw new SchemaValidationException(to, from, - new IllegalArgumentException("Cannot map to record without default value" - + " for new field " + to.getFields().get(i).name())); + new IllegalArgumentException("Cannot map to record without default value" + + " for new field " + to.getFields().get(i).name())); } } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java index 4fe2a5c1..41c56377 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java @@ -124,9 +124,10 @@ public void prepare(ParsedSchemaMetadata keySchema, ParsedSchemaMetadata valueSc @Override public String content(int maxLength) throws IOException { - Buffer buffer = new Buffer(); - writeToSink(buffer, maxLength / 2 - 2); - return "0x" + Strings.bytesToHex( - buffer.readByteArray(Math.min(buffer.size(), maxLength - 2))); + try (Buffer buffer = new Buffer()) { + writeToSink(buffer, maxLength / 2 - 2); + return "0x" + Strings.bytesToHex( + buffer.readByteArray(Math.min(buffer.size(), maxLength - 2))); + } } } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java b/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java index a12c9011..6ce5f98b 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java @@ -56,9 +56,9 @@ public long contentLength() { @Override public void writeTo(BufferedSink sink) throws IOException { - BufferedSink gzipSink = Okio.buffer(new GzipSink(sink)); - body.writeTo(gzipSink); - gzipSink.close(); + try (BufferedSink gzipSink = Okio.buffer(new GzipSink(sink))) { + body.writeTo(gzipSink); + } } }; } @@ -72,4 +72,4 @@ public int hashCode() { public boolean equals(Object obj) { return this == obj || obj != null && getClass() == obj.getClass(); } -} \ No newline at end of file +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java index 1d47ad99..2ae287ce 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java @@ -118,8 +118,9 @@ public void prepare(ParsedSchemaMetadata keySchema, ParsedSchemaMetadata valueSc @Override public String content(int maxLength) throws IOException { - Buffer buffer = new Buffer(); - writeToSink(buffer, maxLength); - return buffer.readString(Math.min(buffer.size(), maxLength), StandardCharsets.UTF_8); + try (Buffer buffer = new Buffer()) { + writeToSink(buffer, maxLength); + return buffer.readString(Math.min(buffer.size(), maxLength), StandardCharsets.UTF_8); + } } } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java index 9bc29727..df1cf8f6 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java @@ -187,11 +187,12 @@ public String toString() { * @throws IOException if the body could not be read as a String. */ public static String responseBody(Response response) throws IOException { - ResponseBody body = response.body(); - if (body == null) { - return null; + try (ResponseBody body = response.body()) { + if (body == null) { + return null; + } + return body.string(); } - return body.string(); } /** Create a new builder with the settings of the current client. */ diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java index 8b609397..074a1ac0 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java @@ -37,6 +37,7 @@ * Retriever of an Avro Schema. Internally, only {@link JSONObject} is used to manage JSON data, * to keep the class as lean as possible. */ +@SuppressWarnings("PMD.GodClass") public class SchemaRetriever { private static final Logger logger = LoggerFactory.getLogger(SchemaRetriever.class); private static final long MAX_VALIDITY = 86400L; @@ -195,15 +196,7 @@ protected ParsedSchemaMetadata getCachedVersion(String subject, int id, Integer version = reportedVersion; if (version == null || version <= 0) { ConcurrentMap versions = subjectVersionCache.get(subject); - if (versions != null) { - for (Map.Entry entry : versions.entrySet()) { - if (!entry.getValue().isExpired() && entry.getKey() != 0 - && entry.getValue().value == id) { - version = entry.getKey(); - break; - } - } - } + version = findCachedVersion(id, versions); if (version == null || version <= 0) { return null; } @@ -211,6 +204,20 @@ protected ParsedSchemaMetadata getCachedVersion(String subject, int id, return new ParsedSchemaMetadata(id, version, schema); } + private Integer findCachedVersion(int id, ConcurrentMap cache) { + if (cache == null) { + return null; + } + for (Map.Entry entry : cache.entrySet()) { + if (!entry.getValue().isExpired() + && entry.getKey() != 0 + && entry.getValue().value == id) { + return entry.getKey(); + } + } + return null; + } + protected void cache(ParsedSchemaMetadata metadata, String subject, boolean latest) { TimedInt id = new TimedInt(metadata.getId(), cacheValidity); schemaCache.put(metadata.getSchema(), id); diff --git a/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java index 2b63875e..ce8cda49 100644 --- a/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java +++ b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java @@ -17,6 +17,7 @@ package org.radarbase.topic; import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; import java.util.List; import java.util.Objects; import org.apache.avro.Schema; @@ -96,7 +97,7 @@ public V newValueInstance() throws ClassCastException { } public Schema.Type[] getValueFieldTypes() { - return valueFieldTypes; + return Arrays.copyOf(valueFieldTypes, valueFieldTypes.length); } /** diff --git a/radar-commons/src/main/java/org/radarbase/util/Base64.java b/radar-commons/src/main/java/org/radarbase/util/Base64.java index 62d630b1..3f23e4a3 100644 --- a/radar-commons/src/main/java/org/radarbase/util/Base64.java +++ b/radar-commons/src/main/java/org/radarbase/util/Base64.java @@ -52,7 +52,8 @@ @SuppressWarnings("PMD.ClassNamingConventions") public class Base64 { - private Base64() {} + private Base64() { + } /** * Returns a {@link Encoder} that encodes using the diff --git a/settings.gradle b/settings.gradle index d6543b68..2c61dca1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,18 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -pluginManagement { - repositories { - gradlePluginPortal() - jcenter() - maven { - name "JCenter Gradle Plugins" - url "https://dl.bintray.com/gradle/gradle-plugins" - } - } -} include ':radar-commons' include ':radar-commons-testing' include ':radar-commons-server' -include ':radar-commons-unsafe'