diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 371db27..dd82318 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -23,7 +23,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: 'zulu' # zulu as it supports a wide version range
- java-version: '11' # Most recent LTS that passes tests
+ java-version: '17' # until Spark 4 per SPARK-43831
- name: Cache local Maven repository
uses: actions/cache@v3
with:
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index bef38da..7c8ba79 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -26,7 +26,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: 'zulu' # zulu as it supports a wide version range
- java-version: '11' # Most recent LTS that passes tests
+ java-version: '17' # until Spark 4 per SPARK-43831
- name: Cache local Maven repository
uses: actions/cache@v3
with:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9abde7e..7761d46 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -31,7 +31,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: 'zulu' # zulu as it supports a wide version range
- java-version: '11' # Most recent LTS that passes tests
+ java-version: '17' # until Spark 4 per SPARK-43831
- name: Test without Docker
run: build-bin/maven/maven_go_offline && build-bin/test -DexcludedGroups=docker
test_docker:
@@ -69,7 +69,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: 'zulu' # zulu as it supports a wide version range
- java-version: '11' # Most recent LTS that passes tests
+ java-version: '17' # until Spark 4 per SPARK-43831
- name: Test with Docker
run: |
build-bin/docker/configure_docker &&
diff --git a/.github/workflows/test_readme.yml b/.github/workflows/test_readme.yml
index 5543df7..626ab9a 100644
--- a/.github/workflows/test_readme.yml
+++ b/.github/workflows/test_readme.yml
@@ -22,7 +22,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: 'zulu' # zulu as it supports a wide version range
- java-version: '11' # Most recent LTS that passes tests
+ java-version: '17' # until Spark 4 per SPARK-43831
- name: Cache local Maven repository
uses: actions/cache@v3
with:
diff --git a/README.md b/README.md
index aeff0e4..e296e55 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ are supported, including Cassandra, MySQL and Elasticsearch.
## Quick-start
-Until [SPARK-43831](https://issues.apache.org/jira/browse/SPARK-43831), Zipkin Dependencies requires Java 11 to run.
+Until [SPARK-43831](https://issues.apache.org/jira/browse/SPARK-43831), Zipkin Dependencies requires Java 17 to run.
The quickest way to get started is to fetch the [latest released job](https://search.maven.org/remote_content?g=io.zipkin.dependencies&a=zipkin-dependencies&v=LATEST) as a self-contained jar. For example:
diff --git a/cassandra3/pom.xml b/cassandra3/pom.xml
index 1c9db1e..00375c3 100644
--- a/cassandra3/pom.xml
+++ b/cassandra3/pom.xml
@@ -11,7 +11,7 @@
io.zipkin.dependencies
zipkin-dependencies-parent
- 3.1.6-SNAPSHOT
+ 3.2.0-SNAPSHOT
zipkin-dependencies-cassandra3
@@ -19,6 +19,7 @@
${project.basedir}/..
+ 3.1.19
@@ -28,6 +29,13 @@
${spark-cassandra-connector.version}
+
+
+ com.github.jnr
+ jnr-posix
+ ${jnr-posix.version}
+
+
io.zipkin.zipkin2
diff --git a/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraDependenciesJob.java b/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraDependenciesJob.java
index 252fcfc..a8161d8 100644
--- a/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraDependenciesJob.java
+++ b/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraDependenciesJob.java
@@ -150,6 +150,9 @@ public CassandraDependenciesJob build() {
df.setTimeZone(TimeZone.getTimeZone("UTC"));
this.dateStamp = df.format(new Date(builder.day));
this.conf = new SparkConf(true).setMaster(builder.sparkMaster).setAppName(getClass().getName());
+ if (builder.sparkMaster.startsWith("local[")) {
+ conf.set("spark.driver.bindAddress", "127.0.0.1");
+ }
conf.set("spark.cassandra.connection.host", parseHosts(builder.contactPoints));
conf.set("spark.cassandra.connection.port", parsePort(builder.contactPoints));
conf.set("spark.cassandra.connection.localDC", builder.localDc);
@@ -232,7 +235,7 @@ static String parseHosts(String contactPoints) {
List result = new ArrayList<>();
for (String contactPoint : contactPoints.split(",", -1)) {
HostAndPort parsed = HostAndPort.fromString(contactPoint);
- result.add(parsed.getHostText());
+ result.add(parsed.getHost());
}
return Joiner.on(',').join(result);
}
diff --git a/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraRowToSpan.java b/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraRowToSpan.java
index c97740a..0095741 100644
--- a/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraRowToSpan.java
+++ b/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraRowToSpan.java
@@ -7,12 +7,12 @@
import com.datastax.spark.connector.japi.CassandraRow;
import com.datastax.spark.connector.japi.UDTValue;
import com.datastax.spark.connector.types.TypeConverter;
+import java.io.Serializable;
import java.util.Map;
import javax.annotation.Nullable;
import org.apache.spark.api.java.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import scala.Serializable;
import zipkin2.Endpoint;
import zipkin2.Span;
diff --git a/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraRowsToDependencyLinks.java b/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraRowsToDependencyLinks.java
index b1a5ade..82add62 100644
--- a/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraRowsToDependencyLinks.java
+++ b/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/CassandraRowsToDependencyLinks.java
@@ -5,12 +5,12 @@
package zipkin2.dependencies.cassandra3;
import com.datastax.spark.connector.japi.CassandraRow;
+import java.io.Serializable;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.spark.api.java.function.FlatMapFunction;
-import scala.Serializable;
import zipkin2.DependencyLink;
import zipkin2.Span;
diff --git a/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/SpansToDependencyLinks.java b/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/SpansToDependencyLinks.java
index 820c722..4ccd2ac 100644
--- a/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/SpansToDependencyLinks.java
+++ b/cassandra3/src/main/java/zipkin2/dependencies/cassandra3/SpansToDependencyLinks.java
@@ -4,13 +4,13 @@
*/
package zipkin2.dependencies.cassandra3;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.spark.api.java.function.FlatMapFunction;
-import scala.Serializable;
import zipkin2.DependencyLink;
import zipkin2.Span;
import zipkin2.internal.DependencyLinker;
diff --git a/cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraContainer.java b/cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraContainer.java
index 2619661..3ea38c2 100644
--- a/cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraContainer.java
+++ b/cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraContainer.java
@@ -50,7 +50,7 @@ final class CassandraContainer extends GenericContainer {
);
CassandraContainer() {
- super(parse("ghcr.io/openzipkin/zipkin-cassandra:3.2.1"));
+ super(parse("ghcr.io/openzipkin/zipkin-cassandra:3.3.0"));
addExposedPort(9042);
waitStrategy = Wait.forHealthcheck();
withLogConsumer(new Slf4jLogConsumer(LOGGER));
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 39ce48a..725ebd2 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -9,8 +9,8 @@
# This is defined in many places because Docker has no "env" script functionality unless you use
# docker-compose: When updating, update everywhere.
#
-# TODO: temporarily 11 until we can update to spark on scala 2.13
-ARG java_version=11.0.22_p7
+# Note: JDK 17 until Spark 4 per SPARK-43831
+ARG java_version=17.0.10_p7
# We copy files from the context into a scratch container first to avoid a problem where docker and
# docker-compose don't share layer hashes https://github.com/docker/compose/issues/883 normally.
@@ -37,9 +37,9 @@ ENV VERSION=$version
ENV MAVEN_PROJECT_BASEDIR=/code
RUN /code/build-bin/maven/maven_build_or_unjar io.zipkin.dependencies zipkin-dependencies ${VERSION}
-# Try -jre again once Spark dependencies are up to date. Spark 3.0 uses old Hadoop in the
-# Elasticsearch driver which needs com.sun.security.auth.module.UnixLoginModule.
-FROM ghcr.io/openzipkin/java:11.0.22_p7 as zipkin-dependencies
+# Try -jre again once Spark 4.0 is released. Spark 3.x uses old Hadoop in the
+# Elasticsearch driver which needs com.sun.security.auth.module.UnixLoginModule
+FROM ghcr.io/openzipkin/java:17.0.10_p7 as zipkin-dependencies
LABEL org.opencontainers.image.description="Zipkin Dependencies Aggregator on OpenJDK and Alpine Linux"
LABEL org.opencontainers.image.source=https://github.com/openzipkin/zipkin-dependencies
diff --git a/docker/bin/start-zipkin-dependencies b/docker/bin/start-zipkin-dependencies
index 9323968..e6c4b3f 100755
--- a/docker/bin/start-zipkin-dependencies
+++ b/docker/bin/start-zipkin-dependencies
@@ -8,4 +8,24 @@
# args override it (Ex. 2020-10-31)
set -eu
-exec java ${JAVA_OPTS} -Djava.io.tmpdir=/tmp -cp classes zipkin2.dependencies.ZipkinDependenciesJob $@
+# Spark 3.4 module config from:
+# https://github.com/apache/spark/blob/branch-3.4/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java#L29
+exec java ${JAVA_OPTS} -Djava.io.tmpdir=/tmp \
+ -XX:+IgnoreUnrecognizedVMOptions \
+ --add-opens=java.base/java.lang=ALL-UNNAMED \
+ --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \
+ --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \
+ --add-opens=java.base/java.io=ALL-UNNAMED \
+ --add-opens=java.base/java.net=ALL-UNNAMED \
+ --add-opens=java.base/java.nio=ALL-UNNAMED \
+ --add-opens=java.base/java.util=ALL-UNNAMED \
+ --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \
+ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \
+ --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \
+ --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
+ --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \
+ --add-opens=java.base/sun.security.action=ALL-UNNAMED \
+ --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \
+ --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
+ -Djdk.reflect.useDirectMethodHandle=false \
+ -cp classes zipkin2.dependencies.ZipkinDependenciesJob $@
diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml
index e84c16c..6a92241 100644
--- a/elasticsearch/pom.xml
+++ b/elasticsearch/pom.xml
@@ -11,7 +11,7 @@
io.zipkin.dependencies
zipkin-dependencies-parent
- 3.1.6-SNAPSHOT
+ 3.2.0-SNAPSHOT
zipkin-dependencies-elasticsearch
diff --git a/elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java b/elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java
index 264617e..4cfd635 100644
--- a/elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java
+++ b/elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java
@@ -154,6 +154,9 @@ private static String getSystemPropertyAsFileResource(String key) {
df.setTimeZone(TimeZone.getTimeZone("UTC"));
this.dateStamp = df.format(new Date(builder.day));
this.conf = new SparkConf(true).setMaster(builder.sparkMaster).setAppName(getClass().getName());
+ if (builder.sparkMaster.startsWith("local[")) {
+ conf.set("spark.driver.bindAddress", "127.0.0.1");
+ }
if (builder.jars != null) conf.setJars(builder.jars);
if (builder.username != null) conf.set(ES_NET_HTTP_AUTH_USER, builder.username);
if (builder.password != null) conf.set(ES_NET_HTTP_AUTH_PASS, builder.password);
@@ -167,33 +170,27 @@ private static String getSystemPropertyAsFileResource(String key) {
}
public void run() {
- run(
- index + "-span-" + dateStamp,
- index + "-dependency-" + dateStamp,
- SpanBytesDecoder.JSON_V2);
-
- log.info("Done");
- }
+ String spanResource = index + "-span-" + dateStamp;
+ String dependencyLinkResource = index + "-dependency-" + dateStamp;
+ SpanBytesDecoder decoder = SpanBytesDecoder.JSON_V2;
- void run(String spanResource, String dependencyLinkResource, SpanBytesDecoder decoder) {
log.info("Processing spans from {}", spanResource);
- JavaSparkContext sc = new JavaSparkContext(conf);
- try {
- JavaRDD