From e478df87df3fc20fca17422063f6bc2e293b8d8f Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Fri, 9 Apr 2021 11:00:57 +0200 Subject: [PATCH] Set spark.driver.bindAddress and SPARK_LOCAL_IP --- .github/workflows/ci.yml | 1 + python/test/spark_common.py | 1 + src/test/scala/uk/co/gresearch/spark/SparkTestSession.scala | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3dd008c0..4d97933f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -269,6 +269,7 @@ jobs: - name: Integration Tests env: + SPARK_LOCAL_IP: 127.0.0.1 SPARK_HOME: ${{ steps.params.outputs.home }}/spark ARTIFACT_ID: ${{ steps.params.outputs.artifact-id }} VERSION: ${{ steps.params.outputs.version }} diff --git a/python/test/spark_common.py b/python/test/spark_common.py index bd890365..16d8fc4f 100644 --- a/python/test/spark_common.py +++ b/python/test/spark_common.py @@ -64,6 +64,7 @@ def get_spark_config(path, dependencies) -> SparkConf: master = 'local[2]' conf = SparkConf().setAppName('unit test').setMaster(master) return conf.setAll([ + ('spark.driver.bindAddress', '127.0.0.1'), ('spark.ui.showConsoleProgress', 'false'), ('spark.test.home', os.environ.get('SPARK_HOME')), ('spark.locality.wait', '0'), diff --git a/src/test/scala/uk/co/gresearch/spark/SparkTestSession.scala b/src/test/scala/uk/co/gresearch/spark/SparkTestSession.scala index f712a59c..eb6994a6 100644 --- a/src/test/scala/uk/co/gresearch/spark/SparkTestSession.scala +++ b/src/test/scala/uk/co/gresearch/spark/SparkTestSession.scala @@ -27,6 +27,7 @@ trait SparkTestSession { .builder() .master(s"local[$cores]") .appName("spark test example") + .config("spark.driver.bindAddress", "127.0.0.1") .config("spark.sql.shuffle.partitions", 2) .config("spark.local.dir", ".") .getOrCreate()