Template Apache Spark containers for cross building (resolves BD2KGen…

…omics#196)

Modifies apache-spark-{common,master,worker} for cross building to
Spark 1.5.2, 1.6.2, and 2.0.1.

apache-spark-common/.gitignore

@@ -0,0 +1 @@
+*.spark/

apache-spark-common/Makefile

@@ -1,33 +1,29 @@
 # Definitions
-build_path = build
-runtime_path = runtime
-build_output = ${runtime_path}/apache-spark
-runtime_fullpath = $(realpath ${runtime_path})
+spark_versions = 1.5.2 1.6.2 2.0.1
+gen_directories = $(spark_versions:=.spark)
 build_tool = runtime-container.DONE
-build_number ?= none
-git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../apache-spark-common | cut -f1 -d " ")
-name = quay.io/ucsc_cgl/apache-spark-common
-tag = 1.5.2--${git_commit}
 
-# Steps
-build: ${build_output} ${build_tool}
+build: $(build_tool)
 
-${build_output}: ${build_path}/Dockerfile
-	cd ${build_path} && docker build -no-cache -t ${name}-build:${tag} . 
-	echo ${runtime_fullpath}
-	docker run -v ${runtime_fullpath}:/data ${name}-build:${tag} cp -r /opt/apache-spark /data/apache-spark
+%.spark: sparkver=$(@:.spark=)
+%.spark: build/Dockerfile build/download.sh.template runtime/Dockerfile.template Makefile.template
+	mkdir -p $@ ${@}/build ${@}/runtime
+	cp build/Dockerfile ${@}/build/Dockerfile
+	sed -e "s:SPARKVER:$(sparkver):g" build/download.sh.template > ${@}/build/download.sh
+	sed -e "s:SPARKVER:$(sparkver):g" runtime/Dockerfile.template > ${@}/runtime/Dockerfile
+	sed -e "s:SPARKVER:$(sparkver):g" Makefile.template > ${@}/Makefile
 
-${build_tool}: ${build_output} ${runtime_path}/Dockerfile 
-	cd ${runtime_path} && docker build -t ${name}:${tag} .
-	touch ${build_tool}
+generate: $(gen_directories)
 
-test: build
-
-clean:
-	rm -rf runtime/apache-spark ${build_tool}
-	docker rmi -f ${name}-build:${tag} ${name}:${tag}
+$(build_tool): generate
+	$(foreach subdir, $(gen_directories), cd $(subdir) && $(MAKE) build && cd ..;)
+	touch $(build_tool)
 
 push: build
-	# Requires ~/.dockercfg 
-	docker push ${name}:${tag}
-	docker push ${name}:latest
+	# Requires ~/.dockercfg
+	$(foreach subdir, $(gen_directories), cd $(subdir) && $(MAKE) push && cd ..;)
+
+test:
+
+clean:
+	-rm -rf $(gen_directories) && rm $(build_tool)

apache-spark-common/Makefile.template

@@ -0,0 +1,33 @@
+# Definitions
+build_path = build
+runtime_path = runtime
+build_output = ${runtime_path}/apache-spark
+runtime_fullpath = $(realpath ${runtime_path})
+build_tool = runtime-container.DONE
+build_number ?= none
+git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../apache-spark-common | cut -f1 -d " ")
+name = quay.io/ucsc_cgl/apache-spark-common
+tag = SPARKVER--${git_commit}
+
+# Steps
+build: ${build_output} ${build_tool}
+
+${build_output}: ${build_path}/Dockerfile
+	cd ${build_path} && docker build -no-cache -t ${name}-build:${tag} . 
+	echo ${runtime_fullpath}
+	docker run -v ${runtime_fullpath}:/data ${name}-build:${tag} cp -r /opt/apache-spark /data/apache-spark
+
+${build_tool}: ${build_output} ${runtime_path}/Dockerfile 
+	cd ${runtime_path} && docker build -t ${name}:${tag} .
+	touch ${build_tool}
+
+test: build
+
+clean:
+	rm -rf runtime/apache-spark ${build_tool}
+	docker rmi -f ${name}-build:${tag} ${name}:${tag}
+
+push: build
+	# Requires ~/.dockercfg 
+	docker push ${name}:${tag}
+	docker push ${name}:latest

apache-spark-common/build/download.sh → apache-spark-common/build/download.sh.template

@@ -1,5 +1,5 @@
 mirror=$(python -c "from urllib2 import urlopen; import json; print json.load( urlopen('http://www.apache.org/dyn/closer.lua?path=$path&asjson=1'))['preferred']")
 
 mkdir /opt/apache-spark && \
-  curl ${mirror}spark/spark-1.5.2/spark-1.5.2.tgz \
+  curl ${mirror}spark/spark-SPARKVER/spark-SPARKVER.tgz \
   | tar --strip-components=1 -xzC /opt/apache-spark

apache-spark-common/runtime/Dockerfile → apache-spark-common/runtime/Dockerfile.template

@@ -10,6 +10,6 @@ RUN apt-get update && \
 
 # add spark to path
 RUN mkdir /opt/apache-spark
-COPY apache-spark/spark-1.5.2-bin-custom-spark.tgz /opt/
-RUN tar --strip-components=1 -xzf /opt/spark-1.5.2-bin-custom-spark.tgz -C /opt/apache-spark
+COPY apache-spark/spark-SPARKVER-bin-custom-spark.tgz /opt/
+RUN tar --strip-components=1 -xzf /opt/spark-SPARKVER-bin-custom-spark.tgz -C /opt/apache-spark
 ENV PATH /opt/apache-spark/bin:$PATH

apache-spark-master/.gitignore

@@ -0,0 +1 @@
+*.spark/

apache-spark-master/Dockerfile → apache-spark-master/Dockerfile.template

@@ -1,4 +1,4 @@
-FROM quay.io/ucsc_cgl/apache-spark-common:1.5.2--9885dd3736e1c59552e2a821b5a4690a5d23f1ab
+FROM quay.io/ucsc_cgl/apache-spark-common:SPARKVER--COMMONCOMMIT
 
 MAINTAINER Audrey Musselman-Brown, [email protected]
 

apache-spark-master/Makefile

@@ -1,24 +1,33 @@
+# Definitions
+spark_versions = 1.5.2 1.6.2 2.0.1
+common_git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../apache-spark-common | cut -f1 -d " ")
+gen_directories = $(spark_versions:=.spark)
 build_tool = runtime-container.DONE
-git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../apache-spark-master | cut -f1 -d " ")
-nametag = quay.io/ucsc_cgl/apache-spark-master:1.5.2--${git_commit}
 
-build: ${build_tool}
+build: $(build_tool)
 
-${build_tool}: Dockerfile run-master.sh
+%.spark: sparkver=$(@:.spark=)
+%.spark: Makefile.template Dockerfile.template
+	mkdir -p $@
+	cp run-master.sh ${@}/
+	sed -e "s:SPARKVER:$(sparkver):g" -e "s:COMMONCOMMIT:$(common_git_commit):g" Dockerfile.template > ${@}/Dockerfile
+	sed -e "s:SPARKVER:$(sparkver):g" Makefile.template > ${@}/Makefile
+
+generate: $(gen_directories)
+
+$(build_tool): generate
 	# has apache-spark-common been built yet? if not, build it
 	# if it has been built, this command won't do anything.
-	cd ../apache-spark-common && make build
+	cd ../apache-spark-common && $(MAKE) build
 
-	# build the runtime container
-	docker build -t ${nametag} .
-	touch ${build_tool}
+	$(foreach subdir, $(gen_directories), cd $(subdir) && $(MAKE) build && cd ..;)
+	touch $(build_tool)
 
-clean:
-	rm -rf ${build_tool}
-	docker rmi -f ${nametag}
+push: build
+	# Requires ~/.dockercfg
+	$(foreach subdir, $(gen_directories), cd $(subdir) && $(MAKE) push && cd ..;)
 
-test: build
+test:
 
-push: build
-	# Requires ~/.dockercfg 
-	docker push ${nametag}
+clean:
+	-rm -rf $(gen_directories) && rm $(build_tool)

apache-spark-master/Makefile.template

@@ -0,0 +1,20 @@
+build_tool = runtime-container.DONE
+git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../apache-spark-master ../apache-spark-common | cut -f1 -d " ")
+nametag = quay.io/ucsc_cgl/apache-spark-master:SPARKVER--${git_commit}
+
+build: ${build_tool}
+
+${build_tool}: Dockerfile run-master.sh
+	# build the runtime container
+	docker build -t ${nametag} .
+	touch ${build_tool}
+
+clean:
+	rm -rf ${build_tool}
+	docker rmi -f ${nametag}
+
+test: build
+
+push: build
+	# Requires ~/.dockercfg 
+	docker push ${nametag}

apache-spark-worker/.gitignore

@@ -0,0 +1 @@
+*.spark/

apache-spark-worker/Dockerfile → apache-spark-worker/Dockerfile.template

@@ -1,4 +1,4 @@
-FROM quay.io/ucsc_cgl/apache-spark-common:1.5.2--9885dd3736e1c59552e2a821b5a4690a5d23f1ab
+FROM quay.io/ucsc_cgl/apache-spark-common:SPARKVER--COMMONCOMMIT
 
 MAINTAINER Audrey Musselman-Brown, [email protected]
 

apache-spark-worker/Makefile

@@ -1,24 +1,33 @@
+# Definitions
+spark_versions = 1.5.2 1.6.2 2.0.1
+common_git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../apache-spark-common | cut -f1 -d " ")
+gen_directories = $(spark_versions:=.spark)
 build_tool = runtime-container.DONE
-git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../apache-spark-worker | cut -f1 -d " ")
-nametag = quay.io/ucsc_cgl/apache-spark-worker:1.5.2--${git_commit}
 
-build: ${build_tool}
+build: $(build_tool)
 
-${build_tool}: Dockerfile run-worker.sh
+%.spark: sparkver=$(@:.spark=)
+%.spark: Makefile.template Dockerfile.template
+	mkdir -p $@
+	cp run-worker.sh ${@}/
+	sed -e "s:SPARKVER:$(sparkver):g" -e "s:COMMONCOMMIT:$(common_git_commit):g" Dockerfile.template > ${@}/Dockerfile
+	sed -e "s:SPARKVER:$(sparkver):g" Makefile.template > ${@}/Makefile
+
+generate: $(gen_directories)
+
+$(build_tool): generate
 	# has apache-spark-common been built yet? if not, build it
 	# if it has been built, this command won't do anything.
-	cd ../apache-spark-common && make build
-
-	# build the runtime container
-	docker build -t ${nametag} .
-	touch ${build_tool}
-
-clean:
-	rm -rf ${build_tool}
-	docker rmi -f ${nametag}
+	cd ../apache-spark-common && $(MAKE) build
 
-test: build
+	$(foreach subdir, $(gen_directories), cd $(subdir) && $(MAKE) build && cd ..;)
+	touch $(build_tool)
 
 push: build
-	# Requires ~/.dockercfg 
-	docker push ${nametag}
+	# Requires ~/.dockercfg
+	$(foreach subdir, $(gen_directories), cd $(subdir) && $(MAKE) push && cd ..;)
+
+test:
+
+clean:
+	-rm -rf $(gen_directories) && rm $(build_tool)

apache-spark-worker/Makefile.template

@@ -0,0 +1,20 @@
+build_tool = runtime-container.DONE
+git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../apache-spark-common ../apache-spark-worker | cut -f1 -d " ")
+nametag = quay.io/ucsc_cgl/apache-spark-worker:SPARKVER--${git_commit}
+
+build: ${build_tool}
+
+${build_tool}: Dockerfile run-worker.sh
+	# build the runtime container
+	docker build -t ${nametag} .
+	touch ${build_tool}
+
+clean:
+	rm -rf ${build_tool}
+	docker rmi -f ${nametag}
+
+test: build
+
+push: build
+	# Requires ~/.dockercfg 
+	docker push ${nametag}