spark: create files for creating build folder opencb#13

roldanx · Dec 13, 2018 · ac84e30 · ac84e30
1 parent 1cb7b85
commit ac84e30
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 3 deletions.
diff --git a/oskar-app/app/pyoskar/pyoskar-nb.sh b/oskar-app/app/pyoskar/pyoskar-nb.sh
@@ -1,2 +1,11 @@
 #!/bin/bash
 
+export PYSPARK_DRIVER_PYTHON=jupyter
+export PYSPARK_DRIVER_PYTHON_OPTS='lab'
+export SPARK_CONF_DIR='../conf'
+
+if [[ -z "$SPARK_HOME" ]]; then
+  '~/soft/spark-2.4.0-bin-hadoop2.7/bin/pyspark'
+else
+  $SPARK_HOME/bin/pyspark
+fi
diff --git a/oskar-app/pom.xml b/oskar-app/pom.xml
@@ -138,14 +138,19 @@
                                 <exec executable="cp">
                                     <arg value="-r"/>
                                     <arg value="${project.basedir}/../oskar-spark/src/main/python/pyoskar"/>
-                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebook/variant_filtering.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/variant_filtering.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/stats.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/facets.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/GWAS.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-app/app/pyoskar/pyoskar-nb.sh"/>
                                     <arg value="${build.dir}/pyoskar"/>
                                 </exec>
 
                                 <echo>Copying configuration files</echo>
                                 <exec executable="cp">
                                     <arg value="${project.basedir}/../oskar-core/target/classes/configuration.yml"/>
-                                    <arg value="${project.basedir}/../oskar-core/target/classes/log4j.properties"/>
+                                    <!--<arg value="${project.basedir}/../oskar-core/target/classes/log4j.properties"/>-->
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/resources/spark-defaults.conf"/>
                                     <arg value="${build.dir}/conf"/>
                                 </exec>
                             </target>

diff --git a/oskar-spark/src/main/resources/spark-defaults.conf b/oskar-spark/src/main/resources/spark-defaults.conf
@@ -27,4 +27,4 @@
 # spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
 
 
-spark.jars			        /PATH_TO_JARS/oskar-spark-0.1.0.jar,/PATH_TO_JARS/oskar-spark-0.1.0-jar-with-dependencies.jar
+spark.jars                              ../libs/oskar-spark-0.1.0.jar,../libs/oskar-spark-0.1.0-jar-with-dependencies.jar