spark: allows notebooks to be saved opencb#13

There is a problem when you set a new conf folder for pyspark beyond the one by default (this is one located in $SPARK_HOME). The problem is that you can use the loaded jars with the build notebooks, but you can't acces the jars from the oskar-spark notebooks. Furthermore, when you apply "maven install", the build notebooks get erased. So I had to edit oskar-app/pom in order to replace those notebooks at "oskar-spark" before we erase everything in the build folder.
roldanx · Dec 14, 2018 · db30061 · db30061
1 parent 81eddd6
commit db30061
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 18 deletions.
diff --git a/oskar-app/app/pyoskar/pyoskar-nb.sh b/oskar-app/app/pyoskar/pyoskar-nb.sh
@@ -5,7 +5,7 @@ export PYSPARK_DRIVER_PYTHON_OPTS='lab'
 export SPARK_CONF_DIR='../conf'
 
 if [[ -z "$SPARK_HOME" ]]; then
-  '~/soft/spark-2.4.0-bin-hadoop2.7/bin/pyspark'
+  ~/soft/spark-2.4.0-bin-hadoop2.7/bin/pyspark
 else
   $SPARK_HOME/bin/pyspark
 fi
diff --git a/oskar-app/pom.xml b/oskar-app/pom.xml
@@ -82,27 +82,31 @@
                 <artifactId>maven-antrun-plugin</artifactId>
                 <version>1.8</version>
                 <executions>
-                    <execution>
-                        <id>clean-libs</id>
-                        <phase>clean</phase>
-                        <configuration>
-                            <target>
-                                <echo>Preparing ${build.dir} folder</echo>
-                                <exec executable="rm">
-                                    <arg value="-rf"/>
-                                    <arg value="${build.dir}"/>
-                                </exec>
-                            </target>
-                        </configuration>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                    </execution>
                     <execution>
                         <id>install-app</id>
                         <phase>install</phase>
                         <configuration>
                             <target>
+
+                                <echo>Removing notebooks from ${build.dir}/../oskar-spark</echo>
+                                <exec executable="rm">
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/variant_filtering.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/variant_filtering_advanced.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/stats.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/facets.ipynb"/>
+                                    <arg value="${project.basedir}/../oskar-spark/src/main/python/notebooks/GWAS.ipynb"/>
+                                </exec>
+
+                                <echo>Copying notebooks to ${build.dir}/../oskar-spark</echo>
+                                <exec executable="cp">
+                                    <arg value="${build.dir}/pyoskar/variant_filtering.ipynb"/>
+                                    <arg value="${build.dir}/pyoskar/variant_filtering_advanced.ipynb"/>
+                                    <arg value="${build.dir}/pyoskar/stats.ipynb"/>
+                                    <arg value="${build.dir}/pyoskar/facets.ipynb"/>
+                                    <arg value="${build.dir}/pyoskar/GWAS.ipynb"/>
+                                    <arg value="${build.dir}/../oskar-spark/src/main/python/notebooks"/>
+                                </exec>
+
                                 <echo>Preparing ${build.dir} folder</echo>
                                 <exec executable="rm">
                                     <arg value="-rf"/>
@@ -121,7 +125,7 @@
                                     <arg value="-r"/>
                                     <arg value="${project.basedir}/target/appassembler/bin"/>
                                     <arg value="${project.basedir}/target/appassembler/libs"/>
-                                    <arg value="${project.basedir}/target/conf"/>
+                                    <!--<arg value="${project.basedir}/target/conf"/>-->
                                     <arg value="${project.basedir}/../LICENSE"/>
                                     <arg value="${project.basedir}/../README.md"/>
                                     <arg value="${build.dir}"/>