[SEDONA-640] Refactor to split spark 3.0 profile to multiple spark versions 3.0, 3.1, 3.2, and 3.3

.github/workflows/java.yml

@@ -100,10 +100,7 @@ jobs:
           SCALA_VERSION: ${{ matrix.scala }}
           SKIP_TESTS: ${{ matrix.skipTests }}
         run: |
-          SPARK_COMPAT_VERSION="3.0"
-          if [ ${SPARK_VERSION:2:1} -gt "3" ]; then
-            SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
-          fi
+          SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
           mvn -q clean install -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dspark.version=${SPARK_VERSION} ${SKIP_TESTS}
       - run: mkdir staging
       - run: cp spark-shaded/target/sedona-*.jar staging

.github/workflows/python.yml

@@ -111,10 +111,7 @@ jobs:
           SPARK_VERSION: ${{ matrix.spark }}
           SCALA_VERSION: ${{ matrix.scala }}
         run: |
-            SPARK_COMPAT_VERSION="3.0"
-            if [ ${SPARK_VERSION:2:1} -gt "3" ]; then
-              SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
-            fi
+            SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
             mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dgeotools
       - env:
           SPARK_VERSION: ${{ matrix.spark }}

.github/workflows/r.yml

@@ -102,10 +102,7 @@ jobs:
           key: apache.sedona-apache-spark-${{ steps.os-name.outputs.os-name }}-${{ env.SPARK_VERSION }}
       - name: Build Sedona libraries
         run: |
-          SPARK_COMPAT_VERSION="3.0"
-          if [ ${SPARK_VERSION:2:1} -gt "3" ]; then
-            SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
-          fi
+          SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
           mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4}
       - name: Run tests
         run: |

pom.xml

@@ -83,7 +83,7 @@
              Setting a default value helps IDE:s that can't make sense of profiles. -->
         <scala.compat.version>2.12</scala.compat.version>
         <spark.version>3.3.0</spark.version>
-        <spark.compat.version>3.0</spark.compat.version>
+        <spark.compat.version>3.3</spark.compat.version>
         <log4j.version>2.17.2</log4j.version>
 
         <flink.version>1.19.0</flink.version>
@@ -674,18 +674,63 @@
             </repositories>
         </profile>
         <profile>
-            <!-- This profile works for Spark 3.0, 3.1, 3.2 and 3.3 -->
+            <!-- This profile works for Spark 3.0 -->
             <id>sedona-spark-3.0</id>
             <activation>
                 <property>
                     <name>spark</name>
                     <value>3.0</value>
                 </property>
+            </activation>
+            <properties>
+                <spark.version>3.0.3</spark.version>
+                <spark.compat.version>3.0</spark.compat.version>
+                <log4j.version>2.17.2</log4j.version>
+            </properties>
+        </profile>
+        <profile>
+            <!-- This profile works for Spark 3.1 -->
+            <id>sedona-spark-3.1</id>
+            <activation>
+                <property>
+                    <name>spark</name>
+                    <value>3.1</value>
+                </property>
+            </activation>
+            <properties>
+                <spark.version>3.1.2</spark.version>
+                <spark.compat.version>3.1</spark.compat.version>
+                <log4j.version>2.17.2</log4j.version>
+            </properties>
+        </profile>
+        <profile>
+            <!-- This profile works for Spark 3.2 -->
+            <id>sedona-spark-3.2</id>
+            <activation>
+                <property>
+                    <name>spark</name>
+                    <value>3.2.3</value>
+                </property>
+            </activation>
+            <properties>
+                <spark.version>3.2.0</spark.version>
+                <spark.compat.version>3.2</spark.compat.version>
+                <log4j.version>2.17.2</log4j.version>
+            </properties>
+        </profile>
+        <profile>
+            <!-- This profile works for Spark 3.3 -->
+            <id>sedona-spark-3.3</id>
+            <activation>
+                <property>
+                    <name>spark</name>
+                    <value>3.3</value>
+                </property>
                 <activeByDefault>true</activeByDefault>
             </activation>
             <properties>
                 <spark.version>3.3.0</spark.version>
-                <spark.compat.version>3.0</spark.compat.version>
+                <spark.compat.version>3.3</spark.compat.version>
                 <log4j.version>2.17.2</log4j.version>
             </properties>
         </profile>

spark/pom.xml

@@ -48,6 +48,10 @@
         <profile>
             <id>enable-all-submodules</id>
             <modules>
+                <module>spark-3.0</module>
+                <module>spark-3.1</module>
+                <module>spark-3.2</module>
+                <module>spark-3.3</module>
                 <module>spark-3.4</module>
                 <module>spark-3.5</module>
             </modules>

spark/spark-3.1/.gitignore

@@ -0,0 +1,12 @@
+/target/
+/.settings/
+/.classpath
+/.project
+/dependency-reduced-pom.xml
+/doc/
+/.idea/
+*.iml
+/latest/
+/spark-warehouse/
+/metastore_db/
+*.log

spark/spark-3.1/pom.xml

@@ -0,0 +1,145 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.sedona</groupId>
+        <artifactId>sedona-spark-parent-${spark.compat.version}_${scala.compat.version}</artifactId>
+        <version>1.6.1-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+	<artifactId>sedona-spark-3.1_${scala.compat.version}</artifactId>
+
+	<name>${project.groupId}:${project.artifactId}</name>
+	<description>A cluster computing system for processing large-scale spatial data: SQL API for Spark 3.1.</description>
+    <url>http://sedona.apache.org/</url>
+	<packaging>jar</packaging>
+
+    <properties>
+        <maven.deploy.skip>false</maven.deploy.skip>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.sedona</groupId>
+            <artifactId>sedona-common</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.sedona</groupId>
+            <artifactId>sedona-spark-common-${spark.compat.version}_${scala.compat.version}</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_${scala.compat.version}</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.compat.version}</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-1.2-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-main</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-referencing</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-epsg-hsql</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-geotiff</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-coverage</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-arcgrid</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.locationtech.jts</groupId>
+            <artifactId>jts-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.wololo</groupId>
+            <artifactId>jts2geojson</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.scala-lang.modules</groupId>
+            <artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_${scala.compat.version}</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-inline</artifactId>
+        </dependency>
+    </dependencies>
+	<build>
+        <sourceDirectory>src/main/scala</sourceDirectory>
+        <plugins>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.scalatest</groupId>
+                <artifactId>scalatest-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.scalastyle</groupId>
+                <artifactId>scalastyle-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+	</build>
+</project>

spark/spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister

@@ -0,0 +1,2 @@
+org.apache.spark.sql.execution.datasources.parquet.GeoParquetFileFormat
+org.apache.spark.sql.execution.datasources.v2.geoparquet.metadata.GeoParquetMetadataDataSource