Merge branch 'develop' into TASK-5659

.github/workflows/check-junit-test.yml

@@ -1,8 +1,36 @@
 name: Check junits
+run-name: "Check Junit. Hadoop ${{ inputs.hadoop }} ${{ inputs.module == 'all' && ' ' || format(':{0}. ', inputs.module ) }}${{ inputs.short_tests && 'Short ' || '' }}${{ inputs.medium_tests && 'Medium ' || '' }}${{ inputs.long_tests && 'Long ' || '' }}${{ ! ( inputs.short_tests || inputs.medium_tests || inputs.long_tests ) && 'Short Medium Long ' || '' }}tests."
 
 on:
   workflow_dispatch:
     inputs:
+      hadoop:
+        type: choice
+        description: 'Hadoop flavour.'
+        required: false
+        default: "hdp3.1"
+        options:
+          - "all"
+          - "hdp3.1"
+          - "hdi5.1"
+          - "emr6.1"
+          - "emr6.13"
+      module:
+        type: choice
+        description: 'OpenCGA module to test.'
+        required: false
+        default: "all"
+        options:
+          - "all"
+          - "opencga-analysis"
+          - "opencga-app"
+          - "opencga-catalog"
+          - "opencga-client"
+          - "opencga-core"
+          - "opencga-master"
+          - "opencga-server"
+          - "opencga-storage"
+          - "opencga-test"
       short_tests:
         type: boolean
         required: false
@@ -25,31 +53,49 @@ jobs:
     name: Test JUnit
     runs-on: ubuntu-22.04
     outputs:
-      profiles: ${{ steps.getter.outputs.profiles }}
+      profiles: ${{ steps.validate.outputs.profiles }}
+      modules: ${{ steps.validate.outputs.modules }}
+      hadoop: ${{ steps.validate.outputs.hadoop }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
       - name: Building string profiles to run
-        id: getter
+        id: validate
         run: |
           if [ -f "./.github/workflows/scripts/get_profiles.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_profiles.sh
-            echo "profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }})" >> $GITHUB_OUTPUT
+            profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }})
+            modules='["${{ inputs.module }}"]'
+            hadoop='["${{ inputs.hadoop }}"]'
+          
+            if [ "${{ inputs.hadoop }}" == "all" ]; then
+              hadoop='["hdp3.1", "hdi5.1", "emr6.1", "emr6.13"]'
+            elif [ "${{ inputs.module }}" == "all" ]; then
+              # Only execute modules with matrix strategy if we are testing one single hadoop profile
+              modules='["opencga-analysis", "opencga-app", "opencga-catalog", "opencga-client", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-test"]'
+            fi
+            echo "profiles=[\"$profiles\"]" >> $GITHUB_OUTPUT
+            echo "modules=$modules" >> $GITHUB_OUTPUT
+            echo "hadoop=$hadoop" >> $GITHUB_OUTPUT
+            echo "Executing testing profiles -> $profiles" >> $GITHUB_STEP_SUMMARY
+            echo "Modules -> $modules" >> $GITHUB_STEP_SUMMARY
+            echo "Hadoop -> $hadoop" >> $GITHUB_STEP_SUMMARY
+            #echo "matrix={\"hadoop\": $hadoop, \"modules\": $modules}" >> $GITHUB_OUTPUT
           fi
-
-  echo_log:
-    needs: get_profiles
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Echo profiles
-        id: log
-        run: echo "Executing testing profiles -> ${{ needs.get_profiles.outputs.profiles }}"
-
   test:
-    needs: [ get_profiles, echo_log ]
+    strategy:
+      fail-fast: false
+      matrix:
+        hadoop: ${{ fromJson(needs.get_profiles.outputs.hadoop) }}
+        module: ${{ fromJson(needs.get_profiles.outputs.modules) }}
+        profile: ${{ fromJson(needs.get_profiles.outputs.profiles) }}
+    needs: [ get_profiles ]
     uses: ./.github/workflows/test-analysis.yml
     with:
-      test_profile: ${{ needs.get_profiles.outputs.profiles }}
+      test_profile: ${{ matrix.profile }}
+      hadoop: ${{ matrix.hadoop }}
+      module: ${{ matrix.module }}
       mvn_opts: ${{ inputs.mvn_opts }}
+      sonar: false
     secrets: inherit

.github/workflows/long-test-analysis.yml

@@ -9,9 +9,13 @@ on:
 
 jobs:
   test:
+    strategy:
+      fail-fast: false
+      matrix:
+        hadoop: [ "hdp3.1", "hdi5.1", "emr6.1", "emr6.13" ]
     uses: ./.github/workflows/test-analysis.yml
     secrets: inherit
     with:
-      test_profile: runShortTests,runMediumTests,runLongTests,
-
+      test_profile: runShortTests,runMediumTests,runLongTests
+      hadoop: ${{ matrix.hadoop }}
 

.github/workflows/manual-deploy-docker.yml

@@ -10,6 +10,11 @@ on:
         description: "The tag for the new docker."
         type: string
         required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
 
 jobs:
   build:
@@ -32,10 +37,10 @@ jobs:
         run: |
           if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
+        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }},RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
       - uses: actions/upload-artifact@v3
         with:
           name: build-folder

.github/workflows/manual-deploy-ext-tools.yml

@@ -10,6 +10,11 @@ on:
         description: "The tag for the new docker."
         type: string
         required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
 
 jobs:
   build:
@@ -32,7 +37,7 @@ jobs:
         run: |
           if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
         run: mvn -T 2 clean install -DskipTests

.github/workflows/release.yml

@@ -23,7 +23,7 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ needs.build.outputs.version }}
     secrets: inherit
 
   deploy-python:

.github/workflows/scripts/get_same_branch.sh

@@ -1,21 +1,25 @@
 #!/bin/bash
 
 BRANCH_NAME=$1
+HADOOP=${2:-hdp3.1}
 
-if [[ -z $BRANCH_NAME  ]]; then
+if [[ -z "$BRANCH_NAME"  ]]; then
   echo "The first parameter is mandatory and must be a valid branch name."
   exit 1
 fi
 
-
 function install(){
   local REPO=$1
   cd /home/runner/work/ || exit 2
   git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME"
   if [ -d "./$REPO" ]; then
     cd "$REPO" || exit 2
     echo "Branch name $BRANCH_NAME already exists."
-    mvn clean install -DskipTests
+    if [[ "$REPO" == "opencga-hadoop-thirdparty" ]]; then
+      ./dev/build.sh "$HADOOP"
+    else
+      mvn clean install -DskipTests
+    fi
   else
     echo "$CURRENT Branch is NOT EQUALS $BRANCH_NAME "
   fi
@@ -24,3 +28,4 @@ function install(){
 install "java-common-libs"
 install "biodata"
 install "cellbase"
+install "opencga-hadoop-thirdparty"

.github/workflows/test-analysis.yml

@@ -1,14 +1,29 @@
 name: Build and test the project
+run-name: "Build and test the project. Hadoop flavour: ${{ inputs.hadoop }}. Test profile: ${{ inputs.test_profile }}. Test module: ${{ inputs.module }}"
 
 on:
   workflow_call:
     inputs:
       test_profile:
+        description: 'Maven test profile. Any combination of : [runShortTests, runMediumTests, runLongTests]'
         type: string
         required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
       mvn_opts:
         type: string
-
+        required: false
+        default: ""
+      sonar:
+        type: boolean
+        required: false
+        default: true
+      module:
+        type: string
+        description: "Maven modules to test. Empty means all. Only top-level modules. Example: 'opencga-storage'"
         required: false
         default: ""
     secrets:
@@ -19,12 +34,13 @@ jobs:
   analysis:
     name: Execute Sonar Analysis
     runs-on: ubuntu-22.04
+    if: ${{ inputs.sonar }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '17'
@@ -33,25 +49,25 @@ jobs:
         run: |
           if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           else
              echo "./.github/workflows/scripts/get_same_branch.sh does not exist."
           fi
       - name: Test and Analyze
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: mvn -T 2 clean install -P hdp3.1,${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga
+        run: mvn -T 2 clean install -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga
 
   test:
     name: Execute JUnit and Jacoco tests
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -60,7 +76,7 @@ jobs:
         run: |
           if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           else
              echo "./.github/workflows/scripts/get_same_branch.sh does not exist."
           fi
@@ -71,8 +87,10 @@ jobs:
         with:
           mongodb-version: 6.0
           mongodb-replica-set: rs-test
+      - name: Maven build
+        run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Run Junit tests
-        run: mvn -B verify surefire-report:report --fail-never -P hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
+        run: mvn -B verify surefire-report:report --fail-never -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Publish Test Report
         uses: scacap/action-surefire-report@v1
         env:
@@ -81,9 +99,10 @@ jobs:
         ## https://docs.github.com/en/actions/learn-github-actions/expressions#cancelled
         if: success() || failure()
         with:
-          check_name: "Surefire tests report"
+          check_name: "Surefire tests report ${{ inputs.hadoop }} ${{ inputs.module }} ${{ inputs.test_profile }}"
           report_paths: './**/surefire-reports/TEST-*.xml'
           commit: '${{ github.sha }}'
           fail_on_test_failures: true
+          fail_if_no_tests: false
 
 

opencga-analysis/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.0.0-SNAPSHOT</version>
+        <version>3.1.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java

@@ -12,7 +12,9 @@
 import org.opencb.opencga.core.models.job.Job;
 import org.opencb.opencga.core.response.OpenCGAResult;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.*;
 
@@ -46,6 +48,26 @@ public static File getBamFileBySampleId(String sampleId, String studyId, FileMan
         return (fileQueryResult.getNumResults() == 0) ? null : fileQueryResult.first();
     }
 
+    public static File getBwFileBySampleId(String sampleId, String studyId, FileManager fileManager, String token) throws ToolException {
+        // Look for the bam file for each sample
+        OpenCGAResult<File> fileQueryResult;
+
+        Query query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), File.Format.BIGWIG)
+                .append(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sampleId);
+        try {
+            fileQueryResult = fileManager.search(studyId, query, QueryOptions.empty(), token);
+        } catch (CatalogException e) {
+            throw new ToolException(e);
+        }
+
+        // Sanity check
+        if (fileQueryResult.getNumResults() > 1) {
+            throw new ToolException("Found more than one BIGWIG files (" + fileQueryResult.getNumResults() + ") for sample " + sampleId);
+        }
+
+        return (fileQueryResult.getNumResults() == 0) ? null : fileQueryResult.first();
+    }
+
     public static File getBamFile(String filename, String sampleId, String studyId, FileManager fileManager, String token) throws ToolException {
         // Look for the bam file for each sample
         OpenCGAResult<File> fileQueryResult;