Merge branch 'release-3.x.x' into TASK-5318

.github/workflows/pull-request-approved.yml

@@ -1,19 +1,41 @@
 name: Pull request approve workflow
+run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}'
 
 on:
   pull_request_review:
-    types: [submitted]
+    types: [ submitted ]
 
 jobs:
-  build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
-    with:
-      maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+  calculate-xetabase-branch:
+    name: Calculate Xetabase branch
+    runs-on: ubuntu-22.04
+    outputs:
+      xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }}
+    steps:
+      - name: Clone project
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found"
+          persist-credentials: false
+      - id: get_xetabase_branch
+        name: "Get current branch for Xetabase from target branch"
+        run: |
+          chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
+          echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}"
+          echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}"
+          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }})
+          echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
+          echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
+        env:
+          ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
 
   test:
-    name: "Run all tests before merging, ie. short, medium and long tests."
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
+    name: "Run all tests before merging"
+    needs: calculate-xetabase-branch
+    uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop
     with:
-      test_profile: runShortTests,runMediumTests,runLongTests
+      branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }}
+      task: ${{ github.event.pull_request.head.ref }}
     secrets: inherit
+

.github/workflows/scripts/get-xetabase-branch.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Function to calculate the corresponding branch of Xetabase project
+get_xetabase_branch() {
+  # Input parameter (branch name)
+  input_branch="$1"
+
+  # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
+  if [[ $input_branch == TASK* ]]; then
+    if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then
+      echo $input_branch;
+      return 0;
+    fi
+  fi
+
+  # Check if the branch name is "develop" in that case return the same branch name
+  if [[ "$input_branch" == "develop" ]]; then
+    echo "develop"
+    return 0
+  fi
+
+  # Check if the branch name starts with "release-" and follows the patterns "release-a.x.x" or "release-a.b.x"
+  if [[ "$input_branch" =~ ^release-([0-9]+)\.x\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]]; then
+    # Extract the MAJOR part of the branch name
+    MAJOR=${BASH_REMATCH[1]}
+    # Calculate the XETABASE_MAJOR by subtracting 1 from MAJOR of opencga
+    XETABASE_MAJOR=$((MAJOR - 1))
+    # Check if the XETABASE_MAJOR is negative
+    if (( XETABASE_MAJOR < 0 )); then
+      echo "Error: 'MAJOR' digit after subtraction results in a negative number."
+      return 1
+    fi
+    # Construct and echo the new branch name
+    echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}"
+    return 0
+  fi
+
+  # If the branch name does not match any of the expected patterns
+  echo "Error: The branch name is not correct."
+  return 1
+}
+
+# Check if the script receives exactly one argument
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <branch-name>"
+  exit 1
+fi
+
+# Call the function with the input branch name
+get_xetabase_branch "$1"

opencga-analysis/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.2.1-SNAPSHOT</version>
+        <version>3.3.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java

@@ -16,9 +16,13 @@
 
 package org.opencb.opencga.analysis;
 
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.commons.utils.FileUtils;
+import org.opencb.opencga.core.config.AnalysisTool;
 import org.opencb.opencga.core.config.Configuration;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
+import org.opencb.opencga.core.exceptions.ToolException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,10 +32,15 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
 
 public class ConfigurationUtils {
     private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class);
 
+    private ConfigurationUtils() {
+        throw new IllegalStateException("Utility class");
+    }
     /**
      * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml.
      *
@@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome)
                     .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml"));
         }
     }
+
+    public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException {
+        List<AnalysisTool> tools = new ArrayList<>();
+        for (AnalysisTool tool : configuration.getAnalysis().getTools()) {
+            if (tool.getId().equals(toolId)) {
+                tools.add(tool);
+            }
+        }
+        if (CollectionUtils.isEmpty(tools)) {
+            throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file");
+        }
+        if (tools.size() == 1) {
+            return tools.get(0).getVersion();
+        }
+        String defaultVersion = null;
+        for (AnalysisTool tool : tools) {
+            if (tool.isDefaultVersion()) {
+                if (!StringUtils.isEmpty(defaultVersion)) {
+                    throw new ToolException("More than one default version found for tool ID '" + toolId + "'");
+                } else {
+                    defaultVersion = tool.getVersion();
+                }
+            }
+        }
+        if (StringUtils.isEmpty(defaultVersion)) {
+            throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true");
+        }
+        return defaultVersion;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java

@@ -30,8 +30,10 @@
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
+import org.opencb.opencga.analysis.ConfigurationUtils;
 import org.opencb.opencga.analysis.clinical.InterpretationAnalysis;
 import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils;
+import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis;
 import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.catalog.utils.ParamUtils;
@@ -44,10 +46,10 @@
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.individual.Individual;
 import org.opencb.opencga.core.response.OpenCGAResult;
-import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
+import org.opencb.opencga.storage.core.variant.query.VariantQueryResult;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -69,6 +71,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis {
     private String clinicalAnalysisId;
     private String sampleId;
     private ClinicalAnalysis.Type clinicalAnalysisType;
+    private String exomiserVersion;
 
     private ClinicalAnalysis clinicalAnalysis;
 
@@ -97,8 +100,7 @@ protected void check() throws Exception {
         try {
             clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager().get(studyId, clinicalAnalysisId, QueryOptions.empty(),
                     token);
-        } catch (
-                CatalogException e) {
+        } catch (CatalogException e) {
             throw new ToolException(e);
         }
         if (clinicalAnalysisQueryResult.getNumResults() != 1) {
@@ -117,6 +119,7 @@ protected void check() throws Exception {
         }
         sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId();
 
+        // Check clinical analysis type
         if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) {
             clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY;
         } else {
@@ -125,6 +128,13 @@ protected void check() throws Exception {
         logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(),
                 clinicalAnalysisType);
 
+        // Check exomiser version
+        if (StringUtils.isEmpty(exomiserVersion)) {
+            // Missing exomiser version use the default one
+            exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration);
+            logger.warn("Missing exomiser version, using the default {}", exomiserVersion);
+        }
+
         // Update executor params with OpenCGA home and session ID
         setUpStorageEngineExecutor(studyId);
     }
@@ -134,25 +144,28 @@ protected void run() throws ToolException {
         step(() -> {
 
             executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID);
-            getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
+            ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class)
                     .setStudyId(studyId)
                     .setSampleId(sampleId)
                     .setClinicalAnalysisType(clinicalAnalysisType)
-                    .execute();
+                    .setExomiserVersion(exomiserVersion);
+
+            exomiserExecutor.execute();
 
-            saveInterpretation(studyId, clinicalAnalysis);
+            saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion());
         });
     }
 
-    protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException,
+    protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion)
+            throws ToolException, StorageEngineException,
             CatalogException, IOException {
         // Interpretation method
         InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(),
                 GitRepositoryState.getInstance().getCommitId(), Collections.singletonList(
                 new Software()
                         .setName("Exomiser")
-                        .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME)
-                        .setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION)));
+                        .setRepository("Docker: " + dockerImage)
+                        .setVersion(dockerImageVersion)));
 
         // Analyst
         ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token);
@@ -274,8 +287,17 @@ private List<ClinicalVariant> getPrimaryFindings() throws IOException, StorageEn
                 // Convert variants to clinical variants
                 for (Variant variant : variantResults.getResults()) {
                     ClinicalVariant clinicalVariant = clinicalVariantCreator.create(variant);
-                    List<ExomiserTranscriptAnnotation> exomiserTranscripts = new ArrayList<>(variantTranscriptMap.get(normalizedToTsv
-                            .get(variant.toStringSimple())));
+                    List<ExomiserTranscriptAnnotation> exomiserTranscripts = new ArrayList<>();
+                    if (normalizedToTsv.containsKey(variant.toStringSimple())) {
+                        if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) {
+                            exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple())));
+                        } else {
+                            logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(),
+                                    normalizedToTsv.get(variant.toStringSimple()));
+                        }
+                    } else {
+                        logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple());
+                    }
                     for (String[] fields : variantTsvMap.get(variant.toStringSimple())) {
                         ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]);
                         Map<String, Object> attributes = getAttributesFromTsv(fields);
@@ -463,4 +485,13 @@ public ExomiserInterpretationAnalysis setClinicalAnalysisId(String clinicalAnaly
         this.clinicalAnalysisId = clinicalAnalysisId;
         return this;
     }
+
+    public String getExomiserVersion() {
+        return exomiserVersion;
+    }
+
+    public ExomiserInterpretationAnalysis setExomiserVersion(String exomiserVersion) {
+        this.exomiserVersion = exomiserVersion;
+        return this;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java

@@ -552,7 +552,7 @@ protected final <T extends OpenCgaToolExecutor> T getToolExecutor(Class<T> clazz
                 toolExecutor.getSource(),
                 toolExecutor.getFramework()));
 
-        toolExecutor.setUp(erm, executorParams, outDir);
+        toolExecutor.setUp(erm, executorParams, outDir, configuration);
         return toolExecutor;
     }
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java

@@ -295,7 +295,7 @@ protected void run() throws ToolException {
         step("gwas", () -> {
             GwasAnalysisExecutor gwasExecutor = getToolExecutor(GwasAnalysisExecutor.class);
 
-            gwasExecutor.setConfiguration(gwasConfiguration)
+            gwasExecutor.setGwasConfiguration(gwasConfiguration)
                     .setStudy(study)
                     .setSampleList1(caseCohortSamples)
                     .setSampleList2(controlCohortSamples)