Merge branch 'develop' into TASK-4372

.github/workflows/develop.yml

@@ -15,7 +15,7 @@ jobs:
 
   test:
     uses: ./.github/workflows/test-analysis.yml
-    needs: deploy-docker-ext-tools
+    needs: build
     with:
       test_profile: runShortTests,runMediumTests
     secrets: inherit

.github/workflows/docker-ext-tools.yml

@@ -8,7 +8,6 @@ on:
       - TASK-*
     paths:
       - opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile
-  workflow_dispatch:
 
 jobs:
   build:
@@ -19,5 +18,4 @@ jobs:
     needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ needs.build.outputs.version }}
-    secrets: inherit
-
+    secrets: inherit

.github/workflows/long-test-analysis.yml

@@ -5,13 +5,13 @@ name: Long test for run only on weekend
 on:
   schedule:
     - cron: '10 1 * * 6'
+  workflow_dispatch:
 
 jobs:
   test:
     uses: ./.github/workflows/test-analysis.yml
     secrets: inherit
     with:
-      test_profile: runLongTests,runMediumTests,runShortTests
-      report_context: development
+      test_profile: runShortTests,runMediumTests,runLongTests,
 
 

.github/workflows/manual-deploy-docker.yml

@@ -0,0 +1,50 @@
+name: Manual deploy Docker base
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch, tag or SHA of the source code to build docker."
+        type: string
+        required: true
+      tag:
+        description: "The tag for the new docker."
+        type: string
+        required: true
+
+jobs:
+  build:
+    name: Build Java app
+    runs-on: ubuntu-22.04
+    outputs:
+      version: ${{ steps.get_project_version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: '10'
+          ref: "${{ inputs.branch }}"
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Install dependencies branches
+        run: |
+          if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
+            chmod +x ./.github/workflows/scripts/get_same_branch.sh
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+          fi
+      - name: Maven Build (skip tests)
+        run: mvn -T 2 clean install -DskipTests
+      - uses: actions/upload-artifact@v3
+        with:
+          name: build-folder
+          path: build
+
+
+  deploy-docker:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ inputs.tag }}
+    secrets: inherit

.github/workflows/manual-deploy-ext-tools.yml

@@ -0,0 +1,50 @@
+name: Manual deploy Docker Ext-Tools
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch, tag or SHA of the source code to build docker."
+        type: string
+        required: true
+      tag:
+        description: "The tag for the new docker."
+        type: string
+        required: true
+
+jobs:
+  build:
+    name: Build Java app
+    runs-on: ubuntu-22.04
+    outputs:
+      version: ${{ steps.get_project_version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: '10'
+          ref: "${{ inputs.branch }}"
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Install dependencies branches
+        run: |
+          if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
+            chmod +x ./.github/workflows/scripts/get_same_branch.sh
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+          fi
+      - name: Maven Build (skip tests)
+        run: mvn -T 2 clean install -DskipTests
+      - uses: actions/upload-artifact@v3
+        with:
+          name: build-folder
+          path: build
+
+
+  deploy-docker-ext-tools:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ inputs.tag }}
+    secrets: inherit

.github/workflows/release.yml

@@ -14,21 +14,21 @@ jobs:
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga
     secrets: inherit
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init
     secrets: inherit
 
   deploy-python:
     uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       cli: ./clients/python/python-build.sh push
       artifact: build-folder
@@ -42,3 +42,4 @@ jobs:
       file: |
         opencga-client-${{ needs.build.outputs.version }}.tar.gz
         clients/R/opencgaR_${{ needs.build.outputs.version }}.tar.gz
+

.github/workflows/test-analysis.yml

@@ -8,6 +8,7 @@ on:
         required: true
       mvn_opts:
         type: string
+
         required: false
         default: ""
     secrets:
@@ -82,3 +83,4 @@ jobs:
           report_paths: './**/surefire-reports/TEST-*.xml'
           commit: '${{ github.sha }}'
           fail_on_test_failures: true
+

opencga-analysis/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.10.0-SNAPSHOT</version>
+        <version>2.11.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java

@@ -230,7 +230,7 @@ public OpenCGAResult<GeneCoverageStats> coverageStats(String studyIdStr, String
         String species = projectQueryResult.first().getOrganism().getScientificName();
         String assembly = projectQueryResult.first().getOrganism().getAssembly();
         String dataRelease = projectQueryResult.first().getCellbase().getDataRelease();
-        String cellbaseToken = projectQueryResult.first().getCellbase().getToken();
+        String cellbaseToken = projectQueryResult.first().getCellbase().getApiKey();
 
         for (String geneName : geneNames) {
 
@@ -451,7 +451,7 @@ public List<Region> mergeRegions(List<Region> regions, List<String> genes, boole
         String species = projectQueryResult.first().getOrganism().getScientificName();
         String assembly = projectQueryResult.first().getOrganism().getAssembly();
         String dataRelease = projectQueryResult.first().getCellbase().getDataRelease();
-        String cellbaseToken = projectQueryResult.first().getCellbase().getToken();
+        String cellbaseToken = projectQueryResult.first().getCellbase().getApiKey();
         CellBaseClient cellBaseClient = new CellBaseClient(species, assembly, dataRelease, cellbaseToken,
                 projectQueryResult.first().getCellbase().toClientConfiguration());
         GeneClient geneClient = cellBaseClient.getGeneClient();

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectAnalysis.java

@@ -59,7 +59,7 @@ public class HRDetectAnalysis extends OpenCgaToolScopeStudy {
     public static final String ID = "hr-detect";
     public static final String DESCRIPTION = "Run HRDetect analysis for a given somatic sample.";
 
-    public final static String HRDETECT_SCORES_FILENAME_DEFAULT = "data_matrix.tsv";
+    public final static String HRDETECT_SCORES_FILENAME_DEFAULT = "hrdetect_output.tsv";
 
     @ToolParams
     private HRDetectAnalysisParams hrdetectParams = new HRDetectAnalysisParams();

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java

@@ -49,6 +49,7 @@
 import org.opencb.opencga.catalog.managers.StudyManager;
 import org.opencb.opencga.core.api.ParamConstants;
 import org.opencb.opencga.core.cellbase.CellBaseValidator;
+import org.opencb.opencga.core.common.ExceptionUtils;
 import org.opencb.opencga.core.common.UriUtils;
 import org.opencb.opencga.core.config.storage.CellBaseConfiguration;
 import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
@@ -564,7 +565,10 @@ public OpenCGAResult<Job> setCellbaseConfiguration(String project, CellBaseConfi
                                                        String annotationSaveId, String token)
             throws CatalogException, StorageEngineException {
         StopWatch stopwatch = StopWatch.createStarted();
-        return secureOperationByProject("configureCellbase", project, new ObjectMap(), token, engine -> {
+        return secureOperationByProject("configureCellbase", project, new ObjectMap()
+                .append("cellbaseConfiguration", cellbaseConfiguration)
+                .append("annotate", annotate)
+                .append("annotationSaveId", annotationSaveId), token, engine -> {
             OpenCGAResult<Job> result = new OpenCGAResult<>();
             result.setResultType(Job.class.getCanonicalName());
             result.setResults(new ArrayList<>());
@@ -1017,8 +1021,9 @@ private void setCellbaseConfiguration(VariantStorageEngine engine, String projec
                 .get(project, new QueryOptions(INCLUDE, ProjectDBAdaptor.QueryParams.CELLBASE.key()), token)
                 .first().getCellbase();
         if (cellbase != null) {
-            if (StringUtils.isEmpty(cellbase.getToken()) || storageConfiguration.getCellbase() != null) {
-                cellbase.setToken(storageConfiguration.getCellbase().getToken());
+            // To ensure that we use the project API key before using the storage API key
+            if (StringUtils.isEmpty(cellbase.getApiKey()) && storageConfiguration.getCellbase() != null) {
+                cellbase.setApiKey(storageConfiguration.getCellbase().getApiKey());
             }
             engine.getConfiguration().setCellbase(cellbase);
             engine.reloadCellbaseConfiguration();
@@ -1225,7 +1230,7 @@ private <R> R secureTool(String toolId, boolean isOperation, ObjectMap params, S
             throw e;
         } catch (Exception e) {
             exception = e;
-            throw new StorageEngineException("Error executing operation " + toolId, e);
+            throw new StorageEngineException("Error executing operation '" + toolId + "' : " + e.getMessage(), e);
         } finally {
             if (result instanceof DataResult) {
                 auditAttributes.append("dbTime", ((DataResult) result).getTime());
@@ -1237,6 +1242,8 @@ private <R> R secureTool(String toolId, boolean isOperation, ObjectMap params, S
             if (exception != null) {
                 auditAttributes.append("errorType", exception.getClass());
                 auditAttributes.append("errorMessage", exception.getMessage());
+                auditAttributes.append("errorMessageFull", ExceptionUtils.prettyExceptionMessage(exception, false, true));
+                auditAttributes.append("exceptionStackTrace", ExceptionUtils.prettyExceptionStackTrace(exception));
                 status = new AuditRecord.Status(AuditRecord.Status.Result.ERROR,
                         new Error(-1, exception.getClass().getName(), exception.getMessage()));
             } else {