Prepare new development branch release-5.6.x

.github/workflows/develop.yml

@@ -10,14 +10,13 @@ on:
 jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      maven_opts: -DCELLBASE.WAR.NAME=cellbase -P default-config-test
 
   test:
     uses: ./.github/workflows/test-analysis.yml
     needs: build
     secrets: inherit
-    with:
-      report_context: development
-      report_dir: ${{ github.ref_name }}/cellbase/${{ github.sha }}
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
@@ -32,4 +31,3 @@ jobs:
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base
     secrets: inherit
-

.github/workflows/pull-request-merge.yml

@@ -1,4 +1,4 @@
-name: "Merge Workflow"
+name: "Pull Request Merge Workflow"
 
 on:
   pull_request:
@@ -13,6 +13,4 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop
     with:
       cli: python3 ./build/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }}
-    secrets:
-      DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
-      DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+    secrets: inherit

.github/workflows/release.yml

@@ -9,32 +9,26 @@ on:
 jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
-
-  test:
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
-    secrets: inherit
     with:
-      report_context: xetabase
-      report_dir: cellbase
+      maven_opts: -DCELLBASE.WAR.NAME=cellbase
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase
     secrets: inherit
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base
     secrets: inherit
 
   deploy-python:
     uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       cli: bash ./clients/python/python-build.sh push
       artifact: build-folder

.github/workflows/task.yml

@@ -9,20 +9,17 @@ on:
 jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      maven_opts: -DCELLBASE.WAR.NAME=cellbase -P default-config-test
 
   test:
     uses: ./.github/workflows/test-analysis.yml
     needs: build
     secrets: inherit
-    with:
-      report_context: development
-      report_dir: ${{ github.ref_name }}/cellbase/${{ github.sha }}
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: test
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag ${{ github.ref_name }}
     secrets: inherit
-
-

.github/workflows/test-analysis.yml

@@ -4,6 +4,7 @@ on:
     secrets:
       SONAR_TOKEN:
         required: true
+<<<<<<< HEAD
       SSH_TESTING_SERVER_HOST:
         required: true
       SSH_TESTING_SERVER_PORT:
@@ -21,6 +22,9 @@ on:
         required: true
 env:
   xb_version: "1.6.0"
+=======
+
+>>>>>>> release-5.6.x
 jobs:
   test:
     name: Test and push Sonar analysis
@@ -52,31 +56,14 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: mvn -B verify surefire-report:report -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_cellbase
-      - name: Upload result dir
-        uses: actions/upload-artifact@v3
+        run: mvn -B verify surefire-report:report --fail-never -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_cellbase
+      - name: Publish Test Report
+        uses: scacap/action-surefire-report@v1
+        ## Skip cancelled()
+        ## https://docs.github.com/en/actions/learn-github-actions/expressions#cancelled
+        if: success() || failure()
         with:
-          name: workdir
-          path: "**/target/site"
-  publish-test:
-    name: Publish test results
-    runs-on: ubuntu-22.04
-    needs: test
-    strategy:
-      matrix:
-        module: ["cellbase-core", "cellbase-app", "cellbase-client", "cellbase-lib", "cellbase-server"]
-    steps:
-      - name: Download result dir
-        uses: actions/download-artifact@v3
-        with:
-          name: workdir
-      - name: Deploy unit tests web recursively to remote
-        uses: garygrossgarten/github-action-scp@release
-        with:
-          local: ${{ matrix.module }}/target/site
-          remote: /var/www/html/reports/${{ inputs.report_context }}/${{ env.xb_version }}/${{ inputs.report_dir }}/unit/${{ matrix.module }}
-          host: ${{ secrets.SSH_TESTING_SERVER_HOST}}
-          port: ${{ secrets.SSH_TESTING_SERVER_PORT}}
-          username: ${{ secrets.SSH_TESTING_SERVER_USER }}
-          password: ${{ secrets.SSH_TESTING_SERVER_PASSWORD }}
-          concurrency: 2
+          check_name: "Surefire tests report"
+          report_paths: './**/surefire-reports/TEST-*.xml'
+          commit: '${{ github.sha }}'
+          fail_on_test_failures: true

cellbase-app/app/cloud/docker/cellbase-base/Dockerfile

@@ -1,5 +1,6 @@
-## Based on Debian 11 (bullseye)
-FROM openjdk:11-jre
+## Based on Ubuntu 22.04 (jammy)
+## We are now using OpenJDK 8u372 to support "cgroup v2", see https://developers.redhat.com/articles/2023/04/19/openjdk-8u372-feature-cgroup-v2-support#
+FROM eclipse-temurin:8u372-b07-jre-jammy
 
 LABEL org.label-schema.vendor="OpenCB" \
       org.label-schema.name="cellbase-base" \

cellbase-app/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.cellbase</groupId>
         <artifactId>cellbase</artifactId>
-        <version>5.5.0</version>
+        <version>5.7.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java

@@ -36,6 +36,7 @@ public class AdminCliOptionsParser extends CliOptionsParser {
     private DataReleaseCommandOptions dataReleaseCommandOptions;
     private DataTokenCommandOptions dataTokenCommandOptions;
     private LoadCommandOptions loadCommandOptions;
+    private ExportCommandOptions exportCommandOptions;
     private CustomiseCommandOptions customiseCommandOptions;
     private IndexCommandOptions indexCommandOptions;
     private InstallCommandOptions installCommandOptions;
@@ -52,6 +53,7 @@ public AdminCliOptionsParser() {
         dataReleaseCommandOptions = new DataReleaseCommandOptions();
         dataTokenCommandOptions = new DataTokenCommandOptions();
         loadCommandOptions = new LoadCommandOptions();
+        exportCommandOptions = new ExportCommandOptions();
         customiseCommandOptions = new CustomiseCommandOptions();
         indexCommandOptions = new IndexCommandOptions();
         installCommandOptions = new InstallCommandOptions();
@@ -63,6 +65,7 @@ public AdminCliOptionsParser() {
         jCommander.addCommand("data-release", dataReleaseCommandOptions);
         jCommander.addCommand("data-token", dataTokenCommandOptions);
         jCommander.addCommand("load", loadCommandOptions);
+        jCommander.addCommand("export", exportCommandOptions);
         jCommander.addCommand("customise", customiseCommandOptions);
         jCommander.addCommand("index", indexCommandOptions);
         jCommander.addCommand("install", installCommandOptions);
@@ -171,9 +174,9 @@ public class LoadCommandOptions {
         @ParametersDelegate
         public CommonCommandOptions commonOptions = commonCommandOptions;
 
-        @Parameter(names = {"-d", "--data"}, description = "Data model type to be loaded: genome, gene, variation, "
-                + "conservation, regulation, protein, clinical_variants, repeats, regulatory_pfm, splice_score, pubmed. 'all' loads everything",
-                required = true, arity = 1)
+        @Parameter(names = {"-d", "--data"}, description = "Data model type to be loaded: genome, gene, variation,"
+                + " conservation, regulation, protein, clinical_variants, repeats, regulatory_pfm, splice_score, pubmed, pharmacogenomics."
+                + " 'all' loads everything", required = true, arity = 1)
         public String data;
 
         @Parameter(names = {"-i", "--input"}, required = true, arity = 1,
@@ -212,6 +215,40 @@ public class LoadCommandOptions {
 
     }
 
+    @Parameters(commandNames = {"export"}, commandDescription = "Export data into JSON files")
+    public class ExportCommandOptions {
+
+        @ParametersDelegate
+        public CommonCommandOptions commonOptions = commonCommandOptions;
+
+        @Parameter(names = {"-d", "--data"}, description = "Data model type to be loaded: genome, gene, variation, "
+                + "conservation, regulation, protein, clinical_variants, repeats, regulatory_pfm, splice_score, pubmed. 'all' "
+                + " loads everything", required = true, arity = 1)
+        public String data;
+
+        @Parameter(names = {"--db", "--database"}, description = "Database name, e.g., cellbase_hsapiens_grch38_v5", required = true,
+                arity = 1)
+        public String database;
+
+        @Parameter(names = {"--data-release"}, description = "Data release for exporting data.", required = true, arity = 1)
+        public int dataRelease;
+
+        @Parameter(names = {"--token"}, description = "Data token to export licensed data.", arity = 1)
+        public String token;
+
+        @Parameter(names = {"--gene"}, description = "List of genes (separated by commas). Exported data will be related to these genes"
+                + " (gene coordinates will be taken into account).", required = true, arity = 1)
+        public String gene;
+
+        @Parameter(names = {"--region"}, description = "List of regions (separated by commas). Exported data will be related to these"
+                + " regions taking into account their coordinates.", arity = 1)
+        public String region;
+
+        @Parameter(names = {"-o", "--output"}, required = true, arity = 1,
+                description = "Output directory where to save the JSON data models.")
+        public String output;
+    }
+
     @Parameters(commandNames = {"load"}, commandDescription = "Load the built data models into the database")
     public class CustomiseCommandOptions {
 
@@ -360,9 +397,9 @@ public DataTokenCommandOptions getDataTokenCommandOptions() {
         return dataTokenCommandOptions;
     }
 
-    public LoadCommandOptions getLoadCommandOptions() {
-        return loadCommandOptions;
-    }
+    public LoadCommandOptions getLoadCommandOptions() { return loadCommandOptions; }
+
+    public ExportCommandOptions getExportCommandOptions() { return exportCommandOptions; }
 
     public IndexCommandOptions getIndexCommandOptions() {
         return indexCommandOptions;

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminMain.java

@@ -72,6 +72,9 @@ public static void main(String[] args) {
                     case "load":
                         commandExecutor = new LoadCommandExecutor(cliOptionsParser.getLoadCommandOptions());
                         break;
+                    case "export":
+                        commandExecutor = new ExportCommandExecutor(cliOptionsParser.getExportCommandOptions());
+                        break;
                     case "index":
                         commandExecutor = new IndexCommandExecutor(cliOptionsParser.getIndexCommandOptions());
                         break;

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/BuildCommandExecutor.java

@@ -38,6 +38,8 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.opencb.cellbase.lib.EtlCommons.PHARMGKB_DATA;
+
 /**
  * Created by imedina on 03/02/15.
  */
@@ -163,6 +165,9 @@ public void execute() {
                         case EtlCommons.PUBMED_DATA:
                             parser = buildPubMed();
                             break;
+                        case EtlCommons.PHARMACOGENOMICS_DATA:
+                            parser = buildPharmacogenomics();
+                            break;
                         default:
                             logger.error("Build option '" + buildCommandOptions.data + "' is not valid");
                             break;
@@ -414,4 +419,22 @@ private CellBaseBuilder buildPubMed() throws IOException {
         CellBaseFileSerializer serializer = new CellBaseJsonFileSerializer(pubmedOutputFolder);
         return new PubMedBuilder(pubmedInputFolder, serializer);
     }
+
+    private CellBaseBuilder buildPharmacogenomics() throws IOException {
+        Path inFolder = downloadFolder.resolve(EtlCommons.PHARMACOGENOMICS_DATA);
+        Path outFolder = buildFolder.resolve(EtlCommons.PHARMACOGENOMICS_DATA);
+        if (!outFolder.toFile().exists()) {
+            outFolder.toFile().mkdirs();
+        }
+
+        logger.info("Copying PharmGKB version file...");
+        if (inFolder.resolve(PHARMGKB_DATA).resolve(EtlCommons.PHARMGKB_VERSION_FILENAME).toFile().exists()) {
+            Files.copy(inFolder.resolve(PHARMGKB_DATA).resolve(EtlCommons.PHARMGKB_VERSION_FILENAME),
+                    outFolder.resolve(EtlCommons.PHARMGKB_VERSION_FILENAME),
+                    StandardCopyOption.REPLACE_EXISTING);
+        }
+
+        CellBaseFileSerializer serializer = new CellBaseJsonFileSerializer(outFolder);
+        return new PharmGKBBuilder(inFolder, serializer);
+    }
 }

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DownloadCommandExecutor.java

@@ -102,6 +102,9 @@ public void execute() {
                     case EtlCommons.PUBMED_DATA:
                         downloadFiles.addAll(downloader.downloadPubMed());
                         break;
+                    case EtlCommons.PHARMACOGENOMICS_DATA:
+                        downloadFiles.addAll(downloader.downloadPharmKGB());
+                        break;
                     default:
                         System.out.println("Value \"" + data + "\" is not allowed for the data parameter. Allowed values"
                                 + " are: {genome, gene, gene_disease_association, variation, variation_functional_score,"