From beede13a3688ebe431486b4b4e2c19cf204513fb Mon Sep 17 00:00:00 2001 From: imedina Date: Mon, 27 Mar 2023 01:59:38 +0100 Subject: [PATCH 01/28] client: replace old URL with the new from Zetta --- .../main/resources/client-configuration.yml | 6 ++--- .../opencb/cellbase/core/ParamConstants.java | 22 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cellbase-client/src/main/resources/client-configuration.yml b/cellbase-client/src/main/resources/client-configuration.yml index f13e1f33fa..5db9eeaca8 100644 --- a/cellbase-client/src/main/resources/client-configuration.yml +++ b/cellbase-client/src/main/resources/client-configuration.yml @@ -1,10 +1,10 @@ --- -version: "v5" +version: "v5.1" defaultSpecies: "hsapiens" ## These are the RESTful configurations parameters rest: hosts: - - "bioinfodev.hpc.cam.ac.uk/cellbase-5.0.0" - timeout: 2000 + - "https://ws.zettagenomics.com/cellbase" + timeout: 10000 diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java index 9e0c5f10ae..ef762ce048 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java @@ -117,7 +117,7 @@ public Type type() { public static final String DEFAULT_DATA_RELEASE = "0"; public static final String ASSEMBLY_DESCRIPTION = "Set the reference genome assembly, e.g. grch38. For a full list of " + "potentially available assemblies, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species"; + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/meta/species"; public static final String COUNT_DESCRIPTION = "Get the total number of results matching the query. "; @@ -136,7 +136,7 @@ public Type type() { public static final String SPECIES_DESCRIPTION = "Name of the species, e.g. hsapiens. For a full list " + "of potentially available species ids, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species"; + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/meta/species"; public static final String DATA_MODEL_DESCRIPTION = "Get JSON specification of the data model"; public static final String REGION_DESCRIPTION = "Comma separated list of genomic regions to be queried, " @@ -174,7 +174,7 @@ public Type type() { + "e.g. Diabetes mellitus,histone kinase activity"; public static final String ONTOLOGY_NAMESPACES = "Comma separated list of namespaces, e.g. human_phenotype,biological_process. " + "For a full list of potentially available namespaces, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/feature/ontology/distinct?field=namespace"; + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/feature/ontology/distinct?field=namespace"; public static final String ONTOLOGY_SOURCES = "Name of ontology, e.g. HP or GO"; public static final String ONTOLOGY_SYNONYMS = "Comma separated list of synonyms, e.g. Cerebellar ataxia"; public static final String ONTOLOGY_XREFS = "Comma separated list of cross references, e.g. MSH:D002524"; @@ -299,7 +299,7 @@ public Type type() { public static final String SOURCE_PARAM = "source"; public static final String SEQUENCE_ONTOLOGY_DESCRIPTION = "Comma separated list of consequence types, " + "e.g. missense_variant. Exact text matches will be returned. A list of searchable consequence types can be" - + " accessed at https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/feature/variation/consequence_types"; + + " accessed at https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/feature/variation/consequence_types"; public static final String SEQUENCE_ONTOLOGY_PARAM = "consequenceType"; public static final String FEATURE_IDS_DESCRIPTION = "Comma separated list of feature ids, which can be either ENSEMBL gene " + "ids, HGNC gene symbols, transcript symbols or ENSEMBL transcript ids, e.g.: BRCA2,ENST00000409047. " @@ -318,24 +318,24 @@ public Type type() { public static final String VARIANT_IDS_PARAM = "id"; public static final String VARIANT_TYPES_DESCRIPTION = "Comma separated list of variant types, e.g. \"SNV\" A list of " + "searchable types can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/type"; + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/clinical/variant/type"; public static final String VARIANT_TYPES_PARAM = "type"; public static final String CONSISTENCY_STATUS_DESCRIPTION = "Comma separated list of consistency labels. A list of searchable " + "consistency labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/consistency_labels"; + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/clinical/variant/consistency_labels"; public static final String CONSISTENCY_STATUS_PARAM = "consistencyStatus"; public static final String CLINICAL_SIGNFICANCE_DESCRIPTION = "Comma separated list of clinical significance labels. " + "A list of searchable clinical significance labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/clinsig_labels" + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/clinical/variant/clinsig_labels" + " WARNING: returned numTotalResults will always be -1 if more than 1 label is provided."; public static final String CLINICAL_SIGNFICANCE_PARAM = "clinicalSignificance"; public static final String MODE_INHERITANCE_DESCRIPTION = "Comma separated list of mode of inheritance labels. A list of " + "searchable mode of inheritance labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/mode_inheritance_labels"; + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/clinical/variant/mode_inheritance_labels"; public static final String MODE_INHERITANCE_PARAM = "modeInheritance"; public static final String ALLELE_ORIGIN_DESCRIPTION = "Comma separated list of allele origin labels. A list of searchable " + "allele origin labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/allele_origin_labels"; + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/clinical/variant/allele_origin_labels"; public static final String ALLELE_ORIGIN_PARAM = "alleleOrigin"; // --------------------------------------------- @@ -384,11 +384,11 @@ public Type type() { // public static final String REGULATION_FEATURE_CLASSES = "Comma separated list of regulatory region classes, e.g.:" // + "Histone,Transcription Factor. Exact text matches will be returned. For a full" // + "list of available regulatory types: " -// + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/regulatory/featureClass"; +// + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/regulatory/featureClass"; public static final String REGULATION_FEATURE_TYPES = "Comma separated list of regulatory region types, e.g.: " + "TF_binding_site,histone_acetylation_site. Exact text matches will be returned. For a full" + "list of available regulatory types: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/regulatory/featureType"; + + "https://ws.zettagenomics.com/cellbase/webservices/rest/v5/hsapiens/regulatory/featureType"; public static final String CELLTYPE = "Type of cell."; From 0970179770094ec6468a76ab6c2d547b5ce58eb6 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 13 Apr 2023 11:02:42 +0200 Subject: [PATCH 02/28] cicd: Added script to install dependencies branches #TASK-4163 --- .github/workflows/scripts/get_same_branch.sh | 29 ++++++++++++++++++++ .github/workflows/task.yml | 26 ------------------ .github/workflows/test-analysis.yml | 8 ++++++ 3 files changed, 37 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/scripts/get_same_branch.sh diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh new file mode 100644 index 0000000000..a5759e8ddf --- /dev/null +++ b/.github/workflows/scripts/get_same_branch.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +BRANCH_NAME=$1 + +if [[ -z $BRANCH_NAME ]]; then + echo "The first parameter is mandatory and must be a valid branch name." + exit 1 +fi + +if [[ $BRANCH_NAME != "TASK-"* ]]; then + echo "No need to check dependencies." + exit 0 +fi + +function install(){ + local REPO=$1 + cd /home/runner/work/ || exit 2 + git clone https://github.com/opencb/"$REPO".git -d "$BRANCH_NAME" + if [ -d "./$REPO" ]; then + cd "$REPO" || exit 2 + echo "Branch name $BRANCH_NAME already exists." + mvn clean install -DskipTests + else + echo "$CURRENT Branch is NOT EQUALS $BRANCH_NAME " + fi +} + +install "java-common-libs" +install "biodata" diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index b75fec23e1..52e48035a7 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -25,30 +25,4 @@ jobs: cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag ${{ github.ref_name }} secrets: inherit - deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop - needs: [ build, test ] - with: - maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase - if: contains( needs.build.outputs.version ,'TASK') - secrets: inherit - - #The following jobs are to see that the previous if does not fail and that the maven deploy is executed because it is true - snapshot-version: - name: Check SNAPSHOT version - needs: [ build, test ] - if: contains(needs.build.outputs.version ,'SNAPSHOT') - runs-on: ubuntu-22.04 - steps: - - name: test-version-from-check - run: echo "Project version is " ${{ needs.build.outputs.version }} - - task-version: - name: Check TASK version - needs: [ build, test ] - if: contains(needs.build.outputs.version ,'TASK') - runs-on: ubuntu-22.04 - steps: - - name: test-version-from-check - run: echo "Project version is " ${{ needs.build.outputs.version }} diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index 7a0ebc58ea..efd6d28158 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -42,6 +42,14 @@ jobs: with: mongodb-version: 5.0 mongodb-replica-set: rs-test + - name: Install dependencies branches + run: | + if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x ./.github/workflows/scripts/get_same_branch.sh + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + else + echo "./.github/workflows/scripts/get_same_branch.sh does not exist." + fi - name: Test and Analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any From 216d9e7ad0ad19054496a9b543b8fe802a19de0a Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 13 Apr 2023 11:56:41 +0200 Subject: [PATCH 03/28] cicd: change script get_same_branch.sh to check if directory exists #TASK-4163 --- .github/workflows/scripts/get_same_branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh index a5759e8ddf..bf0f7bb12d 100644 --- a/.github/workflows/scripts/get_same_branch.sh +++ b/.github/workflows/scripts/get_same_branch.sh @@ -15,7 +15,7 @@ fi function install(){ local REPO=$1 cd /home/runner/work/ || exit 2 - git clone https://github.com/opencb/"$REPO".git -d "$BRANCH_NAME" + git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME" if [ -d "./$REPO" ]; then cd "$REPO" || exit 2 echo "Branch name $BRANCH_NAME already exists." From 1b9890a6624a3631400919fce5ab4cd74a715e2b Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 13 Apr 2023 13:18:52 +0200 Subject: [PATCH 04/28] cicd: testing workflow #TASK-4163 --- .../main/java/org/opencb/cellbase/app/cli/CommandExecutor.java | 2 +- .../lib/builders/clinical/variant/ClinicalIndexer.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java index 0a90cb5ea1..6e8101470a 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java @@ -18,13 +18,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; import org.opencb.cellbase.client.config.ClientConfiguration; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.commons.utils.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.logging.log4j.core.config.Configurator; import java.io.File; import java.io.FileInputStream; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java index 8726219505..bbe33017fd 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java @@ -25,6 +25,7 @@ import org.opencb.biodata.models.variant.avro.*; import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.cellbase.lib.variant.VariantAnnotationUtils; +import org.opencb.commons.utils.PrintUtils; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.slf4j.Logger; @@ -63,6 +64,8 @@ public abstract class ClinicalIndexer { mapper.configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true); objectReader = mapper.readerFor(VariantAnnotation.class); jsonObjectWriter = mapper.writer(); + + PrintUtils.printSpace(); // jsonObjectWriter = mapper.writerFor(VariantAnnotation.class); } From 615971e51a1890d69b2a84a30f16f76255620e48 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 13 Apr 2023 17:42:46 +0200 Subject: [PATCH 05/28] CICD: Test build branch TASK-4163 java-common-libs #TASK-4163 --- .github/workflows/task.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index 52e48035a7..5a1795426b 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -8,7 +8,7 @@ on: jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@TASK-4163 test: uses: ./.github/workflows/test-analysis.yml From a932dbfa783597542283cad625d1b9fdca33f53f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Wed, 26 Apr 2023 17:56:03 +0200 Subject: [PATCH 06/28] lib: support to assign a default data release for each CellBase version, #TASK-4278, #TASK-4157 --- .../app/cli/admin/AdminCliOptionsParser.java | 7 +- .../executors/DataReleaseCommandExecutor.java | 41 ++++---- .../admin/executors/LoadCommandExecutor.java | 95 ++++++++----------- .../cellbase/core/models/DataRelease.java | 30 +++--- .../lib/impl/core/CellBaseDBAdaptor.java | 4 - .../lib/impl/core/ReleaseMongoDBAdaptor.java | 54 ++++++++++- .../lib/managers/DataReleaseManager.java | 56 +++-------- .../lib/managers/DataReleaseManagerTest.java | 12 +-- .../server/rest/GenericRestWSServer.java | 48 ++++------ .../cellbase/server/rest/MetaWSServer.java | 18 +--- 10 files changed, 175 insertions(+), 190 deletions(-) diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java index ec8ea3ea0f..28a65d5f93 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java @@ -142,8 +142,11 @@ public class DataReleaseCommandOptions { @Parameter(names = {"--list"}, description = "List the data releases present in the database", arity = 0) public boolean list; - @Parameter(names = {"--set-active"}, description = "Set the data release active", arity = 1) - public int active; + @Parameter(names = {"--update"}, description = "Data release to be updated by adding CellBase vesions", arity = 1) + public int update; + + @Parameter(names = {"--add-versions"}, description = "CellBase versions separated by commas, e.g.: v5.2,v5.3. This parameter has to be used together to the parameter --update", arity = 1) + public String versions; } @Parameters(commandNames = {"data-token"}, commandDescription = "Manage data access tokens in order to access to restricted/licensed data sources") diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DataReleaseCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DataReleaseCommandExecutor.java index b1ffe28b88..b46c4755c7 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DataReleaseCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DataReleaseCommandExecutor.java @@ -17,12 +17,14 @@ package org.opencb.cellbase.app.cli.admin.executors; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.app.cli.CommandExecutor; import org.opencb.cellbase.app.cli.admin.AdminCliOptionsParser; import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.managers.DataReleaseManager; +import java.util.Arrays; import java.util.List; public class DataReleaseCommandExecutor extends CommandExecutor { @@ -51,29 +53,28 @@ public void execute() { DataReleaseManager dataReleaseManager = new DataReleaseManager(database, configuration); + DataRelease dataRelease; if (dataReleaseCommandOptions.create) { // Create release - DataRelease dataRelease = dataReleaseManager.createRelease(); + dataRelease = dataReleaseManager.createRelease(); System.out.println("\nData release " + dataRelease.getRelease() + " was created."); System.out.println("Data release description (in JSON format):"); System.out.println(new ObjectMapper().writerFor(DataRelease.class).writeValueAsString(dataRelease)); - } else if (dataReleaseCommandOptions.active > 0) { - // Set-active release - CellBaseDataResult results = dataReleaseManager.getReleases(); - for (DataRelease dr : results.getResults()) { - if (dr.isActive() && dr.getRelease() == dataReleaseCommandOptions.active) { - logger.info("Data release " + dataReleaseCommandOptions.active + " is already active"); - return; - } - } - DataRelease dataRelease = dataReleaseManager.active(dataReleaseCommandOptions.active); - if (dataRelease != null) { - System.out.println("\nThe data release " + dataRelease.getRelease() + " is the active one."); - System.out.println("Data release description (in JSON format):"); - System.out.println(new ObjectMapper().writerFor(DataRelease.class).writeValueAsString(dataRelease)); - } else { - logger.error("It could not set to active the data release " + dataReleaseCommandOptions.active); + } else if (dataReleaseCommandOptions.update > 0) { + if (StringUtils.isEmpty(dataReleaseCommandOptions.versions)) { + throw new IllegalArgumentException("Missing CellBase versions to be added when updating data release"); } + + // Versions to be added + List versions = Arrays.asList(dataReleaseCommandOptions.versions.split(",")); + + // Update data release + dataReleaseManager.update(dataReleaseCommandOptions.update, versions); + + dataRelease = dataReleaseManager.get(dataReleaseCommandOptions.update); + System.out.println("\nData release " + dataRelease.getRelease() + " was updated."); + System.out.println("Data release description (in JSON format):"); + System.out.println(new ObjectMapper().writerFor(DataRelease.class).writeValueAsString(dataRelease)); } else if (dataReleaseCommandOptions.list) { // List releases CellBaseDataResult dataReleases = dataReleaseManager.getReleases(); @@ -94,11 +95,11 @@ private void checkParameters() { if (dataReleaseCommandOptions.list) { opts++; } - if (dataReleaseCommandOptions.active > 0) { + if (dataReleaseCommandOptions.update > 0) { opts++; } - if (opts > 1) { - throw new IllegalArgumentException("Please, select only one of these input parameters: create, list or set-active"); + if (opts != 1) { + throw new IllegalArgumentException("Please, select only one of these input parameters: create, update or list"); } } } diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java index 26ceb22cbe..780f5bf39f 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java @@ -17,6 +17,7 @@ package org.opencb.cellbase.app.cli.admin.executors; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.app.cli.CommandExecutor; import org.opencb.cellbase.app.cli.admin.AdminCliOptionsParser; import org.opencb.cellbase.core.exception.CellBaseException; @@ -105,7 +106,7 @@ public void execute() throws CellBaseException { dataReleaseManager = new DataReleaseManager(database, configuration); checkParameters(); - logger.info("Loading in data release " + dataRelease); + logger.info("Loading in data release {}", dataRelease); if (loadCommandOptions.data != null) { // If 'authenticationDatabase' is not passed by argument then we read it from configuration.json @@ -359,49 +360,35 @@ private void checkParameters() throws CellBaseException { if (CollectionUtils.isEmpty(dataReleaseResults.getResults())) { throw new CellBaseException("No data releases are available for database " + database); } - int lastDataRelease = 0; - int defaultDataRelease = 0; + DataRelease lastDataRelease = null; for (DataRelease dataRelease : dataReleaseResults.getResults()) { - if (dataRelease.isActive()) { - defaultDataRelease = dataRelease.getRelease(); - } - if (dataRelease.getRelease() > lastDataRelease) { - lastDataRelease = dataRelease.getRelease(); + if (lastDataRelease == null) { + lastDataRelease = dataRelease; + } else if (dataRelease.getRelease() > lastDataRelease.getRelease()) { + lastDataRelease = dataRelease; } } - if (lastDataRelease == defaultDataRelease) { - throw new CellBaseException("Loading data in the active data release (" + defaultDataRelease + ") is not permitted."); + if (lastDataRelease == null) { + throw new CellBaseException("Loading data is not permitted since no data release is found"); } - dataRelease = lastDataRelease; - -// for (DataRelease dr : dataReleaseResults.getResults()) { -// if (dr.getRelease() == dataRelease) { -// for (String loadOption : loadOptions) { -// if (dr.getCollections().containsKey(loadOption)) { -// String collectionName = CellBaseDBAdaptor.buildCollectionName(loadOption, dataRelease); -// if (dr.getCollections().get(loadOption).equals(collectionName)) { -// throw new CellBaseException("Impossible load data " + loadOption + " with release " + dataRelease + " since it" -// + " has already been done."); -// } -// } -// } -// break; -// } -// } + if (CollectionUtils.isNotEmpty(lastDataRelease.getActiveByDefaultIn())) { + throw new CellBaseException("Loading data is not permitted for data release " + lastDataRelease.getRelease() + " since it has" + + " already assigned CellBase versions:" + StringUtils.join(lastDataRelease.getActiveByDefaultIn(), ",")); + } + dataRelease = lastDataRelease.getRelease(); } private void loadVariationData() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException, LoaderException { + IOException, LoaderException, CellBaseException { // First load data // Common loading process from CellBase variation data models if (field == null) { - DirectoryStream stream = Files.newDirectoryStream(input, entry -> { - return entry.getFileName().toString().startsWith("variation_chr"); - }); + DirectoryStream stream = Files.newDirectoryStream(input, + entry -> entry.getFileName().toString().startsWith("variation_chr")); for (Path entry : stream) { - logger.info("Loading file '{}'", entry.toString()); + logger.info("Loading file '{}'", entry); loadRunner.load(input.resolve(entry.getFileName()), "variation", dataRelease); } @@ -416,20 +403,20 @@ private void loadVariationData() throws NoSuchMethodException, InterruptedExcept // Custom update required e.g. population freqs loading } else { - logger.info("Loading file '{}'", input.toString()); + logger.info("Loading file '{}'", input); loadRunner.load(input, "variation", dataRelease, field, innerFields); } } private void loadConservation() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException { + IOException, CellBaseException { // Load data - DirectoryStream stream = Files.newDirectoryStream(input, entry -> { - return entry.getFileName().toString().startsWith("conservation_"); - }); + DirectoryStream stream = Files.newDirectoryStream(input, + entry -> entry.getFileName().toString().startsWith("conservation_")); + for (Path entry : stream) { - logger.info("Loading file '{}'", entry.toString()); + logger.info("Loading file '{}'", entry); loadRunner.load(input.resolve(entry.getFileName()), "conservation", dataRelease); } @@ -447,13 +434,13 @@ private void loadConservation() throws NoSuchMethodException, InterruptedExcepti private void loadProteinFunctionalPrediction() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException { + IOException, CellBaseException { // Load data - DirectoryStream stream = Files.newDirectoryStream(input, entry -> { - return entry.getFileName().toString().startsWith("prot_func_pred_"); - }); + DirectoryStream stream = Files.newDirectoryStream(input, + entry -> entry.getFileName().toString().startsWith("prot_func_pred_")); + for (Path entry : stream) { - logger.info("Loading file '{}'", entry.toString()); + logger.info("Loading file '{}'", entry); loadRunner.load(input.resolve(entry.getFileName()), "protein_functional_prediction", dataRelease); } @@ -483,11 +470,11 @@ private void loadClinical() throws FileNotFoundException { )); dataReleaseManager.update(dataRelease, "clinical_variants", EtlCommons.CLINICAL_VARIANTS_DATA, sources); } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException - | IllegalAccessException | ExecutionException | IOException | InterruptedException e) { + | IllegalAccessException | ExecutionException | IOException | InterruptedException | CellBaseException e) { logger.error(e.toString()); } } else { - throw new FileNotFoundException("File " + path.toString() + " does not exist"); + throw new FileNotFoundException("File " + path + " does not exist"); } } @@ -510,18 +497,18 @@ private void loadRepeats() { )); dataReleaseManager.update(dataRelease, "repeats", EtlCommons.REPEATS_DATA, sources); } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException - | IllegalAccessException | ExecutionException | IOException | InterruptedException e) { + | IllegalAccessException | ExecutionException | IOException | InterruptedException | CellBaseException e) { logger.error(e.toString()); } } else { - logger.warn("Repeats file {} not found", path.toString()); + logger.warn("Repeats file {} not found", path); logger.warn("No repeats data will be loaded"); } } private void loadSpliceScores() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException { + IOException, CellBaseException { // Load data logger.info("Loading splice scores from '{}'", input); // MMSplice scores @@ -543,17 +530,17 @@ private void loadSpliceScores() throws NoSuchMethodException, InterruptedExcepti private void loadSpliceScores(Path spliceFolder) throws IOException, ExecutionException, InterruptedException, ClassNotFoundException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { // Get files from folder - DirectoryStream stream = Files.newDirectoryStream(spliceFolder, entry -> { - return entry.getFileName().toString().startsWith("splice_score_"); - }); + DirectoryStream stream = Files.newDirectoryStream(spliceFolder, + entry -> entry.getFileName().toString().startsWith("splice_score_")); + // Load from JSON files for (Path entry : stream) { - logger.info("Loading file '{}'", entry.toString()); + logger.info("Loading file '{}'", entry); loadRunner.load(spliceFolder.resolve(entry.getFileName()), "splice_score", dataRelease); } } - private void loadPubMed() { + private void loadPubMed() throws CellBaseException { Path pubmedPath = input.resolve(EtlCommons.PUBMED_DATA); if (Files.exists(pubmedPath)) { @@ -576,7 +563,7 @@ private void loadPubMed() { List sources = Collections.singletonList(pubmedPath.resolve(EtlCommons.PUBMED_VERSION_FILENAME)); dataReleaseManager.update(dataRelease, "pubmed", EtlCommons.REPEATS_DATA, sources); } else { - logger.warn("PubMed folder {} not found", pubmedPath.toString()); + logger.warn("PubMed folder {} not found", pubmedPath); } } @@ -589,7 +576,7 @@ private void createIndex(String collection) { try { indexManager.createMongoDBIndexes(Collections.singletonList(collectionName), true); } catch (IOException e) { - logger.error("Error creating index: " + e.getMessage()); + logger.error("Error creating index: {}", e.getMessage()); } } } diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java index 375542b498..8baefdca68 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java @@ -16,27 +16,31 @@ package org.opencb.cellbase.core.models; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; public class DataRelease { private int release; private String date; + /** + * @deprecated it is maintained to back-compatibility with previous CellBase versions to v5.5 + */ + @Deprecated private boolean active; + private List activeByDefaultIn; private Map collections; private List sources; public DataRelease() { + this.activeByDefaultIn = Collections.emptyList(); this.collections = Collections.emptyMap(); this.sources = Collections.emptyList(); } - public DataRelease(int release, String date, boolean active, Map collections, List sources) { + public DataRelease(int release, String date, List activeByDefaultIn, Map collections, + List sources) { this.release = release; this.date = date; - this.active = active; + this.activeByDefaultIn = activeByDefaultIn; this.collections = collections; this.sources = sources; } @@ -46,7 +50,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("DataRelease{"); sb.append("release=").append(release); sb.append(", date='").append(date).append('\''); - sb.append(", active=").append(active); + sb.append(", activeByDefaultIn=").append(activeByDefaultIn); sb.append(", collections=").append(collections); sb.append(", sources=").append(sources); sb.append('}'); @@ -71,12 +75,12 @@ public DataRelease setDate(String date) { return this; } - public boolean isActive() { - return active; + public List getActiveByDefaultIn() { + return activeByDefaultIn; } - public DataRelease setActive(boolean active) { - this.active = active; + public DataRelease setActiveByDefaultIn(List activeByDefaultIn) { + this.activeByDefaultIn = activeByDefaultIn; return this; } @@ -108,7 +112,7 @@ public boolean equals(Object o) { } DataRelease that = (DataRelease) o; return release == that.release - && active == that.active + && Objects.equals(activeByDefaultIn, that.activeByDefaultIn) && Objects.equals(date, that.date) && Objects.equals(collections, that.collections) && Objects.equals(sources, that.sources); @@ -116,6 +120,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(release, date, active, collections, sources); + return Objects.hash(release, date, activeByDefaultIn, collections, sources); } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java index 5a919ebe33..afbcefc162 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java @@ -45,10 +45,6 @@ public Map buildCollectionByReleaseMap(String data) if (dataRelease.getCollections().containsKey(data)) { String collectionName = dataRelease.getCollections().get(data); collectionMap.put(dataRelease.getRelease(), mongoDataStore.getCollection(collectionName)); - if (dataRelease.isActive()) { - // Associate the default data release to the key 0 in the map - collectionMap.put(0, mongoDataStore.getCollection(collectionName)); - } } } } else { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java index fc2cc31455..001e34e9f7 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java @@ -18,13 +18,17 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.TransactionOptions; +import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; +import org.apache.commons.lang3.StringUtils; import org.bson.BsonDocument; import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.cellbase.core.api.query.AbstractQuery; import org.opencb.cellbase.core.api.query.ProjectionQueryOptions; +import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.iterator.CellBaseIterator; @@ -34,7 +38,10 @@ import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDataStore; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class ReleaseMongoDBAdaptor extends MongoDBAdaptor implements CellBaseCoreDBAdaptor { @@ -62,12 +69,55 @@ public DataResult insert(DataRelease dataRelease) throws JsonProcessingException return mongoDBCollection.insert(document, QueryOptions.empty()); } - public void update(int release, String field, Object value) { + public DataResult update(int release, List versions) throws CellBaseException { + Map toUpdate = new HashMap<>(); + for (String version : versions) { + DataResult result = mongoDBCollection.find(Filters.eq("activeByDefaultInd", version), null, DataRelease.class, + QueryOptions.empty()); + if (result.getNumResults() > 1) { + throw new CellBaseException("There's something wrong in the CellBase MongoDB. CellBase version " + version + " has" + + " multiple data releases: " + StringUtils.join(result.getResults().stream().map(dr -> dr.getRelease()), ",")); + } + if (result.getNumResults() == 1) { + toUpdate.put(version, result.first()); + } + } + + // Start a transaction + ClientSession session = mongoDataStore.startSession(); + try { + session.startTransaction(TransactionOptions.builder().build()); + + // Update data releases by removing versions + for (Map.Entry entry : toUpdate.entrySet()) { + DataRelease dataRelease = entry.getValue(); + List updatedVersions = new ArrayList<>(dataRelease.getActiveByDefaultIn()); + updatedVersions.remove(entry.getKey()); + update(entry.getValue().getRelease(), "activeByDefaultIn", updatedVersions); + } + + // Update data release by adding versions + update(release, "activeByDefaultIn", versions); + + // Commit the transaction + session.commitTransaction(); + } catch (Exception e) { + // Roll back the transaction if any operation fails + session.abortTransaction(); + System.err.println("Transaction rolled back: " + e.getMessage()); + } finally { + session.close(); + } + + return mongoDBCollection.find(Filters.eq("release", release), null, DataRelease.class, QueryOptions.empty()); + } + + public DataResult update(int release, String field, Object value) { Bson query = Filters.eq("release", release); Document projection = new Document(field, true); Bson update = Updates.set(field, value); QueryOptions queryOptions = new QueryOptions("replace", true); - mongoDBCollection.findAndUpdate(query, projection, null, update, queryOptions); + return mongoDBCollection.findAndUpdate(query, projection, null, update, DataRelease.class, queryOptions); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java index 712b44f546..fb0c37dfdb 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java @@ -78,7 +78,6 @@ public DataRelease createRelease() throws JsonProcessingException { // Create the first release, collections and sources are empty lastRelease = new DataRelease() .setRelease(1) - .setActive(false) .setDate(sdf.format(new Date())); releaseDBAdaptor.insert(lastRelease); } else { @@ -86,7 +85,6 @@ public DataRelease createRelease() throws JsonProcessingException { if (MapUtils.isNotEmpty(lastRelease.getCollections())) { // Increment the release number, only if the previous release has collections lastRelease.setRelease(lastRelease.getRelease() + 1) - .setActive(false) .setDate(sdf.format(new Date())); // Write it to the database releaseDBAdaptor.insert(lastRelease); @@ -97,7 +95,7 @@ public DataRelease createRelease() throws JsonProcessingException { return lastRelease; } - public DataRelease get(int release) { + public DataRelease get(int release) throws CellBaseException { CellBaseDataResult result = releaseDBAdaptor.getAll(); if (CollectionUtils.isNotEmpty(result.getResults())) { for (DataRelease dataRelease : result.getResults()) { @@ -106,22 +104,27 @@ public DataRelease get(int release) { } } } - return null; + throw new CellBaseException("Data release '" + release + "' does not exist."); } - public DataRelease getDefault() { + public DataRelease getDefault(String cellBaseVersion) throws CellBaseException { CellBaseDataResult result = releaseDBAdaptor.getAll(); if (CollectionUtils.isNotEmpty(result.getResults())) { for (DataRelease dataRelease : result.getResults()) { - if (dataRelease.isActive()) { + if (dataRelease.getActiveByDefaultIn().contains(cellBaseVersion)) { return dataRelease; } } } - return null; + throw new CellBaseException("No data release found for CellBase " + cellBaseVersion); } - public void update(int release, String collection, String data, List dataSourcePaths) { + public DataRelease update(int release, List versions) throws CellBaseException { + return releaseDBAdaptor.update(release, versions).first(); + } + + public DataRelease update(int release, String collection, String data, List dataSourcePaths) + throws CellBaseException { DataRelease currDataRelease = get(release); if (currDataRelease != null) { // Update collections @@ -163,7 +166,10 @@ public void update(int release, String collection, String data, List dataS // Update data release in the database update(currDataRelease); + + return currDataRelease; } + throw new CellBaseException("Data release '" + release + "' does not exist."); } public void update(DataRelease dataRelase) { @@ -197,39 +203,6 @@ public void update(DataRelease dataRelase) { } } - public DataRelease active(int release) throws JsonProcessingException { - // Gel all releases and check if the input release exists - DataRelease prevActive = null; - DataRelease newActive = null; - CellBaseDataResult releaseResult = getReleases(); - if (CollectionUtils.isEmpty(releaseResult.getResults())) { - // Nothing to do, maybe exception or warning - return null; - } - for (DataRelease dataRelease : releaseResult.getResults()) { - if (dataRelease.isActive()) { - prevActive = dataRelease; - } else if (dataRelease.getRelease() == release) { - newActive = dataRelease; - } - } - if (prevActive != null && newActive != null && newActive.getRelease() == prevActive.getRelease()) { - // Nothing to do - return newActive; - } - - // Change active by default - if (prevActive != null) { - prevActive.setActive(false); - releaseDBAdaptor.update(prevActive.getRelease(), "active", prevActive.isActive()); - } - if (newActive != null) { - newActive.setActive(true); - releaseDBAdaptor.update(newActive.getRelease(), "active", newActive.isActive()); - } - return newActive; - } - public String getMaintenanceFlagFile() { return configuration.getMaintenanceFlagFile(); } @@ -237,5 +210,4 @@ public String getMaintenanceFlagFile() { public String getMaintainerContact() { return configuration.getMaintainerContact(); } - } diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java index cf101d1eb5..edb64d8478 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java @@ -10,6 +10,8 @@ import java.io.IOException; +import static org.junit.jupiter.api.Assertions.assertEquals; + class DataReleaseManagerTest extends GenericMongoDBAdaptorTest { public DataReleaseManagerTest() throws IOException { @@ -27,14 +29,12 @@ public void setUp() throws Exception { @Disabled public void test1() throws JsonProcessingException, CellBaseException { DataReleaseManager dataReleaseManager = new DataReleaseManager(CELLBASE_DBNAME, cellBaseConfiguration); + DataRelease firstRelease = dataReleaseManager.createRelease(); - if (firstRelease != null) { - dataReleaseManager.active(firstRelease.getRelease()); - } + assertEquals(firstRelease.getRelease(), 1); + DataRelease secondRelease = dataReleaseManager.createRelease(); - if (secondRelease != null) { - dataReleaseManager.active(secondRelease.getRelease()); - } + assertEquals(secondRelease.getRelease(), 2); } } \ No newline at end of file diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index 7b52d92267..fba5ca6cf0 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -31,7 +31,6 @@ import org.opencb.cellbase.core.api.query.QueryException; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; -import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.core.utils.SpeciesUtils; @@ -65,7 +64,7 @@ public class GenericRestWSServer implements IWSServer { protected String version; - protected int defaultDataRelease = 0; + protected int defaultDataRelease = -1; protected String species; protected Query query; @@ -170,22 +169,15 @@ private void initQuery() throws CellBaseException { // check version. species is validated later checkVersion(); - // Check data release and retrieve the default data release - if (!DONT_CHECK_SPECIES.equals(species)) { - // Prepare data release (do we need to get the default one?) - if (StringUtils.isEmpty(uriParams.get("dataRelease")) || uriParams.get("dataRelease").equals("0")) { - if (defaultDataRelease == 0) { - if (StringUtils.isEmpty(assembly)) { - assembly = SpeciesUtils.getSpecies(cellBaseConfiguration, species, assembly).getAssembly(); - } - if (StringUtils.isNotEmpty(assembly)) { - DataReleaseManager releaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly); - DataRelease dr = releaseManager.getDefault(); - if (dr != null) { - defaultDataRelease = dr.getRelease(); - } - } - } + // Set default data release if necessary + if (!DONT_CHECK_SPECIES.equals(species) && defaultDataRelease == -1) { + // As the assembly may not be presented in the query, we have to be sure to get it from the CellBase configuration + assembly = SpeciesUtils.getSpecies(cellBaseConfiguration, this.species, assembly).getAssembly(); + + if (StringUtils.isNotEmpty(assembly)) { + DataReleaseManager releaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly); + // getDefault launches an exception if no data release is found for that CellBase version + defaultDataRelease = releaseManager.getDefault(version).getRelease(); } } } @@ -193,23 +185,17 @@ private void initQuery() throws CellBaseException { protected int getDataRelease() throws CellBaseException { if (uriParams.containsKey("dataRelease") && StringUtils.isNotEmpty(uriParams.get("dataRelease"))) { try { - return Integer.parseInt(uriParams.get("dataRelease")); + int dataRelease = Integer.parseInt(uriParams.get("dataRelease")); + // If data release is 0, then use the default data release + return dataRelease == 0 ? defaultDataRelease : dataRelease; } catch (NumberFormatException e) { throw new CellBaseException("Invalid data release number '" + uriParams.get("dataRelease") + "'"); } } - // It means to use the default data release + // If no data release is present in the query, then use the default data release return defaultDataRelease; } - protected int getDataReleaseUsed() throws CellBaseException { - int dataRelease = getDataRelease(); - if (dataRelease == 0) { - return defaultDataRelease; - } - return dataRelease; - } - protected String getToken() { return uriParams.get(DATA_ACCESS_TOKEN); } @@ -312,7 +298,7 @@ protected Response createErrorResponse(Exception e) { queryResponse.setTime(new Long(System.currentTimeMillis() - startTime).intValue()); queryResponse.setApiVersion(version); try { - queryResponse.setDataRelease(getDataReleaseUsed()); + queryResponse.setDataRelease(getDataRelease()); } catch (CellBaseException ex) { logger.warn("Impossible to set the data release used in the query response", e); } @@ -343,7 +329,7 @@ protected Response createOkResponse(Object obj) { queryResponse.setTime(new Long(System.currentTimeMillis() - startTime).intValue()); queryResponse.setApiVersion(version); try { - queryResponse.setDataRelease(getDataReleaseUsed()); + queryResponse.setDataRelease(getDataRelease()); } catch (CellBaseException e) { logger.warn("Impossible to set the data release used in the query response", e); } @@ -417,6 +403,4 @@ private Response buildResponse(ResponseBuilder responseBuilder) { .header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") .build(); } - - } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java index cba0b93a02..46e25ff47d 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java @@ -27,11 +27,11 @@ import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.cellbase.core.token.DataAccessTokenManager; import org.opencb.cellbase.core.token.DataAccessTokenSources; import org.opencb.cellbase.core.utils.SpeciesUtils; import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.cellbase.lib.managers.MetaManager; -import org.opencb.cellbase.core.token.DataAccessTokenManager; import org.opencb.cellbase.server.rest.clinical.ClinicalWSServer; import org.opencb.cellbase.server.rest.feature.GeneWSServer; import org.opencb.cellbase.server.rest.feature.IdWSServer; @@ -121,9 +121,7 @@ public Response getSpecies() { public Response getDataRelease(@PathParam("species") @ApiParam(name = "species", value = ParamConstants.SPECIES_DESCRIPTION, required = true) String species, @ApiParam(name = "assembly", value = ParamConstants.ASSEMBLY_DESCRIPTION) @QueryParam("assembly") - String assembly, - @ApiParam(name = "onlyActive", value = "Set to true if you only want to get the active data relaease") - @QueryParam("onlyActive") @DefaultValue("false") boolean onlyActive) { + String assembly) { try { if (StringUtils.isEmpty(assembly)) { SpeciesConfiguration.Assembly assemblyObject = SpeciesUtils.getDefaultAssembly(cellBaseConfiguration, species); @@ -136,17 +134,7 @@ public Response getDataRelease(@PathParam("species") + assembly + "'"); } DataReleaseManager dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly); - CellBaseDataResult result = dataReleaseManager.getReleases(); - if (onlyActive) { - for (DataRelease release : result.getResults()) { - if (release.isActive()) { - return createOkResponse(new CellBaseDataResult<>(result.getId(), result.getTime(), result.getEvents(), 1, - Collections.singletonList(release), 1)); - } - } - } - - return createOkResponse(result); + return createOkResponse(dataReleaseManager.getReleases()); } catch (CellBaseException e) { return createErrorResponse(e); } From 325bf514fe9cb0422d1d5593d4c22225cba7b1e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 28 Apr 2023 13:25:56 +0200 Subject: [PATCH 07/28] test: add junit test and update code according to, #TASK-4278, #TASK-4157 --- .../cellbase/app/cli/CommandExecutor.java | 3 +- .../executors/CustomiseCommandExecutor.java | 8 +- .../admin/executors/LoadCommandExecutor.java | 67 ++++--- .../executors/ValidationCommandExecutor.java | 4 +- .../VariantAnnotationCommandExecutor.java | 5 +- .../src/test/resources/configuration.yml | 4 +- .../lib/impl/core/ReleaseMongoDBAdaptor.java | 24 ++- .../cellbase/lib/loader/LoadRunner.java | 148 ++++++++------- .../lib/managers/DataReleaseManager.java | 29 +++ .../VariantAnnotationCalculator.java | 7 +- .../lib/variant/hgvs/HgvsCalculator.java | 68 +++---- .../lib/GenericMongoDBAdaptorTest.java | 7 +- .../opencb/cellbase/lib/SpeciesUtilsTest.java | 7 +- .../lib/managers/DataReleaseManagerTest.java | 171 ++++++++++++++++-- .../server/rest/GenericRestWSServer.java | 12 +- 15 files changed, 399 insertions(+), 165 deletions(-) diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java index 0a90cb5ea1..39018bf170 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java @@ -18,13 +18,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; import org.opencb.cellbase.client.config.ClientConfiguration; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.commons.utils.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.logging.log4j.core.config.Configurator; import java.io.File; import java.io.FileInputStream; @@ -204,5 +204,4 @@ protected void makeDir(Path folderPath) throws IOException { Files.createDirectories(folderPath); } } - } diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/CustomiseCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/CustomiseCommandExecutor.java index 9b274cc491..8d7d07d40a 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/CustomiseCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/CustomiseCommandExecutor.java @@ -19,7 +19,9 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.app.cli.CommandExecutor; import org.opencb.cellbase.app.cli.admin.AdminCliOptionsParser; +import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.lib.loader.LoadRunner; +import org.opencb.cellbase.lib.loader.LoaderException; import org.opencb.commons.utils.FileUtils; import java.io.File; @@ -112,9 +114,9 @@ public void execute() { } } - private void loadIfExists(Path path, String collection) throws NoSuchMethodException, InterruptedException, - ExecutionException, InstantiationException, IOException, IllegalAccessException, InvocationTargetException, - ClassNotFoundException { + private void loadIfExists(Path path, String collection) throws NoSuchMethodException, InterruptedException, ExecutionException, + InstantiationException, IOException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, + LoaderException, CellBaseException { File file = new File(path.toString()); if (file.exists()) { if (file.isFile()) { diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java index 780f5bf39f..2eed03257d 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java @@ -114,7 +114,7 @@ public void execute() throws CellBaseException { configuration.getDatabases().getMongodb().getOptions().put("authenticationDatabase", loadCommandOptions.loaderParams.get("authenticationDatabase")); } - loadRunner = new LoadRunner(loader, database, numThreads, configuration); + loadRunner = new LoadRunner(loader, database, numThreads, dataReleaseManager, configuration); if (createIndexes) { Path indexFile = Paths.get(this.appHome).resolve("conf").resolve("mongodb-indexes.json"); indexManager = new IndexManager(database, indexFile, configuration); @@ -317,7 +317,7 @@ public void execute() throws CellBaseException { private void loadIfExists(Path path, String collection) throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IOException, IllegalAccessException, InvocationTargetException, - ClassNotFoundException { + ClassNotFoundException, LoaderException, CellBaseException { File file = new File(path.toString()); if (file.exists()) { if (file.isFile()) { @@ -356,26 +356,7 @@ private void checkParameters() throws CellBaseException { } // Check data release - CellBaseDataResult dataReleaseResults = dataReleaseManager.getReleases(); - if (CollectionUtils.isEmpty(dataReleaseResults.getResults())) { - throw new CellBaseException("No data releases are available for database " + database); - } - DataRelease lastDataRelease = null; - for (DataRelease dataRelease : dataReleaseResults.getResults()) { - if (lastDataRelease == null) { - lastDataRelease = dataRelease; - } else if (dataRelease.getRelease() > lastDataRelease.getRelease()) { - lastDataRelease = dataRelease; - } - } - if (lastDataRelease == null) { - throw new CellBaseException("Loading data is not permitted since no data release is found"); - } - if (CollectionUtils.isNotEmpty(lastDataRelease.getActiveByDefaultIn())) { - throw new CellBaseException("Loading data is not permitted for data release " + lastDataRelease.getRelease() + " since it has" - + " already assigned CellBase versions:" + StringUtils.join(lastDataRelease.getActiveByDefaultIn(), ",")); - } - dataRelease = lastDataRelease.getRelease(); + dataRelease = getDataReleaseForLoading(dataReleaseManager).getRelease(); } private void loadVariationData() throws NoSuchMethodException, InterruptedException, ExecutionException, @@ -410,7 +391,7 @@ private void loadVariationData() throws NoSuchMethodException, InterruptedExcept private void loadConservation() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException, CellBaseException { + IOException, CellBaseException, LoaderException { // Load data DirectoryStream stream = Files.newDirectoryStream(input, entry -> entry.getFileName().toString().startsWith("conservation_")); @@ -434,7 +415,7 @@ private void loadConservation() throws NoSuchMethodException, InterruptedExcepti private void loadProteinFunctionalPrediction() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException, CellBaseException { + IOException, CellBaseException, LoaderException { // Load data DirectoryStream stream = Files.newDirectoryStream(input, entry -> entry.getFileName().toString().startsWith("prot_func_pred_")); @@ -472,6 +453,8 @@ private void loadClinical() throws FileNotFoundException { } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException | IllegalAccessException | ExecutionException | IOException | InterruptedException | CellBaseException e) { logger.error(e.toString()); + } catch (LoaderException e) { + e.printStackTrace(); } } else { throw new FileNotFoundException("File " + path + " does not exist"); @@ -499,6 +482,8 @@ private void loadRepeats() { } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException | IllegalAccessException | ExecutionException | IOException | InterruptedException | CellBaseException e) { logger.error(e.toString()); + } catch (LoaderException e) { + e.printStackTrace(); } } else { logger.warn("Repeats file {} not found", path); @@ -506,9 +491,8 @@ private void loadRepeats() { } } - private void loadSpliceScores() throws NoSuchMethodException, InterruptedException, ExecutionException, - InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException, CellBaseException { + private void loadSpliceScores() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, + IllegalAccessException, InvocationTargetException, ClassNotFoundException, IOException, CellBaseException, LoaderException { // Load data logger.info("Loading splice scores from '{}'", input); // MMSplice scores @@ -528,7 +512,8 @@ private void loadSpliceScores() throws NoSuchMethodException, InterruptedExcepti } private void loadSpliceScores(Path spliceFolder) throws IOException, ExecutionException, InterruptedException, - ClassNotFoundException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + ClassNotFoundException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, + LoaderException, CellBaseException { // Get files from folder DirectoryStream stream = Files.newDirectoryStream(spliceFolder, entry -> entry.getFileName().toString().startsWith("splice_score_")); @@ -551,7 +536,7 @@ private void loadPubMed() throws CellBaseException { try { loadRunner.load(file.toPath(), EtlCommons.PUBMED_DATA, dataRelease); } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException - | IllegalAccessException | ExecutionException | IOException | InterruptedException e) { + | IllegalAccessException | ExecutionException | IOException | InterruptedException | LoaderException e) { logger.error("Error loading file '{}': {}", file.getName(), e.toString()); } } @@ -579,4 +564,28 @@ private void createIndex(String collection) { logger.error("Error creating index: {}", e.getMessage()); } } + + private DataRelease getDataReleaseForLoading(DataReleaseManager dataReleaseManager) throws CellBaseException { + // Check data release + CellBaseDataResult dataReleaseResults = dataReleaseManager.getReleases(); + if (CollectionUtils.isEmpty(dataReleaseResults.getResults())) { + throw new CellBaseException("No data releases are available"); + } + DataRelease lastDataRelease = null; + for (DataRelease dataRelease : dataReleaseResults.getResults()) { + if (lastDataRelease == null) { + lastDataRelease = dataRelease; + } else if (dataRelease.getRelease() > lastDataRelease.getRelease()) { + lastDataRelease = dataRelease; + } + } + if (lastDataRelease == null) { + throw new CellBaseException("Loading data is not permitted since no data release is found"); + } + if (CollectionUtils.isNotEmpty(lastDataRelease.getActiveByDefaultIn())) { + throw new CellBaseException("Loading data is not permitted for data release " + lastDataRelease.getRelease() + " since it has" + + " already assigned CellBase versions:" + StringUtils.join(lastDataRelease.getActiveByDefaultIn(), ",")); + } + return lastDataRelease; + } } diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java index 188820896a..2d7f19b1d1 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java @@ -67,13 +67,13 @@ public ValidationCommandExecutor(AdminCliOptionsParser.ValidationCommandOptions public void execute() { checkFilesExist(); - CellBaseManagerFactory cellbaseManagerFactory = new CellBaseManagerFactory(configuration); + CellBaseManagerFactory cellBaseManagerFactory = new CellBaseManagerFactory(configuration); // dbAdaptorFactory = new MongoDBAdaptorFactory(configuration); VariantAnnotationCalculator variantAnnotationCalculator = null; try { variantAnnotationCalculator = new VariantAnnotationCalculator(validationCommandOptions.species, validationCommandOptions.assembly, validationCommandOptions.dataRelease, validationCommandOptions.token, - cellbaseManagerFactory); + cellBaseManagerFactory); } catch (CellBaseException e) { e.printStackTrace(); return; diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java index a40ff30512..a0c3ca3008 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java @@ -478,9 +478,10 @@ private VariantAnnotator createCellBaseAnnotator() throws CellBaseException { // Normalization should just be performed in one place: before calling the annotation calculator - within the // corresponding *AnnotatorTask since the AnnotatorTasks need that the number of sent variants coincides // equals the number of returned annotations + CellBaseManagerFactory cellBaseManagerFactory = new CellBaseManagerFactory(configuration); return new CellBaseLocalVariantAnnotator(new VariantAnnotationCalculator(species, assembly, - variantAnnotationCommandOptions.dataRelease, variantAnnotationCommandOptions.token, - new CellBaseManagerFactory(configuration)), serverQueryOptions); + variantAnnotationCommandOptions.dataRelease, variantAnnotationCommandOptions.token, cellBaseManagerFactory), + serverQueryOptions); } else { try { ClientConfiguration clientConfiguration = ClientConfiguration.load(getClass() diff --git a/cellbase-core/src/test/resources/configuration.yml b/cellbase-core/src/test/resources/configuration.yml index b72d1318a1..e4625240ef 100644 --- a/cellbase-core/src/test/resources/configuration.yml +++ b/cellbase-core/src/test/resources/configuration.yml @@ -1,8 +1,8 @@ -version: v5 +version: v5.5 apiVersion: "${project.version}" wiki: https://github.com/opencb/cellbase/wiki maintenanceFlagFile: "/tmp/maintenance" -maintainerContact: javier.lopez@genomicsengland.co.uk +maintainerContact: ignacio.medina@zettagenomics.com logLevel: "INFO" logDir: "./logs" # where to output the logs diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java index 001e34e9f7..48fda65d28 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java @@ -70,16 +70,23 @@ public DataResult insert(DataRelease dataRelease) throws JsonProcessingException } public DataResult update(int release, List versions) throws CellBaseException { - Map toUpdate = new HashMap<>(); + DataRelease currDataRelease = mongoDBCollection.find(Filters.eq("release", release), null, DataRelease.class, QueryOptions.empty()) + .first(); + + Map> toUpdate = new HashMap<>(); for (String version : versions) { - DataResult result = mongoDBCollection.find(Filters.eq("activeByDefaultInd", version), null, DataRelease.class, + DataResult result = mongoDBCollection.find(Filters.eq("activeByDefaultIn", version), null, DataRelease.class, QueryOptions.empty()); if (result.getNumResults() > 1) { throw new CellBaseException("There's something wrong in the CellBase MongoDB. CellBase version " + version + " has" + " multiple data releases: " + StringUtils.join(result.getResults().stream().map(dr -> dr.getRelease()), ",")); } if (result.getNumResults() == 1) { - toUpdate.put(version, result.first()); + DataRelease dr = result.first(); + if (!toUpdate.containsKey(dr.getRelease())) { + toUpdate.put(dr.getRelease(), dr.getActiveByDefaultIn()); + } + toUpdate.get(dr.getRelease()).remove(version); } } @@ -89,15 +96,14 @@ public DataResult update(int release, List versions) throws session.startTransaction(TransactionOptions.builder().build()); // Update data releases by removing versions - for (Map.Entry entry : toUpdate.entrySet()) { - DataRelease dataRelease = entry.getValue(); - List updatedVersions = new ArrayList<>(dataRelease.getActiveByDefaultIn()); - updatedVersions.remove(entry.getKey()); - update(entry.getValue().getRelease(), "activeByDefaultIn", updatedVersions); + for (Map.Entry> entry : toUpdate.entrySet()) { + update(entry.getKey(), "activeByDefaultIn", entry.getValue()); } // Update data release by adding versions - update(release, "activeByDefaultIn", versions); + List vers = new ArrayList<>(currDataRelease.getActiveByDefaultIn()); + vers.addAll(versions); + update(release, "activeByDefaultIn", vers); // Commit the transaction session.commitTransaction(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/LoadRunner.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/LoadRunner.java index 790457333d..0fbe3f4f5d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/LoadRunner.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/LoadRunner.java @@ -16,7 +16,12 @@ package org.opencb.cellbase.lib.loader; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.cellbase.core.exception.CellBaseException; +import org.opencb.cellbase.core.models.DataRelease; +import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +49,8 @@ public class LoadRunner { private final int numThreads; private CellBaseConfiguration cellBaseConfiguration; + protected DataReleaseManager dataReleaseManager; + protected BlockingQueue> blockingQueue; private final Logger logger; @@ -53,10 +60,12 @@ public class LoadRunner { public static final List POISON_PILL = new ArrayList<>(); - public LoadRunner(String loader, String database, int numThreads, CellBaseConfiguration cellBaseConfiguration) { + public LoadRunner(String loader, String database, int numThreads, DataReleaseManager dataReleaseManager, + CellBaseConfiguration cellBaseConfiguration) { this.loader = loader; this.database = database; this.numThreads = numThreads; + this.dataReleaseManager = dataReleaseManager; this.cellBaseConfiguration = cellBaseConfiguration; this.blockingQueue = new ArrayBlockingQueue<>(QUEUE_CAPACITY); @@ -66,86 +75,101 @@ public LoadRunner(String loader, String database, int numThreads, CellBaseConfig @Deprecated public void load(Path filePath, String data) throws ClassNotFoundException, NoSuchMethodException, InstantiationException, - IllegalAccessException, InvocationTargetException, ExecutionException, InterruptedException, IOException { + IllegalAccessException, InvocationTargetException, ExecutionException, InterruptedException, IOException, CellBaseException, + LoaderException { load(filePath, data, 0, null, null); } public void load(Path filePath, String data, int dataRelease) throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, ExecutionException, InterruptedException, - IOException { + IOException, CellBaseException, LoaderException { load(filePath, data, dataRelease, null, null); } public void load(Path filePath, String data, int dataRelease, String field, String[] innerFields) throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, - ExecutionException, InterruptedException, IOException { - try { + ExecutionException, InterruptedException, IOException, CellBaseException, LoaderException { - if (filePath == null || !Files.exists(filePath) || Files.isDirectory(filePath)) { - throw new IOException("File '" + filePath + "' does not exist or is a directory"); - } + if (filePath == null || !Files.exists(filePath) || Files.isDirectory(filePath)) { + throw new IOException("File '" + filePath + "' does not exist or is a directory"); + } - // protein_functional_prediction documents are extremely big. Increasing the batch size will probably - // lead to an OutOfMemory error for this collection. Batch size can be much higher for the rest of - // collections though - if (data.equals(PROTEIN_FUNCTIONAL_PREDICTION)) { - batchSize = 50; - } else { - batchSize = 200; - } + // Check data release + checkDataRelease(dataRelease, dataReleaseManager); - // One CellBaseLoader is created for each thread in 'numThreads' variable - List cellBaseLoaders = new ArrayList<>(numThreads); - for (int i = 0; i < numThreads; i++) { - cellBaseLoaders.add((CellBaseLoader) Class.forName(loader) - .getConstructor(BlockingQueue.class, String.class, Integer.class, String.class, String.class, - String[].class, CellBaseConfiguration.class) - .newInstance(blockingQueue, data, dataRelease, database, field, innerFields, cellBaseConfiguration)); - logger.debug("CellBase loader thread '{}' created", i); - } + // protein_functional_prediction documents are extremely big. Increasing the batch size will probably + // lead to an OutOfMemory error for this collection. Batch size can be much higher for the rest of + // collections though + if (data.equals(PROTEIN_FUNCTIONAL_PREDICTION)) { + batchSize = 50; + } else { + batchSize = 200; + } - /* - * ExecutorServices and Futures are created, all CellBaseLoaders are initialized and submitted to them. - * After this the different loaders are blocked waiting for the blockingQueue to be populated. - */ - ExecutorService executorService = Executors.newFixedThreadPool(numThreads); - List> futures = new ArrayList<>(numThreads); - for (int i = 0; i < numThreads; i++) { - cellBaseLoaders.get(i).init(); - futures.add(executorService.submit(cellBaseLoaders.get(i))); - logger.debug("CellBaseLoader '{}' initialized and submitted to the ExecutorService", i); - } + // One CellBaseLoader is created for each thread in 'numThreads' variable + List cellBaseLoaders = new ArrayList<>(numThreads); + for (int i = 0; i < numThreads; i++) { + cellBaseLoaders.add((CellBaseLoader) Class.forName(loader) + .getConstructor(BlockingQueue.class, String.class, Integer.class, String.class, String.class, + String[].class, CellBaseConfiguration.class) + .newInstance(blockingQueue, data, dataRelease, database, field, innerFields, cellBaseConfiguration)); + logger.debug("CellBase loader thread '{}' created", i); + } - /* - * Execution starts by reading the file and loading batches to the blockingQueue. This makes the loaders - * to start fetching and loading batches into the database. The number of records processed is returned. - */ - int processedRecords = readInputJsonFile(filePath); - // Check if all the records have been loaded - int loadedRecords = 0; - for (Future future : futures) { - loadedRecords += future.get(); - } - if (processedRecords == loadedRecords) { - logger.info("All the '{}' records have been loaded into the database", processedRecords); - } else { - logger.warn("Only '{}' out of '{}' have been loaded into the database", loadedRecords, processedRecords); - } + /* + * ExecutorServices and Futures are created, all CellBaseLoaders are initialized and submitted to them. + * After this the different loaders are blocked waiting for the blockingQueue to be populated. + */ + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + List> futures = new ArrayList<>(numThreads); + for (int i = 0; i < numThreads; i++) { + cellBaseLoaders.get(i).init(); + futures.add(executorService.submit(cellBaseLoaders.get(i))); + logger.debug("CellBaseLoader '{}' initialized and submitted to the ExecutorService", i); + } - /* - * For sanity database connection and other resources must be released. This close() call must be - * implemented in the specific data loader. - */ - for (int i = 0; i < numThreads; i++) { - cellBaseLoaders.get(i).close(); - logger.debug("CellBaseLoader '{}' being closed", i); - } + /* + * Execution starts by reading the file and loading batches to the blockingQueue. This makes the loaders + * to start fetching and loading batches into the database. The number of records processed is returned. + */ + int processedRecords = readInputJsonFile(filePath); + // Check if all the records have been loaded + int loadedRecords = 0; + for (Future future : futures) { + loadedRecords += future.get(); + } + if (processedRecords == loadedRecords) { + logger.info("All the '{}' records have been loaded into the database", processedRecords); + } else { + logger.warn("Only '{}' out of '{}' have been loaded into the database", loadedRecords, processedRecords); + } - executorService.shutdown(); - } catch (LoaderException e) { - logger.error("Error executing CellBase Load: " + e.getMessage()); + /* + * For sanity database connection and other resources must be released. This close() call must be + * implemented in the specific data loader. + */ + for (int i = 0; i < numThreads; i++) { + cellBaseLoaders.get(i).close(); + logger.debug("CellBaseLoader '{}' being closed", i); } + executorService.shutdown(); + } + + private void checkDataRelease(int release, DataReleaseManager dataReleaseManager) throws CellBaseException { + // Check data release + if (release < 1) { + throw new CellBaseException("Invalid data release " + release); + } + + DataRelease currDataRelease = dataReleaseManager.get(release); + if (currDataRelease == null) { + throw new CellBaseException("Loading data is not permitted since no data release " + release + " is found"); + } + if (CollectionUtils.isNotEmpty(currDataRelease.getActiveByDefaultIn())) { + throw new CellBaseException("Loading data is not permitted for data release " + currDataRelease.getRelease() + " since it has" + + " already assigned CellBase versions: " + StringUtils.join(currDataRelease.getActiveByDefaultIn(), ",")); + } } private int readInputJsonFile(Path inputFile) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java index fb0c37dfdb..9a81a58b19 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java @@ -22,6 +22,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.opencb.cellbase.core.common.GitRepositoryState; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; @@ -34,6 +35,7 @@ import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.*; +import java.util.stream.Collectors; public class DataReleaseManager extends AbstractManager { private ReleaseMongoDBAdaptor releaseDBAdaptor; @@ -85,6 +87,7 @@ public DataRelease createRelease() throws JsonProcessingException { if (MapUtils.isNotEmpty(lastRelease.getCollections())) { // Increment the release number, only if the previous release has collections lastRelease.setRelease(lastRelease.getRelease() + 1) + .setActiveByDefaultIn(new ArrayList<>()) .setDate(sdf.format(new Date())); // Write it to the database releaseDBAdaptor.insert(lastRelease); @@ -210,4 +213,30 @@ public String getMaintenanceFlagFile() { public String getMaintainerContact() { return configuration.getMaintainerContact(); } + + public int checkDataRelease(int inRelease) throws CellBaseException { + int outRelease = inRelease; + if (outRelease < 0) { + throw new CellBaseException("Invalid data release " + outRelease + ". Data release must be greater or equal to 0"); + } + if (outRelease == 0) { + String[] split = GitRepositoryState.get().getBuildVersion().split("[.-]"); + String version = "v" + split[0] + "." + split[1]; + outRelease = getDefault(version).getRelease(); + logger.info("Using data release 0: it means to take default data release '" + outRelease + "' for CellBase version '" + + version + "'"); + return outRelease; + } + + List dataReleases = getReleases().getResults(); + for (DataRelease dataRelease : dataReleases) { + if (outRelease == dataRelease.getRelease()) { + return outRelease; + } + } + + throw new CellBaseException("Invalid data release " + outRelease + ". Valid data releases are: " + + StringUtils.join(dataReleases.stream().map(dr -> dr.getRelease()).collect(Collectors.toList()), ",") + ". And use 0 to" + + " use the default data release."); + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java index d5433a7327..7eaea65017 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java @@ -101,14 +101,15 @@ public VariantAnnotationCalculator(String species, String assembly, int dataRele this.clinicalManager = cellbaseManagerFactory.getClinicalManager(species, assembly); this.repeatsManager = cellbaseManagerFactory.getRepeatsManager(species, assembly); - this.dataRelease = dataRelease; + // Check data release + this.dataRelease = cellbaseManagerFactory.getDataReleaseManager(species, assembly).checkDataRelease(dataRelease); this.token = token; // Initialises normaliser configuration with default values. HEADS UP: configuration might be updated // at parseQueryParam this.normalizer = new VariantNormalizer(getNormalizerConfig()); - hgvsCalculator = new HgvsCalculator(genomeManager, dataRelease); + hgvsCalculator = new HgvsCalculator(genomeManager, this.dataRelease); logger.debug("VariantAnnotationMongoDBAdaptor: in 'constructor'"); } @@ -1322,6 +1323,7 @@ private boolean[] getRegulatoryRegionOverlaps(String chromosome, Integer positio boolean[] overlapsRegulatoryRegion = {false, false}; RegulationQuery query = new RegulationQuery(); + query.setDataRelease(dataRelease); query.setIncludes(Collections.singletonList(REGULATORY_REGION_FEATURE_TYPE_ATTRIBUTE)); query.setRegions(Collections.singletonList(new Region(chromosome, position))); CellBaseDataResult cellBaseDataResult = regulationManager.search(query); @@ -1772,6 +1774,7 @@ public List> call() throws Exception { List queries = new ArrayList<>(); for (Region region : breakpointsToRegionList(variant)) { RepeatsQuery query = new RepeatsQuery(); + query.setDataRelease(dataRelease); query.setRegions(Collections.singletonList(region)); queries.add(query); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculator.java index 794da50115..df066dbb45 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculator.java @@ -25,8 +25,8 @@ import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.lib.managers.GenomeManager; -import org.opencb.cellbase.lib.variant.annotation.UnsupportedURLVariantFormat; import org.opencb.cellbase.lib.variant.VariantAnnotationUtils; +import org.opencb.cellbase.lib.variant.annotation.UnsupportedURLVariantFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -166,8 +166,8 @@ protected static boolean isCoding(Transcript transcript) { } protected void setRangeCoordsAndAlleles(int genomicStart, int genomicEnd, String genomicReference, - String genomicAlternate, Transcript transcript, - BuildingComponents buildingComponents) { + String genomicAlternate, Transcript transcript, + BuildingComponents buildingComponents) { int start; int end; String reference; @@ -224,7 +224,7 @@ protected String reverseComplementary(String string) { * @param strand String {"+", "-"}. */ protected void justify(Variant variant, int startOffset, int endOffset, String allele, String genomicSequence, - String strand) { + String strand) { StringBuilder stringBuilder = new StringBuilder(allele); // Justify to the left if ("-".equals(strand)) { @@ -236,7 +236,7 @@ protected void justify(Variant variant, int startOffset, int endOffset, String a variant.setStart(variant.getStart() - 1); variant.setEnd(variant.getEnd() - 1); } - // Justify to the right + // Justify to the right } else { while ((endOffset + 1) < genomicSequence.length() && genomicSequence.charAt(endOffset + 1) == stringBuilder.charAt(0)) { stringBuilder.deleteCharAt(0); @@ -250,7 +250,7 @@ protected void justify(Variant variant, int startOffset, int endOffset, String a // Insertion if (variant.getReference().isEmpty()) { variant.setAlternate(stringBuilder.toString()); - // Deletion + // Deletion } else { variant.setReference(stringBuilder.toString()); } @@ -287,14 +287,14 @@ private static CdnaCoord genomicToCdnaCoordInNonCodingTranscript(Transcript tran cdnaCoord.setOffset(genomicPosition - nearestExon.getStart()); // TODO: probably needs +-1 bp adjust cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart())); cdnaCoord.setLandmark(CdnaCoord.Landmark.TRANSCRIPT_START); - // Exonic variant - // -------------S|||p||E------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Exonic variant + // -------------S|||p||E------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else if (genomicPosition - nearestExon.getEnd() < 0) { // no offset cdnaCoord.setReferencePosition(getCdnaPosition(transcript, genomicPosition)); cdnaCoord.setLandmark(CdnaCoord.Landmark.TRANSCRIPT_START); - // Non-exonic variant: intronic, intergenic - // -------------S||||||E-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Non-exonic variant: intronic, intergenic + // -------------S||||||E-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else { // offset must be positive cdnaCoord.setOffset(genomicPosition - nearestExon.getEnd()); // TODO: probably needs +-1 bp adjust @@ -312,14 +312,14 @@ private static CdnaCoord genomicToCdnaCoordInNonCodingTranscript(Transcript tran cdnaCoord.setOffset(nearestExon.getStart() - genomicPosition); // TODO: probably needs +-1 bp adjust cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart())); cdnaCoord.setLandmark(CdnaCoord.Landmark.TRANSCRIPT_START); - // Exonic variant - // -------------E|||p||S------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Exonic variant + // -------------E|||p||S------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else if (genomicPosition - nearestExon.getEnd() < 0) { // no offset cdnaCoord.setReferencePosition(getCdnaPosition(transcript, genomicPosition)); cdnaCoord.setLandmark(CdnaCoord.Landmark.TRANSCRIPT_START); - // Non-exonic variant: intronic, intergenic - // -------------E||||||S-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Non-exonic variant: intronic, intergenic + // -------------E||||||S-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else { // offset must be negative cdnaCoord.setOffset(nearestExon.getEnd() - genomicPosition); // TODO: probably needs +-1 bp adjust @@ -355,49 +355,49 @@ private static CdnaCoord genomicToCdnaCoordInCodingTranscript(Transcript transcr cdnaCoord.setOffset(genomicPosition - nearestExon.getStart()); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart()) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // After coding end + // After coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(genomicPosition - nearestExon.getStart()); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart()) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // Within coding start and end + // Within coding start and end } else { // offset must be negative cdnaCoord.setOffset(genomicPosition - nearestExon.getStart()); // TODO: probably needs +-1 bp adjust cdnaCoord.setReferencePosition(nearestExon.getCdsStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); } - // Exonic variant - // -------------S|||p||E------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Exonic variant + // -------------S|||p||E------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else if (genomicPosition - nearestExon.getEnd() <= 0) { // Before coding start if (genomicPosition < transcript.getGenomicCodingStart()) { cdnaCoord.setOffset(getCdnaPosition(transcript, genomicPosition) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // After coding end + // After coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(getCdnaPosition(transcript, genomicPosition) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // Within coding start and end + // Within coding start and end } else { // no offset cdnaCoord.setReferencePosition(nearestExon.getCdsStart() + (genomicPosition - nearestExon.getGenomicCodingStart())); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); } - // Non-exonic variant: intronic, intergenic - // -------------S||||||E-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Non-exonic variant: intronic, intergenic + // -------------S||||||E-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else { // Before coding start if (genomicPosition < transcript.getGenomicCodingStart()) { cdnaCoord.setOffset(genomicPosition - nearestExon.getEnd()); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getEnd()) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // After coding end + // After coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(genomicPosition - nearestExon.getEnd()); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getEnd()) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // Within coding start and end + // Within coding start and end } else { // offset must be positive cdnaCoord.setOffset(genomicPosition - nearestExon.getEnd()); // TODO: probably needs +-1 bp adjust @@ -417,49 +417,49 @@ private static CdnaCoord genomicToCdnaCoordInCodingTranscript(Transcript transcr cdnaCoord.setOffset(nearestExon.getStart() - genomicPosition); cdnaCoord.setReferencePosition(transcript.getCdnaCodingEnd() - getCdnaPosition(transcript, nearestExon.getStart())); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // After (genomic) coding end + // After (genomic) coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(nearestExon.getStart() - genomicPosition); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart()) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // Within coding start and end + // Within coding start and end } else { // offset must be positive cdnaCoord.setOffset(nearestExon.getStart() - genomicPosition); // TODO: probably needs +-1 bp adjust cdnaCoord.setReferencePosition(nearestExon.getCdsEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); } - // Exonic variant - // -------------E|||p||S------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Exonic variant + // -------------E|||p||S------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else if (genomicPosition - nearestExon.getEnd() <= 0) { // Before (genomic) coding start if (genomicPosition < transcript.getGenomicCodingStart()) { cdnaCoord.setOffset(getCdnaPosition(transcript, genomicPosition) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // After (genomic) coding end + // After (genomic) coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(getCdnaPosition(transcript, genomicPosition) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // Within coding start and end + // Within coding start and end } else { // no offset cdnaCoord.setReferencePosition(nearestExon.getCdsStart() + nearestExon.getGenomicCodingEnd() - genomicPosition); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); } - // Non-exonic variant: intronic, intergenic - // -------------E||||||S-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Non-exonic variant: intronic, intergenic + // -------------E||||||S-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else { // Before (genomic) coding start if (genomicPosition < transcript.getGenomicCodingStart()) { cdnaCoord.setOffset(nearestExon.getEnd() - genomicPosition); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getEnd()) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // After (genomic) coding end + // After (genomic) coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(nearestExon.getEnd() - genomicPosition); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getEnd()) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // Within coding start and end + // Within coding start and end } else { // offset must be negative cdnaCoord.setOffset(nearestExon.getEnd() - genomicPosition); // TODO: probably needs +-1 bp adjust diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java index 2dd2d18974..a3770ca623 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java @@ -24,6 +24,7 @@ import org.opencb.cellbase.lib.db.MongoDBManager; import org.opencb.cellbase.lib.impl.core.CellBaseDBAdaptor; import org.opencb.cellbase.lib.loader.LoadRunner; +import org.opencb.cellbase.lib.loader.LoaderException; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.commons.datastore.core.DataStoreServerAddress; @@ -73,8 +74,9 @@ public GenericMongoDBAdaptorTest() { cellBaseConfiguration = CellBaseConfiguration.load( GenericMongoDBAdaptorTest.class.getClassLoader().getResourceAsStream("configuration.test.yaml"), CellBaseConfiguration.ConfigurationFileFormat.YAML); - loadRunner = new LoadRunner(MONGODB_CELLBASE_LOADER, CELLBASE_DBNAME, 2, cellBaseConfiguration); cellBaseManagerFactory = new CellBaseManagerFactory(cellBaseConfiguration); + dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager("hsapiens", "GRCh37"); + loadRunner = new LoadRunner(MONGODB_CELLBASE_LOADER, CELLBASE_DBNAME, 2, dataReleaseManager, cellBaseConfiguration); // dbAdaptorFactory = new MongoDBAdaptorFactory(cellBaseConfiguration); } catch (Exception e) { e.printStackTrace(); @@ -93,8 +95,7 @@ protected void clearDB(String dbName) throws Exception { protected void initDB() throws IOException, ExecutionException, ClassNotFoundException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, - URISyntaxException, CellBaseException { - dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager("hsapiens", "GRCh37"); + URISyntaxException, CellBaseException, LoaderException { dataRelease = dataReleaseManager.createRelease().getRelease(); Path path = Paths.get(getClass() diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/SpeciesUtilsTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/SpeciesUtilsTest.java index 3b51155bfd..246d41684e 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/SpeciesUtilsTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/SpeciesUtilsTest.java @@ -22,6 +22,7 @@ import org.opencb.cellbase.core.config.SpeciesConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.lib.loader.LoadRunner; +import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,11 +44,13 @@ public class SpeciesUtilsTest { protected Logger logger = LoggerFactory.getLogger(this.getClass()); - public SpeciesUtilsTest() throws IOException { + public SpeciesUtilsTest() throws IOException, CellBaseException { cellBaseConfiguration = CellBaseConfiguration.load( SpeciesUtilsTest.class.getClassLoader().getResourceAsStream("configuration.test.yaml"), CellBaseConfiguration.ConfigurationFileFormat.YAML); - loadRunner = new LoadRunner(MONGODB_CELLBASE_LOADER, GRCH37_DBNAME, 2, cellBaseConfiguration); + + loadRunner = new LoadRunner(MONGODB_CELLBASE_LOADER, GRCH37_DBNAME, 2, + new DataReleaseManager(GRCH37_DBNAME, cellBaseConfiguration), cellBaseConfiguration); } @Test diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java index edb64d8478..802c2e5b55 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java @@ -1,40 +1,189 @@ package org.opencb.cellbase.lib.managers; import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.models.variant.avro.VariantAnnotation; +import org.opencb.cellbase.core.api.query.QueryException; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; +import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.GenericMongoDBAdaptorTest; +import org.opencb.cellbase.lib.loader.LoaderException; +import org.opencb.cellbase.lib.variant.annotation.VariantAnnotationCalculator; +import org.opencb.commons.datastore.core.QueryOptions; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.concurrent.ExecutionException; +import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; class DataReleaseManagerTest extends GenericMongoDBAdaptorTest { + protected DataReleaseManager dataReleaseManager; + private static boolean populated = false; + public DataReleaseManagerTest() throws IOException { super(); } @BeforeEach public void setUp() throws Exception { - clearDB(CELLBASE_DBNAME); -// Path path = Paths.get(getClass().getResource("/gene/gene-test.json.gz").toURI()); -// loadRunner.load(path, "gene"); + if (!populated) { + clearDB(CELLBASE_DBNAME); + initDB(); + } + + dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager(SPECIES, ASSEMBLY); + populated = true; + } + + @Test + public void testCreate() throws JsonProcessingException { + CellBaseDataResult result = dataReleaseManager.getReleases(); + DataRelease dr = dataReleaseManager.createRelease(); + assertEquals(result.getNumResults() + 1, dr.getRelease()); + } + + @Test + public void testAddActiveByDefaultIn() throws CellBaseException, JsonProcessingException { + DataRelease dr = dataReleaseManager.createRelease(); + dataReleaseManager.update(dr.getRelease(), Arrays.asList("v5.1", "v5.2")); + + DataRelease auxDr = dataReleaseManager.get(dr.getRelease()); + assertEquals(dr.getRelease(), auxDr.getRelease()); + assertEquals(2, auxDr.getActiveByDefaultIn().size()); + } + + @Test + public void testChangeActiveByDefaultIn() throws JsonProcessingException, CellBaseException { + String version3 = "v5.3"; + String version4 = "v5.4"; + + int rA = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rA, Arrays.asList(version3, version4)); + + DataRelease auxDr1 = dataReleaseManager.get(rA); + assertEquals(2, auxDr1.getActiveByDefaultIn().size()); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version3)); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version4)); + + int rB = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rB, Arrays.asList(version4)); + + DataRelease auxDr2 = dataReleaseManager.get(rA); + assertEquals(1, auxDr2.getActiveByDefaultIn().size(), 1); + assertEquals(version3, auxDr2.getActiveByDefaultIn().get(0)); + + DataRelease auxDr3 = dataReleaseManager.get(rB); + assertEquals(1, auxDr3.getActiveByDefaultIn().size()); + assertEquals(version4, auxDr3.getActiveByDefaultIn().get(0)); } @Test - @Disabled - public void test1() throws JsonProcessingException, CellBaseException { - DataReleaseManager dataReleaseManager = new DataReleaseManager(CELLBASE_DBNAME, cellBaseConfiguration); + public void failLoading() throws IOException, ExecutionException, ClassNotFoundException, InterruptedException, + InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, URISyntaxException, + CellBaseException, LoaderException { + String version5 = "v5.5"; - DataRelease firstRelease = dataReleaseManager.createRelease(); - assertEquals(firstRelease.getRelease(), 1); + int release = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(release, Arrays.asList(version5)); - DataRelease secondRelease = dataReleaseManager.createRelease(); - assertEquals(secondRelease.getRelease(), 2); + Path path = Paths.get(getClass() + .getResource("/variant-annotation/gene.test.json.gz").toURI()); + + CellBaseException thrown = Assertions.assertThrows(CellBaseException.class, () -> { loadRunner.load(path, "gene", release); }); + Assertions.assertTrue(thrown.getMessage().contains("since it has already assigned CellBase versions: " + version5)); } + @Test + public void testMultipleAddActiveByDefaultIn() throws JsonProcessingException, CellBaseException { + String version6 = "v5.6"; + String version7 = "v5.7"; + + int rA = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rA, Arrays.asList(version6)); + + DataRelease auxDr1 = dataReleaseManager.get(rA); + assertEquals(auxDr1.getActiveByDefaultIn().size(), 1); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version6)); + + dataReleaseManager.update(rA, Arrays.asList(version7)); + + DataRelease auxDr2 = dataReleaseManager.get(rA); + assertEquals(2, auxDr2.getActiveByDefaultIn().size()); + assertTrue(auxDr2.getActiveByDefaultIn().contains(version6)); + assertTrue(auxDr2.getActiveByDefaultIn().contains(version7)); + } + + @Test + public void testRemoveMultipleAddActiveByDefaultIn() throws JsonProcessingException, CellBaseException { + String version8 = "v5.8"; + String version9 = "v5.9"; + + int rA = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rA, Arrays.asList(version8, version9)); + + DataRelease auxDr1 = dataReleaseManager.get(rA); + assertEquals(2, auxDr1.getActiveByDefaultIn().size()); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version8)); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version9)); + + int rB = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rB, Arrays.asList(version8, version9)); + + auxDr1 = dataReleaseManager.get(rA); + assertEquals(0, auxDr1.getActiveByDefaultIn().size()); + + DataRelease auxDr2 = dataReleaseManager.get(rB); + assertEquals(2, auxDr2.getActiveByDefaultIn().size()); + assertTrue(auxDr2.getActiveByDefaultIn().contains(version8)); + assertTrue(auxDr2.getActiveByDefaultIn().contains(version9)); + } + + @Test + public void testAnnotationWithDR0() throws CellBaseException, QueryException, ExecutionException, InterruptedException, + IllegalAccessException { + dataReleaseManager.update(1, Arrays.asList("v5.5")); + + VariantAnnotationCalculator annotator = new VariantAnnotationCalculator(SPECIES, ASSEMBLY, 0, token, cellBaseManagerFactory); + + Variant variant = new Variant("10", 113588287, "G", "A"); + CellBaseDataResult cellBaseDataResult = annotator.getAnnotationByVariant(variant, QueryOptions.empty()); + VariantAnnotation variantAnnotation = cellBaseDataResult.first(); + System.out.println(variantAnnotation); + assertEquals(variant.getChromosome(), variantAnnotation.getChromosome()); + assertEquals(variant.getStart(), variantAnnotation.getStart()); + assertEquals(variant.getReference(), variantAnnotation.getReference()); + assertEquals(variant.getAlternate(), variantAnnotation.getAlternate()); + } + + @Test + public void testAnnotationWithInvalidDR() { + CellBaseException thrown = Assertions.assertThrows(CellBaseException.class, () -> { + VariantAnnotationCalculator annotator = new VariantAnnotationCalculator(SPECIES, ASSEMBLY, -1, token, cellBaseManagerFactory); + Variant variant = new Variant("10", 113588287, "G", "A"); + CellBaseDataResult cellBaseDataResult = annotator.getAnnotationByVariant(variant, QueryOptions.empty()); + }); + Assertions.assertTrue(thrown.getMessage().contains("Data release must be greater or equal to 0")); + } + + @Test + public void testAnnotationWithInvalidDR_1() { + int dr = 12; + CellBaseException thrown = Assertions.assertThrows(CellBaseException.class, () -> { + VariantAnnotationCalculator annotator = new VariantAnnotationCalculator(SPECIES, ASSEMBLY, dr, token, cellBaseManagerFactory); + Variant variant = new Variant("10", 113588287, "G", "A"); + CellBaseDataResult cellBaseDataResult = annotator.getAnnotationByVariant(variant, QueryOptions.empty()); + }); + Assertions.assertTrue(thrown.getMessage().contains("data release "+ dr + ". Valid data releases are:")); + } } \ No newline at end of file diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index fba5ca6cf0..e35a01f9f2 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -172,7 +172,7 @@ private void initQuery() throws CellBaseException { // Set default data release if necessary if (!DONT_CHECK_SPECIES.equals(species) && defaultDataRelease == -1) { // As the assembly may not be presented in the query, we have to be sure to get it from the CellBase configuration - assembly = SpeciesUtils.getSpecies(cellBaseConfiguration, this.species, assembly).getAssembly(); + assembly = SpeciesUtils.getSpecies(cellBaseConfiguration, this.species, assembly).getAssembly(); if (StringUtils.isNotEmpty(assembly)) { DataReleaseManager releaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly); @@ -187,12 +187,20 @@ protected int getDataRelease() throws CellBaseException { try { int dataRelease = Integer.parseInt(uriParams.get("dataRelease")); // If data release is 0, then use the default data release - return dataRelease == 0 ? defaultDataRelease : dataRelease; + if (dataRelease == 0) { + logger.info("Using data release 0 in query: using the default data release '" + defaultDataRelease + "' for CellBase" + + " version '" + version + "'"); + return defaultDataRelease; + } else { + return dataRelease; + } } catch (NumberFormatException e) { throw new CellBaseException("Invalid data release number '" + uriParams.get("dataRelease") + "'"); } } // If no data release is present in the query, then use the default data release + logger.info("No data release present in query: using the default data release '" + defaultDataRelease + "' for CellBase version" + + " '" + version + "'"); return defaultDataRelease; } From caa0109e95ecf9f8e0104e08855fc19d73ff1694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 2 May 2023 16:08:32 +0200 Subject: [PATCH 08/28] server: update endpoints, #TASK-4278, #TASK-4157 --- .../opencb/cellbase/core/api/query/AbstractQuery.java | 5 ----- .../org/opencb/cellbase/core/models/DataRelease.java | 9 +++++++++ cellbase-core/src/test/resources/configuration.yml | 2 +- .../cellbase/lib/managers/TranscriptManager.java | 11 ++++++----- .../cellbase/server/rest/GenericRestWSServer.java | 2 +- .../opencb/cellbase/server/rest/OntologyWSServer.java | 2 ++ .../cellbase/server/rest/PublicationWSServer.java | 1 + .../server/rest/clinical/ClinicalWSServer.java | 1 + .../cellbase/server/rest/feature/GeneWSServer.java | 11 +++++++++++ .../cellbase/server/rest/feature/IdWSServer.java | 1 + .../cellbase/server/rest/feature/ProteinWSServer.java | 3 +++ .../server/rest/feature/TranscriptWSServer.java | 6 +++++- .../cellbase/server/rest/genomic/RegionWSServer.java | 7 +++++++ .../cellbase/server/rest/genomic/VariantWSServer.java | 1 + .../server/rest/regulatory/RegulatoryWSServer.java | 4 ++++ .../cellbase/server/rest/regulatory/TfWSServer.java | 2 ++ 16 files changed, 55 insertions(+), 13 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/query/AbstractQuery.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/query/AbstractQuery.java index b9fc9030c5..50b0451e5f 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/query/AbstractQuery.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/query/AbstractQuery.java @@ -172,11 +172,6 @@ public void updateParams(Map uriParams) { } } objectMapper.updateValue(this, objectHashMap); - - // Initialize the data release (0 means to use the default data release) - if (dataRelease == null) { - dataRelease = 0; - } } catch (JsonProcessingException e) { throw new IllegalArgumentException(e); } diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java index 8baefdca68..5674839aa8 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java @@ -75,6 +75,15 @@ public DataRelease setDate(String date) { return this; } + public boolean isActive() { + return active; + } + + public DataRelease setActive(boolean active) { + this.active = active; + return this; + } + public List getActiveByDefaultIn() { return activeByDefaultIn; } diff --git a/cellbase-core/src/test/resources/configuration.yml b/cellbase-core/src/test/resources/configuration.yml index e4625240ef..c9a5d98743 100644 --- a/cellbase-core/src/test/resources/configuration.yml +++ b/cellbase-core/src/test/resources/configuration.yml @@ -1,4 +1,4 @@ -version: v5.5 +version: v5 apiVersion: "${project.version}" wiki: https://github.com/opencb/cellbase/wiki maintenanceFlagFile: "/tmp/maintenance" diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java index f16bcf8aea..80f5866ebf 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java @@ -60,8 +60,9 @@ public List> info(List ids, ProjectionQue } - public CellBaseDataResult getCdna(String id) throws CellBaseException { + public CellBaseDataResult getCdna(String id, int dataRelease) throws CellBaseException { TranscriptQuery query = new TranscriptQuery(); + query.setDataRelease(dataRelease); query.setTranscriptsXrefs(Collections.singletonList(id)); CellBaseDataResult transcriptCellBaseDataResult = transcriptDBAdaptor.query(query); String cdnaSequence = null; @@ -78,17 +79,17 @@ public CellBaseDataResult getCdna(String id) throws CellBaseException { transcriptCellBaseDataResult.getNumResults(), Collections.singletonList(cdnaSequence), 1); } - private List> getCdna(List idList) throws CellBaseException { + private List> getCdna(List idList, int dataRelease) throws CellBaseException { List> cellBaseDataResults = new ArrayList<>(); for (String id : idList) { - cellBaseDataResults.add(getCdna(id)); + cellBaseDataResults.add(getCdna(id, dataRelease)); } return cellBaseDataResults; } - public List> getSequence(String id) throws CellBaseException { + public List> getSequence(String id, int dataRelease) throws CellBaseException { List transcriptsList = Arrays.asList(id.split(",")); - List> queryResult = getCdna(transcriptsList); + List> queryResult = getCdna(transcriptsList, dataRelease); for (int i = 0; i < transcriptsList.size(); i++) { queryResult.get(i).setId(transcriptsList.get(i)); } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index e35a01f9f2..a9fea682af 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -234,7 +234,7 @@ private void checkVersion() throws CellBaseException { // System.out.println("cellBaseConfiguration.getVersion() = " + cellBaseConfiguration.getVersion()); // System.out.println("version = " + version); // System.out.println("*************************************"); - if (!cellBaseConfiguration.getVersion().equalsIgnoreCase(version)) { + if (!version.startsWith(cellBaseConfiguration.getVersion())) { logger.error("Version '{}' does not match configuration '{}'", this.version, cellBaseConfiguration.getVersion()); throw new CellBaseException("Version not valid: '" + version + "'"); } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/OntologyWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/OntologyWSServer.java index 6ea67fb7dd..68b6d0021a 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/OntologyWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/OntologyWSServer.java @@ -112,6 +112,7 @@ public OntologyWSServer(@PathParam("apiVersion") @ApiParam(name = "apiVersion", public Response getAll() { try { OntologyQuery query = new OntologyQuery(uriParams); + query.setDataRelease(getDataRelease()); logger.info("/search OntologyQuery: " + query.toString()); CellBaseDataResult queryResults = ontologyManager.search(query); return createOkResponse(queryResults); @@ -167,6 +168,7 @@ public Response getUniqueValues(@QueryParam("field") @ApiParam(name = "field", r try { copyToFacet("field", field); OntologyQuery query = new OntologyQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = ontologyManager.distinct(query); return createOkResponse(queryResults); } catch (Exception e) { diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/PublicationWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/PublicationWSServer.java index 2918fec234..4135848cc0 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/PublicationWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/PublicationWSServer.java @@ -105,6 +105,7 @@ public Response getModel() { public Response getAll() { try { PublicationQuery query = new PublicationQuery(uriParams); + query.setDataRelease(getDataRelease()); logger.info("/search PublicationQuery: " + query); CellBaseDataResult queryResults = publicationManager.search(query); return createOkResponse(queryResults); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/clinical/ClinicalWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/clinical/ClinicalWSServer.java index d7a54aee9d..8e057f8b19 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/clinical/ClinicalWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/clinical/ClinicalWSServer.java @@ -117,6 +117,7 @@ public ClinicalWSServer(@PathParam("apiVersion") @ApiParam(name = "apiVersion", public Response getAll() { try { ClinicalVariantQuery query = new ClinicalVariantQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = clinicalManager.search(query); return createOkResponse(queryResults); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/GeneWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/GeneWSServer.java index e6be98fc14..c73e711d75 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/GeneWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/GeneWSServer.java @@ -137,6 +137,7 @@ public Response groupBy(@DefaultValue("") @QueryParam("field") @ApiParam(name = try { copyToFacet("field", field); GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = geneManager.groupBy(geneQuery); return createOkResponse(queryResults); } catch (Exception e) { @@ -185,6 +186,7 @@ public Response getAggregationStats(@DefaultValue("") @QueryParam("field") try { copyToFacet("field", field); GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = geneManager.aggregationStats(geneQuery); return createOkResponse(queryResults); } catch (Exception e) { @@ -283,6 +285,7 @@ public Response getAll(@QueryParam(SPLIT_RESULT_PARAM) @ApiParam(name = SPLIT_RE logger.info("/search identifiers: {} ", identifiers); for (String identifier : identifiers) { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); geneQuery.setTranscriptsXrefs(Collections.singletonList(identifier)); geneQueries.add(geneQuery); logger.info("/search geneQuery: {}", geneQuery.toString()); @@ -291,6 +294,7 @@ public Response getAll(@QueryParam(SPLIT_RESULT_PARAM) @ApiParam(name = SPLIT_RE return createOkResponse(queryResults); } else { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); logger.info("/search GeneQuery: {} ", geneQuery.toString()); CellBaseDataResult queryResults = geneManager.search(geneQuery); return createOkResponse(queryResults); @@ -357,6 +361,7 @@ private String[] getGeneIdentifiers() { public Response getInfo(@PathParam("genes") @ApiParam(name = "genes", value = GENE_IDS, required = true) String genes) { try { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); String source = "ensembl"; if (geneQuery.getSource() != null && !geneQuery.getSource().isEmpty()) { source = geneQuery.getSource().get(0); @@ -416,6 +421,7 @@ public Response getTranscriptsByGenes(@PathParam("genes") @ApiParam(name = "gene String[] ids = genes.split(","); for (String id : ids) { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(new LogicalList(Collections.singletonList(id))); queries.add(query); } @@ -468,6 +474,7 @@ public Response getUniqueValues(@QueryParam("field") @ApiParam(name = "field", r try { copyToFacet("field", field); GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = geneManager.distinct(geneQuery); return createOkResponse(queryResults); } catch (Exception e) { @@ -505,6 +512,7 @@ public Response getSNPByGenes(@PathParam("genes") String[] ids = genes.split(","); for (String id : ids) { VariantQuery query = new VariantQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setGenes(new LogicalList(Collections.singletonList(id))); queries.add(query); } @@ -529,6 +537,7 @@ public Response getAllTfbs(@PathParam("genes") @ApiParam(name = "genes", value = required = true) String genes) { try { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); geneQuery.setIds(Arrays.asList(genes.split(","))); List> queryResults = geneManager.getTfbs(geneQuery); return createOkResponse(queryResults); @@ -551,6 +560,7 @@ public Response getProteinById(@PathParam("genes") @ApiParam(name = "genes", val required = true) String genes) { try { ProteinQuery query = new ProteinQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setGenes(Arrays.asList(genes.split(","))); logger.info("REST proteinQuery: {}", query.toString()); CellBaseDataResult queryResults = proteinManager.search(query); @@ -577,6 +587,7 @@ public Response getSequence(@PathParam("genes") @ApiParam(name = "genes", value String[] identifiers = genes.split(","); for (String identifier : identifiers) { GeneQuery query = new GeneQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(Arrays.asList(identifier)); queries.add(query); logger.info("REST GeneQuery: {} ", query.toString()); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java index 3f3f8f80bf..28c5dafcd4 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java @@ -109,6 +109,7 @@ public Response getInfo(@PathParam("id") @ApiParam(name = "id", value = FEATURE_ public Response getAllXrefs() { try { XrefQuery query = new XrefQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = xrefManager.search(query); return createOkResponse(queryResults); } catch (Exception e) { diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/ProteinWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/ProteinWSServer.java index 47b1d63f49..be2d76db1b 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/ProteinWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/ProteinWSServer.java @@ -137,6 +137,7 @@ public Response getInfo(@PathParam("proteins") @ApiParam(name = "proteins", valu public Response getAll() { try { ProteinQuery query = new ProteinQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = proteinManager.search(query); return createOkResponse(queryResults); } catch (Exception e) { @@ -174,6 +175,7 @@ public Response getSubstitutionScores(@PathParam("proteins") @ApiParam(name = "p required = false) String aa) { try { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(Arrays.asList(id.split(","))); CellBaseDataResult queryResult = proteinManager.getSubstitutionScores(query, position, aa); return createOkResponse(queryResult); @@ -218,6 +220,7 @@ public Response getSequence(@PathParam("proteins") @ApiParam (name = "proteins", required = true) String proteins) throws QueryException { try { ProteinQuery query = new ProteinQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setAccessions(Arrays.asList(proteins.split(","))); CellBaseDataResult queryResult = proteinManager.getSequence(query); return createOkResponse(queryResult); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java index f1812cd27f..11e8de47fd 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java @@ -130,6 +130,7 @@ public Response getGeneById(@PathParam("transcripts") @ApiParam(name = "transcri String[] ids = id.split(","); for (String transcriptId : ids) { GeneQuery query = new GeneQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(Collections.singletonList(transcriptId)); queries.add(query); } @@ -192,6 +193,7 @@ public Response getGeneById(@PathParam("transcripts") @ApiParam(name = "transcri public Response getAll() { try { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); logger.info("/search TranscriptQuery: {}", query.toString()); CellBaseDataResult queryResult = transcriptManager.search(query); return createOkResponse(queryResult); @@ -228,7 +230,7 @@ public Response getSequencesByIdList(@PathParam("transcripts") @ApiParam(name = value = TRANSCRIPT_XREFS_DESCRIPTION, required = true) String id) { try { - List> queryResults = transcriptManager.getSequence(id); + List> queryResults = transcriptManager.getSequence(id, getDataRelease()); return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e); @@ -252,6 +254,7 @@ public Response getProtein(@PathParam("transcripts") @ApiParam(name = "transcrip String[] ids = transcripts.split(","); for (String transcriptId : ids) { ProteinQuery query = new ProteinQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setXrefs(Collections.singletonList(transcriptId)); queries.add(query); } @@ -276,6 +279,7 @@ public Response getProteinFunctionPredictionBytranscripts(@PathParam("transcript required = false) String aa) { try { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(Arrays.asList(id)); CellBaseDataResult queryResults = proteinManager.getSubstitutionScores(query, position, aa); return createOkResponse(queryResults); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/RegionWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/RegionWSServer.java index e0311376da..a12a247614 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/RegionWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/RegionWSServer.java @@ -187,6 +187,7 @@ public Response getGenesByRegion(@PathParam("regions") @ApiParam(name = "regions String[] coordinates = regions.split(","); for (String coordinate : coordinates) { GeneQuery query = new GeneQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Collections.singletonList(Region.parseRegion(coordinate))); queries.add(query); logger.info("REST GeneQuery: {}", query.toString()); @@ -233,6 +234,7 @@ public Response getTranscriptByRegion(@PathParam("regions") @ApiParam(name = "re String[] coordinates = regions.split(","); for (String coordinate : coordinates) { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Region.parseRegions(coordinate)); queries.add(query); logger.info("REST TranscriptQuery: {}", query.toString()); @@ -272,6 +274,7 @@ public Response getRepeatByRegion(@PathParam("regions") @ApiParam(name = "region String[] coordinates = region.split(","); for (String coordinate : coordinates) { RepeatsQuery query = new RepeatsQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Region.parseRegions(coordinate)); queries.add(query); logger.info("REST RepeatsQuery: {}", query.toString()); @@ -317,6 +320,7 @@ public Response getVariantByRegion(@PathParam("regions") @ApiParam(name = "regio List regionList = Region.parseRegions(regions); for (Region region : regionList) { VariantQuery query = new VariantQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Collections.singletonList(region)); queries.add(query); logger.info("REST variantQuery: {}", query.toString()); @@ -358,6 +362,7 @@ public Response getSequenceByRegion(@PathParam("regions") allowableValues = "1,-1", defaultValue = "1", required = true) String strand) { try { GenomeQuery query = new GenomeQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Region.parseRegions(regions)); List> queryResults = genomeManager.getByRegions(query); return createOkResponse(queryResults); @@ -457,6 +462,7 @@ public Response getRegulatoryRegions(@PathParam("regions") @ApiParam(name = "reg String[] regionArray = regions.split(","); for (String regionString : regionArray) { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Region.parseRegions(regionString)); logger.info("REST RegulationQuery: {}", query.toString()); queries.add(query); @@ -493,6 +499,7 @@ public Response getTfByRegion(@PathParam("regions") @ApiParam(name = "regions", String[] regionArray = regions.split(","); for (String regionString : regionArray) { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Collections.singletonList(Region.parseRegion(regionString))); query.setFeatureTypes(Collections.singletonList("TF_binding_site")); logger.info("REST RegulationQuery: {}", query.toString()); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java index 12f2db6e0c..f774d3ba5b 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java @@ -395,6 +395,7 @@ public Response getInfo(@PathParam("variants") @ApiParam(name = "variants", valu public Response search() { try { VariantQuery query = new VariantQuery(uriParams); + query.setDataRelease(getDataRelease()); logger.info("/search VariantQuery: {}", query.toString()); CellBaseDataResult queryResults = variantManager.search(query); return createOkResponse(queryResults); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/RegulatoryWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/RegulatoryWSServer.java index 1289006e5a..d41f06c10d 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/RegulatoryWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/RegulatoryWSServer.java @@ -79,6 +79,7 @@ public Response getUniqueValues(@QueryParam("field") @ApiParam(name = "field", r try { copyToFacet("field", field); RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = regulatoryManager.distinct(query); return createOkResponse(queryResults); } catch (Exception e) { @@ -98,6 +99,7 @@ public Response getUniqueValues(@QueryParam("field") @ApiParam(name = "field", r public Response getFeatureTypes() { try { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setFacet("featureType"); CellBaseDataResult queryResults = regulatoryManager.distinct(query); return createOkResponse(queryResults); @@ -120,6 +122,7 @@ public Response getFeatureTypes() { public Response getFeatureClasses() { try { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setFacet("featureClass"); CellBaseDataResult queryResults = regulatoryManager.distinct(query); return createOkResponse(queryResults); @@ -161,6 +164,7 @@ public Response getFeatureClasses() { public Response getAll() { try { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = regulatoryManager.search(query); return createOkResponse(queryResults); } catch (Exception e) { diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java index 37e40aed0a..4b7b04cca9 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java @@ -102,6 +102,7 @@ public Response getAllByTfbs(@PathParam("tf") @ApiParam(name = "tf", value = TFB String[] identifiers = tf.split(","); for (String identifier : identifiers) { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setNames(Arrays.asList(identifier)); query.setFeatureTypes(Arrays.asList("TF_binding_site")); queries.add(query); @@ -152,6 +153,7 @@ public Response getAllByTfbs(@PathParam("tf") @ApiParam(name = "tf", value = TFB public Response getEnsemblGenes(@PathParam("tf") @ApiParam(name = "tf", value = TFBS_IDS, required = true) String tf) { try { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); LogicalList logicalList = new LogicalList(Arrays.asList(tf.split(","))); geneQuery.setTranscriptsTfbsId(logicalList); CellBaseDataResult queryResults = geneManager.search(geneQuery); From 85cb36ab78893ebfd48ea3256295daa7531e904c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 4 May 2023 16:13:24 +0200 Subject: [PATCH 09/28] server: show default data release from the endpoint 'meta/about', #TASK-4278, #TASK-4157 --- .../opencb/cellbase/core/ParamConstants.java | 64 +++++++------------ .../lib/GenericMongoDBAdaptorTest.java | 4 +- .../lib/variant/hgvs/HgvsCalculatorTest.java | 5 +- .../server/rest/GenericRestWSServer.java | 4 +- .../cellbase/server/rest/MetaWSServer.java | 37 +++++++++-- 5 files changed, 62 insertions(+), 52 deletions(-) diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java index 9167239ad7..3d24a3050d 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java @@ -109,17 +109,15 @@ public Type type() { public static final String DOT_NOTATION_NOTE = "Parameters can be camel case (e.g. transcriptsBiotype) " + "or dot notation (e.g. transcripts.biotype)."; - public static final String VERSION_DESCRIPTION = "Possible values: v5"; - public static final String DEFAULT_VERSION = "v5"; - public static final String DATA_RELEASE_DESCRIPTION = "Data release. To use the default data release, set this to 0. Please," - + " check the web service /metadata/dataReleases in order to know the different data releases and sources, and the default" - + " data release"; - public static final String DEFAULT_DATA_RELEASE = "0"; + public static final String DEFAULT_VERSION = "v5.5"; + public static final String VERSION_DESCRIPTION = "API version, e.g.: " + DEFAULT_VERSION; + public static final String DATA_RELEASE_DESCRIPTION = "Data release. To use the default data release, set this to 0. To get the list" + + " of available data release, please call the endpoint 'meta/dataReleases'"; public static final String DATA_ACCESS_TOKEN_DESCRIPTION = "Data token to allow access to licensed/restricted data sources such as" + " Cosmic or HGMD"; + public static final String DEFAULT_ASSEMBLY = "grch38"; public static final String ASSEMBLY_DESCRIPTION = "Set the reference genome assembly, e.g. grch38. For a full list of " - + "potentially available assemblies, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species"; + + "potentially available assemblies, please call the endpoint 'meta/species'"; public static final String COUNT_DESCRIPTION = "Get the total number of results matching the query. "; @@ -136,9 +134,9 @@ public Type type() { public static final String SORT_DESCRIPTION = "Sort returned results by a certain data model attribute"; public static final String ORDER_DESCRIPTION = "Results are in ascending order by default"; + public static final String DEFAULT_SPECIES = "hsapiens"; public static final String SPECIES_DESCRIPTION = "Name of the species, e.g. hsapiens. For a full list " - + "of potentially available species ids, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species"; + + "of potentially available species ids, please call the endpoint 'meta/species'"; public static final String DATA_MODEL_DESCRIPTION = "Get JSON specification of the data model"; public static final String REGION_DESCRIPTION = "Comma separated list of genomic regions to be queried, " @@ -175,8 +173,7 @@ public Type type() { public static final String ONTOLOGY_NAMES = "Comma separated list of ontology term names, " + "e.g. Diabetes mellitus,histone kinase activity"; public static final String ONTOLOGY_NAMESPACES = "Comma separated list of namespaces, e.g. human_phenotype,biological_process. " - + "For a full list of potentially available namespaces, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/feature/ontology/distinct?field=namespace"; + + "For a full list of potentially available namespaces, please call the endpoint 'feature/ontology/distinct?field=namespace'"; public static final String ONTOLOGY_SOURCES = "Name of ontology, e.g. HP or GO"; public static final String ONTOLOGY_SYNONYMS = "Comma separated list of synonyms, e.g. Cerebellar ataxia"; public static final String ONTOLOGY_XREFS = "Comma separated list of cross references, e.g. MSH:D002524"; @@ -302,8 +299,8 @@ public Type type() { + " Possible values are clinvar or cosmic"; public static final String SOURCE_PARAM = "source"; public static final String SEQUENCE_ONTOLOGY_DESCRIPTION = "Comma separated list of consequence types, " - + "e.g. missense_variant. Exact text matches will be returned. A list of searchable consequence types can be" - + " accessed at https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/feature/variation/consequence_types"; + + "e.g. missense_variant. Exact text matches will be returned. To get list of searchable consequence types, please call" + + " the endpoint 'genomic/variant/consequenceTypes'"; public static final String SEQUENCE_ONTOLOGY_PARAM = "consequenceType"; public static final String FEATURE_IDS_DESCRIPTION = "Comma separated list of feature ids, which can be either ENSEMBL gene " + "ids, HGNC gene symbols, transcript symbols or ENSEMBL transcript ids, e.g.: BRCA2,ENST00000409047. " @@ -320,26 +317,21 @@ public Type type() { public static final String VARIANT_IDS_DESCRIPTION = "Comma separated list of ids, e.g. rs6025,COSM306824. " + "Exact text matches will be returned."; public static final String VARIANT_IDS_PARAM = "id"; - public static final String VARIANT_TYPES_DESCRIPTION = "Comma separated list of variant types, e.g. \"SNV\" A list of " - + "searchable types can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/type"; + public static final String VARIANT_TYPES_DESCRIPTION = "Comma separated list of variant types, e.g. \"SNV\". To get the list of " + + "searchable types, please call the endpoint 'clinical/variant/type'"; public static final String VARIANT_TYPES_PARAM = "type"; - public static final String CONSISTENCY_STATUS_DESCRIPTION = "Comma separated list of consistency labels. A list of searchable " - + "consistency labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/consistency_labels"; + public static final String CONSISTENCY_STATUS_DESCRIPTION = "Comma separated list of consistency labels. To get the list of searchable" + + " consistency labels, please call the endpoint 'clinical/variant/consistencyLabels'"; public static final String CONSISTENCY_STATUS_PARAM = "consistencyStatus"; - public static final String CLINICAL_SIGNFICANCE_DESCRIPTION = "Comma separated list of clinical significance labels. " - + "A list of searchable clinical significance labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/clinsig_labels" + public static final String CLINICAL_SIGNFICANCE_DESCRIPTION = "Comma separated list of clinical significance labels. To get the list of" + + " searchable clinical significance labels, please call the endpoint 'clinical/variant/clinsigLabels'. " + " WARNING: returned numTotalResults will always be -1 if more than 1 label is provided."; public static final String CLINICAL_SIGNFICANCE_PARAM = "clinicalSignificance"; - public static final String MODE_INHERITANCE_DESCRIPTION = "Comma separated list of mode of inheritance labels. A list of " - + "searchable mode of inheritance labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/mode_inheritance_labels"; + public static final String MODE_INHERITANCE_DESCRIPTION = "Comma separated list of mode of inheritance labels. To the the list of " + + "searchable mode of inheritance labels, please call the endpoint 'clinical/variant/modeInheritanceLabels'"; public static final String MODE_INHERITANCE_PARAM = "modeInheritance"; - public static final String ALLELE_ORIGIN_DESCRIPTION = "Comma separated list of allele origin labels. A list of searchable " - + "allele origin labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/allele_origin_labels"; + public static final String ALLELE_ORIGIN_DESCRIPTION = "Comma separated list of allele origin labels. To get the list of searchable " + + "allele origin labels, please call the endpoint 'clinical/variant/alleleOriginLabels'"; public static final String ALLELE_ORIGIN_PARAM = "alleleOrigin"; // --------------------------------------------- @@ -378,21 +370,13 @@ public Type type() { // --------------------------------------------- - public static final String XREF_DBNAMES = "Comma separated list of source DB names" - + " to include in the search, e.g.: ensembl_gene,vega_gene,havana_gene." - + " Available db names are shown by this web service: " - + " https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/#!/Xref/getDBNames"; + public static final String XREF_DBNAMES = "Comma separated list of source DB names to include in the search, e.g.: ensembl_gene," + + "vega_gene,havana_gene."; // --------------------------------------------- - // public static final String REGULATION_FEATURE_CLASSES = "Comma separated list of regulatory region classes, e.g.:" -// + "Histone,Transcription Factor. Exact text matches will be returned. For a full" -// + "list of available regulatory types: " -// + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/regulatory/featureClass"; public static final String REGULATION_FEATURE_TYPES = "Comma separated list of regulatory region types, e.g.: " - + "TF_binding_site,histone_acetylation_site. Exact text matches will be returned. For a full" - + "list of available regulatory types: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/regulatory/featureType"; + + "TF_binding_site,histone_acetylation_site. Exact text matches will be returned."; public static final String CELLTYPE = "Type of cell."; diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java index a3770ca623..3126905331 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java @@ -83,13 +83,15 @@ public GenericMongoDBAdaptorTest() { } } - protected void clearDB(String dbName) throws Exception { + protected void clearDB(String dbName) { logger.info("Cleaning MongoDB {}", dbName); try (MongoDataStoreManager mongoManager = new MongoDataStoreManager(Collections.singletonList(new DataStoreServerAddress("localhost", 27017)))) { MongoDBConfiguration.Builder builder = MongoDBConfiguration.builder(); MongoDBConfiguration mongoDBConfiguration = builder.build(); mongoManager.get(dbName, mongoDBConfiguration); mongoManager.drop(dbName); + } catch (Exception e) { + e.printStackTrace(); } } diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculatorTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculatorTest.java index 3f74978f53..c6abc17041 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculatorTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculatorTest.java @@ -15,6 +15,7 @@ import org.opencb.cellbase.lib.GenericMongoDBAdaptorTest; import org.opencb.cellbase.lib.impl.core.GeneMongoDBAdaptor; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; +import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.cellbase.lib.managers.GeneManager; import java.io.IOException; @@ -41,9 +42,9 @@ public HgvsCalculatorTest() throws IOException { @BeforeAll public void init() throws Exception { - dataRelease = 1; - clearDB(CELLBASE_DBNAME); + DataReleaseManager dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager("hsapiens", "GRCh37"); + dataRelease = dataReleaseManager.createRelease().getRelease(); Path path = Paths.get(getClass() .getResource("/hgvs/gene.test.json.gz").toURI()); loadRunner.load(path, "gene", dataRelease); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index a9fea682af..1fd334565b 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -64,7 +64,7 @@ public class GenericRestWSServer implements IWSServer { protected String version; - protected int defaultDataRelease = -1; + protected int defaultDataRelease = 0; protected String species; protected Query query; @@ -170,7 +170,7 @@ private void initQuery() throws CellBaseException { checkVersion(); // Set default data release if necessary - if (!DONT_CHECK_SPECIES.equals(species) && defaultDataRelease == -1) { + if (!DONT_CHECK_SPECIES.equals(species) && defaultDataRelease < 1) { // As the assembly may not be presented in the query, we have to be sure to get it from the CellBase configuration assembly = SpeciesUtils.getSpecies(cellBaseConfiguration, this.species, assembly).getAssembly(); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java index 46e25ff47d..3ce0ef0d33 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java @@ -17,6 +17,7 @@ package org.opencb.cellbase.server.rest; import io.swagger.annotations.*; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.core.ParamConstants; import org.opencb.cellbase.core.api.query.QueryException; @@ -82,9 +83,10 @@ public MetaWSServer(@PathParam("apiVersion") @ApiOperation(httpMethod = "GET", value = "Returns source version metadata, including source urls from which " + "data files were downloaded.", response = DownloadProperties.class, responseContainer = "QueryResponse") public Response getVersion(@PathParam("species") - @ApiParam(name = "species", value = ParamConstants.SPECIES_DESCRIPTION, required = true) String species, - @ApiParam(name = "assembly", value = ParamConstants.ASSEMBLY_DESCRIPTION) @QueryParam("assembly") - String assembly) { + @ApiParam(name = "species", value = ParamConstants.SPECIES_DESCRIPTION, + defaultValue = ParamConstants.DEFAULT_SPECIES, required = true) String species, + @ApiParam(name = "assembly", value = ParamConstants.ASSEMBLY_DESCRIPTION, + defaultValue = ParamConstants.DEFAULT_ASSEMBLY) @QueryParam("assembly") String assembly) { try { if (StringUtils.isEmpty(assembly)) { SpeciesConfiguration.Assembly assemblyObject = SpeciesUtils.getDefaultAssembly(cellBaseConfiguration, species); @@ -145,7 +147,7 @@ public Response getDataRelease(@PathParam("species") @ApiOperation(httpMethod = "GET", value = "Display the licensed data sources of the input token and their expiration date", response = Map.class, responseContainer = "QueryResponse") public Response getLicensedData(@ApiParam(name = "token", required = true, value = ParamConstants.DATA_ACCESS_TOKEN_DESCRIPTION) - @QueryParam("token") String token) { + @QueryParam("token") String token) { try { DataAccessTokenManager datManager = new DataAccessTokenManager(cellBaseConfiguration.getSecretKey()); DataAccessTokenSources sources = datManager.decode(token); @@ -287,12 +289,33 @@ private Response getAllSpecies() { @ApiOperation(httpMethod = "GET", value = "Returns info about current CellBase code.", response = Map.class, responseContainer = "QueryResponse") public Response getAbout() { - Map info = new HashMap<>(3); + Map info = new LinkedHashMap<>(); info.put("Program", "CellBase (OpenCB)"); - info.put("Version", GitRepositoryState.get().getBuildVersion()); + info.put("Description", "High-Performance NoSQL database and RESTful web services to access the most relevant biological data"); + info.put("Version", version); + info.put("Build version", GitRepositoryState.get().getBuildVersion()); info.put("Git branch", GitRepositoryState.get().getBranch()); info.put("Git commit", GitRepositoryState.get().getCommitId()); - info.put("Description", "High-Performance NoSQL database and RESTful web services to access the most relevant biological data"); + + // Get default data releases + species = "hsapiens"; + SpeciesConfiguration speciesConfiguration = SpeciesUtils.getSpeciesConfiguration(cellBaseConfiguration, species); + List assemblies = speciesConfiguration.getAssemblies(); + if (CollectionUtils.isNotEmpty(assemblies)) { + DataReleaseManager dataReleaseManager; + for (SpeciesConfiguration.Assembly assembly : assemblies) { + String key = "Default data release for " + version + " (" + species + "/" + assembly.getName() + ")"; + try { + dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly.getName()); + DataRelease defaultDataRelease = dataReleaseManager.getDefault(version); + info.put(key, String.valueOf(defaultDataRelease.getRelease())); + } catch (CellBaseException e) { + info.put(key, "ERROR: " + e.getMessage()); + e.printStackTrace(); + } + } + } + CellBaseDataResult queryResult = new CellBaseDataResult(); queryResult.setId("about"); queryResult.setTime(0); From 6a21c61919e108a77dd243ffc74f4ffe13eb461b Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 8 May 2023 15:42:53 +0200 Subject: [PATCH 10/28] CICD: Alert comment#TASK-4163 --- .github/workflows/task.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index 5a1795426b..efa53ce255 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -5,7 +5,8 @@ on: branches: - TASK-* workflow_dispatch: - +###### +##### OJO CAMBIAR A RAMA DEVELOP!!!! jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@TASK-4163 From b752258101e8fab68737de18ad4c048200452ff9 Mon Sep 17 00:00:00 2001 From: Juanfe Sanahuja <85166064+juanfeSanahuja@users.noreply.github.com> Date: Mon, 8 May 2023 17:22:19 +0200 Subject: [PATCH 11/28] Update pom.xml --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 034e406d45..39c0f576aa 100644 --- a/pom.xml +++ b/pom.xml @@ -23,8 +23,8 @@ ${project.version} - 4.8.1-SNAPSHOT - 2.8.1-SNAPSHOT + 4.9.0-SNAPSHOT + 2.9.0-SNAPSHOT 0.1.0 2.11.4 2.30.1 From 8db79fbedd5b00f9f0dfb15bacd132ddadc80dfc Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 17 May 2023 15:14:49 +0200 Subject: [PATCH 12/28] cicd: Fix develop reference #TASK-4163 --- .github/workflows/task.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index efa53ce255..c97f2b9790 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -5,11 +5,10 @@ on: branches: - TASK-* workflow_dispatch: -###### -##### OJO CAMBIAR A RAMA DEVELOP!!!! + jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@TASK-4163 + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop test: uses: ./.github/workflows/test-analysis.yml From 2468d267c48c3fbd24145f52858974bc7eed9eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 19 May 2023 13:49:26 +0200 Subject: [PATCH 13/28] server: minor change in the 'about' message, #TASK-4278, #TASK-4157 --- .../java/org/opencb/cellbase/server/rest/MetaWSServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java index 3ce0ef0d33..f9b8f16764 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java @@ -292,8 +292,8 @@ public Response getAbout() { Map info = new LinkedHashMap<>(); info.put("Program", "CellBase (OpenCB)"); info.put("Description", "High-Performance NoSQL database and RESTful web services to access the most relevant biological data"); - info.put("Version", version); - info.put("Build version", GitRepositoryState.get().getBuildVersion()); + info.put("API version", version); + info.put("Version", GitRepositoryState.get().getBuildVersion()); info.put("Git branch", GitRepositoryState.get().getBranch()); info.put("Git commit", GitRepositoryState.get().getCommitId()); From 2898d8fac0f4ceb4cc1b8a92e6e81a2077dbc6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 19 May 2023 15:35:15 +0200 Subject: [PATCH 14/28] lib: propagate data release when computing substitution scores, #TASK-4278, #TASK-4157 --- .../org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java | 1 + .../lib/variant/annotation/VariantAnnotationCalculator.java | 1 + 2 files changed, 2 insertions(+) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java index be8319c122..1460dfe5e1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java @@ -233,6 +233,7 @@ public CellBaseDataResult getVariantAnnotation(String if (!aaAlternate.equals("STOP") && !aaReference.equals("STOP")) { TranscriptQuery query = new TranscriptQuery(); query.setTranscriptsId(Collections.singletonList(ensemblTranscriptId)); + query.setDataRelease(dataRelease); proteinVariantAnnotation.setSubstitutionScores(getSubstitutionScores(query, position, aaAlternate).getResults()); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java index 7eaea65017..0ab1461591 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java @@ -103,6 +103,7 @@ public VariantAnnotationCalculator(String species, String assembly, int dataRele // Check data release this.dataRelease = cellbaseManagerFactory.getDataReleaseManager(species, assembly).checkDataRelease(dataRelease); + logger.info("Variant annotation calculator using data release {}", this.dataRelease); this.token = token; // Initialises normaliser configuration with default values. HEADS UP: configuration might be updated From a8ebbcb6a58787adf59d5cf00a01beecc1f88ded Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 26 May 2023 15:49:11 +0200 Subject: [PATCH 15/28] Prepare release 5.5.0 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index d4273c4208..48e197d152 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 3ac36256cd..034ba0b665 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index b84dd787b0..0d4d0fba8d 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index c426ebee60..f862a80d5f 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index b9ccff6874..0f5578209b 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/pom.xml b/pom.xml index 39c0f576aa..65383a0679 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.9.0-SNAPSHOT - 2.9.0-SNAPSHOT + 4.9.0 + 2.9.0 0.1.0 2.11.4 2.30.1 From ab8b794415723e9fa491265d6d4ec4210166f9e6 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 26 May 2023 16:07:19 +0200 Subject: [PATCH 16/28] Prepare release 5.5.0 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index d4273c4208..48e197d152 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 3ac36256cd..034ba0b665 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index b84dd787b0..0d4d0fba8d 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index c426ebee60..f862a80d5f 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index b9ccff6874..0f5578209b 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 ../pom.xml diff --git a/pom.xml b/pom.xml index 39c0f576aa..65383a0679 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.5.0 pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.9.0-SNAPSHOT - 2.9.0-SNAPSHOT + 4.9.0 + 2.9.0 0.1.0 2.11.4 2.30.1 From e1873e239790f80698d4ca1c6848c94a609b123a Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 26 May 2023 16:08:02 +0200 Subject: [PATCH 17/28] Prepare new development branch release-5.5.x --- .github/workflows/test-analysis.yml | 2 +- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index 0a5c918b70..869398f86b 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -20,7 +20,7 @@ on: type: string required: true env: - xb_version: "1.6.1" + xb_version: "1.7.0" jobs: test: name: Test and push Sonar analysis diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 48e197d152..0d4682584a 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0 + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 034ba0b665..74791e4ff2 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0 + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 0d4d0fba8d..435b560692 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0 + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index f862a80d5f..b2e20a423b 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0 + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 0f5578209b..88d9d0823c 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0 + 5.6.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 65383a0679..b1e192e27d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0 + 5.6.0-SNAPSHOT pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.9.0 - 2.9.0 + 4.10.0-SNAPSHOT + 2.10.0-SNAPSHOT 0.1.0 2.11.4 2.30.1 From a24dc659387983cb4fb94b025e5cfd6e75a98c69 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 26 May 2023 16:15:31 +0200 Subject: [PATCH 18/28] Fix conflicts --- cellbase-app/pom.xml | 4 ---- cellbase-client/pom.xml | 4 ---- cellbase-core/pom.xml | 4 ---- cellbase-lib/pom.xml | 4 ---- cellbase-server/pom.xml | 4 ---- pom.xml | 9 --------- 6 files changed, 29 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 0fca54033b..0d4682584a 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,11 +6,7 @@ org.opencb.cellbase cellbase -<<<<<<< HEAD - 5.5.0 -======= 5.6.0-SNAPSHOT ->>>>>>> release-5.5.x ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 6acac6e375..74791e4ff2 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,11 +6,7 @@ org.opencb.cellbase cellbase -<<<<<<< HEAD - 5.5.0 -======= 5.6.0-SNAPSHOT ->>>>>>> release-5.5.x ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 7e38d43200..435b560692 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,11 +6,7 @@ org.opencb.cellbase cellbase -<<<<<<< HEAD - 5.5.0 -======= 5.6.0-SNAPSHOT ->>>>>>> release-5.5.x ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 8e08481bbd..b2e20a423b 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,11 +6,7 @@ org.opencb.cellbase cellbase -<<<<<<< HEAD - 5.5.0 -======= 5.6.0-SNAPSHOT ->>>>>>> release-5.5.x ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 6282d25311..88d9d0823c 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,11 +6,7 @@ org.opencb.cellbase cellbase -<<<<<<< HEAD - 5.5.0 -======= 5.6.0-SNAPSHOT ->>>>>>> release-5.5.x ../pom.xml diff --git a/pom.xml b/pom.xml index d93e316920..b1e192e27d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,11 +6,7 @@ org.opencb.cellbase cellbase -<<<<<<< HEAD - 5.5.0 -======= 5.6.0-SNAPSHOT ->>>>>>> release-5.5.x pom CellBase project @@ -27,13 +23,8 @@ ${project.version} -<<<<<<< HEAD - 4.9.0 - 2.9.0 -======= 4.10.0-SNAPSHOT 2.10.0-SNAPSHOT ->>>>>>> release-5.5.x 0.1.0 2.11.4 2.30.1 From cfd84bf09f4b6e1bbdc8b351a48cabae7b44965b Mon Sep 17 00:00:00 2001 From: imedina Date: Tue, 30 May 2023 00:50:26 +0100 Subject: [PATCH 19/28] docker: move docker base image to Ubuntu 22.04 and OpenJDK 8u372 --- .../app/cloud/docker/cellbase-base/Dockerfile | 5 +++-- .../docker/cellbase-base/Dockerfile.alpine | 22 ------------------- 2 files changed, 3 insertions(+), 24 deletions(-) delete mode 100644 cellbase-app/app/cloud/docker/cellbase-base/Dockerfile.alpine diff --git a/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile b/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile index 978c0506f4..0502319f49 100644 --- a/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile +++ b/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile @@ -1,5 +1,6 @@ -## Based on Debian 11 (bullseye) -FROM openjdk:11-jre +## Based on Ubuntu 22.04 (jammy) +## We are now using OpenJDK 8u372 to support "cgroup v2", see https://developers.redhat.com/articles/2023/04/19/openjdk-8u372-feature-cgroup-v2-support# +FROM eclipse-temurin:8u372-b07-jre-jammy LABEL org.label-schema.vendor="OpenCB" \ org.label-schema.name="cellbase-base" \ diff --git a/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile.alpine b/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile.alpine deleted file mode 100644 index 9960039781..0000000000 --- a/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile.alpine +++ /dev/null @@ -1,22 +0,0 @@ -FROM openjdk:8-jre-alpine - -LABEL org.label-schema.vendor="OpenCB" \ - org.label-schema.name="cellbase-base" \ - org.label-schema.url="http://docs.opencb.org/display/cellbase" \ - org.label-schema.description="An Open Computational Genomics Analysis platform for big data processing and analysis in genomics" \ - maintainer="Julie Sullivan " \ - org.label-schema.schema-version="1.0" - -ENV CELLBASE_USER cellbase -ENV CELLBASE_HOME /opt/cellbase/ - -RUN apk update && apk upgrade && apk add ca-certificates openssl wget bash \ - && update-ca-certificates \ - && addgroup -S $CELLBASE_USER && adduser -S $CELLBASE_USER -G $CELLBASE_USER -u 1001 - -USER $CELLBASE_USER - -VOLUME /opt/cellbase/conf - -COPY . /opt/cellbase -WORKDIR /opt/cellbase From c917899ada740bbd1f1b991dfc4447ddb53ff73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Wed, 31 May 2023 18:05:34 +0200 Subject: [PATCH 20/28] lib: fix normalization configuration, #TASK-4530 --- .../annotation/VariantAnnotationCalculator.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java index 57421e30aa..d6b7821e41 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java @@ -117,8 +117,8 @@ private VariantNormalizer.VariantNormalizerConfig getNormalizerConfig() { return (new VariantNormalizer.VariantNormalizerConfig()) .setReuseVariants(false) .setNormalizeAlleles(false) - .setDecomposeMNVs(decompose) - .enableLeftAlign(new CellBaseNormalizerSequenceAdaptor(genomeManager, dataRelease)); + .setDecomposeMNVs(false) + .disableLeftAlign(); } @Deprecated @@ -731,10 +731,10 @@ private void parseQueryParam(QueryOptions queryOptions) { logger.debug("normalize = {}", normalize); // Default behaviour decompose - decompose = (queryOptions.get("skipDecompose") == null || !queryOptions.getBoolean("skipDecompose")); - logger.debug("decompose = {}", decompose); +// decompose = (queryOptions.get("skipDecompose") == null || !queryOptions.getBoolean("skipDecompose")); +// logger.debug("decompose = {}", decompose); // Must update normaliser configuration since normaliser was created on constructor - normalizer.getConfig().setDecomposeMNVs(decompose); +// normalizer.getConfig().setDecomposeMNVs(decompose); // New parameter "ignorePhase" present overrides presence of old "phased" parameter if (queryOptions.get("ignorePhase") != null) { From 316e08e343837d933b83108e71a8154dcbdda9ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 1 Jun 2023 15:06:29 +0200 Subject: [PATCH 21/28] app: add normalization config parameters to the endpoint 'normalization' and add the parameter 'leftAlign' to the endpoint 'annotation', #TASK-4530 --- .../cli/main/CellBaseCliOptionsParser.java | 20 ++-- .../VariantAnnotationCommandExecutor.java | 8 +- .../VariantAnnotationCommandExecutorTest.java | 6 +- .../opencb/cellbase/core/ParamConstants.java | 8 +- .../cellbase/lib/managers/VariantManager.java | 28 ++++- .../VariantAnnotationCalculator.java | 100 ++++++----------- .../core/VariantAnnotationCalculatorTest.java | 2 +- .../lib/variant/VariantManagerTest.java | 2 +- .../server/rest/genomic/VariantWSServer.java | 106 +++++++++++++----- 9 files changed, 159 insertions(+), 121 deletions(-) diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java index b597e5325b..b54a82b656 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java @@ -142,30 +142,30 @@ public class VariantAnnotationCommandOptions { required = false, arity = 1) public String referenceFasta; - @Parameter(names = {"--skip-normalize"}, description = "Skip normalization of input variants. Should not be used" + @Parameter(names = {"normalize"}, description = "Normalize input variants. Should be used" + " when the input (-i, --input-file) is a VCF file. Normalization includes splitting multi-allele positions " + "read from a VCF, allele trimming and decomposing MNVs. Has" + " no effect if reading variants from a CellBase variation collection " + "(\"--input-variation-collection\") or running a variant annotation benchmark (\"--benchmark\"): in" + " these two cases variant normalization is never carried out.", required = false, arity = 0) - public boolean skipNormalize = false; + public boolean normalize = false; - @Parameter(names = {"--skip-decompose"}, description = "Use this flag to avoid decomposition of " + @Parameter(names = {"--decompose"}, description = "Use this flag to decompose of " + "multi-nucleotide-variants (MNVs) / block substitutions as part of the normalization process. If this" - + " flag is NOT activated, as a step during the normalization process reference and alternate alleles" + + " flag is activated, as a step during the normalization process reference and alternate alleles" + " from MNVs/Block substitutions will be aligned and decomposed into their forming simple variants. " - + " This flag has no effect if --skip-normalize is present.", + + " This flag has no effect without normalization.", required = false, arity = 0) - public boolean skipDecompose = false; + public boolean decompose = false; - @Parameter(names = {"--skip-left-align"}, description = "Use this flag to avoid left alignment as part of the" + @Parameter(names = {"--left-align"}, description = "Use this flag left align as part of the" + " normalization process. If this" - + " flag is NOT activated, as a step during the normalization process will left align the variant with" + + " flag is activated, as a step during the normalization process will left align the variant with" + " respect to the reference genome." - + " This flag has no effect if --skip-normalize is present.", + + " This flag has no effect without normalization.", required = false, arity = 0) - public boolean skipLeftAlign = false; + public boolean leftAlign = false; // TODO: remove "phased" CLI parameter in next release. Default behavior from here onwards should be // ignorePhase = false diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java index 771b7919ca..54f4728b1e 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java @@ -562,7 +562,7 @@ private void checkParameters() throws IOException, CellBaseException { FileUtils.checkDirectory(input); normalize = false; } else { - normalize = !variantAnnotationCommandOptions.skipNormalize; + normalize = variantAnnotationCommandOptions.normalize; FileUtils.checkFile(input); inputFormat = getFileFormat(input); } @@ -572,8 +572,8 @@ private void checkParameters() throws IOException, CellBaseException { } parsePhaseConfiguration(); - decompose = !variantAnnotationCommandOptions.skipDecompose; - leftAlign = !variantAnnotationCommandOptions.skipLeftAlign; + decompose = variantAnnotationCommandOptions.decompose; + leftAlign = variantAnnotationCommandOptions.leftAlign; // Update serverQueryOptions serverQueryOptions.put("checkAminoAcidChange", variantAnnotationCommandOptions.checkAminoAcidChange); @@ -619,7 +619,7 @@ private void checkParameters() throws IOException, CellBaseException { // to the server. Actual normalization and decomposition options are set and processed here in the server code // using this.decompose and this.normalize fields. serverQueryOptions.add("normalize", false); - serverQueryOptions.add("skipDecompose", true); + serverQueryOptions.add("decompose", false); if (variantAnnotationCommandOptions.include != null && !variantAnnotationCommandOptions.include.isEmpty()) { serverQueryOptions.add("include", variantAnnotationCommandOptions.include); diff --git a/cellbase-app/src/test/java/org/opencb/cellbase/app/cli/VariantAnnotationCommandExecutorTest.java b/cellbase-app/src/test/java/org/opencb/cellbase/app/cli/VariantAnnotationCommandExecutorTest.java index 403cffeb7b..d64ff0096b 100644 --- a/cellbase-app/src/test/java/org/opencb/cellbase/app/cli/VariantAnnotationCommandExecutorTest.java +++ b/cellbase-app/src/test/java/org/opencb/cellbase/app/cli/VariantAnnotationCommandExecutorTest.java @@ -930,9 +930,9 @@ private void cleanUp() throws IOException { variantAnnotationCommandOptions.benchmark = false; variantAnnotationCommandOptions.phased = true; variantAnnotationCommandOptions.input = inputFilename; - variantAnnotationCommandOptions.skipNormalize = false; - variantAnnotationCommandOptions.skipDecompose = !decompose; - variantAnnotationCommandOptions.skipLeftAlign = false; + variantAnnotationCommandOptions.normalize = true; + variantAnnotationCommandOptions.decompose = decompose; + variantAnnotationCommandOptions.leftAlign = true; variantAnnotationCommandOptions.output = Paths.get(OUTPUT_FILENAME).toString(); variantAnnotationCommandOptions.outputFormat = "json"; variantAnnotationCommandOptions.include = "cytobands"; diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java index ef762ce048..ac98bce229 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java @@ -344,9 +344,11 @@ public Type type() { + "19:45411941:T:C,14:38679764:-:GATCTG,1:6635210:G:-," + "2:114340663:GCTGGGCATCCT:ACTGGGCATCCT,1:816505-825225:"; public static final String NORMALISE = "Boolean to indicate whether input variants shall be " - + "normalized or not. Normalization process does NOT include decomposing "; - public static final String SKIP_DECOMPOSE = "Boolean to indicate whether input MNVs should be " - + "decomposed or not as part of the normalisation step. MNV decomposition is strongly encouraged."; + + "normalized or not. Normalization process does NOT include decomposing nor left alignment."; + public static final String DECOMPOSE = "Boolean to indicate whether input MNVs should be " + + "decomposed or not as part of the normalisation step."; + public static final String LEFT_ALIGN = "Boolean to indicate whether input ambiguous INDELS should be " + + "left aligned or not as part of the normalisation step."; public static final String IGNORE_PHASE = "Boolean to indicate whether phase data should be taken into account."; public static final String PHASED = "DEPRECATED. Will be removed in next release. Please, use ignorePhase instead. " + " Boolean to indicate whether phase should be considered during the annotation process"; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/VariantManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/VariantManager.java index b0403aeee6..0f9c16957f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/VariantManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/VariantManager.java @@ -36,6 +36,7 @@ import org.opencb.cellbase.lib.impl.core.SpliceScoreMongoDBAdaptor; import org.opencb.cellbase.lib.impl.core.VariantMongoDBAdaptor; import org.opencb.cellbase.lib.variant.VariantAnnotationUtils; +import org.opencb.cellbase.lib.variant.annotation.CellBaseNormalizerSequenceAdaptor; import org.opencb.cellbase.lib.variant.annotation.VariantAnnotationCalculator; import org.opencb.cellbase.lib.variant.hgvs.HgvsCalculator; import org.opencb.commons.datastore.core.Query; @@ -107,14 +108,29 @@ public List> getHgvsByVariant(String variants, int da * Normalises a list of variants. * * @param variants list of variant strings + * @param decompose boolean to set the decompose MNV behaviour + * @param leftAlign boolean to set the left alignment behaviour * @param dataRelease data release * @return list of normalised variants * @throws CellBaseException if the species is incorrect */ - public CellBaseDataResult getNormalizationByVariant(String variants, int dataRelease) throws CellBaseException { + public CellBaseDataResult getNormalizationByVariant(String variants, boolean decompose, boolean leftAlign, + int dataRelease) throws CellBaseException { List variantList = parseVariants(variants); VariantAnnotationCalculator variantAnnotationCalculator = new VariantAnnotationCalculator(species, assembly, dataRelease, cellbaseManagerFactory); + + // Set decompose MNV behaviour + variantAnnotationCalculator.getNormalizer().getConfig().setDecomposeMNVs(decompose); + + // Set left alignment behaviour + if (leftAlign) { + variantAnnotationCalculator.getNormalizer().getConfig().enableLeftAlign(new CellBaseNormalizerSequenceAdaptor(genomeManager, + dataRelease)); + } else { + variantAnnotationCalculator.getNormalizer().getConfig().disableLeftAlign(); + } + List normalisedVariants = variantAnnotationCalculator.normalizer(variantList); return new CellBaseDataResult<>(variants, 0, new ArrayList<>(), normalisedVariants.size(), normalisedVariants, -1); } @@ -122,7 +138,8 @@ public CellBaseDataResult getNormalizationByVariant(String variants, in public List> getAnnotationByVariant(QueryOptions queryOptions, String variants, Boolean normalize, - Boolean skipDecompose, + Boolean decompose, + Boolean leftAlign, Boolean ignorePhase, @Deprecated Boolean phased, Boolean imprecise, @@ -151,8 +168,11 @@ public List> getAnnotationByVariant(QueryO if (normalize != null) { queryOptions.put("normalize", normalize); } - if (skipDecompose != null) { - queryOptions.put("skipDecompose", skipDecompose); + if (decompose != null) { + queryOptions.put("decompose", decompose); + } + if (leftAlign != null) { + queryOptions.put("leftAlign", leftAlign); } if (imprecise != null) { queryOptions.put("imprecise", imprecise); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java index d6b7821e41..471ca93b6e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java @@ -75,7 +75,8 @@ public class VariantAnnotationCalculator { private final VariantNormalizer normalizer; private boolean normalize = false; - private boolean decompose = true; + private boolean decompose = false; + private boolean leftAlign = false; private boolean phased = true; private Boolean imprecise = true; private Integer svExtraPadding = 0; @@ -114,11 +115,16 @@ public VariantAnnotationCalculator(String species, String assembly, int dataRele } private VariantNormalizer.VariantNormalizerConfig getNormalizerConfig() { - return (new VariantNormalizer.VariantNormalizerConfig()) + VariantNormalizer.VariantNormalizerConfig variantNormalizerConfig = new VariantNormalizer.VariantNormalizerConfig() .setReuseVariants(false) .setNormalizeAlleles(false) - .setDecomposeMNVs(false) - .disableLeftAlign(); + .setDecomposeMNVs(decompose); + if (leftAlign) { + variantNormalizerConfig.enableLeftAlign(new CellBaseNormalizerSequenceAdaptor(genomeManager, dataRelease)); + } else { + variantNormalizerConfig.disableLeftAlign(); + } + return variantNormalizerConfig; } @Deprecated @@ -181,8 +187,7 @@ private List> generateCellBaseDataResultLi // Return only one result per CellBaseDataResult if either // - size original variant list and normalised one is the same - // - MNV decomposition is switched OFF, i.e. queryOptions.skipDecompose = true and therefore - // this.decompose = false + // - MNV decomposition is switched OFF, i.e. queryOptions.decompose = false and therefore if (!decompose || variantList.size() == normalizedVariantList.size()) { for (int i = 0; i < variantList.size(); i++) { CellBaseDataResult cellBaseDataResult = new CellBaseDataResult<>(variantList.get(i).toString(), @@ -428,37 +433,6 @@ private List getTargets(Gene mirna) throws QueryException, Ille return geneMirnaTargets; } -// private boolean isPhased(Variant variant) { -// return (variant.getStudies() != null && !variant.getStudies().isEmpty()) -// && variant.getStudies().get(0).getSampleDataKeys().contains("PS"); -// } -// -// private String getCachedVariationIncludeFields() { -// StringBuilder stringBuilder = new StringBuilder("annotation.chromosome,annotation.start,annotation.reference"); -// stringBuilder.append(",annotation.alternate,annotation.id"); -// -// if (annotatorSet.contains("variation")) { -// stringBuilder.append(",annotation.id,annotation.additionalAttributes.dgvSpecificAttributes"); -// } -// if (annotatorSet.contains("clinical") || annotatorSet.contains("traitAssociation")) { -// stringBuilder.append(",annotation.variantTraitAssociation"); -// } -// if (annotatorSet.contains("conservation")) { -// stringBuilder.append(",annotation.conservation"); -// } -// if (annotatorSet.contains("functionalScore")) { -// stringBuilder.append(",annotation.functionalScore"); -// } -// if (annotatorSet.contains("consequenceType")) { -// stringBuilder.append(",annotation.consequenceTypes,annotation.displayConsequenceType"); -// } -// if (annotatorSet.contains("populationFrequencies")) { -// stringBuilder.append(",annotation.populationFrequencies"); -// } -// -// return stringBuilder.toString(); -// } - private List runAnnotationProcess(List normalizedVariantList, int dataRelease) throws InterruptedException, ExecutionException, QueryException, IllegalAccessException, CellBaseException { long globalStartTime = System.currentTimeMillis(); @@ -730,11 +704,24 @@ private void parseQueryParam(QueryOptions queryOptions) { normalize = (queryOptions.get("normalize") != null && (Boolean) queryOptions.get("normalize")); logger.debug("normalize = {}", normalize); - // Default behaviour decompose -// decompose = (queryOptions.get("skipDecompose") == null || !queryOptions.getBoolean("skipDecompose")); -// logger.debug("decompose = {}", decompose); + // Default behaviour decompose MNV + if (queryOptions.containsKey("skipDecompose")) { + throw new IllegalArgumentException("Param 'skipDecompose' is not supported. Please, use 'decompose' instead"); + } + decompose = (boolean) queryOptions.getOrDefault("decompose", false); + logger.debug("decompose = {}", decompose); // Must update normaliser configuration since normaliser was created on constructor -// normalizer.getConfig().setDecomposeMNVs(decompose); + normalizer.getConfig().setDecomposeMNVs(decompose); + + // Default behaviour left align + leftAlign = (boolean) queryOptions.getOrDefault("leftAlign", false); + logger.debug("leftAlign = {}", leftAlign); + // Must update normaliser configuration since normaliser was created on constructor + if (leftAlign) { + normalizer.getConfig().enableLeftAlign(new CellBaseNormalizerSequenceAdaptor(genomeManager, dataRelease)); + } else { + normalizer.getConfig().disableLeftAlign(); + } // New parameter "ignorePhase" present overrides presence of old "phased" parameter if (queryOptions.get("ignorePhase") != null) { @@ -773,33 +760,6 @@ private void parseQueryParam(QueryOptions queryOptions) { logger.debug("enable = {}", enable); } -// private void mergeAnnotation(VariantAnnotation destination, VariantAnnotation origin) { -// destination.setChromosome(origin.getChromosome()); -// destination.setStart(origin.getStart()); -// destination.setReference(origin.getReference()); -// destination.setAlternate(origin.getAlternate()); -// -// if (annotatorSet.contains("variation")) { -// destination.setId(origin.getId()); -// } -// if (annotatorSet.contains("consequenceType")) { -// destination.setDisplayConsequenceType(origin.getDisplayConsequenceType()); -// destination.setConsequenceTypes(origin.getConsequenceTypes()); -// } -// if (annotatorSet.contains("conservation")) { -// destination.setConservation(origin.getConservation()); -// } -// if (annotatorSet.contains("populationFrequencies")) { -// destination.setPopulationFrequencies(origin.getPopulationFrequencies()); -// } -// if (annotatorSet.contains("clinical")) { -// destination.setVariantTraitAssociation(origin.getVariantTraitAssociation()); -// } -// if (annotatorSet.contains("functionalScore")) { -// destination.setFunctionalScore(origin.getFunctionalScore()); -// } -// } - private void checkAndAdjustPhasedConsequenceTypes(Variant variant, Queue variantBuffer, int dataRelease) throws CellBaseException { // Only SNVs are currently considered for phase adjustment @@ -1990,5 +1950,9 @@ public void processResults(Future>> spliceF } } } + + public VariantNormalizer getNormalizer() { + return normalizer; + } } diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/VariantAnnotationCalculatorTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/VariantAnnotationCalculatorTest.java index 3f82c4775e..b2621d9504 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/VariantAnnotationCalculatorTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/VariantAnnotationCalculatorTest.java @@ -743,7 +743,7 @@ private ConsequenceType getConsequenceType(List consequenceType public void testCellBaseDataResultGroupingDecomposedMNVs() throws Exception { QueryOptions queryOptions = (new QueryOptions("normalize", true)); - queryOptions.put("skipDecompose", false); + queryOptions.put("decompose", true); // Creating here a local VariantAnnotationCalculator since this test requires setting normalizer decompose // option to true which probably breaks some other tests. diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/VariantManagerTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/VariantManagerTest.java index 56ef6239b1..93ec11085c 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/VariantManagerTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/VariantManagerTest.java @@ -117,7 +117,7 @@ public void setUp() throws Exception { @Test public void testNormalisation() throws Exception { - CellBaseDataResult results = variantManager.getNormalizationByVariant("22:18512237:-:AGTT", dataRelease); + CellBaseDataResult results = variantManager.getNormalizationByVariant("22:18512237:-:AGTT", true, true, dataRelease); assertEquals(1, results.getResults().size()); } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java index ef390cb336..a277f52300 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java @@ -92,10 +92,23 @@ public Response getHgvs(@PathParam("variants") @ApiParam(name = "variants", valu @ApiOperation(httpMethod = "GET", value = "FIXME: description needed", response = Map.class, responseContainer = "QueryResponse") public Response getNormalization(@PathParam("variants") @ApiParam(name = "variants", value = RS_IDS, - required = true) String id) { + required = true) String id, + @QueryParam("decompose") + @ApiParam(name = "decompose", + value = "Boolean to indicate whether input MNVs should be " + + "decomposed or not as part of the normalisation step.", + allowableValues = "false,true", + defaultValue = "false") Boolean decompose, + @QueryParam("leftAlign") + @ApiParam(name = "leftAlign", + value = "Boolean to indicate whether input ambiguous INDELS should be " + + "left aligned or not as part of the normalisation step.", + allowableValues = "false,true", + defaultValue = "false") Boolean leftAlign) { try { - CellBaseDataResult queryResults = variantManager.getNormalizationByVariant(id, getDataRelease()); + CellBaseDataResult queryResults = variantManager.getNormalizationByVariant(id, decompose, leftAlign, + getDataRelease()); return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e); @@ -126,23 +139,28 @@ public Response getNormalization(@PathParam("variants") @ApiParam(name = "varian + " expression, geneDisease, drugInteraction, populationFrequencies, repeats, hgvs, geneConstraints, mirnaTargets}.", response = VariantAnnotation.class, responseContainer = "QueryResponse", hidden = true) public Response getAnnotationByVariantsPOST(@ApiParam(name = "variants", value = "Comma separated list of variants to" - + "annotate, e.g. " - + "19:45411941:T:C,14:38679764:-:GATCTG,1:6635210:G:-," - + "2:114340663:GCTGGGCATCCT:ACTGGGCATCCT", - required = true) String variants, + + "annotate, e.g. " + + "19:45411941:T:C,14:38679764:-:GATCTG,1:6635210:G:-," + + "2:114340663:GCTGGGCATCCT:ACTGGGCATCCT", + required = true) String variants, @QueryParam("normalize") @ApiParam(name = "normalize", value = "Boolean to indicate whether input variants shall be " + "normalized or not. Normalization process does NOT " - + "include decomposing ", allowableValues = "false,true", - defaultValue = "false", required = false) Boolean normalize, - @QueryParam("skipDecompose") - @ApiParam(name = "skipDecompose", + + "include decomposing MNV nor left alignment", + allowableValues = "false,true", defaultValue = "false") Boolean normalize, + @QueryParam("decompose") + @ApiParam(name = "decompose", value = "Boolean to indicate whether input MNVs should be " - + "decomposed or not as part of the normalisation step." - + " MNV decomposition is strongly encouraged.", + + "decomposed or not as part of the normalisation step.", + allowableValues = "false,true", + defaultValue = "false") Boolean decompose, + @QueryParam("leftAlign") + @ApiParam(name = "leftAlign", + value = "Boolean to indicate whether input ambiguous INDELS should be " + + "left aligned or not as part of the normalisation step.", allowableValues = "false,true", - defaultValue = "false", required = false) Boolean skipDecompose, + defaultValue = "false") Boolean leftAlign, @QueryParam("ignorePhase") @ApiParam(name = "ignorePhase", value = "Boolean to indicate whether phase data should be " @@ -176,10 +194,10 @@ public Response getAnnotationByVariantsPOST(@ApiParam(name = "variants", value = defaultValue = "0", required = false) Integer cnvExtraPadding, @QueryParam("checkAminoAcidChange") @ApiParam(name = "checkAminoAcidChange", - value = "true/false to specify whether variant match in the clinical variant collection " - + "should also be performed at the aminoacid change level", - allowableValues = "false,true", - defaultValue = "false", required = false) Boolean checkAminoAcidChange, + value = "true/false to specify whether variant match in the clinical variant " + + "collection should also be performed at the aminoacid change level", + allowableValues = "false,true", + defaultValue = "false", required = false) Boolean checkAminoAcidChange, @QueryParam("consequenceTypeSource") @ApiParam(name = "consequenceTypeSource", value = "Gene set, either ensembl (default) " + "or refSeq", allowableValues = "ensembl,refseq", defaultValue = "ensembl", @@ -187,11 +205,17 @@ public Response getAnnotationByVariantsPOST(@ApiParam(name = "variants", value = @QueryParam("enable") @ApiParam(name = "enable", value = "Enable certain fields that are disabled by default, " + " e.g.: hgdm", hidden = true) String enable -) { + ) { + try { + checkNormalizationConfig(); + } catch (IllegalArgumentException e) { + return createErrorResponse(e); + } return getAnnotationByVariant(variants, normalize, - skipDecompose, + decompose, + leftAlign, ignorePhase, phased, imprecise, @@ -233,10 +257,14 @@ public Response getAnnotationByVariantsGET(@PathParam("variants") @ApiParam(name = "normalize", value = NORMALISE, allowableValues = "false,true", defaultValue = "true", required = false) Boolean normalize, - @QueryParam("skipDecompose") - @ApiParam(name = "skipDecompose", value = SKIP_DECOMPOSE, + @QueryParam("decompose") + @ApiParam(name = "decompose", value = DECOMPOSE, + allowableValues = "false,true", + defaultValue = "false") Boolean decompose, + @QueryParam("leftAlign") + @ApiParam(name = "leftAlign", value = LEFT_ALIGN, allowableValues = "false,true", - defaultValue = "false", required = false) Boolean skipDecompose, + defaultValue = "false") Boolean leftAlign, @QueryParam("ignorePhase") @ApiParam(name = "ignorePhase", value = IGNORE_PHASE, allowableValues = "false,true", @@ -265,15 +293,22 @@ public Response getAnnotationByVariantsGET(@PathParam("variants") Boolean checkAminoAcidChange, @QueryParam("consequenceTypeSource") @ApiParam(name = "consequenceTypeSource", value = "Gene set, either ensembl (default) " - + "or refseq", allowableValues = "ensembl,refseq", allowMultiple = true, + + "or refseq", allowableValues = "ensembl,refseq", allowMultiple = true, defaultValue = "ensembl", required = false) String consequenceTypeSource, @QueryParam("enable") @ApiParam(name = "enable", value = "Enable certain fields that are disabled by default, " - + " e.g.: hgdm", hidden = true) String enable + + " e.g.: hgdm", hidden = true) String enable ) { + try { + checkNormalizationConfig(); + } catch (IllegalArgumentException e) { + return createErrorResponse(e); + } + return getAnnotationByVariant(variants, normalize, - skipDecompose, + decompose, + leftAlign, ignorePhase, phased, imprecise, @@ -284,9 +319,26 @@ public Response getAnnotationByVariantsGET(@PathParam("variants") enable); } + private void checkNormalizationConfig() throws IllegalArgumentException { + if (uriParams.containsKey("skipDecompose")) { + throw new IllegalArgumentException("Param 'skipDecompose' is not supported anymore. Please, use 'decompose' instead"); + } + if (uriParams.containsKey("normalize")) { + if (!Boolean.parseBoolean(uriParams.get("normalize"))) { + if (uriParams.containsKey("decompose") && Boolean.parseBoolean(uriParams.get("decompose"))) { + throw new IllegalArgumentException("Incompatible parameter usage: 'normalize'=false and 'decompose'=true"); + } + if (uriParams.containsKey("leftAlign") && Boolean.parseBoolean(uriParams.get("leftAlign"))) { + throw new IllegalArgumentException("Incompatible parameter usage: 'normalize'=false and 'leftAlign'=true"); + } + } + } + } + private Response getAnnotationByVariant(String variants, Boolean normalize, - Boolean skipDecompose, + Boolean decompose, + Boolean leftAlign, Boolean ignorePhase, @Deprecated Boolean phased, Boolean imprecise, @@ -301,7 +353,7 @@ private Response getAnnotationByVariant(String variants, String consequenceTypeSources = (StringUtils.isEmpty(uriParams.get("consequenceTypeSource")) ? consequenceTypeSource : uriParams.get("consequenceTypeSource")); List> queryResults = variantManager.getAnnotationByVariant(query.toQueryOptions(), - variants, normalize, skipDecompose, ignorePhase, phased, imprecise, svExtraPadding, cnvExtraPadding, + variants, normalize, decompose, leftAlign, ignorePhase, phased, imprecise, svExtraPadding, cnvExtraPadding, checkAminoAcidChange, consequenceTypeSources, enable, getDataRelease()); return createOkResponse(queryResults); } catch (Exception e) { From dc00789f577ca093e9b7103c425e6d6d2804676b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 2 Jun 2023 17:45:10 +0200 Subject: [PATCH 22/28] server: fix NPE in the endpoint normalization when decompose or leftAlign is not set (i.e., null), #TASK-4530 --- .../opencb/cellbase/server/rest/genomic/VariantWSServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java index a277f52300..65558b0a6f 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java @@ -107,8 +107,8 @@ public Response getNormalization(@PathParam("variants") @ApiParam(name = "varian defaultValue = "false") Boolean leftAlign) { try { - CellBaseDataResult queryResults = variantManager.getNormalizationByVariant(id, decompose, leftAlign, - getDataRelease()); + CellBaseDataResult queryResults = variantManager.getNormalizationByVariant(id, Boolean.TRUE.equals(decompose), + Boolean.TRUE.equals(leftAlign), getDataRelease()); return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e); From a01f28859cea26c791809afa4132b1d3c3ea298b Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 6 Jun 2023 18:23:33 +0200 Subject: [PATCH 23/28] Prepare release 5.2.10 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 48fae603bd..c403881133 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 7e482d48d5..706364f3fc 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 09946b1af6..ca04f10742 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index f7fb2721ae..9ecbb35d01 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index b433c77a5c..0f9d409e3c 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/pom.xml b/pom.xml index 7a9b176c8f..4e1a693ccd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 pom CellBase project From d7dd0be4fbd71b8c01901c026ad1f5e1caccc722 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 6 Jun 2023 18:24:00 +0200 Subject: [PATCH 24/28] Prepare next release 5.2.11-SNAPSHOT --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index c403881133..6cfcf544fb 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10 + 5.2.11-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 706364f3fc..0772b9edfb 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10 + 5.2.11-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index ca04f10742..9892a6af3b 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10 + 5.2.11-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 9ecbb35d01..5d38ca7abd 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10 + 5.2.11-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 0f9d409e3c..8cd1faf036 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10 + 5.2.11-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 4e1a693ccd..1377a889de 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10 + 5.2.11-SNAPSHOT pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.6.0 - 2.6.0 + 4.6.1-SNAPSHOT + 2.6.1-SNAPSHOT 0.1.0 2.11.4 2.30.1 From d1d0eda1064ec71276aeaa5da2e03cb54409b230 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 6 Jun 2023 20:06:52 +0200 Subject: [PATCH 25/28] cicd: Reference to java-common-libs release-4.6.x instead develop branch --- .github/workflows/develop.yml | 6 +++--- .github/workflows/release.yml | 12 ++++++------ .github/workflows/task.yml | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index d87b660b36..a6e49b44f9 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -39,7 +39,7 @@ jobs: run: mvn -T 2 clean install -DskipTests -Dcheckstyle.skip sonar: - uses: opencb/java-common-libs/.github/workflows/sonar-analysis-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/sonar-analysis-workflow.yml@release-4.6.x needs: test with: sonarProjectKey: opencb_cellbase @@ -47,7 +47,7 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.6.x needs: test with: maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase @@ -58,7 +58,7 @@ jobs: MAVEN_GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.6.x needs: test with: cli: python3 ./build/cloud/docker/docker-build.py push --images base diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 627dcc12e8..8dd6f6231e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ on: jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.6.x with: maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase @@ -38,7 +38,7 @@ jobs: run: mvn -T 2 clean install -DskipTests -Dcheckstyle.skip sonar: - uses: opencb/java-common-libs/.github/workflows/sonar-analysis-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/sonar-analysis-workflow.yml@release-4.6.x needs: test with: sonarProjectKey: opencb_cellbase @@ -46,7 +46,7 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.6.x needs: test with: maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase @@ -57,7 +57,7 @@ jobs: MAVEN_GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.6.x needs: test with: cli: python3 ./build/cloud/docker/docker-build.py push --images base @@ -66,7 +66,7 @@ jobs: DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} deploy-python: - uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@release-4.6.x needs: test with: cli: bash ./clients/python/python-build.sh push @@ -76,7 +76,7 @@ jobs: PYPI_TWINE_PASSWORD: ${{ secrets.PYPI_TWINE_PASSWORD }} release: - uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@release-4.6.x needs: [deploy-maven, deploy-docker, deploy-python] # deploy: diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index 5b2132453e..d096e9816a 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -8,7 +8,7 @@ on: jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.6.x with: maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase @@ -38,7 +38,7 @@ jobs: run: mvn -T 2 clean install -DskipTests -Dcheckstyle.skip sonar: - uses: opencb/java-common-libs/.github/workflows/sonar-analysis-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/sonar-analysis-workflow.yml@release-4.6.x needs: test with: sonarProjectKey: opencb_cellbase @@ -46,7 +46,7 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.6.x needs: test with: cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag ${{ github.ref_name }} From 26a1f8f341d677656be63f87accc4b1ecca05ac1 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 6 Jun 2023 20:08:58 +0200 Subject: [PATCH 26/28] Prepare release 5.2.10 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 6cfcf544fb..48fae603bd 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.11-SNAPSHOT + 5.2.10-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 0772b9edfb..7e482d48d5 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.11-SNAPSHOT + 5.2.10-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 9892a6af3b..09946b1af6 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.11-SNAPSHOT + 5.2.10-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 5d38ca7abd..f7fb2721ae 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.11-SNAPSHOT + 5.2.10-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 8cd1faf036..b433c77a5c 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.11-SNAPSHOT + 5.2.10-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 1377a889de..358412c195 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.11-SNAPSHOT + 5.2.10-SNAPSHOT pom CellBase project From f4479ee64be570a56e5e5166f1baa5598f898ac9 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 6 Jun 2023 20:10:20 +0200 Subject: [PATCH 27/28] Prepare release 5.2.10 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 48fae603bd..c403881133 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 7e482d48d5..706364f3fc 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 09946b1af6..ca04f10742 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index f7fb2721ae..9ecbb35d01 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index b433c77a5c..0f9d409e3c 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 ../pom.xml diff --git a/pom.xml b/pom.xml index 358412c195..a7675c65e9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.2.10-SNAPSHOT + 5.2.10 pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.6.1-SNAPSHOT - 2.6.1-SNAPSHOT + 4.6.1 + 2.6.0 0.1.0 2.11.4 2.30.1 From d7128ced8f23b32b4ce4a05c30bfc9ad809085c4 Mon Sep 17 00:00:00 2001 From: imedina Date: Thu, 8 Jun 2023 10:31:22 +0100 Subject: [PATCH 28/28] cicd: fix war fine name --- .github/workflows/develop.yml | 2 ++ .github/workflows/release.yml | 2 ++ .github/workflows/task.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index d5a82a1c9a..f392092dd1 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -10,6 +10,8 @@ on: jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -DCELLBASE.WAR.NAME=cellbase test: uses: ./.github/workflows/test-analysis.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05191554c3..b13198c8e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,8 @@ on: jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -DCELLBASE.WAR.NAME=cellbase test: uses: ./.github/workflows/test-analysis.yml diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index c97f2b9790..5044527f44 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -9,6 +9,8 @@ on: jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -DCELLBASE.WAR.NAME=cellbase test: uses: ./.github/workflows/test-analysis.yml