diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index d5a82a1c9a..f5bdedf675 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -10,6 +10,8 @@ on: jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -DCELLBASE.WAR.NAME=cellbase test: uses: ./.github/workflows/test-analysis.yml @@ -20,14 +22,14 @@ jobs: report_dir: ${{ github.ref_name }}/cellbase/${{ github.sha }} deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.6.x needs: test with: maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase secrets: inherit deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.6.x needs: test with: cli: python3 ./build/cloud/docker/docker-build.py push --images base diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05191554c3..d84b955ed0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,8 @@ on: jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -DCELLBASE.WAR.NAME=cellbase test: uses: ./.github/workflows/test-analysis.yml @@ -19,21 +21,21 @@ jobs: report_dir: cellbase deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.6.x needs: test with: maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase secrets: inherit deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.6.x needs: test with: cli: python3 ./build/cloud/docker/docker-build.py push --images base secrets: inherit deploy-python: - uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@release-4.6.x needs: test with: cli: bash ./clients/python/python-build.sh push @@ -41,5 +43,5 @@ jobs: secrets: inherit release: - uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@release-4.6.x needs: [deploy-maven, deploy-docker, deploy-python] diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh new file mode 100644 index 0000000000..bf0f7bb12d --- /dev/null +++ b/.github/workflows/scripts/get_same_branch.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +BRANCH_NAME=$1 + +if [[ -z $BRANCH_NAME ]]; then + echo "The first parameter is mandatory and must be a valid branch name." + exit 1 +fi + +if [[ $BRANCH_NAME != "TASK-"* ]]; then + echo "No need to check dependencies." + exit 0 +fi + +function install(){ + local REPO=$1 + cd /home/runner/work/ || exit 2 + git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME" + if [ -d "./$REPO" ]; then + cd "$REPO" || exit 2 + echo "Branch name $BRANCH_NAME already exists." + mvn clean install -DskipTests + else + echo "$CURRENT Branch is NOT EQUALS $BRANCH_NAME " + fi +} + +install "java-common-libs" +install "biodata" diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index b75fec23e1..03a61e6134 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -5,10 +5,12 @@ on: branches: - TASK-* workflow_dispatch: - + jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -DCELLBASE.WAR.NAME=cellbase test: uses: ./.github/workflows/test-analysis.yml @@ -19,36 +21,10 @@ jobs: report_dir: ${{ github.ref_name }}/cellbase/${{ github.sha }} deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.6.x needs: test with: cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag ${{ github.ref_name }} secrets: inherit - deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop - needs: [ build, test ] - with: - maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase - if: contains( needs.build.outputs.version ,'TASK') - secrets: inherit - - #The following jobs are to see that the previous if does not fail and that the maven deploy is executed because it is true - snapshot-version: - name: Check SNAPSHOT version - needs: [ build, test ] - if: contains(needs.build.outputs.version ,'SNAPSHOT') - runs-on: ubuntu-22.04 - steps: - - name: test-version-from-check - run: echo "Project version is " ${{ needs.build.outputs.version }} - - task-version: - name: Check TASK version - needs: [ build, test ] - if: contains(needs.build.outputs.version ,'TASK') - runs-on: ubuntu-22.04 - steps: - - name: test-version-from-check - run: echo "Project version is " ${{ needs.build.outputs.version }} diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index 7795e57731..869398f86b 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -20,7 +20,7 @@ on: type: string required: true env: - xb_version: "1.6.1" + xb_version: "1.7.0" jobs: test: name: Test and push Sonar analysis @@ -40,6 +40,14 @@ jobs: with: mongodb-version: 6.0 mongodb-replica-set: rs-test + - name: Install dependencies branches + run: | + if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x ./.github/workflows/scripts/get_same_branch.sh + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + else + echo "./.github/workflows/scripts/get_same_branch.sh does not exist." + fi - name: Test and Analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any diff --git a/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile b/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile index 978c0506f4..0502319f49 100644 --- a/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile +++ b/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile @@ -1,5 +1,6 @@ -## Based on Debian 11 (bullseye) -FROM openjdk:11-jre +## Based on Ubuntu 22.04 (jammy) +## We are now using OpenJDK 8u372 to support "cgroup v2", see https://developers.redhat.com/articles/2023/04/19/openjdk-8u372-feature-cgroup-v2-support# +FROM eclipse-temurin:8u372-b07-jre-jammy LABEL org.label-schema.vendor="OpenCB" \ org.label-schema.name="cellbase-base" \ diff --git a/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile.alpine b/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile.alpine deleted file mode 100644 index 9960039781..0000000000 --- a/cellbase-app/app/cloud/docker/cellbase-base/Dockerfile.alpine +++ /dev/null @@ -1,22 +0,0 @@ -FROM openjdk:8-jre-alpine - -LABEL org.label-schema.vendor="OpenCB" \ - org.label-schema.name="cellbase-base" \ - org.label-schema.url="http://docs.opencb.org/display/cellbase" \ - org.label-schema.description="An Open Computational Genomics Analysis platform for big data processing and analysis in genomics" \ - maintainer="Julie Sullivan " \ - org.label-schema.schema-version="1.0" - -ENV CELLBASE_USER cellbase -ENV CELLBASE_HOME /opt/cellbase/ - -RUN apk update && apk upgrade && apk add ca-certificates openssl wget bash \ - && update-ca-certificates \ - && addgroup -S $CELLBASE_USER && adduser -S $CELLBASE_USER -G $CELLBASE_USER -u 1001 - -USER $CELLBASE_USER - -VOLUME /opt/cellbase/conf - -COPY . /opt/cellbase -WORKDIR /opt/cellbase diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index d4273c4208..0d4682584a 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java index 0a90cb5ea1..39018bf170 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/CommandExecutor.java @@ -18,13 +18,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; import org.opencb.cellbase.client.config.ClientConfiguration; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.commons.utils.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.logging.log4j.core.config.Configurator; import java.io.File; import java.io.FileInputStream; @@ -204,5 +204,4 @@ protected void makeDir(Path folderPath) throws IOException { Files.createDirectories(folderPath); } } - } diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java index 8ada7a9140..2224b43425 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/AdminCliOptionsParser.java @@ -145,8 +145,11 @@ public class DataReleaseCommandOptions { @Parameter(names = {"--list"}, description = "List the data releases present in the database", arity = 0) public boolean list; - @Parameter(names = {"--set-active"}, description = "Set the data release active", arity = 1) - public int active; + @Parameter(names = {"--update"}, description = "Data release to be updated by adding CellBase vesions", arity = 1) + public int update; + + @Parameter(names = {"--add-versions"}, description = "CellBase versions separated by commas, e.g.: v5.2,v5.3. This parameter has to be used together to the parameter --update", arity = 1) + public String versions; } @Parameters(commandNames = {"data-token"}, commandDescription = "Manage data access tokens in order to access to restricted/licensed data sources") diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/CustomiseCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/CustomiseCommandExecutor.java index 9b274cc491..8d7d07d40a 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/CustomiseCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/CustomiseCommandExecutor.java @@ -19,7 +19,9 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.app.cli.CommandExecutor; import org.opencb.cellbase.app.cli.admin.AdminCliOptionsParser; +import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.lib.loader.LoadRunner; +import org.opencb.cellbase.lib.loader.LoaderException; import org.opencb.commons.utils.FileUtils; import java.io.File; @@ -112,9 +114,9 @@ public void execute() { } } - private void loadIfExists(Path path, String collection) throws NoSuchMethodException, InterruptedException, - ExecutionException, InstantiationException, IOException, IllegalAccessException, InvocationTargetException, - ClassNotFoundException { + private void loadIfExists(Path path, String collection) throws NoSuchMethodException, InterruptedException, ExecutionException, + InstantiationException, IOException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, + LoaderException, CellBaseException { File file = new File(path.toString()); if (file.exists()) { if (file.isFile()) { diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DataReleaseCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DataReleaseCommandExecutor.java index b1ffe28b88..b46c4755c7 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DataReleaseCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/DataReleaseCommandExecutor.java @@ -17,12 +17,14 @@ package org.opencb.cellbase.app.cli.admin.executors; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.app.cli.CommandExecutor; import org.opencb.cellbase.app.cli.admin.AdminCliOptionsParser; import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.managers.DataReleaseManager; +import java.util.Arrays; import java.util.List; public class DataReleaseCommandExecutor extends CommandExecutor { @@ -51,29 +53,28 @@ public void execute() { DataReleaseManager dataReleaseManager = new DataReleaseManager(database, configuration); + DataRelease dataRelease; if (dataReleaseCommandOptions.create) { // Create release - DataRelease dataRelease = dataReleaseManager.createRelease(); + dataRelease = dataReleaseManager.createRelease(); System.out.println("\nData release " + dataRelease.getRelease() + " was created."); System.out.println("Data release description (in JSON format):"); System.out.println(new ObjectMapper().writerFor(DataRelease.class).writeValueAsString(dataRelease)); - } else if (dataReleaseCommandOptions.active > 0) { - // Set-active release - CellBaseDataResult results = dataReleaseManager.getReleases(); - for (DataRelease dr : results.getResults()) { - if (dr.isActive() && dr.getRelease() == dataReleaseCommandOptions.active) { - logger.info("Data release " + dataReleaseCommandOptions.active + " is already active"); - return; - } - } - DataRelease dataRelease = dataReleaseManager.active(dataReleaseCommandOptions.active); - if (dataRelease != null) { - System.out.println("\nThe data release " + dataRelease.getRelease() + " is the active one."); - System.out.println("Data release description (in JSON format):"); - System.out.println(new ObjectMapper().writerFor(DataRelease.class).writeValueAsString(dataRelease)); - } else { - logger.error("It could not set to active the data release " + dataReleaseCommandOptions.active); + } else if (dataReleaseCommandOptions.update > 0) { + if (StringUtils.isEmpty(dataReleaseCommandOptions.versions)) { + throw new IllegalArgumentException("Missing CellBase versions to be added when updating data release"); } + + // Versions to be added + List versions = Arrays.asList(dataReleaseCommandOptions.versions.split(",")); + + // Update data release + dataReleaseManager.update(dataReleaseCommandOptions.update, versions); + + dataRelease = dataReleaseManager.get(dataReleaseCommandOptions.update); + System.out.println("\nData release " + dataRelease.getRelease() + " was updated."); + System.out.println("Data release description (in JSON format):"); + System.out.println(new ObjectMapper().writerFor(DataRelease.class).writeValueAsString(dataRelease)); } else if (dataReleaseCommandOptions.list) { // List releases CellBaseDataResult dataReleases = dataReleaseManager.getReleases(); @@ -94,11 +95,11 @@ private void checkParameters() { if (dataReleaseCommandOptions.list) { opts++; } - if (dataReleaseCommandOptions.active > 0) { + if (dataReleaseCommandOptions.update > 0) { opts++; } - if (opts > 1) { - throw new IllegalArgumentException("Please, select only one of these input parameters: create, list or set-active"); + if (opts != 1) { + throw new IllegalArgumentException("Please, select only one of these input parameters: create, update or list"); } } } diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java index 155bf1618e..3cf1d8089b 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java @@ -17,6 +17,7 @@ package org.opencb.cellbase.app.cli.admin.executors; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.app.cli.CommandExecutor; import org.opencb.cellbase.app.cli.admin.AdminCliOptionsParser; import org.opencb.cellbase.core.exception.CellBaseException; @@ -105,7 +106,7 @@ public void execute() throws CellBaseException { dataReleaseManager = new DataReleaseManager(database, configuration); checkParameters(); - logger.info("Loading in data release " + dataRelease); + logger.info("Loading in data release {}", dataRelease); if (loadCommandOptions.data != null) { // If 'authenticationDatabase' is not passed by argument then we read it from configuration.json @@ -113,7 +114,7 @@ public void execute() throws CellBaseException { configuration.getDatabases().getMongodb().getOptions().put("authenticationDatabase", loadCommandOptions.loaderParams.get("authenticationDatabase")); } - loadRunner = new LoadRunner(loader, database, numThreads, configuration); + loadRunner = new LoadRunner(loader, database, numThreads, dataReleaseManager, configuration); if (createIndexes) { Path indexFile = Paths.get(this.appHome).resolve("conf").resolve("mongodb-indexes.json"); indexManager = new IndexManager(database, indexFile, configuration); @@ -320,7 +321,7 @@ public void execute() throws CellBaseException { private void loadIfExists(Path path, String collection) throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IOException, IllegalAccessException, InvocationTargetException, - ClassNotFoundException { + ClassNotFoundException, LoaderException, CellBaseException { File file = new File(path.toString()); if (file.exists()) { if (file.isFile()) { @@ -359,53 +360,20 @@ private void checkParameters() throws CellBaseException { } // Check data release - CellBaseDataResult dataReleaseResults = dataReleaseManager.getReleases(); - if (CollectionUtils.isEmpty(dataReleaseResults.getResults())) { - throw new CellBaseException("No data releases are available for database " + database); - } - int lastDataRelease = 0; - int defaultDataRelease = 0; - for (DataRelease dataRelease : dataReleaseResults.getResults()) { - if (dataRelease.isActive()) { - defaultDataRelease = dataRelease.getRelease(); - } - if (dataRelease.getRelease() > lastDataRelease) { - lastDataRelease = dataRelease.getRelease(); - } - } - if (lastDataRelease == defaultDataRelease) { - throw new CellBaseException("Loading data in the active data release (" + defaultDataRelease + ") is not permitted."); - } - dataRelease = lastDataRelease; - -// for (DataRelease dr : dataReleaseResults.getResults()) { -// if (dr.getRelease() == dataRelease) { -// for (String loadOption : loadOptions) { -// if (dr.getCollections().containsKey(loadOption)) { -// String collectionName = CellBaseDBAdaptor.buildCollectionName(loadOption, dataRelease); -// if (dr.getCollections().get(loadOption).equals(collectionName)) { -// throw new CellBaseException("Impossible load data " + loadOption + " with release " + dataRelease + " since it" -// + " has already been done."); -// } -// } -// } -// break; -// } -// } + dataRelease = getDataReleaseForLoading(dataReleaseManager).getRelease(); } private void loadVariationData() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException, LoaderException { + IOException, LoaderException, CellBaseException { // First load data // Common loading process from CellBase variation data models if (field == null) { - DirectoryStream stream = Files.newDirectoryStream(input, entry -> { - return entry.getFileName().toString().startsWith("variation_chr"); - }); + DirectoryStream stream = Files.newDirectoryStream(input, + entry -> entry.getFileName().toString().startsWith("variation_chr")); for (Path entry : stream) { - logger.info("Loading file '{}'", entry.toString()); + logger.info("Loading file '{}'", entry); loadRunner.load(input.resolve(entry.getFileName()), "variation", dataRelease); } @@ -420,20 +388,20 @@ private void loadVariationData() throws NoSuchMethodException, InterruptedExcept // Custom update required e.g. population freqs loading } else { - logger.info("Loading file '{}'", input.toString()); + logger.info("Loading file '{}'", input); loadRunner.load(input, "variation", dataRelease, field, innerFields); } } private void loadConservation() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException { + IOException, CellBaseException, LoaderException { // Load data - DirectoryStream stream = Files.newDirectoryStream(input, entry -> { - return entry.getFileName().toString().startsWith("conservation_"); - }); + DirectoryStream stream = Files.newDirectoryStream(input, + entry -> entry.getFileName().toString().startsWith("conservation_")); + for (Path entry : stream) { - logger.info("Loading file '{}'", entry.toString()); + logger.info("Loading file '{}'", entry); loadRunner.load(input.resolve(entry.getFileName()), "conservation", dataRelease); } @@ -451,13 +419,13 @@ private void loadConservation() throws NoSuchMethodException, InterruptedExcepti private void loadProteinFunctionalPrediction() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException { + IOException, CellBaseException, LoaderException { // Load data - DirectoryStream stream = Files.newDirectoryStream(input, entry -> { - return entry.getFileName().toString().startsWith("prot_func_pred_"); - }); + DirectoryStream stream = Files.newDirectoryStream(input, + entry -> entry.getFileName().toString().startsWith("prot_func_pred_")); + for (Path entry : stream) { - logger.info("Loading file '{}'", entry.toString()); + logger.info("Loading file '{}'", entry); loadRunner.load(input.resolve(entry.getFileName()), "protein_functional_prediction", dataRelease); } @@ -487,11 +455,13 @@ private void loadClinical() throws FileNotFoundException { )); dataReleaseManager.update(dataRelease, "clinical_variants", EtlCommons.CLINICAL_VARIANTS_DATA, sources); } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException - | IllegalAccessException | ExecutionException | IOException | InterruptedException e) { + | IllegalAccessException | ExecutionException | IOException | InterruptedException | CellBaseException e) { logger.error(e.toString()); + } catch (LoaderException e) { + e.printStackTrace(); } } else { - throw new FileNotFoundException("File " + path.toString() + " does not exist"); + throw new FileNotFoundException("File " + path + " does not exist"); } } @@ -514,18 +484,19 @@ private void loadRepeats() { )); dataReleaseManager.update(dataRelease, "repeats", EtlCommons.REPEATS_DATA, sources); } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException - | IllegalAccessException | ExecutionException | IOException | InterruptedException e) { + | IllegalAccessException | ExecutionException | IOException | InterruptedException | CellBaseException e) { logger.error(e.toString()); + } catch (LoaderException e) { + e.printStackTrace(); } } else { - logger.warn("Repeats file {} not found", path.toString()); + logger.warn("Repeats file {} not found", path); logger.warn("No repeats data will be loaded"); } } - private void loadSpliceScores() throws NoSuchMethodException, InterruptedException, ExecutionException, - InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, - IOException { + private void loadSpliceScores() throws NoSuchMethodException, InterruptedException, ExecutionException, InstantiationException, + IllegalAccessException, InvocationTargetException, ClassNotFoundException, IOException, CellBaseException, LoaderException { // Load data logger.info("Loading splice scores from '{}'", input); // MMSplice scores @@ -545,19 +516,20 @@ private void loadSpliceScores() throws NoSuchMethodException, InterruptedExcepti } private void loadSpliceScores(Path spliceFolder) throws IOException, ExecutionException, InterruptedException, - ClassNotFoundException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + ClassNotFoundException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, + LoaderException, CellBaseException { // Get files from folder - DirectoryStream stream = Files.newDirectoryStream(spliceFolder, entry -> { - return entry.getFileName().toString().startsWith("splice_score_"); - }); + DirectoryStream stream = Files.newDirectoryStream(spliceFolder, + entry -> entry.getFileName().toString().startsWith("splice_score_")); + // Load from JSON files for (Path entry : stream) { - logger.info("Loading file '{}'", entry.toString()); + logger.info("Loading file '{}'", entry); loadRunner.load(spliceFolder.resolve(entry.getFileName()), "splice_score", dataRelease); } } - private void loadPubMed() { + private void loadPubMed() throws CellBaseException { Path pubmedPath = input.resolve(EtlCommons.PUBMED_DATA); if (Files.exists(pubmedPath)) { @@ -568,7 +540,7 @@ private void loadPubMed() { try { loadRunner.load(file.toPath(), EtlCommons.PUBMED_DATA, dataRelease); } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException - | IllegalAccessException | ExecutionException | IOException | InterruptedException e) { + | IllegalAccessException | ExecutionException | IOException | InterruptedException | LoaderException e) { logger.error("Error loading file '{}': {}", file.getName(), e.toString()); } } @@ -580,7 +552,7 @@ private void loadPubMed() { List sources = Collections.singletonList(pubmedPath.resolve(EtlCommons.PUBMED_VERSION_FILENAME)); dataReleaseManager.update(dataRelease, "pubmed", EtlCommons.REPEATS_DATA, sources); } else { - logger.warn("PubMed folder {} not found", pubmedPath.toString()); + logger.warn("PubMed folder {} not found", pubmedPath); } } @@ -593,7 +565,31 @@ private void createIndex(String collection) { try { indexManager.createMongoDBIndexes(Collections.singletonList(collectionName), true); } catch (IOException e) { - logger.error("Error creating index: " + e.getMessage()); + logger.error("Error creating index: {}", e.getMessage()); + } + } + + private DataRelease getDataReleaseForLoading(DataReleaseManager dataReleaseManager) throws CellBaseException { + // Check data release + CellBaseDataResult dataReleaseResults = dataReleaseManager.getReleases(); + if (CollectionUtils.isEmpty(dataReleaseResults.getResults())) { + throw new CellBaseException("No data releases are available"); + } + DataRelease lastDataRelease = null; + for (DataRelease dataRelease : dataReleaseResults.getResults()) { + if (lastDataRelease == null) { + lastDataRelease = dataRelease; + } else if (dataRelease.getRelease() > lastDataRelease.getRelease()) { + lastDataRelease = dataRelease; + } + } + if (lastDataRelease == null) { + throw new CellBaseException("Loading data is not permitted since no data release is found"); + } + if (CollectionUtils.isNotEmpty(lastDataRelease.getActiveByDefaultIn())) { + throw new CellBaseException("Loading data is not permitted for data release " + lastDataRelease.getRelease() + " since it has" + + " already assigned CellBase versions:" + StringUtils.join(lastDataRelease.getActiveByDefaultIn(), ",")); } + return lastDataRelease; } } diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java index 188820896a..2d7f19b1d1 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java @@ -67,13 +67,13 @@ public ValidationCommandExecutor(AdminCliOptionsParser.ValidationCommandOptions public void execute() { checkFilesExist(); - CellBaseManagerFactory cellbaseManagerFactory = new CellBaseManagerFactory(configuration); + CellBaseManagerFactory cellBaseManagerFactory = new CellBaseManagerFactory(configuration); // dbAdaptorFactory = new MongoDBAdaptorFactory(configuration); VariantAnnotationCalculator variantAnnotationCalculator = null; try { variantAnnotationCalculator = new VariantAnnotationCalculator(validationCommandOptions.species, validationCommandOptions.assembly, validationCommandOptions.dataRelease, validationCommandOptions.token, - cellbaseManagerFactory); + cellBaseManagerFactory); } catch (CellBaseException e) { e.printStackTrace(); return; diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java index e73260e7b3..91101246b9 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java @@ -145,30 +145,30 @@ public class VariantAnnotationCommandOptions { required = false, arity = 1) public String referenceFasta; - @Parameter(names = {"--skip-normalize"}, description = "Skip normalization of input variants. Should not be used" + @Parameter(names = {"normalize"}, description = "Normalize input variants. Should be used" + " when the input (-i, --input-file) is a VCF file. Normalization includes splitting multi-allele positions " + "read from a VCF, allele trimming and decomposing MNVs. Has" + " no effect if reading variants from a CellBase variation collection " + "(\"--input-variation-collection\") or running a variant annotation benchmark (\"--benchmark\"): in" + " these two cases variant normalization is never carried out.", required = false, arity = 0) - public boolean skipNormalize = false; + public boolean normalize = false; - @Parameter(names = {"--skip-decompose"}, description = "Use this flag to avoid decomposition of " + @Parameter(names = {"--decompose"}, description = "Use this flag to decompose of " + "multi-nucleotide-variants (MNVs) / block substitutions as part of the normalization process. If this" - + " flag is NOT activated, as a step during the normalization process reference and alternate alleles" + + " flag is activated, as a step during the normalization process reference and alternate alleles" + " from MNVs/Block substitutions will be aligned and decomposed into their forming simple variants. " - + " This flag has no effect if --skip-normalize is present.", + + " This flag has no effect without normalization.", required = false, arity = 0) - public boolean skipDecompose = false; + public boolean decompose = false; - @Parameter(names = {"--skip-left-align"}, description = "Use this flag to avoid left alignment as part of the" + @Parameter(names = {"--left-align"}, description = "Use this flag left align as part of the" + " normalization process. If this" - + " flag is NOT activated, as a step during the normalization process will left align the variant with" + + " flag is activated, as a step during the normalization process will left align the variant with" + " respect to the reference genome." - + " This flag has no effect if --skip-normalize is present.", + + " This flag has no effect without normalization.", required = false, arity = 0) - public boolean skipLeftAlign = false; + public boolean leftAlign = false; // TODO: remove "phased" CLI parameter in next release. Default behavior from here onwards should be // ignorePhase = false diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java index a40ff30512..d6e19c61e2 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/VariantAnnotationCommandExecutor.java @@ -478,9 +478,10 @@ private VariantAnnotator createCellBaseAnnotator() throws CellBaseException { // Normalization should just be performed in one place: before calling the annotation calculator - within the // corresponding *AnnotatorTask since the AnnotatorTasks need that the number of sent variants coincides // equals the number of returned annotations + CellBaseManagerFactory cellBaseManagerFactory = new CellBaseManagerFactory(configuration); return new CellBaseLocalVariantAnnotator(new VariantAnnotationCalculator(species, assembly, - variantAnnotationCommandOptions.dataRelease, variantAnnotationCommandOptions.token, - new CellBaseManagerFactory(configuration)), serverQueryOptions); + variantAnnotationCommandOptions.dataRelease, variantAnnotationCommandOptions.token, cellBaseManagerFactory), + serverQueryOptions); } else { try { ClientConfiguration clientConfiguration = ClientConfiguration.load(getClass() @@ -563,7 +564,7 @@ private void checkParameters() throws IOException, CellBaseException { FileUtils.checkDirectory(input); normalize = false; } else { - normalize = !variantAnnotationCommandOptions.skipNormalize; + normalize = variantAnnotationCommandOptions.normalize; FileUtils.checkFile(input); inputFormat = getFileFormat(input); } @@ -573,8 +574,8 @@ private void checkParameters() throws IOException, CellBaseException { } parsePhaseConfiguration(); - decompose = !variantAnnotationCommandOptions.skipDecompose; - leftAlign = !variantAnnotationCommandOptions.skipLeftAlign; + decompose = variantAnnotationCommandOptions.decompose; + leftAlign = variantAnnotationCommandOptions.leftAlign; // Update serverQueryOptions serverQueryOptions.put("checkAminoAcidChange", variantAnnotationCommandOptions.checkAminoAcidChange); @@ -620,7 +621,7 @@ private void checkParameters() throws IOException, CellBaseException { // to the server. Actual normalization and decomposition options are set and processed here in the server code // using this.decompose and this.normalize fields. serverQueryOptions.add("normalize", false); - serverQueryOptions.add("skipDecompose", true); + serverQueryOptions.add("decompose", false); if (variantAnnotationCommandOptions.include != null && !variantAnnotationCommandOptions.include.isEmpty()) { serverQueryOptions.add("include", variantAnnotationCommandOptions.include); diff --git a/cellbase-app/src/test/java/org/opencb/cellbase/app/cli/VariantAnnotationCommandExecutorTest.java b/cellbase-app/src/test/java/org/opencb/cellbase/app/cli/VariantAnnotationCommandExecutorTest.java index 9ce1fcb3d2..22b93d33bd 100644 --- a/cellbase-app/src/test/java/org/opencb/cellbase/app/cli/VariantAnnotationCommandExecutorTest.java +++ b/cellbase-app/src/test/java/org/opencb/cellbase/app/cli/VariantAnnotationCommandExecutorTest.java @@ -935,9 +935,9 @@ private void cleanUp() throws IOException { variantAnnotationCommandOptions.benchmark = false; variantAnnotationCommandOptions.phased = true; variantAnnotationCommandOptions.input = inputFilename; - variantAnnotationCommandOptions.skipNormalize = false; - variantAnnotationCommandOptions.skipDecompose = !decompose; - variantAnnotationCommandOptions.skipLeftAlign = false; + variantAnnotationCommandOptions.normalize = true; + variantAnnotationCommandOptions.decompose = decompose; + variantAnnotationCommandOptions.leftAlign = true; variantAnnotationCommandOptions.output = Paths.get(OUTPUT_FILENAME).toString(); variantAnnotationCommandOptions.outputFormat = "json"; variantAnnotationCommandOptions.include = "cytobands"; diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 3ac36256cd..74791e4ff2 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-client/src/main/resources/client-configuration.yml b/cellbase-client/src/main/resources/client-configuration.yml index f13e1f33fa..5db9eeaca8 100644 --- a/cellbase-client/src/main/resources/client-configuration.yml +++ b/cellbase-client/src/main/resources/client-configuration.yml @@ -1,10 +1,10 @@ --- -version: "v5" +version: "v5.1" defaultSpecies: "hsapiens" ## These are the RESTful configurations parameters rest: hosts: - - "bioinfodev.hpc.cam.ac.uk/cellbase-5.0.0" - timeout: 2000 + - "https://ws.zettagenomics.com/cellbase" + timeout: 10000 diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index b84dd787b0..435b560692 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java index 9167239ad7..a07f6daa64 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/ParamConstants.java @@ -109,17 +109,16 @@ public Type type() { public static final String DOT_NOTATION_NOTE = "Parameters can be camel case (e.g. transcriptsBiotype) " + "or dot notation (e.g. transcripts.biotype)."; - public static final String VERSION_DESCRIPTION = "Possible values: v5"; - public static final String DEFAULT_VERSION = "v5"; - public static final String DATA_RELEASE_DESCRIPTION = "Data release. To use the default data release, set this to 0. Please," - + " check the web service /metadata/dataReleases in order to know the different data releases and sources, and the default" - + " data release"; - public static final String DEFAULT_DATA_RELEASE = "0"; + public static final String DEFAULT_VERSION = "v5.5"; + public static final String VERSION_DESCRIPTION = "API version, e.g.: " + DEFAULT_VERSION; + public static final String DATA_RELEASE_DESCRIPTION = "Data release. To use the default data release, set this to 0. To get the list" + + " of available data release, please call the endpoint 'meta/dataReleases'"; public static final String DATA_ACCESS_TOKEN_DESCRIPTION = "Data token to allow access to licensed/restricted data sources such as" + " Cosmic or HGMD"; + public static final String DEFAULT_ASSEMBLY = "grch38"; public static final String ASSEMBLY_DESCRIPTION = "Set the reference genome assembly, e.g. grch38. For a full list of " - + "potentially available assemblies, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species"; + + "potentially available assemblies, please call the endpoint 'meta/species'"; + public static final String COUNT_DESCRIPTION = "Get the total number of results matching the query. "; @@ -136,9 +135,9 @@ public Type type() { public static final String SORT_DESCRIPTION = "Sort returned results by a certain data model attribute"; public static final String ORDER_DESCRIPTION = "Results are in ascending order by default"; + public static final String DEFAULT_SPECIES = "hsapiens"; public static final String SPECIES_DESCRIPTION = "Name of the species, e.g. hsapiens. For a full list " - + "of potentially available species ids, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species"; + + "of potentially available species ids, please call the endpoint 'meta/species'"; public static final String DATA_MODEL_DESCRIPTION = "Get JSON specification of the data model"; public static final String REGION_DESCRIPTION = "Comma separated list of genomic regions to be queried, " @@ -175,8 +174,8 @@ public Type type() { public static final String ONTOLOGY_NAMES = "Comma separated list of ontology term names, " + "e.g. Diabetes mellitus,histone kinase activity"; public static final String ONTOLOGY_NAMESPACES = "Comma separated list of namespaces, e.g. human_phenotype,biological_process. " - + "For a full list of potentially available namespaces, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/feature/ontology/distinct?field=namespace"; + + "For a full list of potentially available namespaces, please call the endpoint 'feature/ontology/distinct?field=namespace'"; + public static final String ONTOLOGY_SOURCES = "Name of ontology, e.g. HP or GO"; public static final String ONTOLOGY_SYNONYMS = "Comma separated list of synonyms, e.g. Cerebellar ataxia"; public static final String ONTOLOGY_XREFS = "Comma separated list of cross references, e.g. MSH:D002524"; @@ -302,8 +301,9 @@ public Type type() { + " Possible values are clinvar or cosmic"; public static final String SOURCE_PARAM = "source"; public static final String SEQUENCE_ONTOLOGY_DESCRIPTION = "Comma separated list of consequence types, " - + "e.g. missense_variant. Exact text matches will be returned. A list of searchable consequence types can be" - + " accessed at https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/feature/variation/consequence_types"; + + + "e.g. missense_variant. Exact text matches will be returned. To get list of searchable consequence types, please call" + + " the endpoint 'genomic/variant/consequenceTypes'"; public static final String SEQUENCE_ONTOLOGY_PARAM = "consequenceType"; public static final String FEATURE_IDS_DESCRIPTION = "Comma separated list of feature ids, which can be either ENSEMBL gene " + "ids, HGNC gene symbols, transcript symbols or ENSEMBL transcript ids, e.g.: BRCA2,ENST00000409047. " @@ -320,26 +320,22 @@ public Type type() { public static final String VARIANT_IDS_DESCRIPTION = "Comma separated list of ids, e.g. rs6025,COSM306824. " + "Exact text matches will be returned."; public static final String VARIANT_IDS_PARAM = "id"; - public static final String VARIANT_TYPES_DESCRIPTION = "Comma separated list of variant types, e.g. \"SNV\" A list of " - + "searchable types can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/type"; + public static final String VARIANT_TYPES_DESCRIPTION = "Comma separated list of variant types, e.g. \"SNV\". To get the list of " + + "searchable types, please call the endpoint 'clinical/variant/type'"; public static final String VARIANT_TYPES_PARAM = "type"; - public static final String CONSISTENCY_STATUS_DESCRIPTION = "Comma separated list of consistency labels. A list of searchable " - + "consistency labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/consistency_labels"; + public static final String CONSISTENCY_STATUS_DESCRIPTION = "Comma separated list of consistency labels. To get the list of searchable" + + " consistency labels, please call the endpoint 'clinical/variant/consistencyLabels'"; public static final String CONSISTENCY_STATUS_PARAM = "consistencyStatus"; - public static final String CLINICAL_SIGNFICANCE_DESCRIPTION = "Comma separated list of clinical significance labels. " - + "A list of searchable clinical significance labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/clinsig_labels" + public static final String CLINICAL_SIGNFICANCE_DESCRIPTION = "Comma separated list of clinical significance labels. To get the list of" + + " searchable clinical significance labels, please call the endpoint 'clinical/variant/clinsigLabels'. " + " WARNING: returned numTotalResults will always be -1 if more than 1 label is provided."; public static final String CLINICAL_SIGNFICANCE_PARAM = "clinicalSignificance"; - public static final String MODE_INHERITANCE_DESCRIPTION = "Comma separated list of mode of inheritance labels. A list of " - + "searchable mode of inheritance labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/mode_inheritance_labels"; + public static final String MODE_INHERITANCE_DESCRIPTION = "Comma separated list of mode of inheritance labels. To the the list of " + + "searchable mode of inheritance labels, please call the endpoint 'clinical/variant/modeInheritanceLabels'"; public static final String MODE_INHERITANCE_PARAM = "modeInheritance"; - public static final String ALLELE_ORIGIN_DESCRIPTION = "Comma separated list of allele origin labels. A list of searchable " - + "allele origin labels can be accessed at " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/clinical/variant/allele_origin_labels"; + public static final String ALLELE_ORIGIN_DESCRIPTION = "Comma separated list of allele origin labels. To get the list of searchable " + + "allele origin labels, please call the endpoint 'clinical/variant/alleleOriginLabels'"; + public static final String ALLELE_ORIGIN_PARAM = "alleleOrigin"; // --------------------------------------------- @@ -348,9 +344,11 @@ public Type type() { + "19:45411941:T:C,14:38679764:-:GATCTG,1:6635210:G:-," + "2:114340663:GCTGGGCATCCT:ACTGGGCATCCT,1:816505-825225:"; public static final String NORMALISE = "Boolean to indicate whether input variants shall be " - + "normalized or not. Normalization process does NOT include decomposing "; - public static final String SKIP_DECOMPOSE = "Boolean to indicate whether input MNVs should be " - + "decomposed or not as part of the normalisation step. MNV decomposition is strongly encouraged."; + + "normalized or not. Normalization process does NOT include decomposing nor left alignment."; + public static final String DECOMPOSE = "Boolean to indicate whether input MNVs should be " + + "decomposed or not as part of the normalisation step."; + public static final String LEFT_ALIGN = "Boolean to indicate whether input ambiguous INDELS should be " + + "left aligned or not as part of the normalisation step."; public static final String IGNORE_PHASE = "Boolean to indicate whether phase data should be taken into account."; public static final String PHASED = "DEPRECATED. Will be removed in next release. Please, use ignorePhase instead. " + " Boolean to indicate whether phase should be considered during the annotation process"; @@ -378,21 +376,14 @@ public Type type() { // --------------------------------------------- - public static final String XREF_DBNAMES = "Comma separated list of source DB names" - + " to include in the search, e.g.: ensembl_gene,vega_gene,havana_gene." - + " Available db names are shown by this web service: " - + " https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/#!/Xref/getDBNames"; + public static final String XREF_DBNAMES = "Comma separated list of source DB names to include in the search, e.g.: ensembl_gene," + + "vega_gene,havana_gene."; // --------------------------------------------- - // public static final String REGULATION_FEATURE_CLASSES = "Comma separated list of regulatory region classes, e.g.:" -// + "Histone,Transcription Factor. Exact text matches will be returned. For a full" -// + "list of available regulatory types: " -// + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/regulatory/featureClass"; public static final String REGULATION_FEATURE_TYPES = "Comma separated list of regulatory region types, e.g.: " - + "TF_binding_site,histone_acetylation_site. Exact text matches will be returned. For a full" - + "list of available regulatory types: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/hsapiens/regulatory/featureType"; + + "TF_binding_site,histone_acetylation_site. Exact text matches will be returned."; + public static final String CELLTYPE = "Type of cell."; diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/query/AbstractQuery.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/query/AbstractQuery.java index b9fc9030c5..50b0451e5f 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/query/AbstractQuery.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/query/AbstractQuery.java @@ -172,11 +172,6 @@ public void updateParams(Map uriParams) { } } objectMapper.updateValue(this, objectHashMap); - - // Initialize the data release (0 means to use the default data release) - if (dataRelease == null) { - dataRelease = 0; - } } catch (JsonProcessingException e) { throw new IllegalArgumentException(e); } diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java index 375542b498..5674839aa8 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/models/DataRelease.java @@ -16,27 +16,31 @@ package org.opencb.cellbase.core.models; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; public class DataRelease { private int release; private String date; + /** + * @deprecated it is maintained to back-compatibility with previous CellBase versions to v5.5 + */ + @Deprecated private boolean active; + private List activeByDefaultIn; private Map collections; private List sources; public DataRelease() { + this.activeByDefaultIn = Collections.emptyList(); this.collections = Collections.emptyMap(); this.sources = Collections.emptyList(); } - public DataRelease(int release, String date, boolean active, Map collections, List sources) { + public DataRelease(int release, String date, List activeByDefaultIn, Map collections, + List sources) { this.release = release; this.date = date; - this.active = active; + this.activeByDefaultIn = activeByDefaultIn; this.collections = collections; this.sources = sources; } @@ -46,7 +50,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("DataRelease{"); sb.append("release=").append(release); sb.append(", date='").append(date).append('\''); - sb.append(", active=").append(active); + sb.append(", activeByDefaultIn=").append(activeByDefaultIn); sb.append(", collections=").append(collections); sb.append(", sources=").append(sources); sb.append('}'); @@ -80,6 +84,15 @@ public DataRelease setActive(boolean active) { return this; } + public List getActiveByDefaultIn() { + return activeByDefaultIn; + } + + public DataRelease setActiveByDefaultIn(List activeByDefaultIn) { + this.activeByDefaultIn = activeByDefaultIn; + return this; + } + public Map getCollections() { return collections; } @@ -108,7 +121,7 @@ public boolean equals(Object o) { } DataRelease that = (DataRelease) o; return release == that.release - && active == that.active + && Objects.equals(activeByDefaultIn, that.activeByDefaultIn) && Objects.equals(date, that.date) && Objects.equals(collections, that.collections) && Objects.equals(sources, that.sources); @@ -116,6 +129,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(release, date, active, collections, sources); + return Objects.hash(release, date, activeByDefaultIn, collections, sources); } } diff --git a/cellbase-core/src/test/resources/configuration.yml b/cellbase-core/src/test/resources/configuration.yml index b72d1318a1..c9a5d98743 100644 --- a/cellbase-core/src/test/resources/configuration.yml +++ b/cellbase-core/src/test/resources/configuration.yml @@ -2,7 +2,7 @@ version: v5 apiVersion: "${project.version}" wiki: https://github.com/opencb/cellbase/wiki maintenanceFlagFile: "/tmp/maintenance" -maintainerContact: javier.lopez@genomicsengland.co.uk +maintainerContact: ignacio.medina@zettagenomics.com logLevel: "INFO" logDir: "./logs" # where to output the logs diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index c426ebee60..b2e20a423b 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java index 8726219505..bbe33017fd 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java @@ -25,6 +25,7 @@ import org.opencb.biodata.models.variant.avro.*; import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.cellbase.lib.variant.VariantAnnotationUtils; +import org.opencb.commons.utils.PrintUtils; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.slf4j.Logger; @@ -63,6 +64,8 @@ public abstract class ClinicalIndexer { mapper.configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true); objectReader = mapper.readerFor(VariantAnnotation.class); jsonObjectWriter = mapper.writer(); + + PrintUtils.printSpace(); // jsonObjectWriter = mapper.writerFor(VariantAnnotation.class); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java index 5a919ebe33..afbcefc162 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java @@ -45,10 +45,6 @@ public Map buildCollectionByReleaseMap(String data) if (dataRelease.getCollections().containsKey(data)) { String collectionName = dataRelease.getCollections().get(data); collectionMap.put(dataRelease.getRelease(), mongoDataStore.getCollection(collectionName)); - if (dataRelease.isActive()) { - // Associate the default data release to the key 0 in the map - collectionMap.put(0, mongoDataStore.getCollection(collectionName)); - } } } } else { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java index ed30db95a3..c5d2bd599f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java @@ -234,6 +234,7 @@ public CellBaseDataResult getVariantAnnotation(String if (!aaAlternate.equals("STOP") && !aaReference.equals("STOP")) { TranscriptQuery query = new TranscriptQuery(); query.setTranscriptsId(Collections.singletonList(ensemblTranscriptId)); + query.setDataRelease(dataRelease); proteinVariantAnnotation.setSubstitutionScores(getSubstitutionScores(query, position, aaAlternate).getResults()); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java index fc2cc31455..48fda65d28 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java @@ -18,13 +18,17 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.TransactionOptions; +import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; +import org.apache.commons.lang3.StringUtils; import org.bson.BsonDocument; import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.cellbase.core.api.query.AbstractQuery; import org.opencb.cellbase.core.api.query.ProjectionQueryOptions; +import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.iterator.CellBaseIterator; @@ -34,7 +38,10 @@ import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDataStore; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class ReleaseMongoDBAdaptor extends MongoDBAdaptor implements CellBaseCoreDBAdaptor { @@ -62,12 +69,61 @@ public DataResult insert(DataRelease dataRelease) throws JsonProcessingException return mongoDBCollection.insert(document, QueryOptions.empty()); } - public void update(int release, String field, Object value) { + public DataResult update(int release, List versions) throws CellBaseException { + DataRelease currDataRelease = mongoDBCollection.find(Filters.eq("release", release), null, DataRelease.class, QueryOptions.empty()) + .first(); + + Map> toUpdate = new HashMap<>(); + for (String version : versions) { + DataResult result = mongoDBCollection.find(Filters.eq("activeByDefaultIn", version), null, DataRelease.class, + QueryOptions.empty()); + if (result.getNumResults() > 1) { + throw new CellBaseException("There's something wrong in the CellBase MongoDB. CellBase version " + version + " has" + + " multiple data releases: " + StringUtils.join(result.getResults().stream().map(dr -> dr.getRelease()), ",")); + } + if (result.getNumResults() == 1) { + DataRelease dr = result.first(); + if (!toUpdate.containsKey(dr.getRelease())) { + toUpdate.put(dr.getRelease(), dr.getActiveByDefaultIn()); + } + toUpdate.get(dr.getRelease()).remove(version); + } + } + + // Start a transaction + ClientSession session = mongoDataStore.startSession(); + try { + session.startTransaction(TransactionOptions.builder().build()); + + // Update data releases by removing versions + for (Map.Entry> entry : toUpdate.entrySet()) { + update(entry.getKey(), "activeByDefaultIn", entry.getValue()); + } + + // Update data release by adding versions + List vers = new ArrayList<>(currDataRelease.getActiveByDefaultIn()); + vers.addAll(versions); + update(release, "activeByDefaultIn", vers); + + // Commit the transaction + session.commitTransaction(); + } catch (Exception e) { + // Roll back the transaction if any operation fails + session.abortTransaction(); + System.err.println("Transaction rolled back: " + e.getMessage()); + } finally { + session.close(); + } + + return mongoDBCollection.find(Filters.eq("release", release), null, DataRelease.class, QueryOptions.empty()); + } + + public DataResult update(int release, String field, Object value) { Bson query = Filters.eq("release", release); Document projection = new Document(field, true); Bson update = Updates.set(field, value); QueryOptions queryOptions = new QueryOptions("replace", true); - mongoDBCollection.findAndUpdate(query, projection, null, update, queryOptions); + return mongoDBCollection.findAndUpdate(query, projection, null, update, DataRelease.class, queryOptions); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/LoadRunner.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/LoadRunner.java index e4374f46bc..3a6605c4f5 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/LoadRunner.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/LoadRunner.java @@ -16,7 +16,12 @@ package org.opencb.cellbase.lib.loader; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.cellbase.core.exception.CellBaseException; +import org.opencb.cellbase.core.models.DataRelease; +import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +49,8 @@ public class LoadRunner { private final int numThreads; private CellBaseConfiguration cellBaseConfiguration; + protected DataReleaseManager dataReleaseManager; + protected BlockingQueue> blockingQueue; private final Logger logger; @@ -53,10 +60,12 @@ public class LoadRunner { public static final List POISON_PILL = new ArrayList<>(); - public LoadRunner(String loader, String database, int numThreads, CellBaseConfiguration cellBaseConfiguration) { + public LoadRunner(String loader, String database, int numThreads, DataReleaseManager dataReleaseManager, + CellBaseConfiguration cellBaseConfiguration) { this.loader = loader; this.database = database; this.numThreads = numThreads; + this.dataReleaseManager = dataReleaseManager; this.cellBaseConfiguration = cellBaseConfiguration; this.blockingQueue = new ArrayBlockingQueue<>(QUEUE_CAPACITY); @@ -66,86 +75,101 @@ public LoadRunner(String loader, String database, int numThreads, CellBaseConfig @Deprecated public void load(Path filePath, String data) throws ClassNotFoundException, NoSuchMethodException, InstantiationException, - IllegalAccessException, InvocationTargetException, ExecutionException, InterruptedException, IOException { + IllegalAccessException, InvocationTargetException, ExecutionException, InterruptedException, IOException, CellBaseException, + LoaderException { load(filePath, data, 0, null, null); } public void load(Path filePath, String data, int dataRelease) throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, ExecutionException, InterruptedException, - IOException { + IOException, CellBaseException, LoaderException { load(filePath, data, dataRelease, null, null); } public void load(Path filePath, String data, int dataRelease, String field, String[] innerFields) throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, - ExecutionException, InterruptedException, IOException { - try { + ExecutionException, InterruptedException, IOException, CellBaseException, LoaderException { - if (filePath == null || !Files.exists(filePath) || Files.isDirectory(filePath)) { - throw new IOException("File '" + filePath + "' does not exist or is a directory"); - } + if (filePath == null || !Files.exists(filePath) || Files.isDirectory(filePath)) { + throw new IOException("File '" + filePath + "' does not exist or is a directory"); + } - // protein_functional_prediction documents are extremely big. Increasing the batch size will probably - // lead to an OutOfMemory error for this collection. Batch size can be much higher for the rest of - // collections though - if (data.equals(PROTEIN_FUNCTIONAL_PREDICTION)) { - batchSize = 50; - } else { - batchSize = 200; - } + // Check data release + checkDataRelease(dataRelease, dataReleaseManager); - // One CellBaseLoader is created for each thread in 'numThreads' variable - List cellBaseLoaders = new ArrayList<>(numThreads); - for (int i = 0; i < numThreads; i++) { - cellBaseLoaders.add((CellBaseLoader) Class.forName(loader) - .getConstructor(BlockingQueue.class, String.class, Integer.class, String.class, String.class, - String[].class, CellBaseConfiguration.class) - .newInstance(blockingQueue, data, dataRelease, database, field, innerFields, cellBaseConfiguration)); - logger.debug("CellBase loader thread '{}' created", i); - } + // protein_functional_prediction documents are extremely big. Increasing the batch size will probably + // lead to an OutOfMemory error for this collection. Batch size can be much higher for the rest of + // collections though + if (data.equals(PROTEIN_FUNCTIONAL_PREDICTION)) { + batchSize = 50; + } else { + batchSize = 200; + } - /* - * ExecutorServices and Futures are created, all CellBaseLoaders are initialized and submitted to them. - * After this the different loaders are blocked waiting for the blockingQueue to be populated. - */ - ExecutorService executorService = Executors.newFixedThreadPool(numThreads); - List> futures = new ArrayList<>(numThreads); - for (int i = 0; i < numThreads; i++) { - cellBaseLoaders.get(i).init(); - futures.add(executorService.submit(cellBaseLoaders.get(i))); - logger.debug("CellBaseLoader '{}' initialized and submitted to the ExecutorService", i); - } + // One CellBaseLoader is created for each thread in 'numThreads' variable + List cellBaseLoaders = new ArrayList<>(numThreads); + for (int i = 0; i < numThreads; i++) { + cellBaseLoaders.add((CellBaseLoader) Class.forName(loader) + .getConstructor(BlockingQueue.class, String.class, Integer.class, String.class, String.class, + String[].class, CellBaseConfiguration.class) + .newInstance(blockingQueue, data, dataRelease, database, field, innerFields, cellBaseConfiguration)); + logger.debug("CellBase loader thread '{}' created", i); + } - /* - * Execution starts by reading the file and loading batches to the blockingQueue. This makes the loaders - * to start fetching and loading batches into the database. The number of records processed is returned. - */ - int processedRecords = readInputJsonFile(filePath); - // Check if all the records have been loaded - int loadedRecords = 0; - for (Future future : futures) { - loadedRecords += future.get(); - } - if (processedRecords == loadedRecords) { - logger.info("All the '{}' records have been loaded into the database", processedRecords); - } else { - logger.warn("Only '{}' out of '{}' have been loaded into the database", loadedRecords, processedRecords); - } + /* + * ExecutorServices and Futures are created, all CellBaseLoaders are initialized and submitted to them. + * After this the different loaders are blocked waiting for the blockingQueue to be populated. + */ + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + List> futures = new ArrayList<>(numThreads); + for (int i = 0; i < numThreads; i++) { + cellBaseLoaders.get(i).init(); + futures.add(executorService.submit(cellBaseLoaders.get(i))); + logger.debug("CellBaseLoader '{}' initialized and submitted to the ExecutorService", i); + } - /* - * For sanity database connection and other resources must be released. This close() call must be - * implemented in the specific data loader. - */ - for (int i = 0; i < numThreads; i++) { - cellBaseLoaders.get(i).close(); - logger.debug("CellBaseLoader '{}' being closed", i); - } + /* + * Execution starts by reading the file and loading batches to the blockingQueue. This makes the loaders + * to start fetching and loading batches into the database. The number of records processed is returned. + */ + int processedRecords = readInputJsonFile(filePath); + // Check if all the records have been loaded + int loadedRecords = 0; + for (Future future : futures) { + loadedRecords += future.get(); + } + if (processedRecords == loadedRecords) { + logger.info("All the '{}' records have been loaded into the database", processedRecords); + } else { + logger.warn("Only '{}' out of '{}' have been loaded into the database", loadedRecords, processedRecords); + } - executorService.shutdown(); - } catch (LoaderException e) { - logger.error("Error executing CellBase Load: " + e.getMessage()); + /* + * For sanity database connection and other resources must be released. This close() call must be + * implemented in the specific data loader. + */ + for (int i = 0; i < numThreads; i++) { + cellBaseLoaders.get(i).close(); + logger.debug("CellBaseLoader '{}' being closed", i); } + executorService.shutdown(); + } + + private void checkDataRelease(int release, DataReleaseManager dataReleaseManager) throws CellBaseException { + // Check data release + if (release < 1) { + throw new CellBaseException("Invalid data release " + release); + } + + DataRelease currDataRelease = dataReleaseManager.get(release); + if (currDataRelease == null) { + throw new CellBaseException("Loading data is not permitted since no data release " + release + " is found"); + } + if (CollectionUtils.isNotEmpty(currDataRelease.getActiveByDefaultIn())) { + throw new CellBaseException("Loading data is not permitted for data release " + currDataRelease.getRelease() + " since it has" + + " already assigned CellBase versions: " + StringUtils.join(currDataRelease.getActiveByDefaultIn(), ",")); + } } private int readInputJsonFile(Path inputFile) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java index 712b44f546..9a81a58b19 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java @@ -22,6 +22,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.opencb.cellbase.core.common.GitRepositoryState; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; @@ -34,6 +35,7 @@ import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.*; +import java.util.stream.Collectors; public class DataReleaseManager extends AbstractManager { private ReleaseMongoDBAdaptor releaseDBAdaptor; @@ -78,7 +80,6 @@ public DataRelease createRelease() throws JsonProcessingException { // Create the first release, collections and sources are empty lastRelease = new DataRelease() .setRelease(1) - .setActive(false) .setDate(sdf.format(new Date())); releaseDBAdaptor.insert(lastRelease); } else { @@ -86,7 +87,7 @@ public DataRelease createRelease() throws JsonProcessingException { if (MapUtils.isNotEmpty(lastRelease.getCollections())) { // Increment the release number, only if the previous release has collections lastRelease.setRelease(lastRelease.getRelease() + 1) - .setActive(false) + .setActiveByDefaultIn(new ArrayList<>()) .setDate(sdf.format(new Date())); // Write it to the database releaseDBAdaptor.insert(lastRelease); @@ -97,7 +98,7 @@ public DataRelease createRelease() throws JsonProcessingException { return lastRelease; } - public DataRelease get(int release) { + public DataRelease get(int release) throws CellBaseException { CellBaseDataResult result = releaseDBAdaptor.getAll(); if (CollectionUtils.isNotEmpty(result.getResults())) { for (DataRelease dataRelease : result.getResults()) { @@ -106,22 +107,27 @@ public DataRelease get(int release) { } } } - return null; + throw new CellBaseException("Data release '" + release + "' does not exist."); } - public DataRelease getDefault() { + public DataRelease getDefault(String cellBaseVersion) throws CellBaseException { CellBaseDataResult result = releaseDBAdaptor.getAll(); if (CollectionUtils.isNotEmpty(result.getResults())) { for (DataRelease dataRelease : result.getResults()) { - if (dataRelease.isActive()) { + if (dataRelease.getActiveByDefaultIn().contains(cellBaseVersion)) { return dataRelease; } } } - return null; + throw new CellBaseException("No data release found for CellBase " + cellBaseVersion); } - public void update(int release, String collection, String data, List dataSourcePaths) { + public DataRelease update(int release, List versions) throws CellBaseException { + return releaseDBAdaptor.update(release, versions).first(); + } + + public DataRelease update(int release, String collection, String data, List dataSourcePaths) + throws CellBaseException { DataRelease currDataRelease = get(release); if (currDataRelease != null) { // Update collections @@ -163,7 +169,10 @@ public void update(int release, String collection, String data, List dataS // Update data release in the database update(currDataRelease); + + return currDataRelease; } + throw new CellBaseException("Data release '" + release + "' does not exist."); } public void update(DataRelease dataRelase) { @@ -197,39 +206,6 @@ public void update(DataRelease dataRelase) { } } - public DataRelease active(int release) throws JsonProcessingException { - // Gel all releases and check if the input release exists - DataRelease prevActive = null; - DataRelease newActive = null; - CellBaseDataResult releaseResult = getReleases(); - if (CollectionUtils.isEmpty(releaseResult.getResults())) { - // Nothing to do, maybe exception or warning - return null; - } - for (DataRelease dataRelease : releaseResult.getResults()) { - if (dataRelease.isActive()) { - prevActive = dataRelease; - } else if (dataRelease.getRelease() == release) { - newActive = dataRelease; - } - } - if (prevActive != null && newActive != null && newActive.getRelease() == prevActive.getRelease()) { - // Nothing to do - return newActive; - } - - // Change active by default - if (prevActive != null) { - prevActive.setActive(false); - releaseDBAdaptor.update(prevActive.getRelease(), "active", prevActive.isActive()); - } - if (newActive != null) { - newActive.setActive(true); - releaseDBAdaptor.update(newActive.getRelease(), "active", newActive.isActive()); - } - return newActive; - } - public String getMaintenanceFlagFile() { return configuration.getMaintenanceFlagFile(); } @@ -238,4 +214,29 @@ public String getMaintainerContact() { return configuration.getMaintainerContact(); } + public int checkDataRelease(int inRelease) throws CellBaseException { + int outRelease = inRelease; + if (outRelease < 0) { + throw new CellBaseException("Invalid data release " + outRelease + ". Data release must be greater or equal to 0"); + } + if (outRelease == 0) { + String[] split = GitRepositoryState.get().getBuildVersion().split("[.-]"); + String version = "v" + split[0] + "." + split[1]; + outRelease = getDefault(version).getRelease(); + logger.info("Using data release 0: it means to take default data release '" + outRelease + "' for CellBase version '" + + version + "'"); + return outRelease; + } + + List dataReleases = getReleases().getResults(); + for (DataRelease dataRelease : dataReleases) { + if (outRelease == dataRelease.getRelease()) { + return outRelease; + } + } + + throw new CellBaseException("Invalid data release " + outRelease + ". Valid data releases are: " + + StringUtils.join(dataReleases.stream().map(dr -> dr.getRelease()).collect(Collectors.toList()), ",") + ". And use 0 to" + + " use the default data release."); + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java index f16bcf8aea..80f5866ebf 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java @@ -60,8 +60,9 @@ public List> info(List ids, ProjectionQue } - public CellBaseDataResult getCdna(String id) throws CellBaseException { + public CellBaseDataResult getCdna(String id, int dataRelease) throws CellBaseException { TranscriptQuery query = new TranscriptQuery(); + query.setDataRelease(dataRelease); query.setTranscriptsXrefs(Collections.singletonList(id)); CellBaseDataResult transcriptCellBaseDataResult = transcriptDBAdaptor.query(query); String cdnaSequence = null; @@ -78,17 +79,17 @@ public CellBaseDataResult getCdna(String id) throws CellBaseException { transcriptCellBaseDataResult.getNumResults(), Collections.singletonList(cdnaSequence), 1); } - private List> getCdna(List idList) throws CellBaseException { + private List> getCdna(List idList, int dataRelease) throws CellBaseException { List> cellBaseDataResults = new ArrayList<>(); for (String id : idList) { - cellBaseDataResults.add(getCdna(id)); + cellBaseDataResults.add(getCdna(id, dataRelease)); } return cellBaseDataResults; } - public List> getSequence(String id) throws CellBaseException { + public List> getSequence(String id, int dataRelease) throws CellBaseException { List transcriptsList = Arrays.asList(id.split(",")); - List> queryResult = getCdna(transcriptsList); + List> queryResult = getCdna(transcriptsList, dataRelease); for (int i = 0; i < transcriptsList.size(); i++) { queryResult.get(i).setId(transcriptsList.get(i)); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/VariantManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/VariantManager.java index 029ccab1a8..c478e28f45 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/VariantManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/VariantManager.java @@ -39,6 +39,7 @@ import org.opencb.cellbase.lib.impl.core.VariantMongoDBAdaptor; import org.opencb.cellbase.lib.token.DataAccessTokenUtils; import org.opencb.cellbase.lib.variant.VariantAnnotationUtils; +import org.opencb.cellbase.lib.variant.annotation.CellBaseNormalizerSequenceAdaptor; import org.opencb.cellbase.lib.variant.annotation.VariantAnnotationCalculator; import org.opencb.cellbase.lib.variant.hgvs.HgvsCalculator; import org.opencb.commons.datastore.core.Query; @@ -108,14 +109,31 @@ public List> getHgvsByVariant(String variants, int da * Normalises a list of variants. * * @param variants list of variant strings + * @param decompose boolean to set the decompose MNV behaviour + * @param leftAlign boolean to set the left alignment behaviour * @param dataRelease data release * @return list of normalised variants * @throws CellBaseException if the species is incorrect */ - public CellBaseDataResult getNormalizationByVariant(String variants, int dataRelease) throws CellBaseException { + public CellBaseDataResult getNormalizationByVariant(String variants, boolean decompose, boolean leftAlign, + int dataRelease) throws CellBaseException { List variantList = parseVariants(variants); VariantAnnotationCalculator variantAnnotationCalculator = new VariantAnnotationCalculator(species, assembly, dataRelease, "", cellbaseManagerFactory); + + + // Set decompose MNV behaviour + variantAnnotationCalculator.getNormalizer().getConfig().setDecomposeMNVs(decompose); + + // Set left alignment behaviour + if (leftAlign) { + variantAnnotationCalculator.getNormalizer().getConfig().enableLeftAlign(new CellBaseNormalizerSequenceAdaptor(genomeManager, + dataRelease)); + } else { + variantAnnotationCalculator.getNormalizer().getConfig().disableLeftAlign(); + } + + List normalisedVariants = variantAnnotationCalculator.normalizer(variantList); return new CellBaseDataResult<>(variants, 0, new ArrayList<>(), normalisedVariants.size(), normalisedVariants, -1); } @@ -123,7 +141,8 @@ public CellBaseDataResult getNormalizationByVariant(String variants, in public List> getAnnotationByVariant(QueryOptions queryOptions, String variants, Boolean normalize, - Boolean skipDecompose, + Boolean decompose, + Boolean leftAlign, Boolean ignorePhase, @Deprecated Boolean phased, Boolean imprecise, @@ -152,8 +171,11 @@ public List> getAnnotationByVariant(QueryO if (normalize != null) { queryOptions.put("normalize", normalize); } - if (skipDecompose != null) { - queryOptions.put("skipDecompose", skipDecompose); + if (decompose != null) { + queryOptions.put("decompose", decompose); + } + if (leftAlign != null) { + queryOptions.put("leftAlign", leftAlign); } if (imprecise != null) { queryOptions.put("imprecise", imprecise); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java index d5433a7327..8f2021e475 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java @@ -75,7 +75,8 @@ public class VariantAnnotationCalculator { private final VariantNormalizer normalizer; private boolean normalize = false; - private boolean decompose = true; + private boolean decompose = false; + private boolean leftAlign = false; private boolean phased = true; private Boolean imprecise = true; private Integer svExtraPadding = 0; @@ -101,24 +102,31 @@ public VariantAnnotationCalculator(String species, String assembly, int dataRele this.clinicalManager = cellbaseManagerFactory.getClinicalManager(species, assembly); this.repeatsManager = cellbaseManagerFactory.getRepeatsManager(species, assembly); - this.dataRelease = dataRelease; + // Check data release + this.dataRelease = cellbaseManagerFactory.getDataReleaseManager(species, assembly).checkDataRelease(dataRelease); + logger.info("Variant annotation calculator using data release {}", this.dataRelease); this.token = token; // Initialises normaliser configuration with default values. HEADS UP: configuration might be updated // at parseQueryParam this.normalizer = new VariantNormalizer(getNormalizerConfig()); - hgvsCalculator = new HgvsCalculator(genomeManager, dataRelease); + hgvsCalculator = new HgvsCalculator(genomeManager, this.dataRelease); logger.debug("VariantAnnotationMongoDBAdaptor: in 'constructor'"); } private VariantNormalizer.VariantNormalizerConfig getNormalizerConfig() { - return (new VariantNormalizer.VariantNormalizerConfig()) + VariantNormalizer.VariantNormalizerConfig variantNormalizerConfig = new VariantNormalizer.VariantNormalizerConfig() .setReuseVariants(false) .setNormalizeAlleles(false) - .setDecomposeMNVs(decompose) - .enableLeftAlign(new CellBaseNormalizerSequenceAdaptor(genomeManager, dataRelease)); + .setDecomposeMNVs(decompose); + if (leftAlign) { + variantNormalizerConfig.enableLeftAlign(new CellBaseNormalizerSequenceAdaptor(genomeManager, dataRelease)); + } else { + variantNormalizerConfig.disableLeftAlign(); + } + return variantNormalizerConfig; } @Deprecated @@ -181,8 +189,7 @@ private List> generateCellBaseDataResultLi // Return only one result per CellBaseDataResult if either // - size original variant list and normalised one is the same - // - MNV decomposition is switched OFF, i.e. queryOptions.skipDecompose = true and therefore - // this.decompose = false + // - MNV decomposition is switched OFF, i.e. queryOptions.decompose = false and therefore if (!decompose || variantList.size() == normalizedVariantList.size()) { for (int i = 0; i < variantList.size(); i++) { CellBaseDataResult cellBaseDataResult = new CellBaseDataResult<>(variantList.get(i).toString(), @@ -428,37 +435,6 @@ private List getTargets(Gene mirna) throws QueryException, Ille return geneMirnaTargets; } -// private boolean isPhased(Variant variant) { -// return (variant.getStudies() != null && !variant.getStudies().isEmpty()) -// && variant.getStudies().get(0).getSampleDataKeys().contains("PS"); -// } -// -// private String getCachedVariationIncludeFields() { -// StringBuilder stringBuilder = new StringBuilder("annotation.chromosome,annotation.start,annotation.reference"); -// stringBuilder.append(",annotation.alternate,annotation.id"); -// -// if (annotatorSet.contains("variation")) { -// stringBuilder.append(",annotation.id,annotation.additionalAttributes.dgvSpecificAttributes"); -// } -// if (annotatorSet.contains("clinical") || annotatorSet.contains("traitAssociation")) { -// stringBuilder.append(",annotation.variantTraitAssociation"); -// } -// if (annotatorSet.contains("conservation")) { -// stringBuilder.append(",annotation.conservation"); -// } -// if (annotatorSet.contains("functionalScore")) { -// stringBuilder.append(",annotation.functionalScore"); -// } -// if (annotatorSet.contains("consequenceType")) { -// stringBuilder.append(",annotation.consequenceTypes,annotation.displayConsequenceType"); -// } -// if (annotatorSet.contains("populationFrequencies")) { -// stringBuilder.append(",annotation.populationFrequencies"); -// } -// -// return stringBuilder.toString(); -// } - private List runAnnotationProcess(List normalizedVariantList, int dataRelease) throws InterruptedException, ExecutionException, QueryException, IllegalAccessException, CellBaseException { long globalStartTime = System.currentTimeMillis(); @@ -726,12 +702,25 @@ private void parseQueryParam(QueryOptions queryOptions) { normalize = (queryOptions.get("normalize") != null && (Boolean) queryOptions.get("normalize")); logger.debug("normalize = {}", normalize); - // Default behaviour decompose - decompose = (queryOptions.get("skipDecompose") == null || !queryOptions.getBoolean("skipDecompose")); + // Default behaviour decompose MNV + if (queryOptions.containsKey("skipDecompose")) { + throw new IllegalArgumentException("Param 'skipDecompose' is not supported. Please, use 'decompose' instead"); + } + decompose = (boolean) queryOptions.getOrDefault("decompose", false); logger.debug("decompose = {}", decompose); // Must update normaliser configuration since normaliser was created on constructor normalizer.getConfig().setDecomposeMNVs(decompose); + // Default behaviour left align + leftAlign = (boolean) queryOptions.getOrDefault("leftAlign", false); + logger.debug("leftAlign = {}", leftAlign); + // Must update normaliser configuration since normaliser was created on constructor + if (leftAlign) { + normalizer.getConfig().enableLeftAlign(new CellBaseNormalizerSequenceAdaptor(genomeManager, dataRelease)); + } else { + normalizer.getConfig().disableLeftAlign(); + } + // New parameter "ignorePhase" present overrides presence of old "phased" parameter if (queryOptions.get("ignorePhase") != null) { phased = !queryOptions.getBoolean("ignorePhase"); @@ -1322,6 +1311,7 @@ private boolean[] getRegulatoryRegionOverlaps(String chromosome, Integer positio boolean[] overlapsRegulatoryRegion = {false, false}; RegulationQuery query = new RegulationQuery(); + query.setDataRelease(dataRelease); query.setIncludes(Collections.singletonList(REGULATORY_REGION_FEATURE_TYPE_ATTRIBUTE)); query.setRegions(Collections.singletonList(new Region(chromosome, position))); CellBaseDataResult cellBaseDataResult = regulationManager.search(query); @@ -1772,6 +1762,7 @@ public List> call() throws Exception { List queries = new ArrayList<>(); for (Region region : breakpointsToRegionList(variant)) { RepeatsQuery query = new RepeatsQuery(); + query.setDataRelease(dataRelease); query.setRegions(Collections.singletonList(region)); queries.add(query); } @@ -1964,5 +1955,9 @@ public void processResults(Future>> spliceF } } } + + public VariantNormalizer getNormalizer() { + return normalizer; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculator.java index 794da50115..df066dbb45 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculator.java @@ -25,8 +25,8 @@ import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.lib.managers.GenomeManager; -import org.opencb.cellbase.lib.variant.annotation.UnsupportedURLVariantFormat; import org.opencb.cellbase.lib.variant.VariantAnnotationUtils; +import org.opencb.cellbase.lib.variant.annotation.UnsupportedURLVariantFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -166,8 +166,8 @@ protected static boolean isCoding(Transcript transcript) { } protected void setRangeCoordsAndAlleles(int genomicStart, int genomicEnd, String genomicReference, - String genomicAlternate, Transcript transcript, - BuildingComponents buildingComponents) { + String genomicAlternate, Transcript transcript, + BuildingComponents buildingComponents) { int start; int end; String reference; @@ -224,7 +224,7 @@ protected String reverseComplementary(String string) { * @param strand String {"+", "-"}. */ protected void justify(Variant variant, int startOffset, int endOffset, String allele, String genomicSequence, - String strand) { + String strand) { StringBuilder stringBuilder = new StringBuilder(allele); // Justify to the left if ("-".equals(strand)) { @@ -236,7 +236,7 @@ protected void justify(Variant variant, int startOffset, int endOffset, String a variant.setStart(variant.getStart() - 1); variant.setEnd(variant.getEnd() - 1); } - // Justify to the right + // Justify to the right } else { while ((endOffset + 1) < genomicSequence.length() && genomicSequence.charAt(endOffset + 1) == stringBuilder.charAt(0)) { stringBuilder.deleteCharAt(0); @@ -250,7 +250,7 @@ protected void justify(Variant variant, int startOffset, int endOffset, String a // Insertion if (variant.getReference().isEmpty()) { variant.setAlternate(stringBuilder.toString()); - // Deletion + // Deletion } else { variant.setReference(stringBuilder.toString()); } @@ -287,14 +287,14 @@ private static CdnaCoord genomicToCdnaCoordInNonCodingTranscript(Transcript tran cdnaCoord.setOffset(genomicPosition - nearestExon.getStart()); // TODO: probably needs +-1 bp adjust cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart())); cdnaCoord.setLandmark(CdnaCoord.Landmark.TRANSCRIPT_START); - // Exonic variant - // -------------S|||p||E------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Exonic variant + // -------------S|||p||E------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else if (genomicPosition - nearestExon.getEnd() < 0) { // no offset cdnaCoord.setReferencePosition(getCdnaPosition(transcript, genomicPosition)); cdnaCoord.setLandmark(CdnaCoord.Landmark.TRANSCRIPT_START); - // Non-exonic variant: intronic, intergenic - // -------------S||||||E-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Non-exonic variant: intronic, intergenic + // -------------S||||||E-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else { // offset must be positive cdnaCoord.setOffset(genomicPosition - nearestExon.getEnd()); // TODO: probably needs +-1 bp adjust @@ -312,14 +312,14 @@ private static CdnaCoord genomicToCdnaCoordInNonCodingTranscript(Transcript tran cdnaCoord.setOffset(nearestExon.getStart() - genomicPosition); // TODO: probably needs +-1 bp adjust cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart())); cdnaCoord.setLandmark(CdnaCoord.Landmark.TRANSCRIPT_START); - // Exonic variant - // -------------E|||p||S------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Exonic variant + // -------------E|||p||S------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else if (genomicPosition - nearestExon.getEnd() < 0) { // no offset cdnaCoord.setReferencePosition(getCdnaPosition(transcript, genomicPosition)); cdnaCoord.setLandmark(CdnaCoord.Landmark.TRANSCRIPT_START); - // Non-exonic variant: intronic, intergenic - // -------------E||||||S-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Non-exonic variant: intronic, intergenic + // -------------E||||||S-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else { // offset must be negative cdnaCoord.setOffset(nearestExon.getEnd() - genomicPosition); // TODO: probably needs +-1 bp adjust @@ -355,49 +355,49 @@ private static CdnaCoord genomicToCdnaCoordInCodingTranscript(Transcript transcr cdnaCoord.setOffset(genomicPosition - nearestExon.getStart()); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart()) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // After coding end + // After coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(genomicPosition - nearestExon.getStart()); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart()) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // Within coding start and end + // Within coding start and end } else { // offset must be negative cdnaCoord.setOffset(genomicPosition - nearestExon.getStart()); // TODO: probably needs +-1 bp adjust cdnaCoord.setReferencePosition(nearestExon.getCdsStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); } - // Exonic variant - // -------------S|||p||E------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Exonic variant + // -------------S|||p||E------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else if (genomicPosition - nearestExon.getEnd() <= 0) { // Before coding start if (genomicPosition < transcript.getGenomicCodingStart()) { cdnaCoord.setOffset(getCdnaPosition(transcript, genomicPosition) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // After coding end + // After coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(getCdnaPosition(transcript, genomicPosition) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // Within coding start and end + // Within coding start and end } else { // no offset cdnaCoord.setReferencePosition(nearestExon.getCdsStart() + (genomicPosition - nearestExon.getGenomicCodingStart())); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); } - // Non-exonic variant: intronic, intergenic - // -------------S||||||E-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Non-exonic variant: intronic, intergenic + // -------------S||||||E-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else { // Before coding start if (genomicPosition < transcript.getGenomicCodingStart()) { cdnaCoord.setOffset(genomicPosition - nearestExon.getEnd()); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getEnd()) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // After coding end + // After coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(genomicPosition - nearestExon.getEnd()); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getEnd()) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // Within coding start and end + // Within coding start and end } else { // offset must be positive cdnaCoord.setOffset(genomicPosition - nearestExon.getEnd()); // TODO: probably needs +-1 bp adjust @@ -417,49 +417,49 @@ private static CdnaCoord genomicToCdnaCoordInCodingTranscript(Transcript transcr cdnaCoord.setOffset(nearestExon.getStart() - genomicPosition); cdnaCoord.setReferencePosition(transcript.getCdnaCodingEnd() - getCdnaPosition(transcript, nearestExon.getStart())); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // After (genomic) coding end + // After (genomic) coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(nearestExon.getStart() - genomicPosition); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getStart()) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // Within coding start and end + // Within coding start and end } else { // offset must be positive cdnaCoord.setOffset(nearestExon.getStart() - genomicPosition); // TODO: probably needs +-1 bp adjust cdnaCoord.setReferencePosition(nearestExon.getCdsEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); } - // Exonic variant - // -------------E|||p||S------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Exonic variant + // -------------E|||p||S------------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else if (genomicPosition - nearestExon.getEnd() <= 0) { // Before (genomic) coding start if (genomicPosition < transcript.getGenomicCodingStart()) { cdnaCoord.setOffset(getCdnaPosition(transcript, genomicPosition) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // After (genomic) coding end + // After (genomic) coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(getCdnaPosition(transcript, genomicPosition) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // Within coding start and end + // Within coding start and end } else { // no offset cdnaCoord.setReferencePosition(nearestExon.getCdsStart() + nearestExon.getGenomicCodingEnd() - genomicPosition); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); } - // Non-exonic variant: intronic, intergenic - // -------------E||||||S-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd + // Non-exonic variant: intronic, intergenic + // -------------E||||||S-----p------; p = genomicPosition, S = nearestExon.getStart, E = nearestExon.getEnd } else { // Before (genomic) coding start if (genomicPosition < transcript.getGenomicCodingStart()) { cdnaCoord.setOffset(nearestExon.getEnd() - genomicPosition); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getEnd()) - transcript.getCdnaCodingEnd()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_STOP_CODON); - // After (genomic) coding end + // After (genomic) coding end } else if (genomicPosition > transcript.getGenomicCodingEnd()) { cdnaCoord.setOffset(nearestExon.getEnd() - genomicPosition); cdnaCoord.setReferencePosition(getCdnaPosition(transcript, nearestExon.getEnd()) - transcript.getCdnaCodingStart()); cdnaCoord.setLandmark(CdnaCoord.Landmark.CDNA_START_CODON); - // Within coding start and end + // Within coding start and end } else { // offset must be negative cdnaCoord.setOffset(nearestExon.getEnd() - genomicPosition); // TODO: probably needs +-1 bp adjust diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java index bfc4afbf25..0c1bc4113a 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/GenericMongoDBAdaptorTest.java @@ -27,6 +27,7 @@ import org.opencb.cellbase.lib.impl.core.CellBaseDBAdaptor; import org.opencb.cellbase.lib.impl.core.MongoDBAdaptorFactory; import org.opencb.cellbase.lib.loader.LoadRunner; +import org.opencb.cellbase.lib.loader.LoaderException; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.commons.datastore.core.DataStoreServerAddress; @@ -91,8 +92,9 @@ public GenericMongoDBAdaptorTest() { // cellBaseConfiguration.getDatabases().getMongodb().setPassword("cellbase"); // cellBaseConfiguration.getDatabases().getMongodb().getOptions().put("authenticationDatabase", "admin"); // cellBaseConfiguration.getDatabases().getMongodb().getOptions().put("authenticationMechanism", "SCRAM-SHA-256"); - loadRunner = new LoadRunner(MONGODB_CELLBASE_LOADER, CELLBASE_DBNAME, 2, cellBaseConfiguration); cellBaseManagerFactory = new CellBaseManagerFactory(cellBaseConfiguration); + loadRunner = new LoadRunner(MONGODB_CELLBASE_LOADER, CELLBASE_DBNAME, 2, + cellBaseManagerFactory.getDataReleaseManager(SPECIES, ASSEMBLY), cellBaseConfiguration); initDB(); } catch (Exception e) { e.printStackTrace(); @@ -113,7 +115,7 @@ protected void clearDB(String dbName) throws Exception { protected void initDB() throws IOException, ExecutionException, ClassNotFoundException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, - URISyntaxException, CellBaseException { + URISyntaxException, CellBaseException, LoaderException { dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager(SPECIES, ASSEMBLY); CellBaseDataResult results = dataReleaseManager.getReleases(); List dataReleaseList = results.getResults(); @@ -127,7 +129,7 @@ protected void initDB() throws IOException, ExecutionException, ClassNotFoundExc } } - private void downloadAndPopulate() throws IOException, ExecutionException, ClassNotFoundException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, CellBaseException { + private void downloadAndPopulate() throws IOException, ExecutionException, ClassNotFoundException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, CellBaseException, LoaderException { // Download and uncompress dataset URL url = new URL(DATASET_URL + DATASET_BASENAME + DATASET_EXTENSION); Path tmpPath = Paths.get(DATASET_TMP_DIR); @@ -208,13 +210,13 @@ private void downloadAndPopulate() throws IOException, ExecutionException, Class } private void loadData(String collection, String data, Path filePath) throws IOException, ExecutionException, ClassNotFoundException, - InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, LoaderException, CellBaseException { loadData(collection, data, filePath, false); } private void loadData(String collection, String data, Path filePath, boolean skipUpdate) throws IOException, ExecutionException, ClassNotFoundException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, - IllegalAccessException { + IllegalAccessException, LoaderException, CellBaseException { if (filePath.toFile().exists()) { logger.info("Loading (" + collection + ", " + data + ") from file " + filePath); loadRunner.load(filePath, collection, dataRelease); @@ -229,7 +231,7 @@ private void loadData(String collection, String data, Path filePath, boolean ski @Deprecated protected void initDB_OLD() throws IOException, ExecutionException, ClassNotFoundException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, - URISyntaxException, CellBaseException { + URISyntaxException, CellBaseException, LoaderException { dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager("hsapiens", "GRCh37"); dataRelease = dataReleaseManager.createRelease().getRelease(); diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/SpeciesUtilsTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/SpeciesUtilsTest.java index 3b51155bfd..246d41684e 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/SpeciesUtilsTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/SpeciesUtilsTest.java @@ -22,6 +22,7 @@ import org.opencb.cellbase.core.config.SpeciesConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.lib.loader.LoadRunner; +import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,11 +44,13 @@ public class SpeciesUtilsTest { protected Logger logger = LoggerFactory.getLogger(this.getClass()); - public SpeciesUtilsTest() throws IOException { + public SpeciesUtilsTest() throws IOException, CellBaseException { cellBaseConfiguration = CellBaseConfiguration.load( SpeciesUtilsTest.class.getClassLoader().getResourceAsStream("configuration.test.yaml"), CellBaseConfiguration.ConfigurationFileFormat.YAML); - loadRunner = new LoadRunner(MONGODB_CELLBASE_LOADER, GRCH37_DBNAME, 2, cellBaseConfiguration); + + loadRunner = new LoadRunner(MONGODB_CELLBASE_LOADER, GRCH37_DBNAME, 2, + new DataReleaseManager(GRCH37_DBNAME, cellBaseConfiguration), cellBaseConfiguration); } @Test diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/VariantAnnotationCalculatorTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/VariantAnnotationCalculatorTest.java index b8c384f85a..cc6854e564 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/VariantAnnotationCalculatorTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/VariantAnnotationCalculatorTest.java @@ -647,7 +647,7 @@ private ConsequenceType getConsequenceType(List consequenceType public void testCellBaseDataResultGroupingDecomposedMNVs() throws Exception { QueryOptions queryOptions = (new QueryOptions("normalize", true)); - queryOptions.put("skipDecompose", false); + queryOptions.put("decompose", true); // Creating here a local VariantAnnotationCalculator since this test requires setting normalizer decompose // option to true which probably breaks some other tests. diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java index cf101d1eb5..802c2e5b55 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/managers/DataReleaseManagerTest.java @@ -1,40 +1,189 @@ package org.opencb.cellbase.lib.managers; import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.models.variant.avro.VariantAnnotation; +import org.opencb.cellbase.core.api.query.QueryException; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; +import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.GenericMongoDBAdaptorTest; +import org.opencb.cellbase.lib.loader.LoaderException; +import org.opencb.cellbase.lib.variant.annotation.VariantAnnotationCalculator; +import org.opencb.commons.datastore.core.QueryOptions; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.concurrent.ExecutionException; + +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; class DataReleaseManagerTest extends GenericMongoDBAdaptorTest { + protected DataReleaseManager dataReleaseManager; + private static boolean populated = false; + public DataReleaseManagerTest() throws IOException { super(); } @BeforeEach public void setUp() throws Exception { - clearDB(CELLBASE_DBNAME); -// Path path = Paths.get(getClass().getResource("/gene/gene-test.json.gz").toURI()); -// loadRunner.load(path, "gene"); + if (!populated) { + clearDB(CELLBASE_DBNAME); + initDB(); + } + + dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager(SPECIES, ASSEMBLY); + populated = true; } @Test - @Disabled - public void test1() throws JsonProcessingException, CellBaseException { - DataReleaseManager dataReleaseManager = new DataReleaseManager(CELLBASE_DBNAME, cellBaseConfiguration); - DataRelease firstRelease = dataReleaseManager.createRelease(); - if (firstRelease != null) { - dataReleaseManager.active(firstRelease.getRelease()); - } - DataRelease secondRelease = dataReleaseManager.createRelease(); - if (secondRelease != null) { - dataReleaseManager.active(secondRelease.getRelease()); - } + public void testCreate() throws JsonProcessingException { + CellBaseDataResult result = dataReleaseManager.getReleases(); + DataRelease dr = dataReleaseManager.createRelease(); + assertEquals(result.getNumResults() + 1, dr.getRelease()); } + @Test + public void testAddActiveByDefaultIn() throws CellBaseException, JsonProcessingException { + DataRelease dr = dataReleaseManager.createRelease(); + dataReleaseManager.update(dr.getRelease(), Arrays.asList("v5.1", "v5.2")); + + DataRelease auxDr = dataReleaseManager.get(dr.getRelease()); + assertEquals(dr.getRelease(), auxDr.getRelease()); + assertEquals(2, auxDr.getActiveByDefaultIn().size()); + } + + @Test + public void testChangeActiveByDefaultIn() throws JsonProcessingException, CellBaseException { + String version3 = "v5.3"; + String version4 = "v5.4"; + + int rA = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rA, Arrays.asList(version3, version4)); + + DataRelease auxDr1 = dataReleaseManager.get(rA); + assertEquals(2, auxDr1.getActiveByDefaultIn().size()); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version3)); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version4)); + + int rB = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rB, Arrays.asList(version4)); + + DataRelease auxDr2 = dataReleaseManager.get(rA); + assertEquals(1, auxDr2.getActiveByDefaultIn().size(), 1); + assertEquals(version3, auxDr2.getActiveByDefaultIn().get(0)); + + DataRelease auxDr3 = dataReleaseManager.get(rB); + assertEquals(1, auxDr3.getActiveByDefaultIn().size()); + assertEquals(version4, auxDr3.getActiveByDefaultIn().get(0)); + } + + @Test + public void failLoading() throws IOException, ExecutionException, ClassNotFoundException, InterruptedException, + InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, URISyntaxException, + CellBaseException, LoaderException { + String version5 = "v5.5"; + + int release = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(release, Arrays.asList(version5)); + + Path path = Paths.get(getClass() + .getResource("/variant-annotation/gene.test.json.gz").toURI()); + + CellBaseException thrown = Assertions.assertThrows(CellBaseException.class, () -> { loadRunner.load(path, "gene", release); }); + Assertions.assertTrue(thrown.getMessage().contains("since it has already assigned CellBase versions: " + version5)); + } + + @Test + public void testMultipleAddActiveByDefaultIn() throws JsonProcessingException, CellBaseException { + String version6 = "v5.6"; + String version7 = "v5.7"; + + int rA = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rA, Arrays.asList(version6)); + + DataRelease auxDr1 = dataReleaseManager.get(rA); + assertEquals(auxDr1.getActiveByDefaultIn().size(), 1); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version6)); + + dataReleaseManager.update(rA, Arrays.asList(version7)); + + DataRelease auxDr2 = dataReleaseManager.get(rA); + assertEquals(2, auxDr2.getActiveByDefaultIn().size()); + assertTrue(auxDr2.getActiveByDefaultIn().contains(version6)); + assertTrue(auxDr2.getActiveByDefaultIn().contains(version7)); + } + + @Test + public void testRemoveMultipleAddActiveByDefaultIn() throws JsonProcessingException, CellBaseException { + String version8 = "v5.8"; + String version9 = "v5.9"; + + int rA = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rA, Arrays.asList(version8, version9)); + + DataRelease auxDr1 = dataReleaseManager.get(rA); + assertEquals(2, auxDr1.getActiveByDefaultIn().size()); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version8)); + assertTrue(auxDr1.getActiveByDefaultIn().contains(version9)); + + int rB = dataReleaseManager.createRelease().getRelease(); + dataReleaseManager.update(rB, Arrays.asList(version8, version9)); + + auxDr1 = dataReleaseManager.get(rA); + assertEquals(0, auxDr1.getActiveByDefaultIn().size()); + + DataRelease auxDr2 = dataReleaseManager.get(rB); + assertEquals(2, auxDr2.getActiveByDefaultIn().size()); + assertTrue(auxDr2.getActiveByDefaultIn().contains(version8)); + assertTrue(auxDr2.getActiveByDefaultIn().contains(version9)); + } + + @Test + public void testAnnotationWithDR0() throws CellBaseException, QueryException, ExecutionException, InterruptedException, + IllegalAccessException { + dataReleaseManager.update(1, Arrays.asList("v5.5")); + + VariantAnnotationCalculator annotator = new VariantAnnotationCalculator(SPECIES, ASSEMBLY, 0, token, cellBaseManagerFactory); + + Variant variant = new Variant("10", 113588287, "G", "A"); + CellBaseDataResult cellBaseDataResult = annotator.getAnnotationByVariant(variant, QueryOptions.empty()); + VariantAnnotation variantAnnotation = cellBaseDataResult.first(); + System.out.println(variantAnnotation); + assertEquals(variant.getChromosome(), variantAnnotation.getChromosome()); + assertEquals(variant.getStart(), variantAnnotation.getStart()); + assertEquals(variant.getReference(), variantAnnotation.getReference()); + assertEquals(variant.getAlternate(), variantAnnotation.getAlternate()); + } + + @Test + public void testAnnotationWithInvalidDR() { + CellBaseException thrown = Assertions.assertThrows(CellBaseException.class, () -> { + VariantAnnotationCalculator annotator = new VariantAnnotationCalculator(SPECIES, ASSEMBLY, -1, token, cellBaseManagerFactory); + Variant variant = new Variant("10", 113588287, "G", "A"); + CellBaseDataResult cellBaseDataResult = annotator.getAnnotationByVariant(variant, QueryOptions.empty()); + }); + Assertions.assertTrue(thrown.getMessage().contains("Data release must be greater or equal to 0")); + } + + @Test + public void testAnnotationWithInvalidDR_1() { + int dr = 12; + CellBaseException thrown = Assertions.assertThrows(CellBaseException.class, () -> { + VariantAnnotationCalculator annotator = new VariantAnnotationCalculator(SPECIES, ASSEMBLY, dr, token, cellBaseManagerFactory); + Variant variant = new Variant("10", 113588287, "G", "A"); + CellBaseDataResult cellBaseDataResult = annotator.getAnnotationByVariant(variant, QueryOptions.empty()); + }); + Assertions.assertTrue(thrown.getMessage().contains("data release "+ dr + ". Valid data releases are:")); + } } \ No newline at end of file diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/VariantManagerTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/VariantManagerTest.java index b660b9fa29..4d2ffe7db4 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/VariantManagerTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/VariantManagerTest.java @@ -63,7 +63,7 @@ public void setUp() throws Exception { @Test @Disabled public void testNormalisation() throws Exception { - CellBaseDataResult results = variantManager.getNormalizationByVariant("22:18512237:-:AGTT", dataRelease); + CellBaseDataResult results = variantManager.getNormalizationByVariant("22:18512237:-:AGTT", true, true, dataRelease); assertEquals(1, results.getResults().size()); } diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculatorTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculatorTest.java index 3f74978f53..c6abc17041 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculatorTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/variant/hgvs/HgvsCalculatorTest.java @@ -15,6 +15,7 @@ import org.opencb.cellbase.lib.GenericMongoDBAdaptorTest; import org.opencb.cellbase.lib.impl.core.GeneMongoDBAdaptor; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; +import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.cellbase.lib.managers.GeneManager; import java.io.IOException; @@ -41,9 +42,9 @@ public HgvsCalculatorTest() throws IOException { @BeforeAll public void init() throws Exception { - dataRelease = 1; - clearDB(CELLBASE_DBNAME); + DataReleaseManager dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager("hsapiens", "GRCh37"); + dataRelease = dataReleaseManager.createRelease().getRelease(); Path path = Paths.get(getClass() .getResource("/hgvs/gene.test.json.gz").toURI()); loadRunner.load(path, "gene", dataRelease); diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index b9ccff6874..88d9d0823c 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.6.0-SNAPSHOT ../pom.xml diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index 7b52d92267..1fd334565b 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -31,7 +31,6 @@ import org.opencb.cellbase.core.api.query.QueryException; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; -import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.core.utils.SpeciesUtils; @@ -170,22 +169,15 @@ private void initQuery() throws CellBaseException { // check version. species is validated later checkVersion(); - // Check data release and retrieve the default data release - if (!DONT_CHECK_SPECIES.equals(species)) { - // Prepare data release (do we need to get the default one?) - if (StringUtils.isEmpty(uriParams.get("dataRelease")) || uriParams.get("dataRelease").equals("0")) { - if (defaultDataRelease == 0) { - if (StringUtils.isEmpty(assembly)) { - assembly = SpeciesUtils.getSpecies(cellBaseConfiguration, species, assembly).getAssembly(); - } - if (StringUtils.isNotEmpty(assembly)) { - DataReleaseManager releaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly); - DataRelease dr = releaseManager.getDefault(); - if (dr != null) { - defaultDataRelease = dr.getRelease(); - } - } - } + // Set default data release if necessary + if (!DONT_CHECK_SPECIES.equals(species) && defaultDataRelease < 1) { + // As the assembly may not be presented in the query, we have to be sure to get it from the CellBase configuration + assembly = SpeciesUtils.getSpecies(cellBaseConfiguration, this.species, assembly).getAssembly(); + + if (StringUtils.isNotEmpty(assembly)) { + DataReleaseManager releaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly); + // getDefault launches an exception if no data release is found for that CellBase version + defaultDataRelease = releaseManager.getDefault(version).getRelease(); } } } @@ -193,23 +185,25 @@ private void initQuery() throws CellBaseException { protected int getDataRelease() throws CellBaseException { if (uriParams.containsKey("dataRelease") && StringUtils.isNotEmpty(uriParams.get("dataRelease"))) { try { - return Integer.parseInt(uriParams.get("dataRelease")); + int dataRelease = Integer.parseInt(uriParams.get("dataRelease")); + // If data release is 0, then use the default data release + if (dataRelease == 0) { + logger.info("Using data release 0 in query: using the default data release '" + defaultDataRelease + "' for CellBase" + + " version '" + version + "'"); + return defaultDataRelease; + } else { + return dataRelease; + } } catch (NumberFormatException e) { throw new CellBaseException("Invalid data release number '" + uriParams.get("dataRelease") + "'"); } } - // It means to use the default data release + // If no data release is present in the query, then use the default data release + logger.info("No data release present in query: using the default data release '" + defaultDataRelease + "' for CellBase version" + + " '" + version + "'"); return defaultDataRelease; } - protected int getDataReleaseUsed() throws CellBaseException { - int dataRelease = getDataRelease(); - if (dataRelease == 0) { - return defaultDataRelease; - } - return dataRelease; - } - protected String getToken() { return uriParams.get(DATA_ACCESS_TOKEN); } @@ -240,7 +234,7 @@ private void checkVersion() throws CellBaseException { // System.out.println("cellBaseConfiguration.getVersion() = " + cellBaseConfiguration.getVersion()); // System.out.println("version = " + version); // System.out.println("*************************************"); - if (!cellBaseConfiguration.getVersion().equalsIgnoreCase(version)) { + if (!version.startsWith(cellBaseConfiguration.getVersion())) { logger.error("Version '{}' does not match configuration '{}'", this.version, cellBaseConfiguration.getVersion()); throw new CellBaseException("Version not valid: '" + version + "'"); } @@ -312,7 +306,7 @@ protected Response createErrorResponse(Exception e) { queryResponse.setTime(new Long(System.currentTimeMillis() - startTime).intValue()); queryResponse.setApiVersion(version); try { - queryResponse.setDataRelease(getDataReleaseUsed()); + queryResponse.setDataRelease(getDataRelease()); } catch (CellBaseException ex) { logger.warn("Impossible to set the data release used in the query response", e); } @@ -343,7 +337,7 @@ protected Response createOkResponse(Object obj) { queryResponse.setTime(new Long(System.currentTimeMillis() - startTime).intValue()); queryResponse.setApiVersion(version); try { - queryResponse.setDataRelease(getDataReleaseUsed()); + queryResponse.setDataRelease(getDataRelease()); } catch (CellBaseException e) { logger.warn("Impossible to set the data release used in the query response", e); } @@ -417,6 +411,4 @@ private Response buildResponse(ResponseBuilder responseBuilder) { .header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") .build(); } - - } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java index cba0b93a02..f9b8f16764 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java @@ -17,6 +17,7 @@ package org.opencb.cellbase.server.rest; import io.swagger.annotations.*; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.core.ParamConstants; import org.opencb.cellbase.core.api.query.QueryException; @@ -27,11 +28,11 @@ import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.cellbase.core.token.DataAccessTokenManager; import org.opencb.cellbase.core.token.DataAccessTokenSources; import org.opencb.cellbase.core.utils.SpeciesUtils; import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.cellbase.lib.managers.MetaManager; -import org.opencb.cellbase.core.token.DataAccessTokenManager; import org.opencb.cellbase.server.rest.clinical.ClinicalWSServer; import org.opencb.cellbase.server.rest.feature.GeneWSServer; import org.opencb.cellbase.server.rest.feature.IdWSServer; @@ -82,9 +83,10 @@ public MetaWSServer(@PathParam("apiVersion") @ApiOperation(httpMethod = "GET", value = "Returns source version metadata, including source urls from which " + "data files were downloaded.", response = DownloadProperties.class, responseContainer = "QueryResponse") public Response getVersion(@PathParam("species") - @ApiParam(name = "species", value = ParamConstants.SPECIES_DESCRIPTION, required = true) String species, - @ApiParam(name = "assembly", value = ParamConstants.ASSEMBLY_DESCRIPTION) @QueryParam("assembly") - String assembly) { + @ApiParam(name = "species", value = ParamConstants.SPECIES_DESCRIPTION, + defaultValue = ParamConstants.DEFAULT_SPECIES, required = true) String species, + @ApiParam(name = "assembly", value = ParamConstants.ASSEMBLY_DESCRIPTION, + defaultValue = ParamConstants.DEFAULT_ASSEMBLY) @QueryParam("assembly") String assembly) { try { if (StringUtils.isEmpty(assembly)) { SpeciesConfiguration.Assembly assemblyObject = SpeciesUtils.getDefaultAssembly(cellBaseConfiguration, species); @@ -121,9 +123,7 @@ public Response getSpecies() { public Response getDataRelease(@PathParam("species") @ApiParam(name = "species", value = ParamConstants.SPECIES_DESCRIPTION, required = true) String species, @ApiParam(name = "assembly", value = ParamConstants.ASSEMBLY_DESCRIPTION) @QueryParam("assembly") - String assembly, - @ApiParam(name = "onlyActive", value = "Set to true if you only want to get the active data relaease") - @QueryParam("onlyActive") @DefaultValue("false") boolean onlyActive) { + String assembly) { try { if (StringUtils.isEmpty(assembly)) { SpeciesConfiguration.Assembly assemblyObject = SpeciesUtils.getDefaultAssembly(cellBaseConfiguration, species); @@ -136,17 +136,7 @@ public Response getDataRelease(@PathParam("species") + assembly + "'"); } DataReleaseManager dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly); - CellBaseDataResult result = dataReleaseManager.getReleases(); - if (onlyActive) { - for (DataRelease release : result.getResults()) { - if (release.isActive()) { - return createOkResponse(new CellBaseDataResult<>(result.getId(), result.getTime(), result.getEvents(), 1, - Collections.singletonList(release), 1)); - } - } - } - - return createOkResponse(result); + return createOkResponse(dataReleaseManager.getReleases()); } catch (CellBaseException e) { return createErrorResponse(e); } @@ -157,7 +147,7 @@ public Response getDataRelease(@PathParam("species") @ApiOperation(httpMethod = "GET", value = "Display the licensed data sources of the input token and their expiration date", response = Map.class, responseContainer = "QueryResponse") public Response getLicensedData(@ApiParam(name = "token", required = true, value = ParamConstants.DATA_ACCESS_TOKEN_DESCRIPTION) - @QueryParam("token") String token) { + @QueryParam("token") String token) { try { DataAccessTokenManager datManager = new DataAccessTokenManager(cellBaseConfiguration.getSecretKey()); DataAccessTokenSources sources = datManager.decode(token); @@ -299,12 +289,33 @@ private Response getAllSpecies() { @ApiOperation(httpMethod = "GET", value = "Returns info about current CellBase code.", response = Map.class, responseContainer = "QueryResponse") public Response getAbout() { - Map info = new HashMap<>(3); + Map info = new LinkedHashMap<>(); info.put("Program", "CellBase (OpenCB)"); + info.put("Description", "High-Performance NoSQL database and RESTful web services to access the most relevant biological data"); + info.put("API version", version); info.put("Version", GitRepositoryState.get().getBuildVersion()); info.put("Git branch", GitRepositoryState.get().getBranch()); info.put("Git commit", GitRepositoryState.get().getCommitId()); - info.put("Description", "High-Performance NoSQL database and RESTful web services to access the most relevant biological data"); + + // Get default data releases + species = "hsapiens"; + SpeciesConfiguration speciesConfiguration = SpeciesUtils.getSpeciesConfiguration(cellBaseConfiguration, species); + List assemblies = speciesConfiguration.getAssemblies(); + if (CollectionUtils.isNotEmpty(assemblies)) { + DataReleaseManager dataReleaseManager; + for (SpeciesConfiguration.Assembly assembly : assemblies) { + String key = "Default data release for " + version + " (" + species + "/" + assembly.getName() + ")"; + try { + dataReleaseManager = cellBaseManagerFactory.getDataReleaseManager(species, assembly.getName()); + DataRelease defaultDataRelease = dataReleaseManager.getDefault(version); + info.put(key, String.valueOf(defaultDataRelease.getRelease())); + } catch (CellBaseException e) { + info.put(key, "ERROR: " + e.getMessage()); + e.printStackTrace(); + } + } + } + CellBaseDataResult queryResult = new CellBaseDataResult(); queryResult.setId("about"); queryResult.setTime(0); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/OntologyWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/OntologyWSServer.java index 6ea67fb7dd..68b6d0021a 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/OntologyWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/OntologyWSServer.java @@ -112,6 +112,7 @@ public OntologyWSServer(@PathParam("apiVersion") @ApiParam(name = "apiVersion", public Response getAll() { try { OntologyQuery query = new OntologyQuery(uriParams); + query.setDataRelease(getDataRelease()); logger.info("/search OntologyQuery: " + query.toString()); CellBaseDataResult queryResults = ontologyManager.search(query); return createOkResponse(queryResults); @@ -167,6 +168,7 @@ public Response getUniqueValues(@QueryParam("field") @ApiParam(name = "field", r try { copyToFacet("field", field); OntologyQuery query = new OntologyQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = ontologyManager.distinct(query); return createOkResponse(queryResults); } catch (Exception e) { diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/PublicationWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/PublicationWSServer.java index 2918fec234..4135848cc0 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/PublicationWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/PublicationWSServer.java @@ -105,6 +105,7 @@ public Response getModel() { public Response getAll() { try { PublicationQuery query = new PublicationQuery(uriParams); + query.setDataRelease(getDataRelease()); logger.info("/search PublicationQuery: " + query); CellBaseDataResult queryResults = publicationManager.search(query); return createOkResponse(queryResults); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/clinical/ClinicalWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/clinical/ClinicalWSServer.java index d7a54aee9d..8e057f8b19 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/clinical/ClinicalWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/clinical/ClinicalWSServer.java @@ -117,6 +117,7 @@ public ClinicalWSServer(@PathParam("apiVersion") @ApiParam(name = "apiVersion", public Response getAll() { try { ClinicalVariantQuery query = new ClinicalVariantQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = clinicalManager.search(query); return createOkResponse(queryResults); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/GeneWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/GeneWSServer.java index e6be98fc14..c73e711d75 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/GeneWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/GeneWSServer.java @@ -137,6 +137,7 @@ public Response groupBy(@DefaultValue("") @QueryParam("field") @ApiParam(name = try { copyToFacet("field", field); GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = geneManager.groupBy(geneQuery); return createOkResponse(queryResults); } catch (Exception e) { @@ -185,6 +186,7 @@ public Response getAggregationStats(@DefaultValue("") @QueryParam("field") try { copyToFacet("field", field); GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = geneManager.aggregationStats(geneQuery); return createOkResponse(queryResults); } catch (Exception e) { @@ -283,6 +285,7 @@ public Response getAll(@QueryParam(SPLIT_RESULT_PARAM) @ApiParam(name = SPLIT_RE logger.info("/search identifiers: {} ", identifiers); for (String identifier : identifiers) { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); geneQuery.setTranscriptsXrefs(Collections.singletonList(identifier)); geneQueries.add(geneQuery); logger.info("/search geneQuery: {}", geneQuery.toString()); @@ -291,6 +294,7 @@ public Response getAll(@QueryParam(SPLIT_RESULT_PARAM) @ApiParam(name = SPLIT_RE return createOkResponse(queryResults); } else { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); logger.info("/search GeneQuery: {} ", geneQuery.toString()); CellBaseDataResult queryResults = geneManager.search(geneQuery); return createOkResponse(queryResults); @@ -357,6 +361,7 @@ private String[] getGeneIdentifiers() { public Response getInfo(@PathParam("genes") @ApiParam(name = "genes", value = GENE_IDS, required = true) String genes) { try { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); String source = "ensembl"; if (geneQuery.getSource() != null && !geneQuery.getSource().isEmpty()) { source = geneQuery.getSource().get(0); @@ -416,6 +421,7 @@ public Response getTranscriptsByGenes(@PathParam("genes") @ApiParam(name = "gene String[] ids = genes.split(","); for (String id : ids) { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(new LogicalList(Collections.singletonList(id))); queries.add(query); } @@ -468,6 +474,7 @@ public Response getUniqueValues(@QueryParam("field") @ApiParam(name = "field", r try { copyToFacet("field", field); GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = geneManager.distinct(geneQuery); return createOkResponse(queryResults); } catch (Exception e) { @@ -505,6 +512,7 @@ public Response getSNPByGenes(@PathParam("genes") String[] ids = genes.split(","); for (String id : ids) { VariantQuery query = new VariantQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setGenes(new LogicalList(Collections.singletonList(id))); queries.add(query); } @@ -529,6 +537,7 @@ public Response getAllTfbs(@PathParam("genes") @ApiParam(name = "genes", value = required = true) String genes) { try { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); geneQuery.setIds(Arrays.asList(genes.split(","))); List> queryResults = geneManager.getTfbs(geneQuery); return createOkResponse(queryResults); @@ -551,6 +560,7 @@ public Response getProteinById(@PathParam("genes") @ApiParam(name = "genes", val required = true) String genes) { try { ProteinQuery query = new ProteinQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setGenes(Arrays.asList(genes.split(","))); logger.info("REST proteinQuery: {}", query.toString()); CellBaseDataResult queryResults = proteinManager.search(query); @@ -577,6 +587,7 @@ public Response getSequence(@PathParam("genes") @ApiParam(name = "genes", value String[] identifiers = genes.split(","); for (String identifier : identifiers) { GeneQuery query = new GeneQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(Arrays.asList(identifier)); queries.add(query); logger.info("REST GeneQuery: {} ", query.toString()); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java index 3f3f8f80bf..28c5dafcd4 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java @@ -109,6 +109,7 @@ public Response getInfo(@PathParam("id") @ApiParam(name = "id", value = FEATURE_ public Response getAllXrefs() { try { XrefQuery query = new XrefQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = xrefManager.search(query); return createOkResponse(queryResults); } catch (Exception e) { diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/ProteinWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/ProteinWSServer.java index 47b1d63f49..be2d76db1b 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/ProteinWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/ProteinWSServer.java @@ -137,6 +137,7 @@ public Response getInfo(@PathParam("proteins") @ApiParam(name = "proteins", valu public Response getAll() { try { ProteinQuery query = new ProteinQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = proteinManager.search(query); return createOkResponse(queryResults); } catch (Exception e) { @@ -174,6 +175,7 @@ public Response getSubstitutionScores(@PathParam("proteins") @ApiParam(name = "p required = false) String aa) { try { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(Arrays.asList(id.split(","))); CellBaseDataResult queryResult = proteinManager.getSubstitutionScores(query, position, aa); return createOkResponse(queryResult); @@ -218,6 +220,7 @@ public Response getSequence(@PathParam("proteins") @ApiParam (name = "proteins", required = true) String proteins) throws QueryException { try { ProteinQuery query = new ProteinQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setAccessions(Arrays.asList(proteins.split(","))); CellBaseDataResult queryResult = proteinManager.getSequence(query); return createOkResponse(queryResult); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java index f1812cd27f..11e8de47fd 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java @@ -130,6 +130,7 @@ public Response getGeneById(@PathParam("transcripts") @ApiParam(name = "transcri String[] ids = id.split(","); for (String transcriptId : ids) { GeneQuery query = new GeneQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(Collections.singletonList(transcriptId)); queries.add(query); } @@ -192,6 +193,7 @@ public Response getGeneById(@PathParam("transcripts") @ApiParam(name = "transcri public Response getAll() { try { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); logger.info("/search TranscriptQuery: {}", query.toString()); CellBaseDataResult queryResult = transcriptManager.search(query); return createOkResponse(queryResult); @@ -228,7 +230,7 @@ public Response getSequencesByIdList(@PathParam("transcripts") @ApiParam(name = value = TRANSCRIPT_XREFS_DESCRIPTION, required = true) String id) { try { - List> queryResults = transcriptManager.getSequence(id); + List> queryResults = transcriptManager.getSequence(id, getDataRelease()); return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e); @@ -252,6 +254,7 @@ public Response getProtein(@PathParam("transcripts") @ApiParam(name = "transcrip String[] ids = transcripts.split(","); for (String transcriptId : ids) { ProteinQuery query = new ProteinQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setXrefs(Collections.singletonList(transcriptId)); queries.add(query); } @@ -276,6 +279,7 @@ public Response getProteinFunctionPredictionBytranscripts(@PathParam("transcript required = false) String aa) { try { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setTranscriptsXrefs(Arrays.asList(id)); CellBaseDataResult queryResults = proteinManager.getSubstitutionScores(query, position, aa); return createOkResponse(queryResults); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/RegionWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/RegionWSServer.java index e0311376da..a12a247614 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/RegionWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/RegionWSServer.java @@ -187,6 +187,7 @@ public Response getGenesByRegion(@PathParam("regions") @ApiParam(name = "regions String[] coordinates = regions.split(","); for (String coordinate : coordinates) { GeneQuery query = new GeneQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Collections.singletonList(Region.parseRegion(coordinate))); queries.add(query); logger.info("REST GeneQuery: {}", query.toString()); @@ -233,6 +234,7 @@ public Response getTranscriptByRegion(@PathParam("regions") @ApiParam(name = "re String[] coordinates = regions.split(","); for (String coordinate : coordinates) { TranscriptQuery query = new TranscriptQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Region.parseRegions(coordinate)); queries.add(query); logger.info("REST TranscriptQuery: {}", query.toString()); @@ -272,6 +274,7 @@ public Response getRepeatByRegion(@PathParam("regions") @ApiParam(name = "region String[] coordinates = region.split(","); for (String coordinate : coordinates) { RepeatsQuery query = new RepeatsQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Region.parseRegions(coordinate)); queries.add(query); logger.info("REST RepeatsQuery: {}", query.toString()); @@ -317,6 +320,7 @@ public Response getVariantByRegion(@PathParam("regions") @ApiParam(name = "regio List regionList = Region.parseRegions(regions); for (Region region : regionList) { VariantQuery query = new VariantQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Collections.singletonList(region)); queries.add(query); logger.info("REST variantQuery: {}", query.toString()); @@ -358,6 +362,7 @@ public Response getSequenceByRegion(@PathParam("regions") allowableValues = "1,-1", defaultValue = "1", required = true) String strand) { try { GenomeQuery query = new GenomeQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Region.parseRegions(regions)); List> queryResults = genomeManager.getByRegions(query); return createOkResponse(queryResults); @@ -457,6 +462,7 @@ public Response getRegulatoryRegions(@PathParam("regions") @ApiParam(name = "reg String[] regionArray = regions.split(","); for (String regionString : regionArray) { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Region.parseRegions(regionString)); logger.info("REST RegulationQuery: {}", query.toString()); queries.add(query); @@ -493,6 +499,7 @@ public Response getTfByRegion(@PathParam("regions") @ApiParam(name = "regions", String[] regionArray = regions.split(","); for (String regionString : regionArray) { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setRegions(Collections.singletonList(Region.parseRegion(regionString))); query.setFeatureTypes(Collections.singletonList("TF_binding_site")); logger.info("REST RegulationQuery: {}", query.toString()); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java index 12f2db6e0c..4ab8afc52a 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java @@ -94,10 +94,23 @@ public Response getHgvs(@PathParam("variants") @ApiParam(name = "variants", valu @ApiOperation(httpMethod = "GET", value = "FIXME: description needed", response = Map.class, responseContainer = "QueryResponse") public Response getNormalization(@PathParam("variants") @ApiParam(name = "variants", value = RS_IDS, - required = true) String id) { + required = true) String id, + @QueryParam("decompose") + @ApiParam(name = "decompose", + value = "Boolean to indicate whether input MNVs should be " + + "decomposed or not as part of the normalisation step.", + allowableValues = "false,true", + defaultValue = "false") Boolean decompose, + @QueryParam("leftAlign") + @ApiParam(name = "leftAlign", + value = "Boolean to indicate whether input ambiguous INDELS should be " + + "left aligned or not as part of the normalisation step.", + allowableValues = "false,true", + defaultValue = "false") Boolean leftAlign) { try { - CellBaseDataResult queryResults = variantManager.getNormalizationByVariant(id, getDataRelease()); + CellBaseDataResult queryResults = variantManager.getNormalizationByVariant(id, Boolean.TRUE.equals(decompose), + Boolean.TRUE.equals(leftAlign), getDataRelease()); return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e); @@ -136,15 +149,20 @@ public Response getAnnotationByVariantsPOST(@ApiParam(name = "variants", value = @ApiParam(name = "normalize", value = "Boolean to indicate whether input variants shall be " + "normalized or not. Normalization process does NOT " - + "include decomposing ", allowableValues = "false,true", - defaultValue = "false", required = false) Boolean normalize, - @QueryParam("skipDecompose") - @ApiParam(name = "skipDecompose", + + "include decomposing MNV nor left alignment", + allowableValues = "false,true", defaultValue = "false") Boolean normalize, + @QueryParam("decompose") + @ApiParam(name = "decompose", value = "Boolean to indicate whether input MNVs should be " - + "decomposed or not as part of the normalisation step." - + " MNV decomposition is strongly encouraged.", + + "decomposed or not as part of the normalisation step.", + allowableValues = "false,true", + defaultValue = "false") Boolean decompose, + @QueryParam("leftAlign") + @ApiParam(name = "leftAlign", + value = "Boolean to indicate whether input ambiguous INDELS should be " + + "left aligned or not as part of the normalisation step.", allowableValues = "false,true", - defaultValue = "false", required = false) Boolean skipDecompose, + defaultValue = "false") Boolean leftAlign, @QueryParam("ignorePhase") @ApiParam(name = "ignorePhase", value = "Boolean to indicate whether phase data should be " @@ -180,7 +198,7 @@ public Response getAnnotationByVariantsPOST(@ApiParam(name = "variants", value = @ApiParam(name = "checkAminoAcidChange", value = "true/false to specify whether variant match in the clinical variant" + " collection should also be performed at the aminoacid change level", - allowableValues = "false,true", + allowableValues = "false,true", defaultValue = "false", required = false) Boolean checkAminoAcidChange, @QueryParam("consequenceTypeSource") @ApiParam(name = "consequenceTypeSource", value = "Gene set, either ensembl (default) " @@ -188,9 +206,17 @@ public Response getAnnotationByVariantsPOST(@ApiParam(name = "variants", value = required = false) String consequenceTypeSource ) { + try { + checkNormalizationConfig(); + } catch (IllegalArgumentException e) { + return createErrorResponse(e); + } + + return getAnnotationByVariant(variants, normalize, - skipDecompose, + decompose, + leftAlign, ignorePhase, phased, imprecise, @@ -231,10 +257,14 @@ public Response getAnnotationByVariantsGET(@PathParam("variants") @ApiParam(name = "normalize", value = NORMALISE, allowableValues = "false,true", defaultValue = "true", required = false) Boolean normalize, - @QueryParam("skipDecompose") - @ApiParam(name = "skipDecompose", value = SKIP_DECOMPOSE, + @QueryParam("decompose") + @ApiParam(name = "decompose", value = DECOMPOSE, allowableValues = "false,true", - defaultValue = "false", required = false) Boolean skipDecompose, + defaultValue = "false") Boolean decompose, + @QueryParam("leftAlign") + @ApiParam(name = "leftAlign", value = LEFT_ALIGN, + allowableValues = "false,true", + defaultValue = "false") Boolean leftAlign, @QueryParam("ignorePhase") @ApiParam(name = "ignorePhase", value = IGNORE_PHASE, allowableValues = "false,true", @@ -265,10 +295,18 @@ public Response getAnnotationByVariantsGET(@PathParam("variants") @ApiParam(name = "consequenceTypeSource", value = "Gene set, either ensembl (default) " + "or refseq", allowableValues = "ensembl,refseq", allowMultiple = true, defaultValue = "ensembl", required = false) String consequenceTypeSource + ) { + try { + checkNormalizationConfig(); + } catch (IllegalArgumentException e) { + return createErrorResponse(e); + } + return getAnnotationByVariant(variants, normalize, - skipDecompose, + decompose, + leftAlign, ignorePhase, phased, imprecise, @@ -278,9 +316,26 @@ public Response getAnnotationByVariantsGET(@PathParam("variants") consequenceTypeSource); } + private void checkNormalizationConfig() throws IllegalArgumentException { + if (uriParams.containsKey("skipDecompose")) { + throw new IllegalArgumentException("Param 'skipDecompose' is not supported anymore. Please, use 'decompose' instead"); + } + if (uriParams.containsKey("normalize")) { + if (!Boolean.parseBoolean(uriParams.get("normalize"))) { + if (uriParams.containsKey("decompose") && Boolean.parseBoolean(uriParams.get("decompose"))) { + throw new IllegalArgumentException("Incompatible parameter usage: 'normalize'=false and 'decompose'=true"); + } + if (uriParams.containsKey("leftAlign") && Boolean.parseBoolean(uriParams.get("leftAlign"))) { + throw new IllegalArgumentException("Incompatible parameter usage: 'normalize'=false and 'leftAlign'=true"); + } + } + } + } + private Response getAnnotationByVariant(String variants, Boolean normalize, - Boolean skipDecompose, + Boolean decompose, + Boolean leftAlign, Boolean ignorePhase, @Deprecated Boolean phased, Boolean imprecise, @@ -294,8 +349,9 @@ private Response getAnnotationByVariant(String variants, String consequenceTypeSources = (StringUtils.isEmpty(uriParams.get("consequenceTypeSource")) ? consequenceTypeSource : uriParams.get("consequenceTypeSource")); List> queryResults = variantManager.getAnnotationByVariant(query.toQueryOptions(), - variants, normalize, skipDecompose, ignorePhase, phased, imprecise, svExtraPadding, cnvExtraPadding, + variants, normalize, decompose, leftAlign, ignorePhase, phased, imprecise, svExtraPadding, cnvExtraPadding, checkAminoAcidChange, consequenceTypeSources, getDataRelease(), getToken()); + return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e); @@ -395,6 +451,7 @@ public Response getInfo(@PathParam("variants") @ApiParam(name = "variants", valu public Response search() { try { VariantQuery query = new VariantQuery(uriParams); + query.setDataRelease(getDataRelease()); logger.info("/search VariantQuery: {}", query.toString()); CellBaseDataResult queryResults = variantManager.search(query); return createOkResponse(queryResults); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/RegulatoryWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/RegulatoryWSServer.java index 1289006e5a..d41f06c10d 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/RegulatoryWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/RegulatoryWSServer.java @@ -79,6 +79,7 @@ public Response getUniqueValues(@QueryParam("field") @ApiParam(name = "field", r try { copyToFacet("field", field); RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = regulatoryManager.distinct(query); return createOkResponse(queryResults); } catch (Exception e) { @@ -98,6 +99,7 @@ public Response getUniqueValues(@QueryParam("field") @ApiParam(name = "field", r public Response getFeatureTypes() { try { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setFacet("featureType"); CellBaseDataResult queryResults = regulatoryManager.distinct(query); return createOkResponse(queryResults); @@ -120,6 +122,7 @@ public Response getFeatureTypes() { public Response getFeatureClasses() { try { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setFacet("featureClass"); CellBaseDataResult queryResults = regulatoryManager.distinct(query); return createOkResponse(queryResults); @@ -161,6 +164,7 @@ public Response getFeatureClasses() { public Response getAll() { try { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); CellBaseDataResult queryResults = regulatoryManager.search(query); return createOkResponse(queryResults); } catch (Exception e) { diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java index 37e40aed0a..4b7b04cca9 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java @@ -102,6 +102,7 @@ public Response getAllByTfbs(@PathParam("tf") @ApiParam(name = "tf", value = TFB String[] identifiers = tf.split(","); for (String identifier : identifiers) { RegulationQuery query = new RegulationQuery(uriParams); + query.setDataRelease(getDataRelease()); query.setNames(Arrays.asList(identifier)); query.setFeatureTypes(Arrays.asList("TF_binding_site")); queries.add(query); @@ -152,6 +153,7 @@ public Response getAllByTfbs(@PathParam("tf") @ApiParam(name = "tf", value = TFB public Response getEnsemblGenes(@PathParam("tf") @ApiParam(name = "tf", value = TFBS_IDS, required = true) String tf) { try { GeneQuery geneQuery = new GeneQuery(uriParams); + geneQuery.setDataRelease(getDataRelease()); LogicalList logicalList = new LogicalList(Arrays.asList(tf.split(","))); geneQuery.setTranscriptsTfbsId(logicalList); CellBaseDataResult queryResults = geneManager.search(geneQuery); diff --git a/cellbase-test/fitnesse/FitNesseRoot/CellBase/CellBaseCompleteSuite/SuiteSetUp.wiki b/cellbase-test/fitnesse/FitNesseRoot/CellBase/CellBaseCompleteSuite/SuiteSetUp.wiki index 05ad09705e..1684905292 100644 --- a/cellbase-test/fitnesse/FitNesseRoot/CellBase/CellBaseCompleteSuite/SuiteSetUp.wiki +++ b/cellbase-test/fitnesse/FitNesseRoot/CellBase/CellBaseCompleteSuite/SuiteSetUp.wiki @@ -14,13 +14,8 @@ Test |smartrics.rest.fitnesse.fixture| !|Table: Rest Fixture | http://bioinfoint.hpc.cam.ac.uk/| -<<<<<<< HEAD |let|baseURL|const|http://${TEST_HOST}/${CELLBASE_WAR}/webservices/rest/${CELLBASE_VERSION}|http://bioinfodev.hpc.cam.ac.uk/cellbase-4.6.0-SNAPSHOT/webservices/rest/v4| |let|baseURL_hsapiens|const|http://${TEST_HOST}/${CELLBASE_WAR}/webservices/rest/${CELLBASE_VERSION}/hsapiens|http://bioinfodev.hpc.cam.ac.uk/cellbase-4.6.0-SNAPSHOT/webservices/rest/v4/hsapiens| -======= -|let|baseURL|const|http://${TEST_HOST}/${CELLBASE_WAR}/webservices/rest/${CELLBASE_VERSION}|http://bioinfodev.hpc.cam.ac.uk/cellbase-4.5.5/webservices/rest/v4| -|let|baseURL_hsapiens|const|http://${TEST_HOST}/${CELLBASE_WAR}/webservices/rest/${CELLBASE_VERSION}/hsapiens|http://bioinfodev.hpc.cam.ac.uk/cellbase-4.5.5/webservices/rest/v4/hsapiens| ->>>>>>> hotfix-4.5.x |let|useCache|const|false|false| | Table: Rest Fixture Config | with-compareJSON.js | diff --git a/pom.xml b/pom.xml index 74732212de..b1e192e27d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.5.0-SNAPSHOT + 5.6.0-SNAPSHOT pom CellBase project @@ -23,7 +23,7 @@ ${project.version} - 4.8.1-SNAPSHOT + 4.10.0-SNAPSHOT 2.10.0-SNAPSHOT 0.1.0 2.11.4