From 5943afc254a3d4ad479a1c750c4925dff601d5ff Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 5 Jul 2024 09:25:30 +0200 Subject: [PATCH 01/12] Prepare next release 5.8.5-SNAPSHOT --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 482f21352..a2c3dfffc 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 5.8.5-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 6a16f543f..f58841f07 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 5.8.5-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index a0c238ce8..2ede0ee01 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 5.8.5-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index d6a4aacde..51e0e0126 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 5.8.5-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index e9ebc0de6..ff6abf35a 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 5.8.5-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index a88d83275..aa4b5f0a2 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 5.8.5-SNAPSHOT pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.12.0 - 2.12.2 + 4.12.1-SNAPSHOT + 2.12.3-SNAPSHOT 0.1.0 2.11.4 1.9.13 From f9a4f457e04616981279497885e4895f2cd1d1dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 18 Jul 2024 12:33:57 +0200 Subject: [PATCH 02/12] lib: add DataReleaseSingleton and JUnit tests, #TASK-6565 On branch TASK-6565 Changes to be committed: new file: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java new file: cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java --- .../lib/impl/core/DataReleaseSingleton.java | 146 ++++++++++++++++++ .../impl/core/DataReleaseSingletonTest.java | 97 ++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java create mode 100644 cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java new file mode 100644 index 000000000..33fee930e --- /dev/null +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java @@ -0,0 +1,146 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.cellbase.lib.impl.core; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.cellbase.core.config.SpeciesConfiguration; +import org.opencb.cellbase.core.exception.CellBaseException; +import org.opencb.cellbase.core.models.DataRelease; +import org.opencb.cellbase.lib.db.MongoDBManager; +import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; +import org.opencb.commons.datastore.mongodb.MongoDBCollection; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public final class DataReleaseSingleton { + + // {key = dbname, value = species} + private Map speciesMap = new HashMap<>(); + // {key = dbname, value = assembly} + private Map assemblyMap = new HashMap<>(); + // {key = dbname, value = lock} + private Map rwLockMap = new HashMap<>(); + + // {key = dbname, value = { key = release, value = { key = data, value = collection } } } + private Map>> cachedData = new HashMap<>(); + + private CellBaseManagerFactory managerFactory; + + private static DataReleaseSingleton instance; + + public static final String UNKOWN_DATABASE_MSG_PREFIX = "Unknown database "; + public static final String INVALID_RELEASE_MSG_PREFIX = "Invalid release "; + public static final String INVALID_DATA_MSG_PREFIX = "Invalid data "; + + // Private constructor to prevent instantiation + private DataReleaseSingleton(CellBaseManagerFactory managerFactory) { + this.managerFactory = managerFactory; + + // Support multi species and assemblies + CellBaseConfiguration configuration = managerFactory.getConfiguration(); + for (SpeciesConfiguration vertebrate : configuration.getSpecies().getVertebrates()) { + for (SpeciesConfiguration.Assembly assembly : vertebrate.getAssemblies()) { + String databaseName = MongoDBManager.getDatabaseName(vertebrate.getId(), assembly.getName(), configuration.getVersion()); + // This is necessary, before creating the database name the assembly is "cleaned", and we need to get the data release + // manager from the species and the assembly + speciesMap.put(databaseName, vertebrate.getId()); + assemblyMap.put(databaseName, assembly.getName()); + rwLockMap.put(databaseName, new ReentrantReadWriteLock()); + cachedData.put(databaseName, new HashMap<>()); + } + } + } + + // Initialization method to set up the instance with parameters + public static synchronized void initialize(CellBaseManagerFactory managerFactory) { + if (instance == null) { + instance = new DataReleaseSingleton(managerFactory); + } + } + + // Method to get the single instance of the class + public static DataReleaseSingleton getInstance() { + if (instance == null) { + throw new IllegalStateException("Singleton not initialized. Call the function 'initialize' first."); + } + return instance; + } + + // Method to load data from MongoDB and cache it + private void loadData(String dbname) throws CellBaseException { + String species = speciesMap.get(dbname); + String assembly = assemblyMap.get(dbname); + ReleaseMongoDBAdaptor releaseMongoDBAdaptor = managerFactory.getDataReleaseManager(species, assembly).getReleaseDBAdaptor(); + List dataReleases = releaseMongoDBAdaptor.getAll().getResults(); + if (CollectionUtils.isNotEmpty(dataReleases)) { + cachedData.put(dbname, new HashMap<>()); + for (DataRelease dataRelease : dataReleases) { + Map collectionMap = new HashMap<>(); + for (Map.Entry entry : dataRelease.getCollections().entrySet()) { + collectionMap.put(entry.getKey(), releaseMongoDBAdaptor.mongoDataStore.getCollection(entry.getValue())); + } + cachedData.get(dbname).put(dataRelease.getRelease(), collectionMap); + } + } + } + + public void checkDataRelease(String dbname, int release) throws CellBaseException { + checkDataRelease(dbname, release, null); + } + + public void checkDataRelease(String dbname, int release, String data) throws CellBaseException { + // Lock and load data if necessary + if (!cachedData.containsKey(dbname)) { + // If the data release is invalid, throw an exception + String msg = UNKOWN_DATABASE_MSG_PREFIX + dbname; + throw new CellBaseException(msg); + } + rwLockMap.get(dbname).writeLock().lock(); + try { + if (!cachedData.containsKey(dbname) || !cachedData.containsKey(release) + || (StringUtils.isNotEmpty(data) && !cachedData.get(release).containsKey(data))) { + loadData(dbname); + } + } finally { + rwLockMap.get(dbname).writeLock().unlock(); + } + + // Check after loading + if (!cachedData.get(dbname).containsKey(release)) { + // If the data release is invalid, throw an exception + String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " + cachedData.get(dbname).keySet(); + throw new CellBaseException(msg); + } + if (StringUtils.isNotEmpty(data) && !cachedData.get(dbname).get(release).containsKey(data)) { + // If the data release is invalid, throw an exception + String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release + ". The available data are: " + + cachedData.get(dbname).get(release).keySet(); + throw new CellBaseException(msg); + } + } + + // Method to get collection name based on the data and the release + public MongoDBCollection getMongoDBCollection(String dbname, String data, int release) throws CellBaseException { + checkDataRelease(dbname, release, data); + return cachedData.get(dbname).get(release).get(data); + } +} diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java new file mode 100644 index 000000000..cb5df3de4 --- /dev/null +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java @@ -0,0 +1,97 @@ +package org.opencb.cellbase.lib.impl.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.hamcrest.CoreMatchers; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opencb.biodata.models.core.Gene; +import org.opencb.cellbase.core.api.GeneQuery; +import org.opencb.cellbase.core.api.query.QueryException; +import org.opencb.cellbase.core.exception.CellBaseException; +import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.cellbase.lib.GenericMongoDBAdaptorTest; +import org.opencb.cellbase.lib.db.MongoDBManager; +import org.opencb.cellbase.lib.managers.GeneManager; +import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDataStore; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; + +class DataReleaseSingletonTest extends GenericMongoDBAdaptorTest { + + private String dbname; + + @BeforeEach + public void init() throws CellBaseException { + dbname = MongoDBManager.getDatabaseName(SPECIES, ASSEMBLY, cellBaseConfiguration.getVersion()); + } + + @Test + public void testChangeCollectionMap() throws CellBaseException, QueryException, IllegalAccessException, JsonProcessingException { + MongoDBManager mongoDBManager = new MongoDBManager(cellBaseConfiguration); + MongoDataStore mongoDatastore = mongoDBManager.createMongoDBDatastore(SPECIES, ASSEMBLY); + + GeneManager geneManager = cellBaseManagerFactory.getGeneManager(SPECIES, ASSEMBLY); + +// DataReleaseManager dataReleaseManager = new DataReleaseManager(SPECIES, ASSEMBLY, cellBaseConfiguration); +// DataRelease newDataRelease = dataReleaseManager.createRelease(); +// assertEquals(5, newDataRelease.getRelease()); + + Map paramMap = new HashMap<>(); + paramMap.put("id", "ENSG00000248746"); + paramMap.put("include", "id,name,start,end"); + paramMap.put(DATA_RELEASE_PARAM, "1"); + + GeneQuery geneQuery = new GeneQuery(paramMap); + geneQuery.setCount(Boolean.TRUE); + + CellBaseDataResult cellBaseDataResult = geneManager.search(geneQuery); + // WARNING: these values below may slightly change from one data version to another + assertEquals(1, cellBaseDataResult.getNumMatches()); + assertThat(cellBaseDataResult.getResults().stream().map(gene -> gene.getName()).collect(Collectors.toList()), + CoreMatchers.hasItems("ACTN3")); + } + + @Test + public void testDataReleaseSingleton() throws CellBaseException { + MongoDBCollection collection = DataReleaseSingleton.getInstance().getMongoDBCollection(dbname, "gene", 1); + assertTrue(collection != null); + } + + @Test + public void testCheckDatabaseFail() throws CellBaseException { + CellBaseException exception = assertThrows(CellBaseException.class, () -> DataReleaseSingleton.getInstance() + .checkDataRelease("toto", 10)); + assertTrue(exception.getMessage().startsWith(DataReleaseSingleton.UNKOWN_DATABASE_MSG_PREFIX)); + } + + @Test + public void testCheckReleaseFail() throws CellBaseException { + CellBaseException exception = assertThrows(CellBaseException.class, () -> DataReleaseSingleton.getInstance() + .checkDataRelease(dbname, 10)); + assertTrue(exception.getMessage().startsWith(DataReleaseSingleton.INVALID_RELEASE_MSG_PREFIX)); + } + + @Test + public void testCheckReleasePass() throws CellBaseException { + DataReleaseSingleton.getInstance().checkDataRelease(dbname, 1); + } + + @Test + public void testCheckDataFail() throws CellBaseException { + CellBaseException exception = assertThrows(CellBaseException.class, () -> DataReleaseSingleton.getInstance() + .checkDataRelease(dbname, 1, "toto")); + assertTrue(exception.getMessage().startsWith(DataReleaseSingleton.INVALID_DATA_MSG_PREFIX)); + } + + @Test + public void testCheckDataPass() throws CellBaseException { + DataReleaseSingleton.getInstance().checkDataRelease(dbname, 1, "gene"); + } +} \ No newline at end of file From 0cd7e9acc5b23a7c45bf4e71c626ee4f8d3e3ec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 18 Jul 2024 12:35:09 +0200 Subject: [PATCH 03/12] lib: update code to use the DataReleaseSingleton and rename some constants, #TASK-6565 On branch TASK-6565 Changes to be committed: modified: cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java --- .../admin/executors/LoadCommandExecutor.java | 6 +- .../org/opencb/cellbase/lib/EtlCommons.java | 5 +- .../lib/impl/core/CellBaseDBAdaptor.java | 62 ++----------------- .../lib/impl/core/ClinicalMongoDBAdaptor.java | 30 +++------ .../lib/impl/core/GeneMongoDBAdaptor.java | 35 ++++++----- .../lib/impl/core/GenomeMongoDBAdaptor.java | 33 ++++------ .../lib/impl/core/MetaMongoDBAdaptor.java | 5 -- ...ariationFunctionalScoreMongoDBAdaptor.java | 16 ++--- .../lib/impl/core/OntologyMongoDBAdaptor.java | 18 ++---- .../core/PharmacogenomicsMongoDBAdaptor.java | 16 ++--- .../lib/impl/core/ProteinMongoDBAdaptor.java | 28 +++------ .../impl/core/PublicationMongoDBAdaptor.java | 18 ++---- .../impl/core/RegulationMongoDBAdaptor.java | 14 ++--- .../lib/impl/core/RepeatsMongoDBAdaptor.java | 14 +---- .../lib/impl/core/SnpMongoDBAdaptor.java | 19 ++---- .../impl/core/SpliceScoreMongoDBAdaptor.java | 13 +--- .../impl/core/TranscriptMongoDBAdaptor.java | 26 +++----- .../lib/impl/core/VariantMongoDBAdaptor.java | 43 +++++-------- .../lib/impl/core/XRefMongoDBAdaptor.java | 11 +--- .../lib/managers/CellBaseManagerFactory.java | 7 +++ .../lib/managers/DataReleaseManager.java | 9 +++ .../VariantAnnotationCalculator.java | 2 +- 22 files changed, 147 insertions(+), 283 deletions(-) diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java index 97460d5a7..7e9d64edd 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java @@ -411,14 +411,14 @@ private void loadVariationData() throws NoSuchMethodException, InterruptedExcept if (dbSnpFilePath.toFile().exists()) { if (variationPath.resolve(DBSNP_VERSION_FILENAME).toFile().exists()) { logger.info("Loading dbSNP file '{}'", dbSnpFilePath); - loadRunner.load(dbSnpFilePath, SNP_COLLECTION_NAME, dataRelease); + loadRunner.load(dbSnpFilePath, SNP_DATA, dataRelease); // Create index - createIndex(SNP_COLLECTION_NAME); + createIndex(SNP_DATA); // Update release (collection and sources) List sources = Collections.singletonList(variationPath.resolve(DBSNP_VERSION_FILENAME)); - dataReleaseManager.update(dataRelease, SNP_COLLECTION_NAME, EtlCommons.VARIATION_DATA, sources); + dataReleaseManager.update(dataRelease, SNP_DATA, EtlCommons.VARIATION_DATA, sources); } else { logger.warn("In order to load the dbSNP file you need the version file {} within the folder '{}'", DBSNP_VERSION_FILENAME, variationPath); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java index d09291bc3..c2ded07a2 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java @@ -38,6 +38,7 @@ public class EtlCommons { public static final String HOMO_SAPIENS_NAME ="Homo sapiens"; public static final String GENOME_DATA = "genome"; + public static final String GENOME_SEQUENCE_DATA = "genome_sequence"; public static final String GENE_DATA = "gene"; public static final String REFSEQ_DATA = "refseq"; public static final String GENE_DISEASE_ASSOCIATION_DATA = "gene_disease_association"; @@ -69,7 +70,7 @@ public class EtlCommons { public static final String DBSNP_FILE = "GCF_000001405.40.gz"; public static final String DBSNP_NAME = "dbSNP"; public static final String DBSNP_VERSION_FILENAME = DBSNP_NAME + "Version.json"; - public static final String SNP_COLLECTION_NAME = "snp"; + public static final String SNP_DATA = "snp"; public static final String STRUCTURAL_VARIANTS_DATA = "svs"; public static final String REPEATS_DATA = "repeats"; @@ -79,6 +80,8 @@ public class EtlCommons { public static final String DOID_FILE = "doid.obo"; public static final String PFM_DATA = "regulatory_pfm"; + public static final String REGULATORY_REGION_DATA = "regulatory_region"; + // Build specific data options public static final String GENOME_INFO_DATA = "genome_info"; public static final String DISGENET_DATA = "disgenet"; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java index afbcefc16..f8c694a28 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java @@ -16,76 +16,24 @@ package org.opencb.cellbase.lib.impl.core; -import org.apache.commons.collections4.CollectionUtils; import org.opencb.cellbase.core.exception.CellBaseException; -import org.opencb.cellbase.core.models.DataRelease; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDataStore; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - public class CellBaseDBAdaptor extends MongoDBAdaptor { - protected List dataReleases; - protected Map mongoDBCollectionByRelease; - public static final String DATA_RELEASE_SEPARATOR = "__v"; - public static String buildCollectionName(String data, int release) { - String name = data + DATA_RELEASE_SEPARATOR + release; - return name; - } - - public Map buildCollectionByReleaseMap(String data) { - Map collectionMap = new HashMap<>(); - if (CollectionUtils.isNotEmpty(dataReleases)) { - for (DataRelease dataRelease : dataReleases) { - if (dataRelease.getCollections().containsKey(data)) { - String collectionName = dataRelease.getCollections().get(data); - collectionMap.put(dataRelease.getRelease(), mongoDataStore.getCollection(collectionName)); - } - } - } else { - // For backward compatibility (i.e., in case data_release collection is missing) - collectionMap.put(0, mongoDataStore.getCollection(data)); - } - - return collectionMap; - } - - public MongoDBCollection getCollectionByRelease(Map collectionMap, Integer dataRelease) - throws CellBaseException { - int release = dataRelease == null ? 0 : dataRelease; - if (!collectionMap.containsKey(release)) { - // If the data release is invalid, throw an exception - String msg = "Data not found in release " + release + ". " + collectionMap.toString(); - logger.error(msg); - throw new CellBaseException(msg); - } - return collectionMap.get(release); - } - public CellBaseDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - this.dataReleases = new ReleaseMongoDBAdaptor(mongoDataStore).getAll().getResults(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("CellBaseDBAdaptor{"); - sb.append("dataRelease=").append(dataReleases); - sb.append('}'); - return sb.toString(); } - public List getDataReleases() { - return dataReleases; + public static String buildCollectionName(String data, int release) { + String name = data + DATA_RELEASE_SEPARATOR + release; + return name; } - public CellBaseDBAdaptor setDataReleases(List dataReleases) { - this.dataReleases = dataReleases; - return this; + public MongoDBCollection getMongoDBCollection(String data, int release) throws CellBaseException { + return DataReleaseSingleton.getInstance().getMongoDBCollection(mongoDataStore.getDatabaseName(), data, release); } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java index 1b203e387..236dc4737 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java @@ -46,6 +46,7 @@ import java.util.function.Consumer; import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; +import static org.opencb.cellbase.lib.EtlCommons.CLINICAL_VARIANTS_DATA; /** * Created by fjlopez on 06/12/16. @@ -65,14 +66,6 @@ public ClinicalMongoDBAdaptor(MongoDataStore mongoDataStore, GenomeManager genom super(mongoDataStore); this.genomeManager = genomeManager; - - init(); - } - - private void init() { - logger.debug("ClinicalMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("clinical_variants"); } public CellBaseDataResult next(Query query, QueryOptions options) { @@ -103,16 +96,14 @@ public CellBaseDataResult getIntervalFrequencies(Query query, int intervalSize, public CellBaseDataResult count(Query query) throws CellBaseException { Bson bson = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return new CellBaseDataResult<>(mongoDBCollection.count(bson)); } public CellBaseDataResult distinct(Query query, String field) throws CellBaseException { Bson bson = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return new CellBaseDataResult<>(mongoDBCollection.distinct(field, bson)); } @@ -128,8 +119,7 @@ public CellBaseDataResult get(Query query, QueryOptions options) throws logger.debug("query: {}", bson.toBsonDocument().toJson()); logger.debug("queryOptions: {}", options.toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return new CellBaseDataResult<>(mongoDBCollection.find(bson, null, Variant.class, parsedOptions)); } @@ -140,8 +130,7 @@ public CellBaseDataResult nativeGet(Query query, QueryOptions options) throws Ce logger.debug("query: {}", bson.toBsonDocument().toJson()); logger.debug("queryOptions: {}", options.toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return new CellBaseDataResult<>(mongoDBCollection.find(bson, parsedOptions)); } @@ -152,8 +141,7 @@ public Iterator iterator(Query query, QueryOptions options) { public Iterator nativeIterator(Query query, QueryOptions options) throws CellBaseException { Bson bson = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return mongoDBCollection.nativeQuery().find(bson, options); } @@ -355,7 +343,7 @@ private CellBaseDataResult getClinvarPhenotypeGeneRelations(QueryOptions queryOp fields.put("associatedGenes", 1); pipeline.add(new Document("$project", fields)); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, dataRelease); return executeAggregation2("", pipeline, queryOptions, mongoDBCollection); } @@ -377,7 +365,7 @@ private CellBaseDataResult getGwasPhenotypeGeneRelations(QueryOptions queryOptio fields.put("associatedGenes", 1); pipeline.add(new Document("$project", fields)); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, dataRelease); return executeAggregation2("", pipeline, queryOptions, mongoDBCollection); } @@ -466,7 +454,7 @@ public CellBaseIterator iterator(ClinicalVariantQuery query) throws CellBaseExce Bson projection = getProjection(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Variant.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java index 94a686310..528012f41 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java @@ -32,6 +32,7 @@ import org.opencb.cellbase.core.api.query.ProjectionQueryOptions; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.cellbase.lib.EtlCommons; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.cellbase.lib.iterator.CellBaseIterator; import org.opencb.cellbase.lib.iterator.CellBaseMongoDBIterator; @@ -43,13 +44,15 @@ import java.util.*; import java.util.regex.Pattern; +import static org.opencb.cellbase.lib.EtlCommons.GENE_DATA; + /** * Created by imedina on 25/11/15. */ public class GeneMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { private static final Set CONSTRAINT_NAMES = new HashSet<>(); - private Map refseqCollectionByRelease; +// private Map refseqCollectionByRelease; private static final GenericDocumentComplexConverter CONVERTER; @@ -66,15 +69,15 @@ public class GeneMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCor public GeneMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - this.init(); +// this.init(); } - private void init() { - mongoDBCollectionByRelease = buildCollectionByReleaseMap("gene"); - refseqCollectionByRelease = buildCollectionByReleaseMap("refseq"); - - logger.debug("GeneMongoDBAdaptor initialised"); - } +// private void init() { +// mongoDBCollectionByRelease = buildCollectionByReleaseMap("gene"); +// refseqCollectionByRelease = buildCollectionByReleaseMap("refseq"); +// +// logger.debug("GeneMongoDBAdaptor initialised"); +// } @Override public CellBaseDataResult aggregationStats(GeneQuery query) { @@ -97,10 +100,10 @@ public List> info(List ids, ProjectionQueryOpti orBsonList.add(Filters.eq("name", id)); Bson query = Filters.or(orBsonList); if (StringUtils.isEmpty(source) || ParamConstants.QueryParams.ENSEMBL.key().equalsIgnoreCase(source)) { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, dataRelease); results.add(new CellBaseDataResult<>(mongoDBCollection.find(query, projection, CONVERTER, new QueryOptions()))); } else { - MongoDBCollection mongoDBCollection = getCollectionByRelease(refseqCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(EtlCommons.REFSEQ_DATA, dataRelease); results.add(new CellBaseDataResult<>(mongoDBCollection.find(query, projection, CONVERTER, new QueryOptions()))); } } @@ -115,10 +118,10 @@ public CellBaseIterator iterator(GeneQuery query) throws CellBaseException MongoDBIterator iterator; if (query.getSource() != null && !query.getSource().isEmpty() && ParamConstants.QueryParams.REFSEQ.key() .equalsIgnoreCase(query.getSource().get(0))) { - MongoDBCollection mongoDBCollection = getCollectionByRelease(refseqCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(EtlCommons.REFSEQ_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); } else { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); } return new CellBaseMongoDBIterator<>(iterator); @@ -127,7 +130,7 @@ public CellBaseIterator iterator(GeneQuery query) throws CellBaseException @Override public CellBaseDataResult distinct(GeneQuery geneQuery) throws CellBaseException { Bson bsonDocument = parseQuery(geneQuery); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, geneQuery.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, geneQuery.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(geneQuery.getFacet(), bsonDocument, String.class)); } @@ -135,7 +138,7 @@ public CellBaseDataResult distinct(GeneQuery geneQuery) throws CellBaseE public CellBaseDataResult groupBy(GeneQuery geneQuery) throws CellBaseException { Bson bsonQuery = parseQuery(geneQuery); logger.info("geneQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, geneQuery.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, geneQuery.getDataRelease()); return groupBy(bsonQuery, geneQuery, "name", mongoDBCollection); } @@ -157,7 +160,7 @@ public CellBaseDataResult startsWith(String id, QueryOptions options, int projection = Projections.exclude("transcripts", "annotation"); } } - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(regex, projection, CONVERTER, options)); } @@ -355,7 +358,7 @@ public CellBaseDataResult getTfbs(String geneId, QueryOptions qu List pipeline = unwindAndMatchTranscripts(query, queryOptions); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(TranscriptTfbs.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, dataRelease); MongoDBIterator iterator = mongoDBCollection.iterator(pipeline, converter, queryOptions); List tfbs = new ArrayList<>(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java index e8d40c802..7901e34e2 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java @@ -44,6 +44,7 @@ import java.util.*; +import static org.opencb.cellbase.lib.EtlCommons.*; import static org.opencb.cellbase.lib.MongoDBCollectionConfiguration.CONSERVATION_CHUNK_SIZE; import static org.opencb.cellbase.lib.MongoDBCollectionConfiguration.GENOME_SEQUENCE_CHUNK_SIZE; @@ -64,20 +65,10 @@ public class GenomeMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseC public GenomeMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("GenomeMongoDBAdaptor: in 'constructor'"); - - genomeInfoMongoDBCollectionByRelease = buildCollectionByReleaseMap("genome_info"); - mongoDBCollectionByRelease = buildCollectionByReleaseMap("genome_sequence"); - conservationMongoDBCollectionByRelease = buildCollectionByReleaseMap("conservation"); } public CellBaseDataResult getGenomeInfo(QueryOptions queryOptions, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(genomeInfoMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_INFO_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(new Document(), queryOptions)); } @@ -88,7 +79,7 @@ public CellBaseDataResult getChromosomeInfo(String chromosomeId, QueryOptions qu queryOptions.addToListOption("include", "chromosomes.$"); } Document dbObject = new Document("chromosomes", new Document("$elemMatch", new Document("name", chromosomeId))); - MongoDBCollection mongoDBCollection = getCollectionByRelease(genomeInfoMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_INFO_DATA, dataRelease); return executeQuery(chromosomeId, dbObject, queryOptions, mongoDBCollection); } @@ -263,7 +254,7 @@ public List>> getConservation(List< ids.add(region.toString()); } - MongoDBCollection mongoDBCollection = getCollectionByRelease(conservationMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CONSERVATION_DATA, dataRelease); List cellBaseDataResults = executeQueryList2(ids, queries, options, mongoDBCollection); List>> conservationCellBaseDataResults = new ArrayList<>(); @@ -355,7 +346,7 @@ public List> getAllScoresByRegionList(List reg ids.add(region.toString()); } - MongoDBCollection mongoDBCollection = getCollectionByRelease(conservationMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CONSERVATION_DATA, dataRelease); List queryResults = executeQueryList2(ids, queries, options, mongoDBCollection); // List queryResults = executeQueryList(ids, queries, options); @@ -419,7 +410,7 @@ public List> getAllScoresByRegionList(List reg public CellBaseDataResult getGenomeSequenceRawData(List chunkIds, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_SEQUENCE_DATA, dataRelease); CellBaseQueryOptions queryOptions = new CellBaseQueryOptions(); queryOptions.setExcludes(Arrays.asList("_id", "_chunkIds")); @@ -438,7 +429,7 @@ public CellBaseDataResult getGenomeSequenceRawData(List(mongoDBCollection.find(bson, options)); } @@ -461,7 +452,7 @@ public CellBaseIterator iterator(GenomeQuery query) throws CellBaseE QueryOptions queryOptions = query.toQueryOptions(); List pipeline = unwind(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Chromosome.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(genomeInfoMongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_INFO_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(pipeline, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -511,14 +502,14 @@ public CellBaseDataResult aggregationStats(GenomeQuery query) { public CellBaseDataResult groupBy(GenomeQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("query: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_SEQUENCE_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } @Override public CellBaseDataResult distinct(GenomeQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(genomeInfoMongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_INFO_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @@ -526,7 +517,7 @@ public CellBaseDataResult distinct(GenomeQuery query) throws CellBaseExc public List> info(List ids, ProjectionQueryOptions queryOptions, int dataRelease, String apiKey) throws CellBaseException { List> results = new ArrayList<>(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_SEQUENCE_DATA, dataRelease); for (String id : ids) { Bson projection = getProjection(queryOptions); List orBsonList = new ArrayList<>(ids.size()); @@ -580,7 +571,7 @@ public Collection getConservationScoreChunkIds(Region region) { public CellBaseDataResult getConservationScoreRegion(List chunkIds, CellBaseQueryOptions options, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(conservationMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CONSERVATION_DATA, dataRelease); Bson projection = getProjection(options); List orBsonList = new ArrayList<>(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java index e5cd4d38c..4413f4bf4 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java @@ -50,11 +50,6 @@ public class MetaMongoDBAdaptor extends MongoDBAdaptor implements CellBaseCoreDB public MetaMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - init(); - } - - - private void init() { logger.debug("MetaMongoDBAdaptor: in 'constructor'"); mongoDBCollection = mongoDataStore.getCollection("metadata"); apiKeyStatsMongoDBCollection = mongoDataStore.getCollection("apikey_stats", WriteConcern.ACKNOWLEDGED, ReadPreference.primary()); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java index bf649886a..f3968fb2d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java @@ -33,19 +33,13 @@ import java.util.Collections; import java.util.List; +import static org.opencb.cellbase.lib.EtlCommons.MISSENSE_VARIATION_SCORE_DATA; + public class MissenseVariationFunctionalScoreMongoDBAdaptor extends CellBaseDBAdaptor { public MissenseVariationFunctionalScoreMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("MissenseVariationFunctionalScoreMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("missense_variation_functional_score"); } public CellBaseDataResult query(String chromosome, int position, String reference, int dataRelease) @@ -56,7 +50,7 @@ public CellBaseDataResult query(String chromosom andBsonList.add(Filters.eq("reference", reference)); Bson query = Filters.and(andBsonList); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(MISSENSE_VARIATION_SCORE_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(query, null, MissenseVariantFunctionalScore.class, new QueryOptions())); } @@ -73,7 +67,7 @@ public CellBaseDataResult getScores(St final String id = chromosome + ":" + position + ":" + reference + ":" + alternate; - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(MISSENSE_VARIATION_SCORE_DATA, dataRelease); DataResult missenseVariantFunctionalScoreDataResult = mongoDBCollection.find(query, null, MissenseVariantFunctionalScore.class, new QueryOptions()); @@ -97,7 +91,7 @@ public CellBaseDataResult getScores(St public CellBaseDataResult getScores(String chromosome, List positions, CellBaseQueryOptions options, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(MISSENSE_VARIATION_SCORE_DATA, dataRelease); Bson projection = getProjection(options); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java index f1e664a50..00b55506c 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java @@ -38,6 +38,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.OBO_DATA; + public class OntologyMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { private static final GenericDocumentComplexConverter CONVERTER; @@ -48,14 +50,6 @@ public class OntologyMongoDBAdaptor extends CellBaseDBAdaptor implements CellBas public OntologyMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - logger.debug("OntologyMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("ontology"); } @Override @@ -64,7 +58,7 @@ public CellBaseIterator iterator(OntologyQuery query) throws CellB Bson projection = getProjection(query); QueryOptions queryOptions = query.toQueryOptions(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(OBO_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -74,7 +68,7 @@ public List> info(List ids, ProjectionQ String apiKey) throws CellBaseException { List> results = new ArrayList<>(); Bson projection = getProjection(queryOptions); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(OBO_DATA, dataRelease); for (String id : ids) { List orBsonList = new ArrayList<>(ids.size()); orBsonList.add(Filters.eq("id", id)); @@ -88,7 +82,7 @@ public List> info(List ids, ProjectionQ @Override public CellBaseDataResult distinct(OntologyQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(OBO_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @@ -101,7 +95,7 @@ public CellBaseDataResult aggregationStats(OntologyQuery query) { public CellBaseDataResult groupBy(OntologyQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("geneQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(OBO_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java index f0a601a13..10a05a11f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java @@ -39,6 +39,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.PHARMACOGENOMICS_DATA; + /** * Created by jtarraga on 9/4/23. */ @@ -53,14 +55,6 @@ public class PharmacogenomicsMongoDBAdaptor extends CellBaseDBAdaptor public PharmacogenomicsMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - mongoDBCollectionByRelease = buildCollectionByReleaseMap("pharmacogenomics"); - - logger.debug("PharmacogenomicsMongoDBAdaptor initialised"); } @Override @@ -79,7 +73,7 @@ public List> info(List ids, Projectio orBsonList.add(Filters.eq("id", id)); orBsonList.add(Filters.eq("name", id)); Bson query = Filters.or(orBsonList); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PHARMACOGENOMICS_DATA, dataRelease); results.add(new CellBaseDataResult<>(mongoDBCollection.find(query, projection, CONVERTER, new QueryOptions()))); } return results; @@ -91,7 +85,7 @@ public CellBaseIterator iterator(PharmaChemicalQuery query) thro QueryOptions queryOptions = query.toQueryOptions(); Bson projection = getProjection(query); MongoDBIterator iterator; - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PHARMACOGENOMICS_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -99,7 +93,7 @@ public CellBaseIterator iterator(PharmaChemicalQuery query) thro @Override public CellBaseDataResult distinct(PharmaChemicalQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PHARMACOGENOMICS_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java index 353b4042c..c1646f2db 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java @@ -42,6 +42,9 @@ import java.util.*; +import static org.opencb.cellbase.lib.EtlCommons.PROTEIN_DATA; +import static org.opencb.cellbase.lib.EtlCommons.PROTEIN_FUNCTIONAL_PREDICTION_DATA; + /** * Created by imedina on 01/12/15. */ @@ -79,18 +82,8 @@ public class ProteinMongoDBAdaptor extends CellBaseDBAdaptor implements CellBase aaShortNameMap.put("VAL", "V"); } - public ProteinMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("ProteinMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("protein"); - proteinSubstitutionMongoDBCollectionByRelease = buildCollectionByReleaseMap("protein_functional_prediction"); } public CellBaseDataResult getSubstitutionScores(TranscriptQuery query, Integer position, String aa) throws CellBaseException { @@ -100,8 +93,7 @@ public CellBaseDataResult getSubstitutionScores(TranscriptQuery query, In if (query.getTranscriptsId() != null && query.getTranscriptsId().get(0) != null) { String transcriptId = query.getTranscriptsId().get(0).split("\\.")[0]; Bson transcript = Filters.eq("transcriptId", transcriptId); - MongoDBCollection mongoDBCollection = getCollectionByRelease(proteinSubstitutionMongoDBCollectionByRelease, - query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_FUNCTIONAL_PREDICTION_DATA, query.getDataRelease()); String aaShortName = null; // If position and aa change are provided we create a 'projection' to return only the required data from the database @@ -272,7 +264,7 @@ public CellBaseDataResult getVariantAnnotation(String groupFields.put("feature", new Document("$addToSet", "$feature")); pipeline.add(new Document("$group", groupFields)); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, dataRelease); proteinVariantData = executeAggregation2(ensemblTranscriptId + "_" + String.valueOf(position) + "_" + aaAlternate, pipeline, new QueryOptions(), mongoDBCollection); if (proteinVariantData.getNumResults() > 0) { @@ -294,7 +286,7 @@ public CellBaseIterator iterator(ProteinQuery query) throws CellBaseExcep QueryOptions queryOptions = query.toQueryOptions(); Bson projection = getProjection(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Entry.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -303,7 +295,7 @@ public CellBaseIterator iterator(ProteinQuery query) throws CellBaseExcep public List> info(List ids, ProjectionQueryOptions queryOptions, int dataRelease, String apiKey) throws CellBaseException { List> results = new ArrayList<>(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, dataRelease); for (String id : ids) { Bson projection = getProjection(queryOptions); List orBsonList = new ArrayList<>(ids.size()); @@ -324,14 +316,14 @@ public CellBaseDataResult aggregationStats(ProteinQuery query) { public CellBaseDataResult groupBy(ProteinQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("proteinQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } @Override public CellBaseDataResult distinct(ProteinQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @@ -490,7 +482,7 @@ private ProteinVariantAnnotation processProteinVariantData(ProteinVariantAnnotat public CellBaseDataResult getProteinSubstitutionRawData(List transcriptIds, CellBaseQueryOptions options, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(proteinSubstitutionMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_FUNCTIONAL_PREDICTION_DATA, dataRelease); // Be sure to exclude the internal field "_id" Bson projection; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java index 5c8fcb571..91087b66c 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java @@ -40,6 +40,8 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.opencb.cellbase.lib.EtlCommons.PUBMED_DATA; + public class PublicationMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { private static final GenericDocumentComplexConverter CONVERTER; @@ -50,14 +52,6 @@ public class PublicationMongoDBAdaptor extends CellBaseDBAdaptor implements Cell public PublicationMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - logger.debug("PublicationMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("pubmed"); } @Override @@ -66,7 +60,7 @@ public CellBaseIterator iterator(PublicationQuery query) throws C Bson projection = getProjection(query); QueryOptions queryOptions = query.toQueryOptions(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PUBMED_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -76,7 +70,7 @@ public List> info(List ids, Projection String apiKey) throws CellBaseException { List> results = new ArrayList<>(); Bson projection = getProjection(queryOptions); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PUBMED_DATA, dataRelease); for (String id : ids) { List orBsonList = new ArrayList<>(ids.size()); orBsonList.add(Filters.eq("id", id)); @@ -89,7 +83,7 @@ public List> info(List ids, Projection @Override public CellBaseDataResult distinct(PublicationQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PUBMED_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @@ -102,7 +96,7 @@ public CellBaseDataResult aggregationStats(PublicationQuery query public CellBaseDataResult groupBy(PublicationQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("geneQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PUBMED_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java index e7373b57b..098b5fa4f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java @@ -38,6 +38,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.REGULATORY_REGION_DATA; + /** * Created by imedina on 07/12/15. */ @@ -51,14 +53,6 @@ public class RegulationMongoDBAdaptor extends CellBaseDBAdaptor implements CellB public RegulationMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - logger.debug("RegulationMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("regulatory_region"); } public Bson parseQuery(RegulationQuery query) { @@ -98,7 +92,7 @@ public CellBaseIterator iterator(RegulationQuery query) throw Bson bson = parseQuery(query); QueryOptions queryOptions = query.toQueryOptions(); Bson projection = getProjection(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REGULATORY_REGION_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -112,7 +106,7 @@ public List> info(List ids, Projec @Override public CellBaseDataResult distinct(RegulationQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REGULATORY_REGION_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java index 7105be25b..04bb2ea5d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java @@ -38,23 +38,15 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.REPEATS_DATA; + /** * Created by fjlopez on 10/05/17. */ public class RepeatsMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { - private static final String REPEAT_COLLECTION = "repeats"; - public RepeatsMongoDBAdaptor(MongoDataStore mongoDatastore) { super(mongoDatastore); - - init(); - } - - private void init() { - logger.debug("RepeatsMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap(REPEAT_COLLECTION); } public Bson parseQuery(RepeatsQuery query) { @@ -95,7 +87,7 @@ public CellBaseIterator iterator(RepeatsQuery query) throws CellBaseException { QueryOptions queryOptions = query.toQueryOptions(); Bson projection = getProjection(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Repeat.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REPEATS_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java index 6b3d78ce8..ee69e9818 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java @@ -41,6 +41,7 @@ import static org.opencb.cellbase.core.ParamConstants.API_KEY_PARAM; import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; +import static org.opencb.cellbase.lib.EtlCommons.SNP_DATA; public class SnpMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { @@ -52,14 +53,6 @@ public class SnpMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCore public SnpMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - logger.debug("SnpMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("snp"); } @Override @@ -68,7 +61,7 @@ public CellBaseIterator iterator(SnpQuery query) throws CellBaseException { Bson projection = getProjection(query); QueryOptions queryOptions = query.toQueryOptions(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -78,7 +71,7 @@ public List> info(List ids, ProjectionQueryOptio throws CellBaseException { List> results = new ArrayList<>(); Bson projection = getProjection(queryOptions); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, dataRelease); for (String id : ids) { List orBsonList = new ArrayList<>(); orBsonList.add(Filters.eq("id", id)); @@ -92,7 +85,7 @@ public List> info(List ids, ProjectionQueryOptio public CellBaseDataResult distinct(SnpQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("snpQuery distinct: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonQuery, String.class)); } @@ -105,7 +98,7 @@ public CellBaseDataResult aggregationStats(SnpQuery query) { public CellBaseDataResult groupBy(SnpQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("snpQuery groupBy: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } @@ -118,7 +111,7 @@ public CellBaseDataResult startsWith(String id, QueryOptions options, int d projection = Projections.exclude(options.getAsStringList(QueryOptions.EXCLUDE)); } - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(regex, projection, CONVERTER, options)); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java index c4dd2d465..553f579ff 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java @@ -23,7 +23,6 @@ import org.opencb.biodata.models.core.SpliceScoreAlternate; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.result.CellBaseDataResult; -import org.opencb.cellbase.lib.EtlCommons; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.mongodb.MongoDBCollection; @@ -33,18 +32,12 @@ import java.util.Collections; import java.util.List; +import static org.opencb.cellbase.lib.EtlCommons.SPLICE_SCORE_DATA; + public class SpliceScoreMongoDBAdaptor extends CellBaseDBAdaptor { public SpliceScoreMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("SpliceScoreMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap(EtlCommons.SPLICE_SCORE_DATA); } public CellBaseDataResult getScores(String chromosome, int position, String reference, String alternate) @@ -67,7 +60,7 @@ public CellBaseDataResult getScores(String chromosome, int position final String id = chromosome + ":" + position + ":" + ref + ":" + alt; - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SPLICE_SCORE_DATA, dataRelease); DataResult spliceScoreDataResult = mongoDBCollection.find(query, null, SpliceScore.class, new QueryOptions()); List results = new ArrayList<>(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java index 9fc26d1c5..0bbf49e09 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java @@ -43,13 +43,14 @@ import java.util.*; +import static org.opencb.cellbase.lib.EtlCommons.GENE_DATA; +import static org.opencb.cellbase.lib.EtlCommons.REFSEQ_DATA; + /** * Created by swaathi on 27/11/15. */ public class TranscriptMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { - private Map refseqCollectionByRelease = null; - private static final GenericDocumentComplexConverter CONVERTER; static { @@ -58,13 +59,6 @@ public class TranscriptMongoDBAdaptor extends CellBaseDBAdaptor implements CellB public TranscriptMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - mongoDBCollectionByRelease = buildCollectionByReleaseMap("gene"); - refseqCollectionByRelease = buildCollectionByReleaseMap("refseq"); } @Override @@ -73,10 +67,10 @@ public CellBaseIterator iterator(TranscriptQuery query) throws CellB List pipeline = unwindAndMatchTranscripts(query, queryOptions); MongoDBIterator iterator; if (query.getSource() != null && !query.getSource().isEmpty() && "RefSeq".equalsIgnoreCase(query.getSource().get(0))) { - MongoDBCollection mongoDBCollection = getCollectionByRelease(refseqCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REFSEQ_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(pipeline, CONVERTER, queryOptions); } else { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(pipeline, CONVERTER, queryOptions); } return new CellBaseMongoDBIterator<>(iterator); @@ -110,10 +104,10 @@ public List> info(List ids, ProjectionQue List pipeline = unwindAndMatchTranscripts(bson, queryOptions); MongoDBIterator iterator; if (StringUtils.isNotEmpty(source) && ParamConstants.QueryParams.REFSEQ.key().equalsIgnoreCase(source)) { - MongoDBCollection mongoDBCollection = getCollectionByRelease(refseqCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REFSEQ_DATA, dataRelease); iterator = mongoDBCollection.iterator(pipeline, CONVERTER, queryOptions); } else { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, dataRelease); iterator = mongoDBCollection.iterator(pipeline, CONVERTER, queryOptions); } @@ -160,7 +154,7 @@ public CellBaseDataResult count(TranscriptQuery query) throws CellBaseExce List projections = unwind(query); Bson group = Aggregates.group("transcripts", Accumulators.sum("count", 1)); projections.add(group); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); CellBaseDataResult cellBaseDataResult = new CellBaseDataResult(mongoDBCollection.aggregate(projections, null)); Number number = (Number) cellBaseDataResult.first().get("count"); Long count = number.longValue(); @@ -177,7 +171,7 @@ public CellBaseDataResult aggregationStats(TranscriptQuery query) { public CellBaseDataResult groupBy(TranscriptQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("transcriptQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } @@ -185,7 +179,7 @@ public CellBaseDataResult groupBy(TranscriptQuery query) throws Cell public CellBaseDataResult distinct(TranscriptQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); logger.info("transcriptQuery: {}", bsonDocument.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java index 3c33266f7..85e1c0878 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java @@ -58,6 +58,7 @@ import static org.opencb.cellbase.core.ParamConstants.API_KEY_PARAM; import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; +import static org.opencb.cellbase.lib.EtlCommons.*; import static org.opencb.cellbase.lib.MongoDBCollectionConfiguration.VARIATION_FUNCTIONAL_SCORE_CHUNK_SIZE; /** @@ -79,16 +80,6 @@ public class VariantMongoDBAdaptor extends CellBaseDBAdaptor implements CellBase public VariantMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("VariationMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("variation"); - caddDBCollectionByRelease = buildCollectionByReleaseMap("variation_functional_score"); - snpDBCollectionByRelease = buildCollectionByReleaseMap("snp"); } public CellBaseDataResult next(Query query, QueryOptions options) { @@ -104,7 +95,7 @@ public CellBaseDataResult getIntervalFrequencies(Query query, int intervalSize, if (query.getString(ParamConstants.QueryParams.REGION.key()) != null) { Region region = Region.parseRegion(query.getString(ParamConstants.QueryParams.REGION.key())); Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return getIntervalFrequencies(bsonDocument, region, intervalSize, options, mongoDBCollection); } return null; @@ -129,13 +120,13 @@ public CellBaseDataResult update(List objectList, String field, String[] i public CellBaseDataResult count(Query query, int dataRelease) throws CellBaseException { Bson document = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new CellBaseDataResult(mongoDBCollection.count(document)); } public CellBaseDataResult distinct(Query query, String field, int dataRelease) throws CellBaseException { Bson document = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new CellBaseDataResult(mongoDBCollection.distinct(field, document)); } @@ -150,7 +141,7 @@ public CellBaseDataResult get(Query query, QueryOptions inputOptions, i // options = addPrivateExcludeOptions(options); logger.debug("query: {}", bson.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(bson, null, Variant.class, options)); } @@ -178,20 +169,20 @@ public CellBaseDataResult nativeGet(Query query, QueryOptions options, int dataR Bson bson = parseQuery(query); // options.put(MongoDBCollection.SKIP_COUNT, true); logger.debug("query: {}", bson.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new CellBaseDataResult(mongoDBCollection.find(bson, options)); } public Iterator iterator(Query query, QueryOptions inputOptions, int dataRelease) throws CellBaseException { Bson bson = parseQuery(query); QueryOptions options = addPrivateExcludeOptions(new QueryOptions(inputOptions)); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new VariantMongoDBIterator(mongoDBCollection.nativeQuery().find(bson, options)); } public Iterator nativeIterator(Query query, QueryOptions options, int dataRelease) throws CellBaseException { Bson bson = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return mongoDBCollection.nativeQuery().find(bson, options); } @@ -205,13 +196,13 @@ public void forEach(Query query, Consumer action, QueryOptions o public CellBaseDataResult groupBy(Query query, String field, QueryOptions options, int dataRelease) throws CellBaseException { Bson bsonQuery = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return groupBy(bsonQuery, field, "name", options, mongoDBCollection); } public CellBaseDataResult groupBy(Query query, List fields, QueryOptions options, int dataRelease) throws CellBaseException { Bson bsonQuery = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return groupBy(bsonQuery, fields, "name", options, mongoDBCollection); } @@ -481,7 +472,7 @@ private CellBaseDataResult updateAnnotation(List variantDocument QueryOptions options = new QueryOptions("upsert", false); options.put("multi", false); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); bulkWriteResult = new CellBaseDataResult<>(mongoDBCollection.update(queries, updates, options)); logger.info("{} object updated", bulkWriteResult.first().getModifiedCount()); @@ -534,7 +525,7 @@ private CellBaseDataResult updatePopulationFrequencies(List vari QueryOptions options = new QueryOptions("upsert", true); options.put("multi", false); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); bulkWriteResult = new CellBaseDataResult<>(mongoDBCollection.update(queries, updates, options)); logger.info("{} object updated", bulkWriteResult.first().getUpserts().size() + bulkWriteResult.first().getModifiedCount()); @@ -576,7 +567,7 @@ public CellBaseDataResult getFunctionalScoreVariant(Variant variant, Quer // QueryBuilder builder = QueryBuilder.start("_chunkIds").is(chunkId); Document query = new Document("_chunkIds", chunkId); - MongoDBCollection mongoDBCollection = getCollectionByRelease(caddDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_FUNCTIONAL_SCORE_DATA, dataRelease); CellBaseDataResult result = executeQuery(chromosome + "_" + position + "_" + reference + "_" + alternate, query, queryOptions, mongoDBCollection); @@ -731,7 +722,7 @@ public CellBaseIterator iterator(VariantQuery query) throws CellBaseExc Bson projection = getProjection(query); VariantConverter converter = new VariantConverter(); logger.info("query: {}", bson.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -755,7 +746,7 @@ public CellBaseDataResult distinct(VariantQuery query) { public List> info(List ids, ProjectionQueryOptions queryOptions, int dataRelease, String apiKey) throws CellBaseException { List> results = new ArrayList<>(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); Bson projection = getProjection(queryOptions); List variantIds = getVariantIds(ids, dataRelease); List orBsonList = new ArrayList<>(variantIds.size()); @@ -777,7 +768,7 @@ public List getFunctionalScoreChunkIds(Region region) { public CellBaseDataResult getFunctionalScoreRegion(List chunkIds, CellBaseQueryOptions options, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(caddDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_FUNCTIONAL_SCORE_DATA, dataRelease); Bson projection = getProjection(options); List orBsonList = new ArrayList<>(); @@ -811,7 +802,7 @@ private List getVariantIds(List ids, int dataRelease) throws Cel Bson query = Filters.or(orBsonList); // 2. We must exclude as much information as possible to improve performance - MongoDBCollection mongoDBCollection = getCollectionByRelease(snpDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, dataRelease); DataResult snpDataResult = mongoDBCollection.find(query, Projections.exclude(ANNOTATION_FIELD), Snp.class, new QueryOptions()); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java index 42889fb02..6c578eabf 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java @@ -40,6 +40,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.GENE_DATA; + /** * Created by imedina on 07/12/15. */ @@ -47,13 +49,6 @@ public class XRefMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCor public XRefMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("XRefMongoDBAdaptor: in 'constructor'"); - mongoDBCollectionByRelease = buildCollectionByReleaseMap("gene"); } @Override @@ -61,7 +56,7 @@ public CellBaseIterator iterator(XrefQuery query) throws CellBaseException QueryOptions queryOptions = query.toQueryOptions(); List pipeline = unwind(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Xref.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(pipeline, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java index ba6e90e15..6063ee565 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java @@ -20,6 +20,7 @@ import org.opencb.cellbase.core.config.SpeciesConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.utils.SpeciesUtils; +import org.opencb.cellbase.lib.impl.core.DataReleaseSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +68,8 @@ public CellBaseManagerFactory(CellBaseConfiguration configuration) { ontologyManagers = new HashMap<>(); dataReleaseManagers = new HashMap<>(); pharmacogenomicsManagers = new HashMap<>(); + + DataReleaseSingleton.initialize(this); } private String getMultiKey(String species, String assembly) { @@ -374,4 +377,8 @@ public PharmacogenomicsManager getPharmacogenomicsManager(String species, String } return pharmacogenomicsManagers.get(multiKey); } + + public CellBaseConfiguration getConfiguration() { + return configuration; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java index c768cb15d..ac5c27865 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java @@ -240,4 +240,13 @@ public DataRelease checkDataRelease(int inRelease) throws CellBaseException { + ". Valid data releases are: " + StringUtils.join(dataReleases.stream().map(dr -> dr.getRelease()) .collect(Collectors.toList()), ",")); } + + public ReleaseMongoDBAdaptor getReleaseDBAdaptor() { + return releaseDBAdaptor; + } + + public DataReleaseManager setReleaseDBAdaptor(ReleaseMongoDBAdaptor releaseDBAdaptor) { + this.releaseDBAdaptor = releaseDBAdaptor; + return this; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java index 1b86b4936..cc64afe6b 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java @@ -469,7 +469,7 @@ private List runAnnotationProcess(List normalizedVar FutureSnpAnnotator futureSnpAnnotator = null; Future>> snpFuture = null; - if (annotatorSet.contains("xrefs") && dataRelease.getCollections().containsKey(EtlCommons.SNP_COLLECTION_NAME)) { + if (annotatorSet.contains("xrefs") && dataRelease.getCollections().containsKey(EtlCommons.SNP_DATA)) { futureSnpAnnotator = new FutureSnpAnnotator(normalizedVariantList, dataRelease.getRelease(), variantManager, logger); snpFuture = CACHED_THREAD_POOL.submit(futureSnpAnnotator); } From c8ebf7fa3d26ad3f7064b61df7ab6b0c28567a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 19 Jul 2024 10:54:37 +0200 Subject: [PATCH 04/12] server: check data release in URL in the GenericRestWSServer, #TASK-6565 On branch TASK-6565 Changes to be committed: modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java modified: cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java --- .../opencb/cellbase/lib/impl/core/DataReleaseSingleton.java | 4 ++++ .../org/opencb/cellbase/server/rest/GenericRestWSServer.java | 2 ++ 2 files changed, 6 insertions(+) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java index 33fee930e..bc33a5cdb 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java @@ -103,6 +103,10 @@ private void loadData(String dbname) throws CellBaseException { } } + public void checkDataRelease(String species, String assembly, String version, int release) throws CellBaseException { + checkDataRelease(MongoDBManager.getDatabaseName(species, assembly, version), release); + } + public void checkDataRelease(String dbname, int release) throws CellBaseException { checkDataRelease(dbname, release, null); } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index fc961bb9a..4d8f69462 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -36,6 +36,7 @@ import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.core.utils.SpeciesUtils; +import org.opencb.cellbase.lib.impl.core.DataReleaseSingleton; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.cellbase.lib.managers.MetaManager; @@ -224,6 +225,7 @@ protected int getDataRelease() throws CellBaseException { if (dataRelease == 0) { logger.info("Using data release 0 in query: using the default data release '" + defaultDataRelease.getRelease() + "' for CellBase version '" + version + "'"); + DataReleaseSingleton.getInstance().checkDataRelease(species, assembly, version, dataRelease); return defaultDataRelease.getRelease(); } else { return dataRelease; From 4737a29998ba93bbeb6cbbd4afc4e7d27b3df1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 19 Jul 2024 13:17:05 +0200 Subject: [PATCH 05/12] server: fix check data release in the GenericRestWSServer, #TASK-6565 On branch TASK-6565 Changes to be committed: modified: cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java --- .../org/opencb/cellbase/server/rest/GenericRestWSServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index 4d8f69462..6d58f8b4a 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -225,9 +225,9 @@ protected int getDataRelease() throws CellBaseException { if (dataRelease == 0) { logger.info("Using data release 0 in query: using the default data release '" + defaultDataRelease.getRelease() + "' for CellBase version '" + version + "'"); - DataReleaseSingleton.getInstance().checkDataRelease(species, assembly, version, dataRelease); return defaultDataRelease.getRelease(); } else { + DataReleaseSingleton.getInstance().checkDataRelease(species, assembly, version, dataRelease); return dataRelease; } } catch (NumberFormatException e) { From 32c5b07b0766cc4d584685ca347cfe07e6f38c8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 19 Jul 2024 13:25:08 +0200 Subject: [PATCH 06/12] lib: fix check in the singlenton, #TASK-6565 On branch TASK-6565 Changes to be committed: modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java --- .../lib/impl/core/DataReleaseSingleton.java | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java index bc33a5cdb..be23082ca 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java @@ -120,26 +120,27 @@ public void checkDataRelease(String dbname, int release, String data) throws Cel } rwLockMap.get(dbname).writeLock().lock(); try { - if (!cachedData.containsKey(dbname) || !cachedData.containsKey(release) - || (StringUtils.isNotEmpty(data) && !cachedData.get(release).containsKey(data))) { + if (!cachedData.get(dbname).containsKey(release) + || (StringUtils.isNotEmpty(data) && !cachedData.get(dbname).get(release).containsKey(data))) { + // Load the data releases from the MongoDB collection for that database name loadData(dbname); + + // Check after loading + if (!cachedData.get(dbname).containsKey(release)) { + // If the release is invalid, throw an exception + String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " + cachedData.get(dbname).keySet(); + throw new CellBaseException(msg); + } + if (StringUtils.isNotEmpty(data) && !cachedData.get(dbname).get(release).containsKey(data)) { + // If the data is invalid, throw an exception + String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release + ". The available data are: " + + cachedData.get(dbname).get(release).keySet(); + throw new CellBaseException(msg); + } } } finally { rwLockMap.get(dbname).writeLock().unlock(); } - - // Check after loading - if (!cachedData.get(dbname).containsKey(release)) { - // If the data release is invalid, throw an exception - String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " + cachedData.get(dbname).keySet(); - throw new CellBaseException(msg); - } - if (StringUtils.isNotEmpty(data) && !cachedData.get(dbname).get(release).containsKey(data)) { - // If the data release is invalid, throw an exception - String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release + ". The available data are: " - + cachedData.get(dbname).get(release).keySet(); - throw new CellBaseException(msg); - } } // Method to get collection name based on the data and the release From dc2785685a71ba83beb0bef72b0ee1056c46f72e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 19 Jul 2024 13:27:01 +0200 Subject: [PATCH 07/12] lib: fix checkstyle, #TASK-6565 On branch TASK-6565 Changes to be committed: modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java --- .../cellbase/lib/impl/core/DataReleaseSingleton.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java index be23082ca..67bf0c926 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java @@ -128,13 +128,14 @@ public void checkDataRelease(String dbname, int release, String data) throws Cel // Check after loading if (!cachedData.get(dbname).containsKey(release)) { // If the release is invalid, throw an exception - String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " + cachedData.get(dbname).keySet(); + String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " + + cachedData.get(dbname).keySet(); throw new CellBaseException(msg); } if (StringUtils.isNotEmpty(data) && !cachedData.get(dbname).get(release).containsKey(data)) { // If the data is invalid, throw an exception - String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release + ". The available data are: " - + cachedData.get(dbname).get(release).keySet(); + String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release + + ". The available data are: " + cachedData.get(dbname).get(release).keySet(); throw new CellBaseException(msg); } } From f0d9599a0a82375f0ba5f05958f45f10a0ed533a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 22 Jul 2024 17:06:13 +0200 Subject: [PATCH 08/12] lib: add MongoDB listener to check updates in the data_release collection, #TASK-6565 --- .../executors/ValidationCommandExecutor.java | 2 +- .../lib/impl/core/DataReleaseSingleton.java | 51 ++++++++++++++++++- .../lib/impl/core/ReleaseMongoDBAdaptor.java | 2 +- .../lib/managers/AbstractManager.java | 4 ++ .../lib/managers/CellBaseManagerFactory.java | 4 +- 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java index 612e8d6a3..611c9eb93 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java @@ -66,7 +66,7 @@ public ValidationCommandExecutor(AdminCliOptionsParser.ValidationCommandOptions } @Override - public void execute() { + public void execute() throws CellBaseException { checkFilesExist(); VariantAnnotationCalculator variantAnnotationCalculator; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java index 67bf0c926..89ecfc11d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java @@ -16,8 +16,14 @@ package org.opencb.cellbase.lib.impl.core; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import com.mongodb.client.model.changestream.FullDocument; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.bson.Document; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.config.SpeciesConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; @@ -25,6 +31,8 @@ import org.opencb.cellbase.lib.db.MongoDBManager; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.List; @@ -47,12 +55,14 @@ public final class DataReleaseSingleton { private static DataReleaseSingleton instance; + private static final Logger LOGGER = LoggerFactory.getLogger(DataReleaseSingleton.class); + public static final String UNKOWN_DATABASE_MSG_PREFIX = "Unknown database "; public static final String INVALID_RELEASE_MSG_PREFIX = "Invalid release "; public static final String INVALID_DATA_MSG_PREFIX = "Invalid data "; // Private constructor to prevent instantiation - private DataReleaseSingleton(CellBaseManagerFactory managerFactory) { + private DataReleaseSingleton(CellBaseManagerFactory managerFactory) throws CellBaseException { this.managerFactory = managerFactory; // Support multi species and assemblies @@ -66,12 +76,29 @@ private DataReleaseSingleton(CellBaseManagerFactory managerFactory) { assemblyMap.put(databaseName, assembly.getName()); rwLockMap.put(databaseName, new ReentrantReadWriteLock()); cachedData.put(databaseName, new HashMap<>()); + + MongoClient mongoClient = managerFactory.getDataReleaseManager(vertebrate.getId(), assembly.getName()).getMongoDatastore() + .getMongoClient(); + MongoDatabase database = mongoClient.getDatabase(databaseName); + MongoCollection collection = database.getCollection(ReleaseMongoDBAdaptor.DATA_RELEASE_COLLECTION_NAME); + LOGGER.info("Setting listener for database {} and collection {}", database.getName(), collection.getNamespace() + .getCollectionName()); + // Set up the change stream for the collection + new Thread(() -> { + collection.watch().fullDocument(FullDocument.UPDATE_LOOKUP).forEach(changeStreamDocument -> { + try { + handleDocumentChange(changeStreamDocument); + } catch (CellBaseException e) { + LOGGER.warn("Exception from handle document change function: {}", e.getStackTrace()); + } + }); + }).start(); } } } // Initialization method to set up the instance with parameters - public static synchronized void initialize(CellBaseManagerFactory managerFactory) { + public static synchronized void initialize(CellBaseManagerFactory managerFactory) throws CellBaseException { if (instance == null) { instance = new DataReleaseSingleton(managerFactory); } @@ -149,4 +176,24 @@ public MongoDBCollection getMongoDBCollection(String dbname, String data, int re checkDataRelease(dbname, release, data); return cachedData.get(dbname).get(release).get(data); } + + private void handleDocumentChange(ChangeStreamDocument changeStreamDocument) throws CellBaseException { + // Get database name + String dbname = changeStreamDocument.getNamespace().getDatabaseName(); + String collectionName = changeStreamDocument.getNamespace().getCollectionName(); + LOGGER.info("Collection {} of database {} has been updated", collectionName, dbname); + + // Handle the change event + if (!cachedData.containsKey(dbname)) { + // If the data release is invalid, throw an exception + String msg = UNKOWN_DATABASE_MSG_PREFIX + dbname; + throw new CellBaseException(msg); + } + rwLockMap.get(dbname).writeLock().lock(); + try { + loadData(dbname); + } finally { + rwLockMap.get(dbname).writeLock().unlock(); + } + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java index 4098f9c53..747e821c0 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java @@ -47,7 +47,7 @@ public class ReleaseMongoDBAdaptor extends MongoDBAdaptor implements CellBaseCor private MongoDBCollection mongoDBCollection; - private final String DATA_RELEASE_COLLECTION_NAME = "data_release"; + public static final String DATA_RELEASE_COLLECTION_NAME = "data_release"; public ReleaseMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java index 07c5a0f86..3ee8e7959 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java @@ -107,4 +107,8 @@ protected List createQueries(Query query, String csvField, String queryKe public void close() { mongoDBManager.close(); } + + public MongoDataStore getMongoDatastore() { + return mongoDatastore; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java index 6063ee565..a4b5a03de 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java @@ -51,7 +51,7 @@ public class CellBaseManagerFactory { private Logger logger; - public CellBaseManagerFactory(CellBaseConfiguration configuration) { + public CellBaseManagerFactory(CellBaseConfiguration configuration) throws CellBaseException { this.configuration = configuration; logger = LoggerFactory.getLogger(this.getClass()); @@ -326,7 +326,7 @@ public OntologyManager getOntologyManager(String species, String assembly) throw return ontologyManagers.get(multiKey); } - public DataReleaseManager getDataRelesaseManager(String species) throws CellBaseException { + public DataReleaseManager getDataReleaseManager(String species) throws CellBaseException { if (species == null) { throw new CellBaseException("Species is required."); } From 8665fa5f7817088bccdbfc4d7b5868fd9d645707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 8 Aug 2024 12:21:09 +0200 Subject: [PATCH 09/12] lib: improve code and exception management in DataReleaseSingleton, #TASK-6565 --- .../lib/impl/core/CellBaseDBAdaptor.java | 1 + .../lib/impl/core/MongoDBAdaptor.java | 4 + .../{ => singleton}/DataReleaseSingleton.java | 109 ++++++++++-------- .../lib/impl/core/singleton/DatabaseInfo.java | 102 ++++++++++++++++ .../lib/managers/CellBaseManagerFactory.java | 2 +- .../impl/core/DataReleaseSingletonTest.java | 1 + .../server/rest/GenericRestWSServer.java | 2 +- 7 files changed, 174 insertions(+), 47 deletions(-) rename cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/{ => singleton}/DataReleaseSingleton.java (66%) create mode 100644 cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java index f8c694a28..f25f93721 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java @@ -17,6 +17,7 @@ package org.opencb.cellbase.lib.impl.core; import org.opencb.cellbase.core.exception.CellBaseException; +import org.opencb.cellbase.lib.impl.core.singleton.DataReleaseSingleton; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDataStore; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java index bcf9fcaa1..d0e55e360 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java @@ -715,4 +715,8 @@ protected List getIntervalFeatureFrequencies(Region re // public void setAssembly(String assembly) { // this.assembly = assembly; // } + + public MongoDataStore getMongoDataStore() { + return mongoDataStore; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java similarity index 66% rename from cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java rename to cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java index 89ecfc11d..b0b401b27 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingleton.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.opencb.cellbase.lib.impl.core; +package org.opencb.cellbase.lib.impl.core.singleton; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; @@ -29,11 +29,13 @@ import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.lib.db.MongoDBManager; +import org.opencb.cellbase.lib.impl.core.ReleaseMongoDBAdaptor; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,15 +43,8 @@ public final class DataReleaseSingleton { - // {key = dbname, value = species} - private Map speciesMap = new HashMap<>(); - // {key = dbname, value = assembly} - private Map assemblyMap = new HashMap<>(); - // {key = dbname, value = lock} - private Map rwLockMap = new HashMap<>(); - - // {key = dbname, value = { key = release, value = { key = data, value = collection } } } - private Map>> cachedData = new HashMap<>(); + // Map where the key is dbname and the value is the DatabaseInfo + private Map dbInfoMap = new HashMap<>(); private CellBaseManagerFactory managerFactory; @@ -72,10 +67,9 @@ private DataReleaseSingleton(CellBaseManagerFactory managerFactory) throws CellB String databaseName = MongoDBManager.getDatabaseName(vertebrate.getId(), assembly.getName(), configuration.getVersion()); // This is necessary, before creating the database name the assembly is "cleaned", and we need to get the data release // manager from the species and the assembly - speciesMap.put(databaseName, vertebrate.getId()); - assemblyMap.put(databaseName, assembly.getName()); - rwLockMap.put(databaseName, new ReentrantReadWriteLock()); - cachedData.put(databaseName, new HashMap<>()); + DatabaseInfo dbInfo = new DatabaseInfo(databaseName, vertebrate.getId(), assembly.getName(), new ReentrantReadWriteLock(), + new HashMap<>()); + dbInfoMap.put(databaseName, dbInfo); MongoClient mongoClient = managerFactory.getDataReleaseManager(vertebrate.getId(), assembly.getName()).getMongoDatastore() .getMongoClient(); @@ -85,13 +79,31 @@ private DataReleaseSingleton(CellBaseManagerFactory managerFactory) throws CellB .getCollectionName()); // Set up the change stream for the collection new Thread(() -> { - collection.watch().fullDocument(FullDocument.UPDATE_LOOKUP).forEach(changeStreamDocument -> { + while (true) { try { - handleDocumentChange(changeStreamDocument); - } catch (CellBaseException e) { - LOGGER.warn("Exception from handle document change function: {}", e.getStackTrace()); + collection.watch().fullDocument(FullDocument.UPDATE_LOOKUP).forEach(changeStreamDocument -> { + try { + handleDocumentChange(changeStreamDocument); + } catch (CellBaseException e) { + LOGGER.warn("Exception from handle document change function (database = {}, collection = {}): {}", + collection.getNamespace().getDatabaseName(), collection.getNamespace().getCollectionName(), + Arrays.toString(e.getStackTrace())); + } + }); + } catch (Exception e) { + LOGGER.error("Failed to watch collection (database = {}, collection = {}): {}", + collection.getNamespace().getDatabaseName(), collection.getNamespace().getCollectionName(), + Arrays.toString(e.getStackTrace())); + try { + // Sleep 5 sec before retrying to avoid tight loop + Thread.sleep(5000); + } catch (InterruptedException ie) { + // Restore interrupt status + Thread.currentThread().interrupt(); + break; // Exit loop if interrupted + } } - }); + } }).start(); } } @@ -114,18 +126,19 @@ public static DataReleaseSingleton getInstance() { // Method to load data from MongoDB and cache it private void loadData(String dbname) throws CellBaseException { - String species = speciesMap.get(dbname); - String assembly = assemblyMap.get(dbname); + DatabaseInfo dbInfo = getDatabaseInfo(dbname); + + String species = dbInfo.getSpecies(); + String assembly = dbInfo.getAssembly(); ReleaseMongoDBAdaptor releaseMongoDBAdaptor = managerFactory.getDataReleaseManager(species, assembly).getReleaseDBAdaptor(); List dataReleases = releaseMongoDBAdaptor.getAll().getResults(); if (CollectionUtils.isNotEmpty(dataReleases)) { - cachedData.put(dbname, new HashMap<>()); for (DataRelease dataRelease : dataReleases) { Map collectionMap = new HashMap<>(); for (Map.Entry entry : dataRelease.getCollections().entrySet()) { - collectionMap.put(entry.getKey(), releaseMongoDBAdaptor.mongoDataStore.getCollection(entry.getValue())); + collectionMap.put(entry.getKey(), releaseMongoDBAdaptor.getMongoDataStore().getCollection(entry.getValue())); } - cachedData.get(dbname).put(dataRelease.getRelease(), collectionMap); + dbInfo.getCacheData().put(dataRelease.getRelease(), collectionMap); } } } @@ -139,42 +152,41 @@ public void checkDataRelease(String dbname, int release) throws CellBaseExceptio } public void checkDataRelease(String dbname, int release, String data) throws CellBaseException { + DatabaseInfo dbInfo = getDatabaseInfo(dbname); + // Lock and load data if necessary - if (!cachedData.containsKey(dbname)) { - // If the data release is invalid, throw an exception - String msg = UNKOWN_DATABASE_MSG_PREFIX + dbname; - throw new CellBaseException(msg); - } - rwLockMap.get(dbname).writeLock().lock(); + dbInfo.getRwLock().writeLock().lock(); try { - if (!cachedData.get(dbname).containsKey(release) - || (StringUtils.isNotEmpty(data) && !cachedData.get(dbname).get(release).containsKey(data))) { + if (!dbInfo.getCacheData().containsKey(release) + || (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data))) { // Load the data releases from the MongoDB collection for that database name loadData(dbname); // Check after loading - if (!cachedData.get(dbname).containsKey(release)) { + if (!dbInfo.getCacheData().containsKey(release)) { // If the release is invalid, throw an exception String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " - + cachedData.get(dbname).keySet(); + + dbInfo.getCacheData().keySet(); throw new CellBaseException(msg); } - if (StringUtils.isNotEmpty(data) && !cachedData.get(dbname).get(release).containsKey(data)) { + if (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data)) { // If the data is invalid, throw an exception String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release - + ". The available data are: " + cachedData.get(dbname).get(release).keySet(); + + ". The available data are: " + dbInfo.getCacheData().get(release).keySet(); throw new CellBaseException(msg); } } } finally { - rwLockMap.get(dbname).writeLock().unlock(); + dbInfo.getRwLock().writeLock().unlock(); } } // Method to get collection name based on the data and the release public MongoDBCollection getMongoDBCollection(String dbname, String data, int release) throws CellBaseException { checkDataRelease(dbname, release, data); - return cachedData.get(dbname).get(release).get(data); + + DatabaseInfo dbInfo = getDatabaseInfo(dbname); + return dbInfo.getCacheData().get(release).get(data); } private void handleDocumentChange(ChangeStreamDocument changeStreamDocument) throws CellBaseException { @@ -183,17 +195,24 @@ private void handleDocumentChange(ChangeStreamDocument changeStreamDoc String collectionName = changeStreamDocument.getNamespace().getCollectionName(); LOGGER.info("Collection {} of database {} has been updated", collectionName, dbname); + DatabaseInfo dbInfo = getDatabaseInfo(dbname); + // Handle the change event - if (!cachedData.containsKey(dbname)) { - // If the data release is invalid, throw an exception - String msg = UNKOWN_DATABASE_MSG_PREFIX + dbname; - throw new CellBaseException(msg); - } - rwLockMap.get(dbname).writeLock().lock(); + dbInfo.getRwLock().writeLock().lock(); try { loadData(dbname); } finally { - rwLockMap.get(dbname).writeLock().unlock(); + dbInfo.getRwLock().writeLock().unlock(); + } + } + + private DatabaseInfo getDatabaseInfo(String dbname) throws CellBaseException { + if (!dbInfoMap.containsKey(dbname)) { + // Unknown database + String msg = UNKOWN_DATABASE_MSG_PREFIX + dbname; + throw new CellBaseException(msg); } + + return dbInfoMap.get(dbname); } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java new file mode 100644 index 000000000..85eb32eb1 --- /dev/null +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.cellbase.lib.impl.core.singleton; + +import org.opencb.commons.datastore.mongodb.MongoDBCollection; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class DatabaseInfo { + private String dbName; + private String species; + private String assembly; + private ReentrantReadWriteLock rwLock; + private Map> cacheData; + + public DatabaseInfo() { + this.rwLock = new ReentrantReadWriteLock(); + this.cacheData = new HashMap<>(); + } + + public DatabaseInfo(String dbName, String species, String assembly, ReentrantReadWriteLock rwLock, Map> cacheData) { + this.dbName = dbName; + this.species = species; + this.assembly = assembly; + this.rwLock = rwLock; + this.cacheData = cacheData; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("DatabaseInfo{"); + sb.append("dbName='").append(dbName).append('\''); + sb.append(", species='").append(species).append('\''); + sb.append(", assembly='").append(assembly).append('\''); + sb.append(", rwLock=").append(rwLock); + sb.append(", cacheData=").append(cacheData); + sb.append('}'); + return sb.toString(); + } + + public String getDbName() { + return dbName; + } + + public DatabaseInfo setDbName(String dbName) { + this.dbName = dbName; + return this; + } + + public String getSpecies() { + return species; + } + + public DatabaseInfo setSpecies(String species) { + this.species = species; + return this; + } + + public String getAssembly() { + return assembly; + } + + public DatabaseInfo setAssembly(String assembly) { + this.assembly = assembly; + return this; + } + + public ReentrantReadWriteLock getRwLock() { + return rwLock; + } + + public DatabaseInfo setRwLock(ReentrantReadWriteLock rwLock) { + this.rwLock = rwLock; + return this; + } + + public Map> getCacheData() { + return cacheData; + } + + public DatabaseInfo setCacheData(Map> cacheData) { + this.cacheData = cacheData; + return this; + } +} diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java index a4b5a03de..9752b0300 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java @@ -20,7 +20,7 @@ import org.opencb.cellbase.core.config.SpeciesConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.utils.SpeciesUtils; -import org.opencb.cellbase.lib.impl.core.DataReleaseSingleton; +import org.opencb.cellbase.lib.impl.core.singleton.DataReleaseSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java index cb5df3de4..ce4873795 100644 --- a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java @@ -11,6 +11,7 @@ import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.GenericMongoDBAdaptorTest; import org.opencb.cellbase.lib.db.MongoDBManager; +import org.opencb.cellbase.lib.impl.core.singleton.DataReleaseSingleton; import org.opencb.cellbase.lib.managers.GeneManager; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDataStore; diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index 6d58f8b4a..775cdd8d3 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -36,7 +36,7 @@ import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.core.utils.SpeciesUtils; -import org.opencb.cellbase.lib.impl.core.DataReleaseSingleton; +import org.opencb.cellbase.lib.impl.core.singleton.DataReleaseSingleton; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.cellbase.lib.managers.MetaManager; From 62dd9022d07c20d6c7e9a5572668b800198b03f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 13 Aug 2024 14:08:09 +0200 Subject: [PATCH 10/12] lib: improve thread management in data release singleton, #TASK-6565 --- .../core/singleton/DataReleaseSingleton.java | 90 +++++++++++-------- .../lib/impl/core/singleton/DatabaseInfo.java | 22 ++--- 2 files changed, 66 insertions(+), 46 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java index b0b401b27..3b884242d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java @@ -39,13 +39,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; public final class DataReleaseSingleton { // Map where the key is dbname and the value is the DatabaseInfo private Map dbInfoMap = new HashMap<>(); + private ExecutorService executorService; private CellBaseManagerFactory managerFactory; private static DataReleaseSingleton instance; @@ -58,6 +62,21 @@ public final class DataReleaseSingleton { // Private constructor to prevent instantiation private DataReleaseSingleton(CellBaseManagerFactory managerFactory) throws CellBaseException { + // Create the executor service with a 4-thread pool and add shutdown hook to terminate thread pool + this.executorService = Executors.newFixedThreadPool(4); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + LOGGER.info("Shutting down thread pool..."); + executorService.shutdownNow(); + try { + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.info("Thread pool did not terminate in the specified time."); + } + } catch (InterruptedException e) { + LOGGER.info("Thread pool termination interrupted."); + Thread.currentThread().interrupt(); + } + })); + this.managerFactory = managerFactory; // Support multi species and assemblies @@ -67,7 +86,7 @@ private DataReleaseSingleton(CellBaseManagerFactory managerFactory) throws CellB String databaseName = MongoDBManager.getDatabaseName(vertebrate.getId(), assembly.getName(), configuration.getVersion()); // This is necessary, before creating the database name the assembly is "cleaned", and we need to get the data release // manager from the species and the assembly - DatabaseInfo dbInfo = new DatabaseInfo(databaseName, vertebrate.getId(), assembly.getName(), new ReentrantReadWriteLock(), + DatabaseInfo dbInfo = new DatabaseInfo(databaseName, vertebrate.getId(), assembly.getName(), new ReentrantLock(), new HashMap<>()); dbInfoMap.put(databaseName, dbInfo); @@ -78,33 +97,31 @@ private DataReleaseSingleton(CellBaseManagerFactory managerFactory) throws CellB LOGGER.info("Setting listener for database {} and collection {}", database.getName(), collection.getNamespace() .getCollectionName()); // Set up the change stream for the collection - new Thread(() -> { - while (true) { + executorService.submit(() -> { + while (!Thread.currentThread().isInterrupted()) { try { collection.watch().fullDocument(FullDocument.UPDATE_LOOKUP).forEach(changeStreamDocument -> { try { handleDocumentChange(changeStreamDocument); } catch (CellBaseException e) { LOGGER.warn("Exception from handle document change function (database = {}, collection = {}): {}", - collection.getNamespace().getDatabaseName(), collection.getNamespace().getCollectionName(), - Arrays.toString(e.getStackTrace())); + collection.getNamespace().getDatabaseName(), collection.getNamespace().getCollectionName(), e); } }); } catch (Exception e) { LOGGER.error("Failed to watch collection (database = {}, collection = {}): {}", - collection.getNamespace().getDatabaseName(), collection.getNamespace().getCollectionName(), - Arrays.toString(e.getStackTrace())); + collection.getNamespace().getDatabaseName(), collection.getNamespace().getCollectionName(), e); try { // Sleep 5 sec before retrying to avoid tight loop Thread.sleep(5000); } catch (InterruptedException ie) { - // Restore interrupt status + // Restore interrupt status, and exit loop if interrupted Thread.currentThread().interrupt(); - break; // Exit loop if interrupted + break; } } } - }).start(); + }); } } } @@ -154,30 +171,33 @@ public void checkDataRelease(String dbname, int release) throws CellBaseExceptio public void checkDataRelease(String dbname, int release, String data) throws CellBaseException { DatabaseInfo dbInfo = getDatabaseInfo(dbname); - // Lock and load data if necessary - dbInfo.getRwLock().writeLock().lock(); - try { - if (!dbInfo.getCacheData().containsKey(release) - || (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data))) { - // Load the data releases from the MongoDB collection for that database name - loadData(dbname); - - // Check after loading - if (!dbInfo.getCacheData().containsKey(release)) { - // If the release is invalid, throw an exception - String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " - + dbInfo.getCacheData().keySet(); - throw new CellBaseException(msg); - } - if (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data)) { - // If the data is invalid, throw an exception - String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release - + ". The available data are: " + dbInfo.getCacheData().get(release).keySet(); - throw new CellBaseException(msg); + if (!dbInfo.getCacheData().containsKey(release) + || (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data))) { + try { + // Lock and load data if necessary + dbInfo.getLock().lock(); + if (!dbInfo.getCacheData().containsKey(release) + || (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data))) { + // Load the data releases from the MongoDB collection for that database name + loadData(dbname); } + } finally { + dbInfo.getLock().unlock(); + } + + // Check after loading + if (!dbInfo.getCacheData().containsKey(release)) { + // If the release is invalid, throw an exception + String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " + + dbInfo.getCacheData().keySet(); + throw new CellBaseException(msg); + } + if (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data)) { + // If the data is invalid, throw an exception + String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release + + ". The available data are: " + dbInfo.getCacheData().get(release).keySet(); + throw new CellBaseException(msg); } - } finally { - dbInfo.getRwLock().writeLock().unlock(); } } @@ -198,11 +218,11 @@ private void handleDocumentChange(ChangeStreamDocument changeStreamDoc DatabaseInfo dbInfo = getDatabaseInfo(dbname); // Handle the change event - dbInfo.getRwLock().writeLock().lock(); + dbInfo.getLock().lock(); try { loadData(dbname); } finally { - dbInfo.getRwLock().writeLock().unlock(); + dbInfo.getLock().unlock(); } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java index 85eb32eb1..d953f278d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java @@ -20,26 +20,26 @@ import java.util.HashMap; import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; public class DatabaseInfo { private String dbName; private String species; private String assembly; - private ReentrantReadWriteLock rwLock; + private ReentrantLock lock; private Map> cacheData; public DatabaseInfo() { - this.rwLock = new ReentrantReadWriteLock(); + this.lock = new ReentrantLock(); this.cacheData = new HashMap<>(); } - public DatabaseInfo(String dbName, String species, String assembly, ReentrantReadWriteLock rwLock, Map> cacheData) { + public DatabaseInfo(String dbName, String species, String assembly, ReentrantLock lock, + Map> cacheData) { this.dbName = dbName; this.species = species; this.assembly = assembly; - this.rwLock = rwLock; + this.lock = lock; this.cacheData = cacheData; } @@ -49,7 +49,7 @@ public String toString() { sb.append("dbName='").append(dbName).append('\''); sb.append(", species='").append(species).append('\''); sb.append(", assembly='").append(assembly).append('\''); - sb.append(", rwLock=").append(rwLock); + sb.append(", lock=").append(lock); sb.append(", cacheData=").append(cacheData); sb.append('}'); return sb.toString(); @@ -82,12 +82,12 @@ public DatabaseInfo setAssembly(String assembly) { return this; } - public ReentrantReadWriteLock getRwLock() { - return rwLock; + public ReentrantLock getLock() { + return lock; } - public DatabaseInfo setRwLock(ReentrantReadWriteLock rwLock) { - this.rwLock = rwLock; + public DatabaseInfo setRwLock(ReentrantLock lock) { + this.lock = lock; return this; } From f86423c0e7d3fcd83dc13971fbdc61cc4a0ad4b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 13 Aug 2024 14:11:58 +0200 Subject: [PATCH 11/12] lib: fix checkstyle issue, #TASK-6565 --- .../cellbase/lib/impl/core/singleton/DataReleaseSingleton.java | 1 - 1 file changed, 1 deletion(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java index 3b884242d..99280240f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java @@ -35,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; From e5ccdabe6ae2024e4389090c83aa12017797f01f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 1 Oct 2024 10:24:57 +0200 Subject: [PATCH 12/12] core: fix configuration file by removing 'conflicts' messages, #TASK-6565 --- .../src/main/resources/configuration.yml | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/cellbase-core/src/main/resources/configuration.yml b/cellbase-core/src/main/resources/configuration.yml index ea500a24a..84378507b 100644 --- a/cellbase-core/src/main/resources/configuration.yml +++ b/cellbase-core/src/main/resources/configuration.yml @@ -106,13 +106,6 @@ download: clinvar: # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2021-07.xml.gz # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-02.xml.gz -<<<<<<< HEAD - host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-11.xml.gz - clinvarVariation: -# host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2021-07.xml.gz -# host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-02.xml.gz - host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-11.xml.gz -======= # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-11.xml.gz host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/RCV_xml_old_format/ClinVarFullRelease_2024-05.xml.gz version: 2024-05 @@ -122,10 +115,6 @@ download: # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-11.xml.gz host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/VCV_xml_old_format/ClinVarVariationRelease_2024-05.xml.gz version: 2024-05 -<<<<<<< HEAD -======= ->>>>>>> release-6.2.x ->>>>>>> release-6.x.x clinvarSummary: host: http://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz clinvarVariationAllele: @@ -150,21 +139,10 @@ download: genomicSuperDups: host: http://hgdownload.cse.ucsc.edu/goldenPath gwasCatalog: -<<<<<<< HEAD -======= -<<<<<<< HEAD - host: http://resources.opencb.org/opencb/cellbase/data/gwas/gwas_catalog_v1.0.2-associations_e106_r2022-05-17.tsv - version: "1.0.2 associations_e106_r2022-05-17" -======= ->>>>>>> release-6.x.x #host: http://resources.opencb.org/opencb/cellbase/data/gwas/gwas_catalog_v1.0.2-associations_e106_r2022-05-17.tsv host: "https://ftp.ebi.ac.uk/pub/databases/gwas/releases/2024/05/20/gwas-catalog-associations_ontology-annotated.tsv" #version: "1.0.2 associations_e106_r2022-05-17" version: "2024-05-20" -<<<<<<< HEAD -======= ->>>>>>> release-6.2.x ->>>>>>> release-6.x.x hpo: host: https://ci.monarchinitiative.org/view/hpo/job/hpo.annotations/lastSuccessfulBuild/artifact/rare-diseases/util/annotation/phenotype_to_genes.txt disgenet: