diff --git a/src/airflow/dags/eqtl_preprocess.py b/src/airflow/dags/eqtl_preprocess.py index 5433d8968..aef70085d 100644 --- a/src/airflow/dags/eqtl_preprocess.py +++ b/src/airflow/dags/eqtl_preprocess.py @@ -15,9 +15,9 @@ AUTOSCALING = "eqtl-preprocess" PROJECT_ID = "open-targets-genetics-dev" -EQTL_CATALOG_SUSIE_LOCATION = "gs://eqtl_catalogue_data/ebi_ftp/susie" -TEMP_DECOMPRESS_LOCATION = "gs://eqtl_catalogue_data/susie_decompressed_tmp" -DECOMPRESS_FAILED_LOG = f"{TEMP_DECOMPRESS_LOCATION}.log" +EQTL_CATALOGUE_SUSIE_LOCATION = "gs://eqtl_catalogue_data/ebi_ftp/susie" +TEMP_DECOMPRESS_LOCATION = f"{EQTL_CATALOGUE_SUSIE_LOCATION}_decompressed_tmp" +DECOMPRESS_FAILED_LOG = f"{TEMP_DECOMPRESS_LOCATION}/logs.log" STUDY_INDEX_PATH = "gs://eqtl_catalogue_data/study_index" CREDIBLE_SET_PATH = "gs://eqtl_catalogue_data/credible_set_datasets/susie" @@ -35,7 +35,7 @@ location="europe-west1", project_id=PROJECT_ID, parameters={ - "inputFilePattern": f"{EQTL_CATALOG_SUSIE_LOCATION}/**/*.gz", + "inputFilePattern": f"{EQTL_CATALOGUE_SUSIE_LOCATION}/**/*.gz", "outputDirectory": TEMP_DECOMPRESS_LOCATION, "outputFailureFile": DECOMPRESS_FAILED_LOG, }, diff --git a/src/gentropy/assets/schemas/study_index.json b/src/gentropy/assets/schemas/study_index.json index e529fafd2..de87fc2fa 100644 --- a/src/gentropy/assets/schemas/study_index.json +++ b/src/gentropy/assets/schemas/study_index.json @@ -42,7 +42,7 @@ "metadata": {} }, { - "name": "tissueFromSourceId", + "name": "biosampleFromSourceId", "type": "string", "nullable": true, "metadata": {} diff --git a/src/gentropy/datasource/eqtl_catalogue/finemapping.py b/src/gentropy/datasource/eqtl_catalogue/finemapping.py index ff55ead72..c6cf58326 100644 --- a/src/gentropy/datasource/eqtl_catalogue/finemapping.py +++ b/src/gentropy/datasource/eqtl_catalogue/finemapping.py @@ -1,4 +1,5 @@ """Process SuSIE finemapping results from eQTL Catalogue.""" + from __future__ import annotations from dataclasses import dataclass @@ -190,7 +191,7 @@ def parse_susie_results( f.col("sample_group"), f.col("molecular_trait_id"), ).alias("studyId"), - f.col("tissue_id").alias("tissueFromSourceId"), + f.col("tissue_id").alias("biosampleFromSourceId"), EqtlCatalogueStudyIndex._identify_study_type( f.col("quant_method") ).alias("studyType"), diff --git a/src/gentropy/datasource/eqtl_catalogue/study_index.py b/src/gentropy/datasource/eqtl_catalogue/study_index.py index 71cec1ec0..628d69a4b 100644 --- a/src/gentropy/datasource/eqtl_catalogue/study_index.py +++ b/src/gentropy/datasource/eqtl_catalogue/study_index.py @@ -43,7 +43,7 @@ class EqtlCatalogueStudyIndex: StructField("quant_method", StringType(), True), ] ) - raw_studies_metadata_path = "https://raw.githubusercontent.com/eQTL-Catalogue/eQTL-Catalogue-resources/19929ff6a99bf402194292a14f96f9615b35f65f/data_tables/dataset_metadata.tsv" + raw_studies_metadata_path = "https://raw.githubusercontent.com/eQTL-Catalogue/eQTL-Catalogue-resources/4c8ca340e3eb2878073b290785cb8ff1a4c788f8/data_tables/dataset_metadata_upcoming.tsv" @classmethod def _identify_study_type( @@ -85,7 +85,8 @@ def _identify_study_type( @classmethod def get_studies_of_interest( - cls: type[EqtlCatalogueStudyIndex], studies_metadata: DataFrame + cls: type[EqtlCatalogueStudyIndex], + studies_metadata: DataFrame, ) -> list[str]: """Filter studies of interest from the raw studies metadata.