Skip to content

Commit

Permalink
small fix
Browse files Browse the repository at this point in the history
  • Loading branch information
ravenac95 committed Dec 12, 2024
1 parent 7ff84a2 commit 472a38b
Showing 1 changed file with 8 additions and 2 deletions.
10 changes: 8 additions & 2 deletions warehouse/metrics_tools/compute/result.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import abc
import logging
import os
import typing as t
from datetime import datetime

Expand Down Expand Up @@ -147,13 +148,18 @@ async def import_reference(self, reference: ExportReference) -> ExportReference:
raise NotImplementedError(f"Unsupported reference type {reference.type}")

# Import the table from gcs into trino using the hive catalog
gcs_path = reference.payload["gcs_path"]
import_path = reference.payload["gcs_path"]
# If we are using a wildcard path, we need to remove the wildcard for
# trino and keep a trailing slash
if os.path.basename(import_path) == "*.parquet":
import_path = f"{os.path.dirname(import_path)}/"

base_create_query = f"""
CREATE table "{self.hive_catalog}"."{self.hive_schema}"."{reference.table_name}" (
placeholder VARCHAR,
) WITH (
format = 'PARQUET',
external_location = '{gcs_path}'
external_location = '{import_path}/'
)
"""
create_query = parse_one(base_create_query)
Expand Down

0 comments on commit 472a38b

Please sign in to comment.