diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index b7fdf787d..507085931 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -57,7 +57,10 @@ def relation_factory(self): relation_changesets={ RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelationChangeset, }, - relation_can_be_renamed={RelationType.MaterializedView}, + relation_can_be_renamed={ + RelationType.Table, + RelationType.View, + }, render_policy=relation_models.RedshiftRenderPolicy, ) diff --git a/dbt/adapters/redshift/relation/models/__init__.py b/dbt/adapters/redshift/relation/models/__init__.py index aa21a54dc..bf98b30ad 100644 --- a/dbt/adapters/redshift/relation/models/__init__.py +++ b/dbt/adapters/redshift/relation/models/__init__.py @@ -1,5 +1,8 @@ from dbt.adapters.redshift.relation.models.database import RedshiftDatabaseRelation -from dbt.adapters.redshift.relation.models.dist import RedshiftDistRelation +from dbt.adapters.redshift.relation.models.dist import ( + RedshiftDistRelation, + RedshiftDistStyle, +) from dbt.adapters.redshift.relation.models.materialized_view import ( RedshiftMaterializedViewRelation, RedshiftMaterializedViewRelationChangeset, @@ -9,4 +12,7 @@ MAX_CHARACTERS_IN_IDENTIFIER, ) from dbt.adapters.redshift.relation.models.schema import RedshiftSchemaRelation -from dbt.adapters.redshift.relation.models.sort import RedshiftSortRelation +from dbt.adapters.redshift.relation.models.sort import ( + RedshiftSortRelation, + RedshiftSortStyle, +) diff --git a/dbt/adapters/redshift/relation/models/dist.py b/dbt/adapters/redshift/relation/models/dist.py index a8523f052..0f85f6004 100644 --- a/dbt/adapters/redshift/relation/models/dist.py +++ b/dbt/adapters/redshift/relation/models/dist.py @@ -1,3 +1,4 @@ +from copy import deepcopy from dataclasses import dataclass from typing import Optional, Set @@ -12,6 +13,8 @@ from dbt.dataclass_schema import StrEnum from dbt.exceptions import DbtRuntimeError +from dbt.adapters.redshift.relation.models.policy import RedshiftRenderPolicy + class RedshiftDistStyle(StrEnum): auto = "auto" @@ -35,9 +38,13 @@ class RedshiftDistRelation(RelationComponent, ValidationMixin): - distkey: the column to use for the dist key if `dist_style` is `key` """ + # attribution diststyle: Optional[RedshiftDistStyle] = RedshiftDistStyle.default() distkey: Optional[str] = None + # configuration + render = RedshiftRenderPolicy + @property def validation_rules(self) -> Set[ValidationRule]: # index rules get run by default with the mixin @@ -64,11 +71,14 @@ def validation_rules(self) -> Set[ValidationRule]: @classmethod def from_dict(cls, config_dict) -> "RedshiftDistRelation": - kwargs_dict = { - "diststyle": config_dict.get("diststyle"), - "distkey": config_dict.get("distkey"), - } - dist: "RedshiftDistRelation" = super().from_dict(kwargs_dict) # type: ignore + # don't alter the incoming config + kwargs_dict = deepcopy(config_dict) + + if diststyle := config_dict.get("diststyle"): + kwargs_dict.update({"diststyle": RedshiftDistStyle(diststyle)}) + + dist = super().from_dict(kwargs_dict) + assert isinstance(dist, RedshiftDistRelation) return dist @classmethod @@ -118,7 +128,7 @@ def parse_describe_relation_results(cls, describe_relation_results: agate.Row) - Returns: a standard dictionary describing this `RedshiftDistConfig` instance """ - dist: str = describe_relation_results.get("diststyle") + dist: str = describe_relation_results.get("dist") try: # covers `AUTO`, `ALL`, `EVEN`, `KEY`, '', diff --git a/dbt/adapters/redshift/relation/models/materialized_view.py b/dbt/adapters/redshift/relation/models/materialized_view.py index e17f3ffac..f0345ff40 100644 --- a/dbt/adapters/redshift/relation/models/materialized_view.py +++ b/dbt/adapters/redshift/relation/models/materialized_view.py @@ -5,10 +5,10 @@ import agate from dbt.adapters.relation.models import ( MaterializedViewRelation, + MaterializedViewRelationChangeset, Relation, RelationChange, RelationChangeAction, - RelationChangeset, ) from dbt.adapters.validation import ValidationMixin, ValidationRule from dbt.contracts.graph.nodes import ModelNode @@ -61,16 +61,14 @@ class RedshiftMaterializedViewRelation(MaterializedViewRelation, ValidationMixin schema: RedshiftSchemaRelation query: str backup: Optional[bool] = True - dist: RedshiftDistRelation = RedshiftDistRelation( - diststyle=RedshiftDistStyle.even, render=RedshiftRenderPolicy - ) - sort: RedshiftSortRelation = RedshiftSortRelation(render=RedshiftRenderPolicy) + dist: RedshiftDistRelation = RedshiftDistRelation.from_dict({"diststyle": "even"}) + sort: RedshiftSortRelation = RedshiftSortRelation.from_dict({}) autorefresh: Optional[bool] = False # configuration render = RedshiftRenderPolicy SchemaParser = RedshiftSchemaRelation # type: ignore - can_be_renamed = True + can_be_renamed = False @property def validation_rules(self) -> Set[ValidationRule]: @@ -119,8 +117,8 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: config_dict.update( { - "backup": model_node.config.get("backup"), - "autorefresh": model_node.config.get("auto_refresh"), + "backup": model_node.config.extra.get("backup"), + "autorefresh": model_node.config.extra.get("autorefresh"), } ) @@ -145,11 +143,11 @@ def parse_describe_relation_results( { "materialized_view": agate.Table( agate.Row({ - "database": "", - "schema": "", - "table": "", - "diststyle": "", # e.g. EVEN | KEY(column1) | AUTO(ALL) | AUTO(KEY(id)), - "sortkey1": "", + "database_name": "", + "schema_name": "", + "name": "", + "dist": "", # e.g. EVEN | KEY(column1) | AUTO(ALL) | AUTO(KEY(id)), + "sortkey": "", "autorefresh: any("t", "f"), }) ), @@ -162,33 +160,45 @@ def parse_describe_relation_results( Returns: a standard dictionary describing this `RedshiftMaterializedViewConfig` instance """ + # merge these because the base class assumes `query` is on the same record as `name`, `schema_name` and + # `database_name` + describe_relation_results = cls._combine_describe_relation_results_tables( + describe_relation_results + ) config_dict = super().parse_describe_relation_results(describe_relation_results) materialized_view: agate.Row = describe_relation_results["materialized_view"].rows[0] - query: agate.Row = describe_relation_results["query"].rows[0] - config_dict.update( { - "autorefresh": {"t": True, "f": False}.get(materialized_view.get("autorefresh")), - "query": cls._parse_query(query.get("definition")), + "autorefresh": materialized_view.get("autorefresh"), + "query": cls._parse_query(materialized_view.get("query")), } ) # the default for materialized views differs from the default for diststyle in general # only set it if we got a value - if materialized_view.get("diststyle"): + if materialized_view.get("dist"): config_dict.update( {"dist": RedshiftDistRelation.parse_describe_relation_results(materialized_view)} ) # TODO: this only shows the first column in the sort key - if materialized_view.get("sortkey1"): + if materialized_view.get("sortkey"): config_dict.update( {"sort": RedshiftSortRelation.parse_describe_relation_results(materialized_view)} ) return config_dict + @classmethod + def _combine_describe_relation_results_tables( + cls, describe_relation_results: Dict[str, agate.Table] + ) -> Dict[str, agate.Table]: + materialized_view_table: agate.Table = describe_relation_results["materialized_view"] + query_table: agate.Table = describe_relation_results["query"] + combined_table: agate.Table = materialized_view_table.join(query_table, full_outer=True) + return {"materialized_view": combined_table} + @classmethod def _parse_query(cls, query: str) -> str: """ @@ -211,9 +221,10 @@ def _parse_query(cls, query: str) -> str: select * from my_base_table """ - open_paren = query.find("as (") + len("as (") - close_paren = query.find(");") - return query[open_paren:close_paren].strip() + return query + # open_paren = query.find("as (") + # close_paren = query.find(");") + # return query[open_paren:close_paren].strip() @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -235,7 +246,7 @@ def requires_full_refresh(self) -> bool: @dataclass -class RedshiftMaterializedViewRelationChangeset(RelationChangeset): +class RedshiftMaterializedViewRelationChangeset(MaterializedViewRelationChangeset): backup: Optional[RedshiftBackupRelationChange] = None dist: Optional[RedshiftDistRelationChange] = None sort: Optional[RedshiftSortRelationChange] = None diff --git a/dbt/adapters/redshift/relation/models/sort.py b/dbt/adapters/redshift/relation/models/sort.py index c07b3b71a..26939d4fd 100644 --- a/dbt/adapters/redshift/relation/models/sort.py +++ b/dbt/adapters/redshift/relation/models/sort.py @@ -1,5 +1,5 @@ from copy import deepcopy -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, FrozenSet, Set import agate @@ -43,15 +43,16 @@ class RedshiftSortRelation(RelationComponent, ValidationMixin): - sort_key: the column(s) to use for the sort key; cannot be combined with `sort_type=auto` """ + # attribution sortstyle: Optional[RedshiftSortStyle] = None - sortkey: Optional[FrozenSet[str]] = None + sortkey: Optional[FrozenSet[str]] = field(default_factory=frozenset) # type: ignore # configuration render = RedshiftRenderPolicy def __post_init__(self): # maintains `frozen=True` while allowing for a variable default on `sort_type` - if self.sortstyle is None and self.sortkey is None: + if self.sortstyle is None and self.sortkey == frozenset(): object.__setattr__(self, "sortstyle", RedshiftSortStyle.default()) elif self.sortstyle is None: object.__setattr__(self, "sortstyle", RedshiftSortStyle.default_with_columns()) @@ -63,7 +64,7 @@ def validation_rules(self) -> Set[ValidationRule]: return { ValidationRule( validation_check=not ( - self.sortstyle == RedshiftSortStyle.auto and self.sortkey is not None + self.sortstyle == RedshiftSortStyle.auto and self.sortkey != frozenset() ), validation_error=DbtRuntimeError( "A `RedshiftSortConfig` that specifies a `sortkey` does not support the `sortstyle` of `auto`." @@ -72,7 +73,7 @@ def validation_rules(self) -> Set[ValidationRule]: ValidationRule( validation_check=not ( self.sortstyle in (RedshiftSortStyle.compound, RedshiftSortStyle.interleaved) - and self.sortkey is None + and self.sortkey == frozenset() ), validation_error=DbtRuntimeError( "A `sortstyle` of `compound` or `interleaved` requires a `sortkey` to be provided." @@ -105,12 +106,11 @@ def from_dict(cls, config_dict) -> "RedshiftSortRelation": # don't alter the incoming config kwargs_dict = deepcopy(config_dict) - kwargs_dict.update( - { - "sortstyle": config_dict.get("sortstyle"), - "sortkey": frozenset(column for column in config_dict.get("sortkey", {})), - } - ) + if sortstyle := config_dict.get("sortstyle"): + kwargs_dict.update({"sortstyle": RedshiftSortStyle(sortstyle)}) + + if sortkey := config_dict.get("sortkey"): + kwargs_dict.update({"sortkey": frozenset(column for column in sortkey)}) sort = super().from_dict(kwargs_dict) assert isinstance(sort, RedshiftSortRelation) @@ -165,7 +165,7 @@ def parse_describe_relation_results(cls, describe_relation_results: agate.Row) - Returns: a standard dictionary describing this `RedshiftSortConfig` instance """ - if sortkey := describe_relation_results.get("sortkey1"): + if sortkey := describe_relation_results.get("sortkey"): return {"sortkey": {sortkey}} return {} diff --git a/dbt/include/redshift/macros/materializations/table.sql b/dbt/include/redshift/macros/materializations/table.sql new file mode 100644 index 000000000..a3b8bacdb --- /dev/null +++ b/dbt/include/redshift/macros/materializations/table.sql @@ -0,0 +1,77 @@ +{% /* + + Ideally we don't overwrite materializations from dbt-core. However, the implementation of materialized views + requires this, at least for now. There are two issues that lead to this. First, Redshift does not support + the renaming of materialized views. That means we cannot back them up when replacing them. If the relation + that's replacing it is another materialized view, we can control for that since the materialization for + materialized views in dbt-core is flexible. That brings us to the second issue. The materialization for table + has the backup/deploy portion built into it; it's one single macro; replacing that has two options. We + can either break apart the macro in dbt-core, which could have unintended downstream effects for all + adapters. Or we can copy this here and keep it up to date with dbt-core until we resolve the larger issue. + We chose to go with the latter. + +*/ %} + +{% materialization table, adapter='redshift', supported_languages=['sql'] %} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') %} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + See ../view/view.sql for more information about this relation. + */ + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup -- this should be the only piece that differs from dbt-core + {% if existing_relation is not none %} + {% if existing_relation.type == 'materialized_view' %} + {{ drop_relation_if_exists(existing_relation) }} + {% else %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% do create_indexes(target_relation) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + -- finally, drop the existing/backup relation after the commit + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} diff --git a/dbt/include/redshift/macros/materializations/view.sql b/dbt/include/redshift/macros/materializations/view.sql new file mode 100644 index 000000000..1edd01d5b --- /dev/null +++ b/dbt/include/redshift/macros/materializations/view.sql @@ -0,0 +1,72 @@ +-- see the table materialization for notes +{%- materialization view, adapter='redshift', supported_languages=['sql'] -%} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='view') -%} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "existing_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the existing_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the existing_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_view_as_sql(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way - unless it's a materialized view, then drop it + {% if existing_relation is not none %} + {% if existing_relation.type == 'materialized_view' %} + {{ drop_relation_if_exists(existing_relation) }} + {% else %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/dbt/include/redshift/macros/relations/materialized_view.sql b/dbt/include/redshift/macros/relations/materialized_view.sql index cd20bec9a..987772155 100644 --- a/dbt/include/redshift/macros/relations/materialized_view.sql +++ b/dbt/include/redshift/macros/relations/materialized_view.sql @@ -51,10 +51,10 @@ diststyle {{ materialized_view.dist.diststyle }} {% if materialized_view.dist.distkey %}distkey ({{ materialized_view.dist.distkey }}){% endif %} {% if materialized_view.sort.sortkey %}sortkey ({{ ','.join(materialized_view.sort.sortkey) }}){% endif %} - auto refresh {% if materialized_view.auto_refresh %}yes{% else %}no{% endif %} + auto refresh {% if materialized_view.autorefresh %}yes{% else %}no{% endif %} as ( {{ materialized_view.query }} - ); + ) {% endmacro %} @@ -67,11 +67,11 @@ {%- set _materialized_view_sql -%} select - tb.database, - tb.schema, - tb.table, - tb.diststyle, - tb.sortkey1, + tb.database as database_name, + tb.schema as schema_name, + tb.table as name, + tb.diststyle as dist, + tb.sortkey1 as sortkey, mv.autorefresh from svv_table_info tb left join stv_mv_info mv @@ -86,7 +86,7 @@ {%- set _query_sql -%} select - vw.definition + vw.definition as query from pg_views vw where vw.viewname = '{{ materialized_view.name }}' and vw.schemaname = '{{ materialized_view.schema_name }}' @@ -109,14 +109,8 @@ {% endmacro %} -{%- macro postgres__rename_materialized_view_template(materialized_view, new_name) -%} - - {%- if adapter.is_relation_model(materialized_view) -%} - {%- set fully_qualified_path = materialized_view.fully_qualified_path -%} - {%- else -%} - {%- set fully_qualified_path = materialized_view -%} - {%- endif -%} - - alter materialized view {{ fully_qualified_path }} rename to {{ new_name }} - +{%- macro redshift__rename_materialized_view_template(materialized_view, new_name) -%} + {{- exceptions.raise_compiler_error( + "Redshift does not support the renaming of materialized views. This macro was called by: " ~ materialized_view + ) -}} {%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/table.sql b/dbt/include/redshift/macros/relations/table.sql index 924592d05..845773faf 100644 --- a/dbt/include/redshift/macros/relations/table.sql +++ b/dbt/include/redshift/macros/relations/table.sql @@ -1,4 +1,4 @@ -{%- macro redshift_drop_table_template(table) -%} +{%- macro redshift__drop_table_template(table) -%} drop table if exists {{ table.fully_qualified_path }} cascade {%- endmacro -%} diff --git a/dbt/include/redshift/macros/relations/view.sql b/dbt/include/redshift/macros/relations/view.sql index 5fe65fe60..efddb63f9 100644 --- a/dbt/include/redshift/macros/relations/view.sql +++ b/dbt/include/redshift/macros/relations/view.sql @@ -3,8 +3,9 @@ {%- endmacro -%} +{# /* Redshift uses `table` here even for a view. Replacing this with `view` will break. */ #} {%- macro redshift__rename_view_template(view, new_name) -%} - alter view {{ view.fully_qualified_path }} rename to {{ new_name }} + alter table {{ view.fully_qualified_path }} rename to {{ new_name }} {%- endmacro -%} diff --git a/tests/functional/adapter/materialized_view_tests/conftest.py b/tests/functional/adapter/materialized_view_tests/conftest.py new file mode 100644 index 000000000..4883ee20f --- /dev/null +++ b/tests/functional/adapter/materialized_view_tests/conftest.py @@ -0,0 +1,59 @@ +import pytest + +from dbt.adapters.relation.models import RelationRef +from dbt.adapters.relation.factory import RelationFactory +from dbt.contracts.relation import RelationType + +from dbt.adapters.redshift.relation import models as relation_models + + +@pytest.fixture(scope="class") +def relation_factory(): + return RelationFactory( + relation_models={ + RelationType.MaterializedView: relation_models.RedshiftMaterializedViewRelation, + }, + relation_can_be_renamed={RelationType.Table, RelationType.View}, + render_policy=relation_models.RedshiftRenderPolicy, + ) + + +@pytest.fixture(scope="class") +def my_materialized_view(project, relation_factory) -> RelationRef: + relation_ref = relation_factory.make_ref( + name="my_materialized_view", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.MaterializedView, + ) + return relation_ref + + +@pytest.fixture(scope="class") +def my_view(project, relation_factory) -> RelationRef: + return relation_factory.make_ref( + name="my_view", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.View, + ) + + +@pytest.fixture(scope="class") +def my_table(project, relation_factory) -> RelationRef: + return relation_factory.make_ref( + name="my_table", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.Table, + ) + + +@pytest.fixture(scope="class") +def my_seed(project, relation_factory) -> RelationRef: + return relation_factory.make_ref( + name="my_seed", + schema_name=project.test_schema, + database_name=project.database, + relation_type=RelationType.Table, + ) diff --git a/tests/functional/adapter/materialized_view_tests/files.py b/tests/functional/adapter/materialized_view_tests/files.py new file mode 100644 index 000000000..c76045084 --- /dev/null +++ b/tests/functional/adapter/materialized_view_tests/files.py @@ -0,0 +1,33 @@ +MY_SEED = """ +id,value +1,100 +2,200 +3,300 +""".strip() + + +MY_TABLE = """ +{{ config( + materialized='table', +) }} +select * from {{ ref('my_seed') }} +""" + + +MY_VIEW = """ +{{ config( + materialized='view', +) }} +select * from {{ ref('my_seed') }} +""" + + +MY_MATERIALIZED_VIEW = """ +{{ config( + materialized='materialized_view', + sort_type='compound', + sort=['id'], + dist='id' +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/tests/functional/adapter/materialized_view_tests/fixtures.py b/tests/functional/adapter/materialized_view_tests/fixtures.py deleted file mode 100644 index 785931c1b..000000000 --- a/tests/functional/adapter/materialized_view_tests/fixtures.py +++ /dev/null @@ -1,85 +0,0 @@ -import pytest - -from dbt.tests.adapter.materialized_view.base import Base -from dbt.tests.adapter.materialized_view.on_configuration_change import ( - OnConfigurationChangeBase, - get_model_file, - set_model_file, -) -from dbt.tests.util import relation_from_name, run_sql_with_adapter - - -def refresh_materialized_view(project, name: str): - sql = f"refresh materialized view {relation_from_name(project.adapter, name)}" - run_sql_with_adapter(project.adapter, sql) - - -class RedshiftBasicBase(Base): - @pytest.fixture(scope="class") - def models(self): - base_table = """ - {{ config(materialized='table') }} - select 1 as base_column - """ - base_materialized_view = """ - {{ config(materialized='materialized_view') }} - select * from {{ ref('base_table') }} - """ - return {"base_table.sql": base_table, "base_materialized_view.sql": base_materialized_view} - - -class RedshiftOnConfigurationChangeBase(OnConfigurationChangeBase): - @pytest.fixture(scope="class") - def models(self): - base_table = """ - {{ config( - materialized='table', - ) }} - select - 1 as id, - 100 as value - """ - base_materialized_view = """ - {{ config( - materialized='materialized_view', - sort='id' - ) }} - select * from {{ ref('base_table') }} - """ - return {"base_table.sql": base_table, "base_materialized_view.sql": base_materialized_view} - - @pytest.fixture(scope="function") - def configuration_changes_apply(self, project): - initial_model = get_model_file(project, "base_materialized_view") - - # turn on auto_refresh - new_model = initial_model.replace( - "materialized='materialized_view',", - "materialized='materialized_view', auto_refresh='yes',", - ) - set_model_file(project, "base_materialized_view", new_model) - - yield - - # set this back for the next test - set_model_file(project, "base_materialized_view", initial_model) - - @pytest.fixture(scope="function") - def configuration_changes_refresh(self, project): - initial_model = get_model_file(project, "base_materialized_view") - - # add a sort_key - new_model = initial_model.replace( - "sort='id'", - "sort='value'", - ) - set_model_file(project, "base_materialized_view", new_model) - - yield - - # set this back for the next test - set_model_file(project, "base_materialized_view", initial_model) - - @pytest.fixture(scope="function") - def update_auto_refresh_message(self, project): - return f"Applying UPDATE AUTOREFRESH to: {relation_from_name(project.adapter, 'base_materialized_view')}" diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index ff63f1e01..65c4d9471 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -1,239 +1,277 @@ import pytest from dbt.contracts.graph.model_config import OnConfigurationChangeOption -from dbt.contracts.relation import RelationType -from dbt.contracts.results import RunStatus -from dbt.tests.adapter.materialized_view.base import ( - run_model, - assert_model_exists_and_is_correct_type, - insert_record, - get_row_count, +from dbt.tests.util import ( + assert_message_in_logs, + get_model_file, + run_dbt, + run_dbt_and_capture, + set_model_file, ) -from dbt.tests.adapter.materialized_view.on_configuration_change import ( - assert_proper_scenario, +from tests.functional.adapter.materialized_view_tests.files import ( + MY_MATERIALIZED_VIEW, + MY_SEED, + MY_TABLE, + MY_VIEW, ) - -from tests.functional.adapter.materialized_view_tests.fixtures import ( - RedshiftBasicBase, - RedshiftOnConfigurationChangeBase, - refresh_materialized_view, +from tests.functional.adapter.materialized_view_tests.utils import ( + query_autorefresh, + query_relation_type, + query_row_count, + query_sort, + swap_autorefresh, + swap_materialized_view_to_table, + swap_materialized_view_to_view, + swap_sortkey, ) -class TestBasic(RedshiftBasicBase): - def test_relation_is_materialized_view_on_initial_creation(self, project): - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) - assert_model_exists_and_is_correct_type(project, "base_table", RelationType.Table) +@pytest.fixture(scope="class", autouse=True) +def seeds(): + return {"my_seed.csv": MY_SEED} - def test_relation_is_materialized_view_when_rerun(self, project): - run_model("base_materialized_view") - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) - def test_relation_is_materialized_view_on_full_refresh(self, project): - run_model("base_materialized_view", full_refresh=True) - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) +@pytest.fixture(scope="class", autouse=True) +def models(): + yield { + "my_table.sql": MY_TABLE, + "my_view.sql": MY_VIEW, + "my_materialized_view.sql": MY_MATERIALIZED_VIEW, + } - def test_relation_is_materialized_view_on_update(self, project): - run_model("base_materialized_view", run_args=["--vars", "quoting: {identifier: True}"]) - assert_model_exists_and_is_correct_type( - project, "base_materialized_view", RelationType.MaterializedView - ) - def test_updated_base_table_data_only_shows_in_materialized_view_after_refresh(self, project): - # poll database - table_start = get_row_count(project, "base_table") - view_start = get_row_count(project, "base_materialized_view") - assert view_start == table_start +@pytest.fixture(scope="class", autouse=True) +def setup(project): + run_dbt(["seed"]) + yield - # insert new record in table - new_record = (2,) - insert_record(project, new_record, "base_table", ["base_column"]) - # poll database - table_mid = get_row_count(project, "base_table") - view_mid = get_row_count(project, "base_materialized_view") +def test_materialized_view_create(project, my_materialized_view): + assert query_relation_type(project, my_materialized_view) is None + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" - # refresh the materialized view - refresh_materialized_view(project, "base_materialized_view") - # poll database - table_end = get_row_count(project, "base_table") - view_end = get_row_count(project, "base_materialized_view") - assert view_end == table_end +def test_materialized_view_create_idempotent(project, my_materialized_view): + assert query_relation_type(project, my_materialized_view) is None + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" - # new records were inserted in the table but didn't show up in the view until it was refreshed - assert table_start < table_mid == table_end - assert view_start == view_mid < view_end +def test_materialized_view_full_refresh(project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name]) + _, logs = run_dbt_and_capture( + ["--debug", "run", "--models", my_materialized_view.name, "--full-refresh"] + ) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs + ) -class TestOnConfigurationChangeApply(RedshiftOnConfigurationChangeBase): - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, - configuration_changes_apply, - configuration_changes_refresh, - replace_message, - configuration_change_message, - ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], - ) - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], - ) +def test_materialized_view_replaces_table(project, my_materialized_view, my_table): + run_dbt(["run", "--models", my_table.name]) + sql = f""" + alter table {my_table.fully_qualified_path} + rename to {my_materialized_view.name} + """ + project.run_sql(sql) + assert query_relation_type(project, my_materialized_view) == "table" + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" - def test_model_applies_changes_with_small_configuration_changes( - self, configuration_changes_apply, alter_message, update_auto_refresh_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[alter_message, update_auto_refresh_message], + +def test_materialized_view_replaces_view(project, my_materialized_view, my_view): + run_dbt(["run", "--models", my_view.name]) + sql = f""" + alter table {my_view.fully_qualified_path} + rename to {my_materialized_view.name} + """ + project.run_sql(sql) + assert query_relation_type(project, my_materialized_view) == "view" + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + + +def test_table_replaces_materialized_view(project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + swap_materialized_view_to_table(project, my_materialized_view) + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "table" + + +def test_view_replaces_materialized_view(project, my_materialized_view, my_view): + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" + swap_materialized_view_to_view(project, my_materialized_view) + run_dbt(["run", "--models", my_materialized_view.name]) + assert query_relation_type(project, my_materialized_view) == "view" + + +def test_materialized_view_only_updates_after_refresh(project, my_materialized_view, my_seed): + run_dbt(["run", "--models", my_materialized_view.name]) + + # poll database + table_start = query_row_count(project, my_seed) + view_start = query_row_count(project, my_materialized_view) + + # insert new record in table + project.run_sql(f"insert into {my_seed.fully_qualified_path} (id, value) values (4, 400);") + + # poll database + table_mid = query_row_count(project, my_seed) + view_mid = query_row_count(project, my_materialized_view) + + # refresh the materialized view + project.run_sql(f"refresh materialized view {my_materialized_view.fully_qualified_path};") + + # poll database + table_end = query_row_count(project, my_seed) + view_end = query_row_count(project, my_materialized_view) + + # new records were inserted in the table but didn't show up in the view until it was refreshed + assert table_start < table_mid == table_end + assert view_start == view_mid < view_end + + +class OnConfigurationChangeBase: + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield {"my_materialized_view.sql": MY_MATERIALIZED_VIEW} + + @pytest.fixture(scope="function", autouse=True) + def setup(self, project, my_materialized_view): + run_dbt(["seed"]) + + # make sure the model in the data reflects the files each time + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + + # the tests touch these files, store their contents in memory + initial_model = get_model_file(project, my_materialized_view) + + yield + + # and then reset them after the test runs + set_model_file(project, my_materialized_view, initial_model) + + +class TestOnConfigurationChangeApply(OnConfigurationChangeBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": OnConfigurationChangeOption.Apply.value}} + + def test_autorefresh_change_is_applied_with_alter(self, project, my_materialized_view): + assert query_autorefresh(project, my_materialized_view) is False + swap_autorefresh(project, my_materialized_view) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_autorefresh(project, my_materialized_view) is True + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying UPDATE AUTOREFRESH to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs, False ) - def test_model_rebuilds_with_large_configuration_changes( - self, configuration_changes_refresh, alter_message, replace_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, - logs, - RunStatus.Success, - messages_in_logs=[alter_message, replace_message], + def test_sort_change_is_applied_with_replace(self, project, my_materialized_view): + assert query_sort(project, my_materialized_view) == "id" + swap_sortkey(project, my_materialized_view) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_sort(project, my_materialized_view) == "value" + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs ) - def test_model_only_rebuilds_with_large_configuration_changes( - self, - configuration_changes_apply, - configuration_changes_refresh, - alter_message, - replace_message, - update_auto_refresh_message, + def test_autorefresh_change_is_applied_with_replace_when_run_with_sort_change( + self, project, my_materialized_view ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Apply, - results, + assert query_autorefresh(project, my_materialized_view) is False + swap_autorefresh(project, my_materialized_view) + swap_sortkey(project, my_materialized_view) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_autorefresh(project, my_materialized_view) is True + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs + ) + assert_message_in_logs( + f"Applying UPDATE AUTOREFRESH to: {my_materialized_view.fully_qualified_path}", logs, - RunStatus.Success, - messages_in_logs=[alter_message, replace_message], - messages_not_in_logs=[update_auto_refresh_message], + False, ) -class TestOnConfigurationChangeContinue(RedshiftOnConfigurationChangeBase): +class TestOnConfigurationChangeContinue(OnConfigurationChangeBase): @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"on_configuration_change": OnConfigurationChangeOption.Continue.value}} - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, - configuration_changes_apply, - configuration_changes_refresh, - replace_message, - configuration_change_message, - ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, + def test_autorefresh_change_is_not_applied(self, project, my_materialized_view): + assert query_autorefresh(project, my_materialized_view) is False + swap_autorefresh(project, my_materialized_view) + _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_materialized_view.name]) + assert query_autorefresh(project, my_materialized_view) is False + assert_message_in_logs( + f"Configuration changes were identified and `on_configuration_change` was set" + f" to `continue` for `{my_materialized_view.fully_qualified_path}`", logs, - RunStatus.Success, - messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], ) - - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, - logs, - RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs, False ) - - def test_model_is_skipped_with_configuration_changes( - self, configuration_changes_apply, configuration_change_continue_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Continue, - results, + assert_message_in_logs( + f"Applying UPDATE AUTOREFRESH to: {my_materialized_view.fully_qualified_path}", logs, - RunStatus.Success, - messages_in_logs=[configuration_change_continue_message], + False, ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs, False + ) + + def test_full_refresh_still_occurs_with_changes(self, project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" -class TestOnConfigurationChangeFail(RedshiftOnConfigurationChangeBase): +class TestOnConfigurationChangeFail(OnConfigurationChangeBase): @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"on_configuration_change": OnConfigurationChangeOption.Fail.value}} - def test_full_refresh_takes_precedence_over_any_configuration_changes( - self, - configuration_changes_apply, - configuration_changes_refresh, - replace_message, - configuration_change_message, - ): - results, logs = run_model("base_materialized_view", full_refresh=True) - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, - logs, - RunStatus.Success, - messages_in_logs=[replace_message], - messages_not_in_logs=[configuration_change_message], + def test_autorefresh_change_is_not_applied(self, project, my_materialized_view): + assert query_autorefresh(project, my_materialized_view) is False + swap_autorefresh(project, my_materialized_view) + _, logs = run_dbt_and_capture( + ["--debug", "run", "--models", my_materialized_view.name], expect_pass=False ) - - def test_model_is_refreshed_with_no_configuration_changes( - self, refresh_message, configuration_change_message - ): - results, logs = run_model("base_materialized_view") - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, + assert query_autorefresh(project, my_materialized_view) is False + assert_message_in_logs( + f"Configuration changes were identified and `on_configuration_change` was set" + f" to `fail` for `{my_materialized_view.fully_qualified_path}`", logs, - RunStatus.Success, - messages_in_logs=[refresh_message, configuration_change_message], ) - - def test_run_fails_with_configuration_changes( - self, configuration_changes_apply, configuration_change_fail_message - ): - results, logs = run_model("base_materialized_view", expect_pass=False) - assert_proper_scenario( - OnConfigurationChangeOption.Fail, - results, + assert_message_in_logs( + f"Applying ALTER to: {my_materialized_view.fully_qualified_path}", logs, False + ) + assert_message_in_logs( + f"Applying UPDATE AUTOREFRESH to: {my_materialized_view.fully_qualified_path}", logs, - RunStatus.Error, - messages_in_logs=[configuration_change_fail_message], + False, + ) + assert_message_in_logs( + f"Applying REPLACE to: {my_materialized_view.fully_qualified_path}", logs, False ) + + def test_full_refresh_still_occurs_with_changes(self, project, my_materialized_view): + run_dbt(["run", "--models", my_materialized_view.name, "--full-refresh"]) + assert query_relation_type(project, my_materialized_view) == "materialized_view" diff --git a/tests/functional/adapter/materialized_view_tests/utils.py b/tests/functional/adapter/materialized_view_tests/utils.py new file mode 100644 index 000000000..5f7980cbe --- /dev/null +++ b/tests/functional/adapter/materialized_view_tests/utils.py @@ -0,0 +1,84 @@ +from typing import Optional + +from dbt.adapters.relation.models import Relation +from dbt.tests.util import get_model_file, set_model_file + + +def swap_sortkey(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("sort=['id']", "sort=['value']") + set_model_file(project, my_materialized_view, new_model) + + +def swap_autorefresh(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("dist='id'", "dist='id', autorefresh=True") + set_model_file(project, my_materialized_view, new_model) + + +def swap_materialized_view_to_table(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("materialized='materialized_view'", "materialized='table'") + set_model_file(project, my_materialized_view, new_model) + + +def swap_materialized_view_to_view(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("materialized='materialized_view'", "materialized='view'") + set_model_file(project, my_materialized_view, new_model) + + +def query_relation_type(project, relation: Relation) -> Optional[str]: + sql = f""" + select + 'table' as relation_type + from pg_tables + where schemaname = '{relation.schema_name}' + and tablename = '{relation.name}' + union all + select + case + when definition ilike '%create materialized view%' + then 'materialized_view' + else 'view' + end as relation_type + from pg_views + where schemaname = '{relation.schema_name}' + and viewname = '{relation.name}' + """ + results = project.run_sql(sql, fetch="all") + if len(results) == 0: + return None + elif len(results) > 1: + raise ValueError(f"More than one instance of {relation.name} found!") + else: + return results[0][0] + + +def query_row_count(project, relation: Relation) -> int: + sql = f"select count(*) from {relation.fully_qualified_path};" + return project.run_sql(sql, fetch="one")[0] + + +def query_sort(project, relation: Relation) -> bool: + sql = f""" + select + tb.sortkey1 as sortkey + from svv_table_info tb + where tb.table ilike '{ relation.name }' + and tb.schema ilike '{ relation.schema_name }' + and tb.database ilike '{ relation.database_name }' + """ + return project.run_sql(sql, fetch="one")[0] + + +def query_autorefresh(project, relation: Relation) -> bool: + sql = f""" + select + case mv.autorefresh when 't' then True when 'f' then False end as autorefresh + from stv_mv_info mv + where trim(mv.name) ilike '{ relation.name }' + and trim(mv.schema) ilike '{ relation.schema_name }' + and trim(mv.db_name) ilike '{ relation.database_name }' + """ + return project.run_sql(sql, fetch="one")[0] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5833cfa4a..6b0f8a487 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -43,8 +43,8 @@ def materialization_factory(relation_factory): @pytest.fixture -def materialized_view_stub(relation_factory): - return relation_factory.make_stub( +def materialized_view_ref(relation_factory): + return relation_factory.make_ref( name="my_materialized_view", schema_name="my_schema", database_name="my_database", @@ -53,8 +53,8 @@ def materialized_view_stub(relation_factory): @pytest.fixture -def view_stub(relation_factory): - return relation_factory.make_stub( +def view_ref(relation_factory): + return relation_factory.make_ref( name="my_view", schema_name="my_schema", database_name="my_database", @@ -71,11 +71,22 @@ def materialized_view_describe_relation_results(): "name": "my_materialized_view", "schema_name": "my_schema", "database_name": "my_database", - "query": "select 42 from meaning_of_life", + "dist": """KEY("id")""", + "sortkey": "other_id", + "autorefresh": "t", } ] ) - return {"materialized_view": materialized_view_agate} + + query_agate = agate.Table.from_object( + [ + { + "query": "select 4 as id, 2 as other_id from meaning_of_life", + } + ] + ) + + return {"materialized_view": materialized_view_agate, "query": query_agate} @pytest.fixture @@ -104,11 +115,11 @@ def materialized_view_model_node(): "quoting": {}, "column_types": {}, "tags": [], - # TODO replace this with sort/dist info - "indexes": [ - {"columns": ["id", "value"], "type": "hash"}, - {"columns": ["id"], "unique": True}, - ], + "autorefresh": True, + "dist": "id", + "sort": ["other_id"], + "sort_type": "compound", + "backup": False, } ), tags=[], @@ -175,8 +186,8 @@ def test_materialization_factory(materialization_factory): assert redshift_parser == models.RedshiftMaterializedViewRelation -def test_materialized_view_stub(materialized_view_stub): - assert materialized_view_stub.name == "my_materialized_view" +def test_materialized_view_ref(materialized_view_ref): + assert materialized_view_ref.name == "my_materialized_view" def test_materialized_view_model_node(materialized_view_model_node): diff --git a/tests/unit/materialization_tests/test_materialization.py b/tests/unit/materialization_tests/test_materialization.py index f8c51675c..8f2a01441 100644 --- a/tests/unit/materialization_tests/test_materialization.py +++ b/tests/unit/materialization_tests/test_materialization.py @@ -5,6 +5,8 @@ MaterializationBuildStrategy, ) +from dbt.adapters.redshift.relation import models + def test_materialized_view_create(materialized_view_runtime_config, relation_factory): materialization = MaterializedViewMaterialization.from_runtime_config( @@ -14,9 +16,9 @@ def test_materialized_view_create(materialized_view_runtime_config, relation_fac assert materialization.should_revoke_grants is False -def test_materialized_view_replace(materialized_view_runtime_config, relation_factory, view_stub): +def test_materialized_view_replace(materialized_view_runtime_config, relation_factory, view_ref): materialization = MaterializedViewMaterialization.from_runtime_config( - materialized_view_runtime_config, relation_factory, view_stub + materialized_view_runtime_config, relation_factory, view_ref ) assert materialization.build_strategy == MaterializationBuildStrategy.Replace assert materialization.should_revoke_grants is True @@ -25,7 +27,9 @@ def test_materialized_view_replace(materialized_view_runtime_config, relation_fa def test_materialized_view_alter( materialized_view_runtime_config, relation_factory, materialized_view_relation ): - altered_materialized_view = replace(materialized_view_relation, indexes={}) + altered_materialized_view = replace( + materialized_view_relation, sort=models.RedshiftSortRelation.from_dict({}) + ) materialization = MaterializedViewMaterialization.from_runtime_config( materialized_view_runtime_config, relation_factory, altered_materialized_view diff --git a/tests/unit/materialization_tests/test_materialization_factory.py b/tests/unit/materialization_tests/test_materialization_factory.py index f176dca4a..f2bb0a93a 100644 --- a/tests/unit/materialization_tests/test_materialization_factory.py +++ b/tests/unit/materialization_tests/test_materialization_factory.py @@ -1,14 +1,14 @@ from dbt.adapters.materialization.models import MaterializationType from dbt.contracts.relation import RelationType -from dbt.adapters.redshift.relation import models as relation_models +from dbt.adapters.redshift.relation import models def test_make_from_runtime_config(materialization_factory, materialized_view_runtime_config): materialization = materialization_factory.make_from_runtime_config( runtime_config=materialized_view_runtime_config, materialization_type=MaterializationType.MaterializedView, - existing_relation_stub=None, + existing_relation_ref=None, ) assert materialization.type == MaterializationType.MaterializedView @@ -19,18 +19,16 @@ def test_make_from_runtime_config(materialization_factory, materialized_view_run assert materialized_view.schema_name == "my_schema" assert materialized_view.database_name == "my_database" assert materialized_view.query == "select 42 from meaning_of_life" - - index_1 = relation_models.RedshiftIndexRelation( - column_names=frozenset({"id", "value"}), - method=relation_models.RedshiftIndexMethod.hash, - unique=False, - render=relation_models.RedshiftRenderPolicy, + sort = models.RedshiftSortRelation( + sortstyle=models.RedshiftSortStyle.compound, + sortkey=frozenset({"other_id"}), + render=models.RedshiftRenderPolicy, ) - index_2 = relation_models.RedshiftIndexRelation( - column_names=frozenset({"id"}), - method=relation_models.RedshiftIndexMethod.btree, - unique=True, - render=relation_models.RedshiftRenderPolicy, + assert materialized_view.sort == sort + dist = models.RedshiftDistRelation( + diststyle=models.RedshiftDistStyle.key, + distkey="id", + render=models.RedshiftRenderPolicy, ) - assert index_1 in materialized_view.indexes - assert index_2 in materialized_view.indexes + assert materialized_view.dist == dist + assert materialized_view.autorefresh is True diff --git a/tests/unit/relation_tests/model_tests/test_dist.py b/tests/unit/relation_tests/model_tests/test_dist.py new file mode 100644 index 000000000..2297bf033 --- /dev/null +++ b/tests/unit/relation_tests/model_tests/test_dist.py @@ -0,0 +1,30 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import RedshiftDistRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({"diststyle": "auto"}, None), + ({"diststyle": "auto", "distkey": "id"}, DbtRuntimeError), + ({"diststyle": "even"}, None), + ({"diststyle": "even", "distkey": "id"}, DbtRuntimeError), + ({"diststyle": "all"}, None), + ({"diststyle": "all", "distkey": "id"}, DbtRuntimeError), + ({"diststyle": "key"}, DbtRuntimeError), + ({"diststyle": "key", "distkey": "id"}, None), + ({}, None), + ], +) +def test_create_index(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + RedshiftDistRelation.from_dict(config_dict) + else: + my_dist = RedshiftDistRelation.from_dict(config_dict) + assert my_dist.diststyle == config_dict.get("diststyle", "auto") + assert my_dist.distkey == config_dict.get("distkey") diff --git a/tests/unit/relation_tests/model_tests/test_index.py b/tests/unit/relation_tests/model_tests/test_index.py deleted file mode 100644 index 11cd355b2..000000000 --- a/tests/unit/relation_tests/model_tests/test_index.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Type - -import pytest -from dbt.exceptions import DbtRuntimeError - -from dbt.adapters.redshift.relation.models import RedshiftIndexRelation - - -@pytest.mark.parametrize( - "config_dict,exception", - [ - ({"column_names": frozenset({"id", "value"}), "method": "hash", "unique": False}, None), - ({"column_names": frozenset("id"), "method": "btree", "unique": True}, None), - ({}, DbtRuntimeError), - ({"method": "btree", "unique": True}, DbtRuntimeError), - ], -) -# TODO replace this with sort/dist stuff -def test_create_index(config_dict: dict, exception: Type[Exception]): - if exception: - with pytest.raises(exception): - RedshiftIndexRelation.from_dict(config_dict) - else: - my_index = RedshiftIndexRelation.from_dict(config_dict) - assert my_index.column_names == config_dict.get("column_names") diff --git a/tests/unit/relation_tests/model_tests/test_materialized_view.py b/tests/unit/relation_tests/model_tests/test_materialized_view.py index 7166154f8..fd2e5c5d8 100644 --- a/tests/unit/relation_tests/model_tests/test_materialized_view.py +++ b/tests/unit/relation_tests/model_tests/test_materialized_view.py @@ -6,8 +6,10 @@ from dbt.exceptions import DbtRuntimeError from dbt.adapters.redshift.relation.models import ( + RedshiftDistRelation, RedshiftMaterializedViewRelation, RedshiftMaterializedViewRelationChangeset, + RedshiftSortRelation, ) @@ -33,14 +35,9 @@ "database": {"name": "my_database"}, }, "query": "select 42 from meaning_of_life", - "indexes": [ - { - "column_names": frozenset({"id", "value"}), - "method": "hash", - "unique": False, - }, - {"column_names": frozenset({"id"}), "method": "btree", "unique": True}, - ], + "dist": {"diststyle": "key", "distkey": "id"}, + "sort": {"sortstyle": "compound", "sortkey": ["id", "value"]}, + "autorefresh": True, }, None, ), @@ -51,6 +48,31 @@ "name": "my_schema", "database": {"name": "my_database"}, }, + # missing "query" + }, + DbtRuntimeError, + ), + ( + { + "name": "my_materialized_view", + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + "query": "select 1 from my_favoriate_table", + "dist": {"diststyle": "auto"}, # "auto" not supported for Redshift MVs + }, + DbtRuntimeError, + ), + ( + { + "name": "my_super_long_named_materialized_view" + * 10, # names must be <= 127 characters + "schema": { + "name": "my_schema", + "database": {"name": "my_database"}, + }, + "query": "select 1 from my_favoriate_table", }, DbtRuntimeError, ), @@ -62,27 +84,62 @@ def test_create_materialized_view(config_dict: dict, exception: Type[Exception]) RedshiftMaterializedViewRelation.from_dict(config_dict) else: my_materialized_view = RedshiftMaterializedViewRelation.from_dict(config_dict) + assert my_materialized_view.name == config_dict.get("name") - assert my_materialized_view.schema_name == config_dict.get("schema").get("name") - assert my_materialized_view.database_name == config_dict.get("schema").get("database").get( - "name" - ) + assert my_materialized_view.schema_name == config_dict.get("schema", {}).get("name") + assert my_materialized_view.database_name == config_dict.get("schema", {}).get( + "database", {} + ).get("name") assert my_materialized_view.query == config_dict.get("query") - if indexes := config_dict.get("indexes"): - parsed = {(index.method, index.unique) for index in my_materialized_view.indexes} - raw = {(index.get("method"), index.get("unique")) for index in indexes} - assert parsed == raw + assert my_materialized_view.backup == config_dict.get("backup", True) + + default_dist = RedshiftDistRelation.from_dict({"diststyle": "even"}) + default_diststyle = default_dist.diststyle + default_distkey = default_dist.distkey + assert my_materialized_view.dist.diststyle == config_dict.get("dist", {}).get( + "diststyle", default_diststyle + ) + assert my_materialized_view.dist.distkey == config_dict.get("dist", {}).get( + "distkey", default_distkey + ) + + default_sort = RedshiftSortRelation.from_dict({}) + default_sortstyle = default_sort.sortstyle + default_sortkey = default_sort.sortkey + assert my_materialized_view.sort.sortstyle == config_dict.get("sort", {}).get( + "sortstyle", default_sortstyle + ) + assert my_materialized_view.sort.sortkey == frozenset( + config_dict.get("sort", {}).get("sortkey", default_sortkey) + ) + + assert my_materialized_view.autorefresh == config_dict.get("autorefresh", False) + assert my_materialized_view.can_be_renamed is False -def test_create_materialized_view_changeset(materialized_view_relation): +@pytest.mark.parametrize( + "changes,is_empty,requires_full_refresh", + [ + ( + {"autorefresh": "f"}, + False, + False, + ), + ({"sort": RedshiftSortRelation.from_dict({"sortkey": "id"})}, False, True), + ({}, True, False), + ], +) +def test_create_materialized_view_changeset( + materialized_view_relation, changes, is_empty, requires_full_refresh +): existing_materialized_view = replace(materialized_view_relation) # pulled from `./dbt_postgres_tests/conftest.py` # TODO update with sort/dist/autorefresh/backup stuff - target_materialized_view = replace(existing_materialized_view, indexes=frozenset({})) + target_materialized_view = replace(existing_materialized_view, **changes) changeset = RedshiftMaterializedViewRelationChangeset.from_relations( existing_materialized_view, target_materialized_view ) - assert changeset.is_empty is False - assert changeset.requires_full_refresh is False + assert changeset.is_empty is is_empty + assert changeset.requires_full_refresh is requires_full_refresh diff --git a/tests/unit/relation_tests/model_tests/test_sort.py b/tests/unit/relation_tests/model_tests/test_sort.py new file mode 100644 index 000000000..1aa016815 --- /dev/null +++ b/tests/unit/relation_tests/model_tests/test_sort.py @@ -0,0 +1,30 @@ +from typing import Type + +import pytest +from dbt.exceptions import DbtRuntimeError + +from dbt.adapters.redshift.relation.models import RedshiftSortRelation + + +@pytest.mark.parametrize( + "config_dict,exception", + [ + ({}, None), + ({"sortstyle": "auto", "sortkey": "id"}, DbtRuntimeError), + ({"sortstyle": "compound", "sortkey": ["id"]}, None), + ({"sortstyle": "interleaved", "sortkey": ["id"]}, None), + ({"sortstyle": "auto"}, None), + ({"sortstyle": "compound"}, DbtRuntimeError), + ({"sortstyle": "interleaved"}, DbtRuntimeError), + ({"sortstyle": "compound", "sortkey": ["id", "value"]}, None), + ], +) +def test_create_sort(config_dict: dict, exception: Type[Exception]): + if exception: + with pytest.raises(exception): + RedshiftSortRelation.from_dict(config_dict) + else: + my_sortkey = RedshiftSortRelation.from_dict(config_dict) + default_sortstyle = "compound" if "sortkey" in config_dict else "auto" + assert my_sortkey.sortstyle == config_dict.get("sortstyle", default_sortstyle) + assert my_sortkey.sortkey == frozenset(config_dict.get("sortkey", {})) diff --git a/tests/unit/relation_tests/test_relation_factory.py b/tests/unit/relation_tests/test_relation_factory.py index 3a35152f1..009f14343 100644 --- a/tests/unit/relation_tests/test_relation_factory.py +++ b/tests/unit/relation_tests/test_relation_factory.py @@ -1,58 +1,47 @@ """ Uses the following fixtures in `unit/dbt_redshift_tests/conftest.py`: - `relation_factory` -- `materialized_view_stub` +- `materialized_view_ref` """ +from dbt.adapters.redshift.relation import models -from dbt.contracts.relation import RelationType -from dbt.adapters.postgres.relation import models +def test_make_ref(materialized_view_ref): + assert materialized_view_ref.name == "my_materialized_view" + assert materialized_view_ref.schema_name == "my_schema" + assert materialized_view_ref.database_name == "my_database" + assert materialized_view_ref.type == "materialized_view" + assert materialized_view_ref.can_be_renamed is True -def test_make_stub(materialized_view_stub): - assert materialized_view_stub.name == "my_materialized_view" - assert materialized_view_stub.schema_name == "my_schema" - assert materialized_view_stub.database_name == "my_database" - assert materialized_view_stub.type == "materialized_view" - assert materialized_view_stub.can_be_renamed is True +def test_make_backup_ref(relation_factory, materialized_view_ref): + backup_ref = relation_factory.make_backup_ref(materialized_view_ref) + assert backup_ref.name == '"my_materialized_view__dbt_backup"' -def test_make_backup_stub(relation_factory, materialized_view_stub): - backup_stub = relation_factory.make_backup_stub(materialized_view_stub) - assert backup_stub.name == '"my_materialized_view__dbt_backup"' - - -def test_make_intermediate(relation_factory, materialized_view_stub): - intermediate_relation = relation_factory.make_intermediate(materialized_view_stub) +def test_make_intermediate(relation_factory, materialized_view_ref): + intermediate_relation = relation_factory.make_intermediate(materialized_view_ref) assert intermediate_relation.name == '"my_materialized_view__dbt_tmp"' -def test_make_from_describe_relation_results( - relation_factory, materialized_view_describe_relation_results -): - materialized_view = relation_factory.make_from_describe_relation_results( - materialized_view_describe_relation_results, RelationType.MaterializedView - ) - - assert materialized_view.name == "my_materialized_view" - assert materialized_view.schema_name == "my_schema" - assert materialized_view.database_name == "my_database" - assert materialized_view.query == "select 42 from meaning_of_life" - - index_1 = models.RedshiftIndexRelation( - column_names=frozenset({"id", "value"}), - method=models.RedshiftIndexMethod.hash, - unique=False, +def test_make_from_describe_relation_results(relation_factory, materialized_view_relation): + assert materialized_view_relation.name == "my_materialized_view" + assert materialized_view_relation.schema_name == "my_schema" + assert materialized_view_relation.database_name == "my_database" + assert materialized_view_relation.query == "select 4 as id, 2 as other_id from meaning_of_life" + sort = models.RedshiftSortRelation( + sortstyle=models.RedshiftSortStyle.compound, + sortkey=frozenset({"other_id"}), render=models.RedshiftRenderPolicy, ) - index_2 = models.RedshiftIndexRelation( - column_names=frozenset({"id"}), - method=models.RedshiftIndexMethod.btree, - unique=True, + assert materialized_view_relation.sort == sort + dist = models.RedshiftDistRelation( + diststyle=models.RedshiftDistStyle.key, + distkey='"id"', render=models.RedshiftRenderPolicy, ) - assert index_1 in materialized_view.indexes - assert index_2 in materialized_view.indexes + assert materialized_view_relation.dist == dist + assert materialized_view_relation.autorefresh is True def test_make_from_model_node(relation_factory, materialized_view_model_node): @@ -62,18 +51,16 @@ def test_make_from_model_node(relation_factory, materialized_view_model_node): assert materialized_view.schema_name == "my_schema" assert materialized_view.database_name == "my_database" assert materialized_view.query == "select 42 from meaning_of_life" - - index_1 = models.RedshiftIndexRelation( - column_names=frozenset({"id", "value"}), - method=models.RedshiftIndexMethod.hash, - unique=False, + sort = models.RedshiftSortRelation( + sortstyle=models.RedshiftSortStyle.compound, + sortkey=frozenset({"other_id"}), render=models.RedshiftRenderPolicy, ) - index_2 = models.RedshiftIndexRelation( - column_names=frozenset({"id"}), - method=models.RedshiftIndexMethod.btree, - unique=True, + assert materialized_view.sort == sort + dist = models.RedshiftDistRelation( + diststyle=models.RedshiftDistStyle.key, + distkey="id", render=models.RedshiftRenderPolicy, ) - assert index_1 in materialized_view.indexes - assert index_2 in materialized_view.indexes + assert materialized_view.dist == dist + assert materialized_view.autorefresh is True