diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index d97989e183a..33594ada5dc 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -72,7 +72,7 @@ jobs: name: dbt Plugin Tests strategy: matrix: - dbt-version: [ dbt110, dbt120, dbt130, dbt140, dbt150, dbt160, dbt170 ] + dbt-version: [ dbt110, dbt120, dbt130, dbt140, dbt150, dbt160, dbt170, dbt180 ] include: # Default to python 3.11 for dbt tests. dbt doesn't support py 3.12 yet. - python-version: "3.11" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b603abefe38..1a144f35f4d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,9 +13,9 @@ repos: test/fixtures/linter/sqlfluffignore/[^/]*/[^/]*.sql| test/fixtures/config/inheritance_b/(nested/)?example.sql| (.*)/trailing_newlines.sql| - plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/my_new_project/multiple_trailing_newline.sql| - plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sql| - plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/(dbt_utils_0.8.0/)?last_day.sql| + plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/dbt_project/models/my_new_project/multiple_trailing_newline.sql| + plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/macro_in_macro.sql| + plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/(dbt_utils_0.8.0/)?last_day.sql| test/fixtures/linter/indentation_errors.sql| test/fixtures/templater/jinja_d_roundtrip/test.sql )$ @@ -26,9 +26,9 @@ repos: test/fixtures/templater/jinja_d_roundtrip/test.sql| test/fixtures/config/inheritance_b/example.sql| test/fixtures/config/inheritance_b/nested/example.sql| - plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sql| - plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/last_day.sql| - plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/dbt_utils_0.8.0/last_day.sql| + plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/macro_in_macro.sql| + plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/last_day.sql| + plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/dbt_utils_0.8.0/last_day.sql| test/fixtures/linter/sqlfluffignore/ )$ - repo: https://github.com/psf/black diff --git a/constraints/dbt180.txt b/constraints/dbt180.txt new file mode 100644 index 00000000000..6f44ff7d5fe --- /dev/null +++ b/constraints/dbt180.txt @@ -0,0 +1,2 @@ +dbt-core~=1.8.0 +dbt-postgres~=1.8.0 diff --git a/plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev b/plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev index 33f587e8622..e9ea71c7d1a 100644 --- a/plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev +++ b/plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev @@ -1,5 +1,7 @@ FROM python:3.9-slim-bullseye +RUN apt update \ +&& apt -y install libpq-dev gcc # Set separate working directory for easier debugging. WORKDIR /app @@ -8,7 +10,7 @@ RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip setupto # Install requirements separately # to take advantage of layer caching. COPY requirements*.txt . -RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade -r requirements.txt -r requirements_dev.txt +RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade -r requirements_dev.txt # Set up dbt-related dependencies. RUN --mount=type=cache,target=/root/.cache/pip pip install dbt-postgres diff --git a/plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py b/plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py index ea9d175e0cc..cbf89a8b539 100644 --- a/plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py +++ b/plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py @@ -66,6 +66,8 @@ class DbtConfigArgs: # https://github.com/sqlfluff/sqlfluff/issues/4861 # https://github.com/sqlfluff/sqlfluff/issues/4965 which: Optional[str] = "compile" + # NOTE: As of dbt 1.8, the following is required to exist. + REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES: Optional[bool] = None class DbtTemplater(JinjaTemplater): @@ -147,6 +149,12 @@ def dbt_config(self): from dbt.adapters.factory import register_adapter from dbt.config.runtime import RuntimeConfig as DbtRuntimeConfig + if self.dbt_version_tuple >= (1, 8): + from dbt_common.clients.system import get_env + from dbt_common.context import set_invocation_context + + set_invocation_context(get_env()) + # Attempt to silence internal logging at this point. # https://github.com/sqlfluff/sqlfluff/issues/5054 self.try_silence_dbt_logs() @@ -178,7 +186,7 @@ def dbt_config(self): ), user_config, ) - self.dbt_config = DbtRuntimeConfig.from_args( + _dbt_config = DbtRuntimeConfig.from_args( DbtConfigArgs( project_dir=self.project_dir, profiles_dir=self.profiles_dir, @@ -188,16 +196,22 @@ def dbt_config(self): threads=1, ) ) - register_adapter(self.dbt_config) - return self.dbt_config + + if self.dbt_version_tuple >= (1, 8): + from dbt.mp_context import get_mp_context + + register_adapter(_dbt_config, get_mp_context()) + else: + register_adapter(_dbt_config) + + return _dbt_config @cached_property def dbt_compiler(self): """Loads the dbt compiler.""" from dbt.compilation import Compiler as DbtCompiler - self.dbt_compiler = DbtCompiler(self.dbt_config) - return self.dbt_compiler + return DbtCompiler(self.dbt_config) @cached_property def dbt_manifest(self): @@ -233,14 +247,14 @@ def dbt_manifest(self): # and before. if self.dbt_version_tuple < (1, 4): os.chdir(self.project_dir) - self.dbt_manifest = ManifestLoader.get_full_manifest(self.dbt_config) + _dbt_manifest = ManifestLoader.get_full_manifest(self.dbt_config) except summary_errors as err: # pragma: no cover raise SQLFluffUserError(f"{err.__class__.__name__}: {err}") finally: if self.dbt_version_tuple < (1, 4): os.chdir(old_cwd) - return self.dbt_manifest + return _dbt_manifest @cached_property def dbt_selector_method(self): @@ -260,7 +274,7 @@ def dbt_selector_method(self): selector_methods_manager = DbtSelectorMethodManager( self.dbt_manifest, previous_state=None ) - self.dbt_selector_method = selector_methods_manager.get_method( + _dbt_selector_method = selector_methods_manager.get_method( DbtMethodName.Path, method_arguments=[] ) @@ -269,7 +283,7 @@ def dbt_selector_method(self): "dbt templater", "Project Compiled." ) - return self.dbt_selector_method + return _dbt_selector_method def _get_profiles_dir(self): """Get the dbt profiles directory from the configuration. @@ -459,7 +473,12 @@ def process( try: # These are the names in dbt-core 1.4.1+ # https://github.com/dbt-labs/dbt-core/pull/6539 - from dbt.exceptions import CompilationError, FailedToConnectError + from dbt.exceptions import CompilationError + + if self.dbt_version_tuple >= (1, 8): + from dbt.adapters.exceptions import FailedToConnectError + else: + from dbt.exceptions import FailedToConnectError except ImportError: # These are the historic names for older dbt-core versions from dbt.exceptions import CompilationException as CompilationError @@ -592,6 +611,9 @@ def from_string(*args, **kwargs): def render_func(in_str): env.add_extension(SnapshotExtension) template = env.from_string(in_str, globals=globals) + if self.dbt_version_tuple >= (1, 8): + # dbt 1.8 requires a context for rendering the template. + return template.render(globals) return template.render() return old_from_string(*args, **kwargs) @@ -628,7 +650,10 @@ def render_func(in_str): try: # These are the names in dbt-core 1.4.1+ # https://github.com/dbt-labs/dbt-core/pull/6539 - from dbt.exceptions import UndefinedMacroError + if self.dbt_version_tuple >= (1, 8): + from dbt_common.exceptions import UndefinedMacroError + else: + from dbt.exceptions import UndefinedMacroError except ImportError: # These are the historic names for older dbt-core versions from dbt.exceptions import UndefinedMacroException as UndefinedMacroError @@ -791,7 +816,16 @@ def connection(self): adapter = get_adapter(self.dbt_config) self.adapters[self.project_dir] = adapter adapter.acquire_connection("master") - adapter.set_relations_cache(self.dbt_manifest) + if self.dbt_version_tuple >= (1, 8): + # See notes from https://github.com/dbt-labs/dbt-adapters/discussions/87 + # about the decoupling of the adapters from core. + from dbt.context.providers import generate_runtime_macro_context + + adapter.set_macro_resolver(self.dbt_manifest) + adapter.set_macro_context_generator(generate_runtime_macro_context) + adapter.set_relations_cache(self.dbt_manifest.nodes.values()) + else: + adapter.set_relations_cache(self.dbt_manifest) yield # :TRICKY: Once connected, we never disconnect. Making multiple diff --git a/plugins/sqlfluff-templater-dbt/test/conftest.py b/plugins/sqlfluff-templater-dbt/test/conftest.py index a3c47a04346..9df47b7470a 100644 --- a/plugins/sqlfluff-templater-dbt/test/conftest.py +++ b/plugins/sqlfluff-templater-dbt/test/conftest.py @@ -1,9 +1,15 @@ """pytest fixtures.""" import os +import shutil +import subprocess +from pathlib import Path import pytest +from sqlfluff.core import FluffConfig +from sqlfluff_templater_dbt.templater import DbtTemplater + @pytest.fixture(scope="session", autouse=True) def dbt_flags(): @@ -13,3 +19,69 @@ def dbt_flags(): # We've seen occasional runtime errors from that code: # TypeError: cannot pickle '_thread.RLock' object os.environ["DBT_USE_EXPERIMENTAL_PARSER"] = "True" + + +@pytest.fixture() +def dbt_fluff_config(dbt_project_folder): + """Returns SQLFluff dbt configuration dictionary.""" + return { + "core": { + "templater": "dbt", + "dialect": "postgres", + }, + "templater": { + "dbt": { + "profiles_dir": f"{dbt_project_folder}/profiles_yml", + "project_dir": f"{dbt_project_folder}/dbt_project", + }, + }, + } + + +@pytest.fixture() +def project_dir(dbt_fluff_config): + """Returns the dbt project directory.""" + return dbt_fluff_config["templater"]["dbt"]["project_dir"] + + +@pytest.fixture() +def profiles_dir(dbt_fluff_config): + """Returns the dbt project directory.""" + return dbt_fluff_config["templater"]["dbt"]["profiles_dir"] + + +@pytest.fixture() +def dbt_templater(): + """Returns an instance of the DbtTemplater.""" + return FluffConfig(overrides={"dialect": "ansi"}).get_templater("dbt") + + +@pytest.fixture(scope="session") +def dbt_project_folder(): + """Fixture for a temporary dbt project directory.""" + src = Path("plugins/sqlfluff-templater-dbt/test/fixtures/dbt") + tmp = Path("plugins/sqlfluff-templater-dbt/test/temp_dbt_project") + tmp.mkdir(exist_ok=True) + shutil.copytree(src, tmp, dirs_exist_ok=True) + if DbtTemplater().dbt_version_tuple >= (1, 8): + # Configuration overrides for dbt 1.8+ + dbt180_fixtures = src.with_name("dbt180") + shutil.copytree(dbt180_fixtures, tmp, dirs_exist_ok=True) + + subprocess.Popen( + [ + "dbt", + "deps", + "--project-dir", + f"{tmp}/dbt_project", + "--profiles-dir", + f"{tmp}/profiles_yml", + ] + ).wait(10) + + # Placeholder value for testing + os.environ["passed_through_env"] = "_" + + yield tmp + + shutil.rmtree(tmp) diff --git a/plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/vars_from_env.sql b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/vars_from_env.sql new file mode 100644 index 00000000000..0642eaba3d9 --- /dev/null +++ b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/vars_from_env.sql @@ -0,0 +1 @@ +SELECT {{ env_var('passed_through_env') }} diff --git a/plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templater.py b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templater.py deleted file mode 100644 index 32ee2f2509d..00000000000 --- a/plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templater.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Fixtures for dbt templating tests.""" - -import pytest - -from sqlfluff.core import FluffConfig - -DBT_FLUFF_CONFIG = { - "core": { - "templater": "dbt", - "dialect": "postgres", - }, - "templater": { - "dbt": { - "profiles_dir": ( - "plugins/sqlfluff-templater-dbt/test/fixtures/dbt/profiles_yml" - ), - "project_dir": ( - "plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project" - ), - }, - }, -} - - -@pytest.fixture() -def project_dir(): - """Returns the dbt project directory.""" - return DBT_FLUFF_CONFIG["templater"]["dbt"]["project_dir"] - - -@pytest.fixture() -def dbt_templater(): - """Returns an instance of the DbtTemplater.""" - return FluffConfig(overrides={"dialect": "ansi"}).get_templater("dbt") diff --git a/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/dbt_project.yml b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/dbt_project.yml new file mode 100644 index 00000000000..c80933f6d0e --- /dev/null +++ b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/dbt_project.yml @@ -0,0 +1,20 @@ +name: 'my_new_project' +version: '1.0.0' +config-version: 2 + +profile: 'default' + +test-paths: ["tests"] + +models: + my_new_project: + materialized: view + +vars: + my_new_project: + # Default date stamp of run + ds: "2020-01-01" + # passed_through_cli: testing for vars passed through cli('--vars' option) rather than dbt_project + +flags: + send_anonymous_usage_stats: false diff --git a/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/profiles_yml/profiles.yml b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/profiles_yml/profiles.yml new file mode 100644 index 00000000000..3127a6ebf8e --- /dev/null +++ b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/profiles_yml/profiles.yml @@ -0,0 +1,12 @@ +default: + target: dev + outputs: + dev: + type: postgres + host: "{{ env_var('POSTGRES_HOST', 'localhost') }}" + user: postgres + pass: password + port: 5432 + dbname: postgres + schema: dbt_alice + threads: 4 diff --git a/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/profiles_yml_fail/profiles.yml b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/profiles_yml_fail/profiles.yml new file mode 100644 index 00000000000..5e7536e8c6f --- /dev/null +++ b/plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/profiles_yml_fail/profiles.yml @@ -0,0 +1,12 @@ +default: + target: dev + outputs: + dev: + type: postgres + host: localhost + user: postgres + pass: password + port: 2345 + dbname: postgres + schema: dbt_alice + threads: 4 diff --git a/plugins/sqlfluff-templater-dbt/test/linter_test.py b/plugins/sqlfluff-templater-dbt/test/linter_test.py index 1fadc57ec5f..c55b163f1f3 100644 --- a/plugins/sqlfluff-templater-dbt/test/linter_test.py +++ b/plugins/sqlfluff-templater-dbt/test/linter_test.py @@ -9,15 +9,14 @@ from sqlfluff.cli.commands import lint from sqlfluff.core import FluffConfig, Linter from sqlfluff.utils.testing.cli import invoke_assert_code -from test.fixtures.dbt.templater import DBT_FLUFF_CONFIG, project_dir # noqa: F401 @pytest.mark.parametrize( "path", ["models/my_new_project/disabled_model.sql", "macros/echo.sql"] ) -def test__linter__skip_file(path, project_dir): # noqa +def test__linter__skip_file(path, project_dir, dbt_fluff_config): # noqa """Test that the linter skips disabled dbt models and macros.""" - conf = FluffConfig(configs=DBT_FLUFF_CONFIG) + conf = FluffConfig(configs=dbt_fluff_config) lntr = Linter(config=conf) model_file_path = os.path.join(project_dir, path) linted_path = lntr.lint_path(path=model_file_path) @@ -30,19 +29,19 @@ def test__linter__skip_file(path, project_dir): # noqa assert not linted_file.tree -def test__linter__lint_ephemeral_3_level(project_dir): # noqa +def test__linter__lint_ephemeral_3_level(project_dir, dbt_fluff_config): """Test linter can lint a project with 3-level ephemeral dependencies.""" # This was previously crashing inside dbt, in a function named # inject_ctes_into_sql(). (issue 2671). - conf = FluffConfig(configs=DBT_FLUFF_CONFIG) + conf = FluffConfig(configs=dbt_fluff_config) lntr = Linter(config=conf) model_file_path = os.path.join(project_dir, "models/ephemeral_3_level") lntr.lint_path(path=model_file_path) -def test__linter__config_pairs(project_dir): # noqa +def test__linter__config_pairs(dbt_fluff_config): # noqa """Test that the dbt templater returns version information in it's config.""" - conf = FluffConfig(configs=DBT_FLUFF_CONFIG) + conf = FluffConfig(configs=dbt_fluff_config) lntr = Linter(config=conf) # NOTE: This method is called within the config readout. assert lntr.templater.config_pairs() == [ @@ -51,17 +50,17 @@ def test__linter__config_pairs(project_dir): # noqa ] -def test_dbt_target_dir(tmpdir): +def test_dbt_target_dir(tmpdir, dbt_project_folder, profiles_dir): """Test with dbt project in subdir that target/ is created in the correct place. https://github.com/sqlfluff/sqlfluff/issues/2895 """ tmp_base_dir = str(tmpdir) tmp_dbt_dir = os.path.join(tmp_base_dir, "dir1", "dir2", "dbt") - # tmp_project_dir = os.path.join(tmp_dbt_dir, "dbt_project") + tmp_project_dir = os.path.join(tmp_dbt_dir, "dbt_project") os.makedirs(os.path.dirname(tmp_dbt_dir)) shutil.copytree( - "plugins/sqlfluff-templater-dbt/test/fixtures/dbt", + dbt_project_folder, tmp_dbt_dir, ) os.unlink(os.path.join(tmp_dbt_dir, ".sqlfluff")) @@ -73,16 +72,14 @@ def test_dbt_target_dir(tmpdir): os.chdir(tmp_base_dir) with open(".sqlfluff", "w") as f: print( - """[sqlfluff] + f"""[sqlfluff] templater = dbt dialect = postgres [sqlfluff:templater:dbt] -project_dir = {tmp_base_dir}/dir1/dir2/dbt/dbt_project -profiles_dir = {old_cwd}/plugins/sqlfluff-templater-dbt/test/fixtures/dbt/profiles_yml -""".format( - old_cwd=old_cwd, tmp_base_dir=tmp_base_dir - ), +project_dir = {tmp_project_dir} +profiles_dir = {old_cwd}/{profiles_dir} +""", file=f, ) try: diff --git a/plugins/sqlfluff-templater-dbt/test/rules_test.py b/plugins/sqlfluff-templater-dbt/test/rules_test.py index f14613300d7..421e2ca1d82 100644 --- a/plugins/sqlfluff-templater-dbt/test/rules_test.py +++ b/plugins/sqlfluff-templater-dbt/test/rules_test.py @@ -9,11 +9,6 @@ from sqlfluff.core import Linter from sqlfluff.core.config import FluffConfig from sqlfluff.utils.testing.rules import assert_rule_raises_violations_in_file -from test.fixtures.dbt.templater import ( # noqa - DBT_FLUFF_CONFIG, - dbt_templater, - project_dir, -) @pytest.mark.parametrize( @@ -25,22 +20,24 @@ ("LT12", "models/my_new_project/multiple_trailing_newline.sql", [(3, 1)]), ], ) -def test__rules__std_file_dbt(rule, path, violations, project_dir): # noqa +def test__rules__std_file_dbt( + rule, path, violations, project_dir, dbt_fluff_config +): # noqa """Test linter finds the given errors in (and only in) the right places (DBT).""" assert_rule_raises_violations_in_file( rule=rule, fpath=os.path.join(project_dir, path), violations=violations, - fluff_config=FluffConfig(configs=DBT_FLUFF_CONFIG, overrides=dict(rules=rule)), + fluff_config=FluffConfig(configs=dbt_fluff_config, overrides=dict(rules=rule)), ) -def test__rules__fix_utf8(project_dir): # noqa +def test__rules__fix_utf8(project_dir, dbt_fluff_config): # noqa """Verify that non-ASCII characters are preserved by 'fix'.""" rule = "CP01" path = "models/my_new_project/utf8/test.sql" linter = Linter( - config=FluffConfig(configs=DBT_FLUFF_CONFIG, overrides=dict(rules=rule)) + config=FluffConfig(configs=dbt_fluff_config, overrides=dict(rules=rule)) ) result = linter.lint_path(os.path.join(project_dir, path), fix=True) # Check that we did actually find issues. @@ -66,12 +63,12 @@ def test__rules__fix_utf8(project_dir): # noqa os.unlink(fixed_path) -def test__rules__order_by(project_dir): # noqa +def test__rules__order_by(project_dir, dbt_fluff_config): # noqa """Verify that rule AM03 works with dbt.""" rule = "AM03" path = "models/my_new_project/AM03_test.sql" lntr = Linter( - config=FluffConfig(configs=DBT_FLUFF_CONFIG, overrides=dict(rules=rule)) + config=FluffConfig(configs=dbt_fluff_config, overrides=dict(rules=rule)) ) lnt = lntr.lint_path(os.path.join(project_dir, path)) diff --git a/plugins/sqlfluff-templater-dbt/test/templater_test.py b/plugins/sqlfluff-templater-dbt/test/templater_test.py index 3aba3425de6..ce486cf52ea 100644 --- a/plugins/sqlfluff-templater-dbt/test/templater_test.py +++ b/plugins/sqlfluff-templater-dbt/test/templater_test.py @@ -17,14 +17,9 @@ from sqlfluff.utils.testing.cli import invoke_assert_code from sqlfluff.utils.testing.logging import fluff_log_catcher from sqlfluff_templater_dbt.templater import DbtTemplater -from test.fixtures.dbt.templater import ( # noqa: F401 - DBT_FLUFF_CONFIG, - dbt_templater, - project_dir, -) -def test__templater_dbt_missing(dbt_templater, project_dir): # noqa: F811 +def test__templater_dbt_missing(dbt_templater, project_dir, dbt_fluff_config): """Check that a nice error is returned when dbt module is missing.""" try: import dbt # noqa: F401 @@ -37,11 +32,11 @@ def test__templater_dbt_missing(dbt_templater, project_dir): # noqa: F811 dbt_templater.process( in_str="", fname=os.path.join(project_dir, "models/my_new_project/test.sql"), - config=FluffConfig(configs=DBT_FLUFF_CONFIG), + config=FluffConfig(configs=dbt_fluff_config), ) -def test__templater_dbt_profiles_dir_expanded(dbt_templater): # noqa: F811 +def test__templater_dbt_profiles_dir_expanded(dbt_templater): """Check that the profiles_dir is expanded.""" dbt_templater.sqlfluff_config = FluffConfig( configs={ @@ -91,35 +86,59 @@ def test__templater_dbt_profiles_dir_expanded(dbt_templater): # noqa: F811 ], ) def test__templater_dbt_templating_result( - project_dir, dbt_templater, fname # noqa: F811 + project_dir, + dbt_templater, + fname, + dbt_fluff_config, + dbt_project_folder, ): """Test that input sql file gets templated into output sql file.""" - _run_templater_and_verify_result(dbt_templater, project_dir, fname) + _run_templater_and_verify_result( + dbt_templater, + project_dir, + fname, + dbt_fluff_config, + dbt_project_folder, + ) def test_dbt_profiles_dir_env_var_uppercase( - project_dir, dbt_templater, tmpdir, monkeypatch # noqa: F811 + project_dir, + dbt_templater, + tmpdir, + monkeypatch, + dbt_fluff_config, + dbt_project_folder, + profiles_dir, ): """Tests specifying the dbt profile dir with env var.""" - profiles_dir = tmpdir.mkdir("SUBDIR") # Use uppercase to test issue 2253 - monkeypatch.setenv("DBT_PROFILES_DIR", str(profiles_dir)) - shutil.copy( - os.path.join(project_dir, "../profiles_yml/profiles.yml"), str(profiles_dir) + sub_profiles_dir = tmpdir.mkdir("SUBDIR") # Use uppercase to test issue 2253 + monkeypatch.setenv("DBT_PROFILES_DIR", str(sub_profiles_dir)) + shutil.copy(os.path.join(profiles_dir, "profiles.yml"), str(sub_profiles_dir)) + _run_templater_and_verify_result( + dbt_templater, + project_dir, + "use_dbt_utils.sql", + dbt_fluff_config, + dbt_project_folder, ) - _run_templater_and_verify_result(dbt_templater, project_dir, "use_dbt_utils.sql") -def _run_templater_and_verify_result(dbt_templater, project_dir, fname): # noqa: F811 +def _run_templater_and_verify_result( + dbt_templater, + project_dir, + fname, + dbt_fluff_config, + dbt_project_folder, +): path = Path(project_dir) / "models/my_new_project" / fname - config = FluffConfig(configs=DBT_FLUFF_CONFIG) + config = FluffConfig(configs=dbt_fluff_config) templated_file, _ = dbt_templater.process( in_str=path.read_text(), fname=str(path), config=config, ) - template_output_folder_path = Path( - "plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/" - ) + template_output_folder_path = dbt_project_folder / "templated_output/" fixture_path = _get_fixture_path(template_output_folder_path, fname) assert str(templated_file) == fixture_path.read_text() # Check we can lex the output too. @@ -180,12 +199,16 @@ def _get_fixture_path(template_output_folder_path, fname): ], ) def test__templater_dbt_sequence_files_ephemeral_dependency( - project_dir, dbt_templater, fnames_input, fnames_expected_sequence # noqa: F811 + project_dir, + dbt_templater, + fnames_input, + fnames_expected_sequence, + dbt_fluff_config, ): """Test that dbt templater sequences files based on dependencies.""" result = dbt_templater.sequence_files( [str(Path(project_dir) / fn) for fn in fnames_input], - config=FluffConfig(configs=DBT_FLUFF_CONFIG), + config=FluffConfig(configs=dbt_fluff_config), ) pd = Path(project_dir) expected = [str(pd / fn) for fn in fnames_expected_sequence] @@ -210,7 +233,11 @@ def test__templater_dbt_sequence_files_ephemeral_dependency( ], ) def test__templater_dbt_slice_file_wrapped_test( - raw_file, templated_file, result, dbt_templater, caplog # noqa: F811 + raw_file, + templated_file, + result, + dbt_templater, + caplog, ): """Test that wrapped queries are sliced safely using _check_for_wrapped().""" @@ -242,14 +269,17 @@ def _render_func(in_str) -> str: ], ) def test__templater_dbt_templating_test_lex( - project_dir, dbt_templater, fname # noqa: F811 + project_dir, + dbt_templater, + fname, + dbt_fluff_config, ): """Demonstrate the lexer works on both dbt models and dbt tests. Handle any number of newlines. """ path = Path(project_dir) / fname - config = FluffConfig(configs=DBT_FLUFF_CONFIG) + config = FluffConfig(configs=dbt_fluff_config) source_dbt_sql = path.read_text() # Count the newlines. n_trailing_newlines = len(source_dbt_sql) - len(source_dbt_sql.rstrip("\n")) @@ -291,24 +321,28 @@ def test__templater_dbt_templating_test_lex( ], ) def test__templater_dbt_skips_file( - path, reason, dbt_templater, project_dir # noqa: F811 + path, + reason, + dbt_templater, + project_dir, + dbt_fluff_config, ): """A disabled dbt model should be skipped.""" with pytest.raises(SQLFluffSkipFile, match=reason): dbt_templater.process( in_str="", fname=os.path.join(project_dir, path), - config=FluffConfig(configs=DBT_FLUFF_CONFIG), + config=FluffConfig(configs=dbt_fluff_config), ) -def test_dbt_fails_stdin(dbt_templater): # noqa: F811 +def test_dbt_fails_stdin(dbt_templater, dbt_fluff_config): """Reading from stdin is not supported with dbt templater.""" with pytest.raises(SQLFluffUserError): dbt_templater.process( in_str="", fname="stdin", - config=FluffConfig(configs=DBT_FLUFF_CONFIG), + config=FluffConfig(configs=dbt_fluff_config), ) @@ -322,10 +356,13 @@ def test_dbt_fails_stdin(dbt_templater): # noqa: F811 ], ) def test__dbt_templated_models_do_not_raise_lint_error( - project_dir, fname, caplog # noqa: F811 + project_dir, + fname, + caplog, + dbt_fluff_config, ): """Test that templated dbt models do not raise a linting error.""" - linter = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG)) + linter = Linter(config=FluffConfig(configs=dbt_fluff_config)) # Log rules output. with caplog.at_level(logging.DEBUG, logger="sqlfluff.rules"): lnt = linter.lint_path( @@ -356,12 +393,15 @@ def _clean_path(glob_expression): "path", ["models/my_new_project/issue_1608.sql", "snapshots/issue_1771.sql"] ) def test__dbt_templated_models_fix_does_not_corrupt_file( - project_dir, path, caplog # noqa: F811 + project_dir, + path, + caplog, + dbt_fluff_config, ): """Test issues where previously "sqlfluff fix" corrupted the file.""" test_glob = os.path.join(project_dir, os.path.dirname(path), "*FIXED.sql") _clean_path(test_glob) - lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG)) + lntr = Linter(config=FluffConfig(configs=dbt_fluff_config)) with caplog.at_level(logging.INFO, logger="sqlfluff.linter"): lnt = lntr.lint_path(os.path.join(project_dir, path), fix=True) try: @@ -376,7 +416,9 @@ def test__dbt_templated_models_fix_does_not_corrupt_file( def test__templater_dbt_templating_absolute_path( - project_dir, dbt_templater # noqa: F811 + project_dir, + dbt_templater, + dbt_fluff_config, ): """Test that absolute path of input path does not cause RuntimeError.""" try: @@ -385,7 +427,7 @@ def test__templater_dbt_templating_absolute_path( fname=os.path.abspath( os.path.join(project_dir, "models/my_new_project/use_var.sql") ), - config=FluffConfig(configs=DBT_FLUFF_CONFIG), + config=FluffConfig(configs=dbt_fluff_config), ) except Exception as e: pytest.fail(f"Unexpected RuntimeError: {e}") @@ -418,29 +460,34 @@ def test__templater_dbt_templating_absolute_path( ], ) def test__templater_dbt_handle_exceptions( - project_dir, dbt_templater, fname, exception_msg # noqa: F811 + project_dir, + dbt_templater, + dbt_fluff_config, + dbt_project_folder, + fname, + exception_msg, ): """Test that exceptions during compilation are returned as violation.""" from dbt.adapters.factory import get_adapter - src_fpath = "plugins/sqlfluff-templater-dbt/test/fixtures/dbt/error_models/" + fname + src_fpath = dbt_project_folder / "error_models" / fname target_fpath = os.path.abspath( os.path.join(project_dir, "models/my_new_project/", fname) ) # We move the file that throws an error in and out of the project directory # as dbt throws an error if a node fails to parse while computing the DAG - os.rename(src_fpath, target_fpath) + shutil.move(src_fpath, target_fpath) try: with pytest.raises(SQLTemplaterError) as excinfo: dbt_templater.process( in_str="", fname=target_fpath, config=FluffConfig( - configs=DBT_FLUFF_CONFIG, overrides={"dialect": "ansi"} + configs=dbt_fluff_config, overrides={"dialect": "ansi"} ), ) finally: - os.rename(target_fpath, src_fpath) + shutil.move(target_fpath, src_fpath) get_adapter(dbt_templater.dbt_config).connections.release() # NB: Replace slashes to deal with different platform paths being returned. assert exception_msg in excinfo.value.desc().replace("\\", "/") @@ -448,19 +495,27 @@ def test__templater_dbt_handle_exceptions( @mock.patch("dbt.adapters.postgres.impl.PostgresAdapter.set_relations_cache") def test__templater_dbt_handle_database_connection_failure( - set_relations_cache, project_dir, dbt_templater # noqa: F811 + set_relations_cache, + project_dir, + dbt_templater, + dbt_fluff_config, ): """Test the result of a failed database connection.""" from dbt.adapters.factory import get_adapter try: - from dbt.exceptions import ( - FailedToConnectException as DbtFailedToConnectException, - ) - except ImportError: - from dbt.exceptions import ( + from dbt.adapters.exceptions import ( FailedToConnectError as DbtFailedToConnectException, ) + except ImportError: + try: + from dbt.exceptions import ( + FailedToConnectError as DbtFailedToConnectException, + ) + except ImportError: + from dbt.exceptions import ( + FailedToConnectException as DbtFailedToConnectException, + ) # Clear the adapter cache to force this test to create a new connection. DbtTemplater.adapters.clear() @@ -476,22 +531,22 @@ def test__templater_dbt_handle_database_connection_failure( project_dir, "models/my_new_project/exception_connect_database.sql" ) ) - dbt_fluff_config_fail = deepcopy(DBT_FLUFF_CONFIG) + dbt_fluff_config_fail = deepcopy(dbt_fluff_config) dbt_fluff_config_fail["templater"]["dbt"][ "profiles_dir" ] = "plugins/sqlfluff-templater-dbt/test/fixtures/dbt/profiles_yml_fail" # We move the file that throws an error in and out of the project directory # as dbt throws an error if a node fails to parse while computing the DAG - os.rename(src_fpath, target_fpath) + shutil.move(src_fpath, target_fpath) try: with pytest.raises(SQLTemplaterError) as excinfo: dbt_templater.process( in_str="", fname=target_fpath, - config=FluffConfig(configs=DBT_FLUFF_CONFIG), + config=FluffConfig(configs=dbt_fluff_config), ) finally: - os.rename(target_fpath, src_fpath) + shutil.move(target_fpath, src_fpath) get_adapter(dbt_templater.dbt_config).connections.release() # NB: Replace slashes to deal with different platform paths being returned. assert ( @@ -501,7 +556,7 @@ def test__templater_dbt_handle_database_connection_failure( ) -def test__project_dir_does_not_exist_error(dbt_templater): # noqa: F811 +def test__project_dir_does_not_exist_error(dbt_templater): """Test an error is logged if the given dbt project directory doesn't exist.""" dbt_templater.sqlfluff_config = FluffConfig( configs={ @@ -525,12 +580,16 @@ def test__project_dir_does_not_exist_error(dbt_templater): # noqa: F811 ], ) def test__context_in_config_is_loaded( - project_dir, dbt_templater, model_path, var_value # noqa: F811 + project_dir, + dbt_templater, + model_path, + var_value, + dbt_fluff_config, ): """Test that variables inside .sqlfluff are passed to dbt.""" context = {"passed_through_cli": var_value} if var_value else {} - config_dict = deepcopy(DBT_FLUFF_CONFIG) + config_dict = deepcopy(dbt_fluff_config) config_dict["templater"]["dbt"]["context"] = context config = FluffConfig(config_dict) @@ -544,7 +603,34 @@ def test__context_in_config_is_loaded( assert str(var_value) in processed.templated_str -def test__dbt_log_supression(): +@pytest.mark.parametrize( + ("model_path", "var_value"), + [ + ("models/vars_from_env.sql", "expected_value"), + ], +) +def test__context_in_env_is_loaded( + project_dir, + dbt_templater, + model_path, + var_value, + dbt_fluff_config, +): + """Test that variables inside env are passed to dbt.""" + os.environ["passed_through_env"] = var_value + + config = FluffConfig(dbt_fluff_config) + path = Path(project_dir) / model_path + + processed, violations = dbt_templater.process( + in_str=path.read_text(), fname=str(path), config=config + ) + + assert violations == [] + assert str(var_value) in processed.templated_str + + +def test__dbt_log_supression(dbt_project_folder): """Test that when we try and parse in JSON format we get JSON. This actually tests that we can successfully suppress unwanted @@ -552,7 +638,7 @@ def test__dbt_log_supression(): """ oldcwd = os.getcwd() try: - os.chdir("plugins/sqlfluff-templater-dbt/test/fixtures/dbt") + os.chdir(dbt_project_folder) result = invoke_assert_code( ret_code=1, args=[ diff --git a/tox.ini b/tox.ini index 665c7485dda..96cb92088a3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = generate-fixture-yml, linting, doclinting, ruleslinting, docbuild, cov-init, doctests, py{38,39,310,311,312}, dbt{110,120,130,140,150,160,170}, cov-report, mypy, winpy, dbt{130,150}-winpy, yamllint +envlist = generate-fixture-yml, linting, doclinting, ruleslinting, docbuild, cov-init, doctests, py{38,39,310,311,312}, dbt{110,120,130,140,150,160,170,180}, cov-report, mypy, winpy, dbt{130,150}-winpy, yamllint min_version = 4.0 # Require 4.0+ for proper pyproject.toml support [testenv] @@ -20,7 +20,7 @@ deps = # we force the right installation version up front in each environment. # NOTE: This is a bit of a hack around tox, but it does achieve reasonably # consistent results. - dbt{110,120,130,140,150,160,170,}: -r {toxinidir}/constraints/{envname}.txt + dbt{110,120,130,140,150,160,170,180,}: -r {toxinidir}/constraints/{envname}.txt # Include any other steps necessary for testing below. # {posargs} is there to allow us to specify specific tests, which # can then be invoked from tox by calling e.g. @@ -32,15 +32,13 @@ commands = # number pinned to the same version number of the main sqlfluff library # so it _must_ be installed second in the context of a version which isn't # yet released (and so not available on pypi). - dbt{110,120,130,140,150,160,170,}: python -m pip install {toxinidir}/plugins/sqlfluff-templater-dbt + dbt{110,120,130,140,150,160,170,180,}: python -m pip install {toxinidir}/plugins/sqlfluff-templater-dbt # Add the example plugin. # NOTE: The trailing comma is important because in the github test suite # the python version is not specified and instead the "py" environment # is invoked. Leaving the trailing comma ensures that this environment # still installs the relevant plugins. {py,winpy}{38,39,310,311,312,}: python -m pip install {toxinidir}/plugins/sqlfluff-plugin-example - # For the dbt test cases install dependencies. - dbt{110,120,130,140,150,160,170,}: dbt deps --project-dir {toxinidir}/plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project --profiles-dir {toxinidir}/plugins/sqlfluff-templater-dbt/test/fixtures/dbt # Clean up from previous tests python {toxinidir}/util.py clean-tests # Run tests