dbt Templater Plugin: dbt 1.8 support (sqlfluff#5892)

Co-authored-by: Kazuhito Osabe <[email protected]>

.github/workflows/ci-tests.yml

@@ -72,7 +72,7 @@ jobs:
     name: dbt Plugin Tests
     strategy:
       matrix:
-        dbt-version: [ dbt110, dbt120, dbt130, dbt140, dbt150, dbt160, dbt170 ]
+        dbt-version: [ dbt110, dbt120, dbt130, dbt140, dbt150, dbt160, dbt170, dbt180 ]
         include:
           # Default to python 3.11 for dbt tests. dbt doesn't support py 3.12 yet.
           - python-version: "3.11"

.pre-commit-config.yaml

@@ -13,9 +13,9 @@ repos:
             test/fixtures/linter/sqlfluffignore/[^/]*/[^/]*.sql|
             test/fixtures/config/inheritance_b/(nested/)?example.sql|
             (.*)/trailing_newlines.sql|
-            plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/my_new_project/multiple_trailing_newline.sql|
-            plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sql|
-            plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/(dbt_utils_0.8.0/)?last_day.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/dbt_project/models/my_new_project/multiple_trailing_newline.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/macro_in_macro.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/(dbt_utils_0.8.0/)?last_day.sql|
             test/fixtures/linter/indentation_errors.sql|
             test/fixtures/templater/jinja_d_roundtrip/test.sql
           )$
@@ -26,9 +26,9 @@ repos:
             test/fixtures/templater/jinja_d_roundtrip/test.sql|
             test/fixtures/config/inheritance_b/example.sql|
             test/fixtures/config/inheritance_b/nested/example.sql|
-            plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sql|
-            plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/last_day.sql|
-            plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/dbt_utils_0.8.0/last_day.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/macro_in_macro.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/last_day.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/dbt_utils_0.8.0/last_day.sql|
             test/fixtures/linter/sqlfluffignore/
           )$
   - repo: https://github.com/psf/black

constraints/dbt180.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.8.0
+dbt-postgres~=1.8.0

plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev

@@ -1,5 +1,7 @@
 FROM python:3.9-slim-bullseye
 
+RUN apt update \
+&& apt -y install libpq-dev gcc
 # Set separate working directory for easier debugging.
 WORKDIR /app
 
@@ -8,7 +10,7 @@ RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip setupto
 # Install requirements separately
 # to take advantage of layer caching.
 COPY requirements*.txt .
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade -r requirements.txt -r requirements_dev.txt
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade -r requirements_dev.txt
 
 # Set up dbt-related dependencies.
 RUN --mount=type=cache,target=/root/.cache/pip pip install dbt-postgres

plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/templater.py

@@ -66,6 +66,8 @@ class DbtConfigArgs:
     # https://github.com/sqlfluff/sqlfluff/issues/4861
     # https://github.com/sqlfluff/sqlfluff/issues/4965
     which: Optional[str] = "compile"
+    # NOTE: As of dbt 1.8, the following is required to exist.
+    REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES: Optional[bool] = None
 
 
 class DbtTemplater(JinjaTemplater):
@@ -147,6 +149,12 @@ def dbt_config(self):
         from dbt.adapters.factory import register_adapter
         from dbt.config.runtime import RuntimeConfig as DbtRuntimeConfig
 
+        if self.dbt_version_tuple >= (1, 8):
+            from dbt_common.clients.system import get_env
+            from dbt_common.context import set_invocation_context
+
+            set_invocation_context(get_env())
+
         # Attempt to silence internal logging at this point.
         # https://github.com/sqlfluff/sqlfluff/issues/5054
         self.try_silence_dbt_logs()
@@ -178,7 +186,7 @@ def dbt_config(self):
             ),
             user_config,
         )
-        self.dbt_config = DbtRuntimeConfig.from_args(
+        _dbt_config = DbtRuntimeConfig.from_args(
             DbtConfigArgs(
                 project_dir=self.project_dir,
                 profiles_dir=self.profiles_dir,
@@ -188,16 +196,22 @@ def dbt_config(self):
                 threads=1,
             )
         )
-        register_adapter(self.dbt_config)
-        return self.dbt_config
+
+        if self.dbt_version_tuple >= (1, 8):
+            from dbt.mp_context import get_mp_context
+
+            register_adapter(_dbt_config, get_mp_context())
+        else:
+            register_adapter(_dbt_config)
+
+        return _dbt_config
 
     @cached_property
     def dbt_compiler(self):
         """Loads the dbt compiler."""
         from dbt.compilation import Compiler as DbtCompiler
 
-        self.dbt_compiler = DbtCompiler(self.dbt_config)
-        return self.dbt_compiler
+        return DbtCompiler(self.dbt_config)
 
     @cached_property
     def dbt_manifest(self):
@@ -233,14 +247,14 @@ def dbt_manifest(self):
             # and before.
             if self.dbt_version_tuple < (1, 4):
                 os.chdir(self.project_dir)
-            self.dbt_manifest = ManifestLoader.get_full_manifest(self.dbt_config)
+            _dbt_manifest = ManifestLoader.get_full_manifest(self.dbt_config)
         except summary_errors as err:  # pragma: no cover
             raise SQLFluffUserError(f"{err.__class__.__name__}: {err}")
         finally:
             if self.dbt_version_tuple < (1, 4):
                 os.chdir(old_cwd)
 
-        return self.dbt_manifest
+        return _dbt_manifest
 
     @cached_property
     def dbt_selector_method(self):
@@ -260,7 +274,7 @@ def dbt_selector_method(self):
         selector_methods_manager = DbtSelectorMethodManager(
             self.dbt_manifest, previous_state=None
         )
-        self.dbt_selector_method = selector_methods_manager.get_method(
+        _dbt_selector_method = selector_methods_manager.get_method(
             DbtMethodName.Path, method_arguments=[]
         )
 
@@ -269,7 +283,7 @@ def dbt_selector_method(self):
                 "dbt templater", "Project Compiled."
             )
 
-        return self.dbt_selector_method
+        return _dbt_selector_method
 
     def _get_profiles_dir(self):
         """Get the dbt profiles directory from the configuration.
@@ -459,7 +473,12 @@ def process(
         try:
             # These are the names in dbt-core 1.4.1+
             # https://github.com/dbt-labs/dbt-core/pull/6539
-            from dbt.exceptions import CompilationError, FailedToConnectError
+            from dbt.exceptions import CompilationError
+
+            if self.dbt_version_tuple >= (1, 8):
+                from dbt.adapters.exceptions import FailedToConnectError
+            else:
+                from dbt.exceptions import FailedToConnectError
         except ImportError:
             # These are the historic names for older dbt-core versions
             from dbt.exceptions import CompilationException as CompilationError
@@ -592,6 +611,9 @@ def from_string(*args, **kwargs):
                         def render_func(in_str):
                             env.add_extension(SnapshotExtension)
                             template = env.from_string(in_str, globals=globals)
+                            if self.dbt_version_tuple >= (1, 8):
+                                # dbt 1.8 requires a context for rendering the template.
+                                return template.render(globals)
                             return template.render()
 
             return old_from_string(*args, **kwargs)
@@ -628,7 +650,10 @@ def render_func(in_str):
         try:
             # These are the names in dbt-core 1.4.1+
             # https://github.com/dbt-labs/dbt-core/pull/6539
-            from dbt.exceptions import UndefinedMacroError
+            if self.dbt_version_tuple >= (1, 8):
+                from dbt_common.exceptions import UndefinedMacroError
+            else:
+                from dbt.exceptions import UndefinedMacroError
         except ImportError:
             # These are the historic names for older dbt-core versions
             from dbt.exceptions import UndefinedMacroException as UndefinedMacroError
@@ -791,7 +816,16 @@ def connection(self):
             adapter = get_adapter(self.dbt_config)
             self.adapters[self.project_dir] = adapter
             adapter.acquire_connection("master")
-            adapter.set_relations_cache(self.dbt_manifest)
+            if self.dbt_version_tuple >= (1, 8):
+                # See notes from https://github.com/dbt-labs/dbt-adapters/discussions/87
+                # about the decoupling of the adapters from core.
+                from dbt.context.providers import generate_runtime_macro_context
+
+                adapter.set_macro_resolver(self.dbt_manifest)
+                adapter.set_macro_context_generator(generate_runtime_macro_context)
+                adapter.set_relations_cache(self.dbt_manifest.nodes.values())
+            else:
+                adapter.set_relations_cache(self.dbt_manifest)
 
         yield
         # :TRICKY: Once connected, we never disconnect. Making multiple

plugins/sqlfluff-templater-dbt/test/conftest.py

@@ -1,9 +1,15 @@
 """pytest fixtures."""
 
 import os
+import shutil
+import subprocess
+from pathlib import Path
 
 import pytest
 
+from sqlfluff.core import FluffConfig
+from sqlfluff_templater_dbt.templater import DbtTemplater
+
 
 @pytest.fixture(scope="session", autouse=True)
 def dbt_flags():
@@ -13,3 +19,69 @@ def dbt_flags():
     # We've seen occasional runtime errors from that code:
     # TypeError: cannot pickle '_thread.RLock' object
     os.environ["DBT_USE_EXPERIMENTAL_PARSER"] = "True"
+
+
+@pytest.fixture()
+def dbt_fluff_config(dbt_project_folder):
+    """Returns SQLFluff dbt configuration dictionary."""
+    return {
+        "core": {
+            "templater": "dbt",
+            "dialect": "postgres",
+        },
+        "templater": {
+            "dbt": {
+                "profiles_dir": f"{dbt_project_folder}/profiles_yml",
+                "project_dir": f"{dbt_project_folder}/dbt_project",
+            },
+        },
+    }
+
+
+@pytest.fixture()
+def project_dir(dbt_fluff_config):
+    """Returns the dbt project directory."""
+    return dbt_fluff_config["templater"]["dbt"]["project_dir"]
+
+
+@pytest.fixture()
+def profiles_dir(dbt_fluff_config):
+    """Returns the dbt project directory."""
+    return dbt_fluff_config["templater"]["dbt"]["profiles_dir"]
+
+
+@pytest.fixture()
+def dbt_templater():
+    """Returns an instance of the DbtTemplater."""
+    return FluffConfig(overrides={"dialect": "ansi"}).get_templater("dbt")
+
+
+@pytest.fixture(scope="session")
+def dbt_project_folder():
+    """Fixture for a temporary dbt project directory."""
+    src = Path("plugins/sqlfluff-templater-dbt/test/fixtures/dbt")
+    tmp = Path("plugins/sqlfluff-templater-dbt/test/temp_dbt_project")
+    tmp.mkdir(exist_ok=True)
+    shutil.copytree(src, tmp, dirs_exist_ok=True)
+    if DbtTemplater().dbt_version_tuple >= (1, 8):
+        # Configuration overrides for dbt 1.8+
+        dbt180_fixtures = src.with_name("dbt180")
+        shutil.copytree(dbt180_fixtures, tmp, dirs_exist_ok=True)
+
+    subprocess.Popen(
+        [
+            "dbt",
+            "deps",
+            "--project-dir",
+            f"{tmp}/dbt_project",
+            "--profiles-dir",
+            f"{tmp}/profiles_yml",
+        ]
+    ).wait(10)
+
+    # Placeholder value for testing
+    os.environ["passed_through_env"] = "_"
+
+    yield tmp
+
+    shutil.rmtree(tmp)

plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/vars_from_env.sql

@@ -0,0 +1 @@
+SELECT {{ env_var('passed_through_env') }}

plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/dbt_project.yml

@@ -0,0 +1,20 @@
+name: 'my_new_project'
+version: '1.0.0'
+config-version: 2
+
+profile: 'default'
+
+test-paths: ["tests"]
+
+models:
+  my_new_project:
+    materialized: view
+
+vars:
+  my_new_project:
+    # Default date stamp of run
+    ds: "2020-01-01"
+    # passed_through_cli: testing for vars passed through cli('--vars' option) rather than dbt_project
+
+flags:
+  send_anonymous_usage_stats: false

plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/profiles_yml/profiles.yml

@@ -0,0 +1,12 @@
+default:
+  target: dev
+  outputs:
+    dev:
+      type: postgres
+      host: "{{ env_var('POSTGRES_HOST', 'localhost') }}"
+      user: postgres
+      pass: password
+      port: 5432
+      dbname: postgres
+      schema: dbt_alice
+      threads: 4

plugins/sqlfluff-templater-dbt/test/fixtures/dbt180/profiles_yml_fail/profiles.yml

@@ -0,0 +1,12 @@
+default:
+  target: dev
+  outputs:
+    dev:
+      type: postgres
+      host: localhost
+      user: postgres
+      pass: password
+      port: 2345
+      dbname: postgres
+      schema: dbt_alice
+      threads: 4