Skip to content

Enhance scan script, split execute to smaller functions. #1978

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Enhance scan script, split execute to smaller functions. #1978

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6d69e94
Enhance scan
duynguyenhoang Dec 15, 2023
d83a1b2
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 30 additions & 4 deletions soda/core/soda/scan.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def build_scan_results(self) -> dict:
"hasErrors": self.has_error_logs(),
"hasWarnings": self.has_check_warns(),
"hasFailures": self.has_check_fails(),
"metrics": [metric.get_dict() for metric in self._metrics],
"metrics": self.get_computed_metrics_val(),
# If archetype is not None, it means that check is automated monitoring
"checks": checks,
"queries": query_list,
Expand Down Expand Up @@ -398,9 +398,13 @@ def disable_telemetry(self):
"""
self._configuration.telemetry = None

def execute(self) -> int:
self._logs.debug("Scan execution starts")
exit_value = 0
def __create_checks(self):
"""
Create checks from configurations
"""
# Reset checks
self._checks = []
self._logs.debug("Create checks from configurations")
try:
from soda.execution.column import Column
from soda.execution.metric.column_metrics import ColumnMetrics
Expand Down Expand Up @@ -470,7 +474,16 @@ def execute(self) -> int:
partition,
column_metrics.column,
)
except Exception as e:
self._logs.error("Error occurred while building checks.", exception=e)

def compute_metrics_val(self):
"""
Calculate metrics from checks
"""
self._logs.debug("Calculate metrics value starts")
try:
self.__create_checks()
# Handle check attributes before proceeding.
invalid_check_attributes = None
invalid_checks = []
Expand Down Expand Up @@ -517,7 +530,17 @@ def execute(self) -> int:
# are associated with the derived metric as well.
for metric_dep in metric.derived_formula.metric_dependencies.values():
metric.queries += metric_dep.queries
except Exception as e:
self._logs.error("Error occurred while calculating metrics.", exception=e)

return invalid_checks

def execute(self) -> int:
self._logs.debug("Scan execution starts")
exit_value = 0
try:
invalid_checks = self.compute_metrics_val()
if not invalid_checks:
# Run profiling, data samples, automated monitoring, sample tables
try:
self.run_data_source_scan()
Expand Down Expand Up @@ -856,6 +879,9 @@ def __log_check_group(self, checks, indent, check_outcome, outcome_text):
for diagnostic in check.get_log_diagnostic_lines():
self._logs.info(f"{indent} {diagnostic}")

def get_computed_metrics_val(self) -> list[dict]:
return [metric.get_dict() for metric in self._metrics]

def get_variable(self, variable_name: str, default_value: str | None = None) -> str | None:
# Note: ordering here must be the same as in Jinja.OsContext.resolve_or_missing: First env vars, then scan vars
if variable_name in os.environ:
Expand Down