Merge branch 'dev'

.github/workflows/fetch-metdata.yaml

@@ -38,4 +38,4 @@ jobs:
 
       # --config ENV:   tells CS Tools to pull the information from environment variables.
       - name: Refresh Metadata from ThoughtSpot
-        run: "cs_tools tools searchable metadata --syncer --syncer '${{ secrets.SYNCER_DECLARATIVE_TRUNCATE }}' --config ENV:"
+        run: "cs_tools tools searchable metadata --syncer '${{ secrets.SYNCER_DECLARATIVE_TRUNCATE }}' --config ENV:"

cs_tools/__project__.py

@@ -1,4 +1,4 @@
-__version__ = "1.5.5"
+__version__ = "1.5.6"
 __docs__ = "https://thoughtspot.github.io/cs_tools/"
 __repo__ = "https://github.com/thoughtspot/cs_tools"
 __help__ = f"{__repo__}/discussions/"

cs_tools/api/middlewares/metadata.py

@@ -4,6 +4,7 @@
 import logging
 
 from cs_tools import utils
+from cs_tools.api import _utils
 from cs_tools.errors import CSToolsError
 from cs_tools.types import (
     GUID,
@@ -131,11 +132,7 @@ def get(self, guids: list[GUID]) -> TableRowsFormat:
                 break
 
         if guids:
-            raise CSToolsError(
-                title=f"failed to find content for guids: {guids}",
-                reason="GUIDs not found in ThoughtSpot",
-                suggestion="check the GUIDs passed to the function and verify they exist.",
-            )
+            raise CSToolsError(f"failed to find content for guids: {guids}")
 
         return content
 
@@ -149,6 +146,7 @@ def find(
         include_subtypes: Optional[list[str]] = None,
         exclude_types: Optional[list[str]] = None,
         exclude_subtypes: Optional[list[str]] = None,
+        exclude_system_content: bool = True,
     ) -> TableRowsFormat:
         """
         Find all object which meet the predicates in the keyword args.
@@ -193,6 +191,9 @@ def find(
                 for header in data["headers"]:
                     subtype = header.get("type", None)
 
+                    if exclude_system_content and header.get("authorName") in _utils.SYSTEM_USERS:
+                        continue
+
                     # All subtypes will be retrieved, so need to filter the subtype appropriately.
                     # Mainly applies to LOGICAL_TABLE.
                     if include_subtypes and subtype and (subtype not in include_subtypes):
@@ -238,18 +239,28 @@ def fetch_header_and_extras(self, metadata_type: MetadataObjectType, guids: list
                 log.warning(f"Failed to fetch details for {guid} ({metadata_type})")
                 continue
 
-            d = r.json()["storables"][0]
+            j = r.json()
+
+            if not j["storables"]:
+                log.warning(f"Failed to fetch details for {guid} ({metadata_type})")
+                continue
+
+            d = j["storables"][0]
 
+            # fmt: off
             header_and_extras = {
                 "metadata_type": metadata_type,
                 "header": d["header"],
                 "type": d.get("type"),  # READ: .subtype  (eg. ONE_TO_ONE_LOGICAL, WORKSHEET, etc..)
+
                 # LOGICAL_TABLE extras
                 "dataSourceId": d.get("dataSourceId"),
                 "columns": d.get("columns"),
+
                 # VIZ extras (answer, liveboard)
                 "reportContent": d.get("reportContent"),
             }
+            # fmt: on
 
             self._details_cache[guid] = header_and_extras
             data.append(header_and_extras)

cs_tools/cli/commands/self.py

@@ -77,7 +77,7 @@ def update(
             raise typer.Exit(0)
 
     log.info("Upgrading CS Tools and its dependencies.")
-    cs_tools_venv.pip("install", requires, "--upgrade", "--upgrade-strategy", "eager")
+    cs_tools_venv.pip("install", requires, "--upgrade", "--upgrade-strategy", "eager", with_system_python=True)
 
 
 @app.command(cls=CSToolsCommand)

cs_tools/cli/tools/scriptability/_export.py

@@ -14,7 +14,7 @@
 from thoughtspot_tml.utils import determine_tml_type, disambiguate
 
 from cs_tools.cli.ux import rich_console
-from cs_tools.errors import CSToolsError
+from cs_tools.errors import CSToolsError, CSToolsCLIError
 from cs_tools.types import GUID, TMLSupportedContent, TMLSupportedContentSubtype
 
 from .tmlfs import ExportTMLFS
@@ -46,7 +46,7 @@ def is_success(self) -> bool:
 
 def export(ts, path, guids, tags, author, include_types, exclude_types, pattern, export_associated, org):
     if guids and (tags or author or include_types or exclude_types or pattern):
-        raise CSToolsError(
+        raise CSToolsCLIError(
             title="GUID cannot be used with other filters.",
             reason="You can only specify GUIDs or a combination of other filters, such as author and tag.",
             mitigation=(

cs_tools/cli/tools/scriptability/_import.py

@@ -22,7 +22,7 @@
 from cs_tools import utils
 from cs_tools.cli.tools.scriptability.util import GUIDMapping
 from cs_tools.cli.ux import rich_console
-from cs_tools.errors import CSToolsError
+from cs_tools.errors import CSToolsCLIError, CSToolsCLIError
 from cs_tools.thoughtspot import ThoughtSpot
 from cs_tools.types import (
     GUID,
@@ -176,21 +176,21 @@ def _check_parameters(
     """
     if (source or dest) and not (source and dest):
         error_msg = "source specified, but not destination" if source else "destination specified, but not source"
-        raise CSToolsError(
+        raise CSToolsCLIError(
             title=error_msg,
             reason="Source and destination must both be specified.",
             mitigation="Specify both source and destination when using mapping.",
         )
 
     path_mitigation = "The path must exist and be a valid TML file system."
     if not path.exists():
-        raise CSToolsError(title=f"Path {path} does not exist.", mitigation=path_mitigation)
+        raise CSToolsCLIError(title=f"Path {path} does not exist.", mitigation=path_mitigation)
 
     if not (path / ".tmlfs"):
-        raise CSToolsError(title=f"Path {path} does not appear to be a TML file system.", mitigation=path_mitigation)
+        raise CSToolsCLIError(title=f"Path {path} does not appear to be a TML file system.", mitigation=path_mitigation)
 
     if guid and (include_types or exclude_types):
-        raise CSToolsError(
+        raise CSToolsCLIError(
             title="A guid and include/exclude specified.",
             reason="Cannot specify both a file and include/exclude types.",
             mitigation="Specify either a file or include/exclude types, but not both.",
@@ -307,7 +307,7 @@ def _verify_connection_passwords(connection_tml: [TML]) -> None:
             if p.key == "password" and p.value:
                 break
         else:
-            raise CSToolsError(
+            raise CSToolsCLIError(
                 title=f'Connection "{tml.connection.name}" missing password',
                 reason="Connections require a valid password to create tables.",
                 mitigation="Add a password to the connection file and try again.",
@@ -494,13 +494,13 @@ def _get_connection(ts: ThoughtSpot, connection_guid: GUID) -> Optional[Connecti
         if e.response.status_code == 404:  # the API will return a 400 if the connection doesn't exist.
             return None
         else:
-            raise CSToolsError(
+            raise CSToolsCLIError(
                 title=f"Unknown error checking for connection {connection_guid}: {e}",
                 reason=str(e.response.content),
                 mitigation="Verify TS connection and GUID",
             ) from None
     except TMLDecodeError as e:
-        raise CSToolsError(
+        raise CSToolsCLIError(
             title=f"Error decoding connection {connection_guid}: {e}", mitigation="Verify TS connection and GUID"
         ) from None
 

cs_tools/cli/tools/scriptability/tmlfs.py

@@ -18,7 +18,7 @@
 from cs_tools.api import _utils
 from cs_tools.cli.tools.scriptability.util import EnvName, GUIDMapping
 from cs_tools.cli.ux import rich_console
-from cs_tools.errors import CSToolsError
+from cs_tools.errors import CSToolsCLIError
 from cs_tools.types import GUID
 
 TMLFS_FILENAME = ".tmlfs"
@@ -46,7 +46,7 @@ def __init__(self, path: pathlib.Path, log_for: str, logger: logging.Logger):
         :param log_for: The type of log file to create.  Must be "export" or "import".
         """
         if log_for not in ["export", "import"]:
-            raise CSToolsError(
+            raise CSToolsCLIError(
                 title=f"Invalid log_for value: {log_for}.  Must be 'export' or 'import'.",
                 mitigation="Specify a valid log_for value.",
             )
@@ -89,11 +89,12 @@ def create_tml_file_system(cls, path: pathlib.Path) -> None:
         """
         if path.exists():
             if not path.is_dir():
-                raise CSToolsError(title=f"TML Path {path} exists but is not a directory.")
+                raise CSToolsCLIError(title=f"TML Path {path} exists but is not a directory.")
             else:
                 fsfile = path / TMLFS_FILENAME
                 if not fsfile.exists():
                     rich_console.log(f"Creating TML File System at {path}")
+                    fsfile.touch()
 
         # now create the sub-folders.  If they already exist, then the creation will be ignored.
         (path / "logs").mkdir(exist_ok=True)
@@ -110,7 +111,7 @@ def _create_log_path(self, log_for: str) -> pathlib.Path:
 
         # make sure the path already exists.
         if not self.path.exists():
-            raise CSToolsError(title=f"Path {self.path} does not exist.", mitigation="Specify a valid path to the FS.")
+            raise CSToolsCLIError(title=f"Path {self.path} does not exist.", mitigation="Specify a valid path to the FS.")
 
         # create the log folder and file.
         log_name = f"{log_for}-{self.start_time.strftime('%Y.%m.%d-%H.%M.%S')}"
@@ -167,7 +168,7 @@ def write_tml(self, tml: TML) -> None:
             directory = self.path / tml_type
             tml.dump(directory / f"{tml.guid}.{tml_type}.tml")
         except Exception as e:
-            raise CSToolsError(
+            raise CSToolsCLIError(
                 title=f"Error writing TML {tml.name} to {directory}: {e}",
                 mitigation="Check write permissions in the file system..",
             ) from None
@@ -216,7 +217,7 @@ def load_tml_for_guid(self, guid: GUID) -> TML:
                     tml_cls = determine_tml_type(path=f)
                     return tml_cls.load(f)
 
-        raise CSToolsError(title=f"Could not find TML with GUID {guid}.", mitigation="Check the GUID and try again.")
+        raise CSToolsCLIError(title=f"Could not find TML with GUID {guid}.", mitigation="Check the GUID and try again.")
 
 
 app = typer.Typer(
@@ -251,14 +252,14 @@ def scriptability_fs_cleanup(
     Performs cleanup of the TML file system.  It removes log files older than the number of days specified.
     """
     if not directory.exists():
-        raise CSToolsError(
+        raise CSToolsCLIError(
             title=f"{directory} does not exist.",
             reason="An invalid directory was provided.",
             mitigation="Verify the the directory exists.",
         )
 
     if not is_tmlfs(directory):
-        raise CSToolsError(
+        raise CSToolsCLIError(
             title=f"{directory} is not a TML file system.",
             reason="Only TML file systems can be cleaned up.",
             mitigation="Verify the the directory is the root of a TML file system.",

cs_tools/cli/tools/searchable/static/TS_METADATA_COLUMN.table.tml

@@ -135,7 +135,7 @@ table:
     properties:
       column_type: ATTRIBUTE
     db_column_properties:
-      data_type: VARCHAR
+      data_type: BOOL
   - name: Default Calendar
     description: The default calendar to align to when using date keywords.
     db_column_name: CALENDAR_TYPE

cs_tools/cli/tools/user-management/app.py

@@ -13,7 +13,7 @@
 from cs_tools.cli.layout import LiveTasks
 from cs_tools.cli.types import MetadataType, MultipleChoiceType, SyncerProtocolType
 from cs_tools.cli.ux import CSToolsApp, rich_console
-from cs_tools.errors import CSToolsError
+from cs_tools.errors import CSToolsError, CSToolsCLIError
 
 from . import layout, models, work
 
@@ -90,7 +90,7 @@ def transfer(
                 ts.api.v1.user_transfer_ownership(from_username=from_username, to_username=to_username, **extra)
             except httpx.HTTPStatusError as e:
                 log.debug(e, exc_info=True)
-                raise CSToolsError(
+                raise CSToolsCLIError(
                     title=f"Could not transfer ownership from {from_username} to {to_username}",
                     reason="See logs for additional details..",
                 ) from None

cs_tools/sync/bigquery/MANIFEST.json

@@ -1,5 +1,7 @@
 {
     "name": "bigquery",
     "syncer_class": "BigQuery",
-    "requirements": ["sqlalchemy-bigquery>=1.4.1"]
+    "requirements": [
+        "sqlalchemy-bigquery[bqstorage] >= 1.11.0"
+    ]
 }

cs_tools/sync/bigquery/sanitize.py

@@ -4,13 +4,11 @@
 import datetime as dt
 import json
 
-from .const import BIG_QUERY_DATETIME_FORMAT
-
 
 def clean_datetime(o: Any) -> str:
-    """ """
+    """Convert the datatime into a string."""
     if isinstance(o, dt.datetime):
-        return o.strftime(BIG_QUERY_DATETIME_FORMAT)
+        return o.strftime("%Y-%m-%dT%H:%M:%S.%f")
     return o