From d825a5e27e5676b064d7ffbb8563e4fd7fb317fd Mon Sep 17 00:00:00 2001 From: boonhapus Date: Tue, 21 May 2024 21:32:45 -0500 Subject: [PATCH 1/5] allow VARCHAR expansion on .name fields --- cs_tools/cli/tools/searchable/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cs_tools/cli/tools/searchable/models.py b/cs_tools/cli/tools/searchable/models.py index ee86ab86..7dd9a25e 100644 --- a/cs_tools/cli/tools/searchable/models.py +++ b/cs_tools/cli/tools/searchable/models.py @@ -140,7 +140,7 @@ class MetadataObject(ValidatedSQLModel, table=True): cluster_guid: str = Field(primary_key=True) org_id: int = Field(primary_key=True) object_guid: str = Field(primary_key=True) - name: str + name: str = Field(sa_column=Column(Text, info={"length_override": "MAX"})) description: Optional[str] = Field(sa_column=Column(Text, info={"length_override": "MAX"})) author_guid: str created: dt.datetime @@ -218,7 +218,7 @@ class DependentObject(ValidatedSQLModel, table=True): cluster_guid: str = Field(primary_key=True) dependent_guid: str = Field(primary_key=True) column_guid: str = Field(primary_key=True) - name: str + name: str = Field(sa_column=Column(Text, info={"length_override": "MAX"})) description: Optional[str] = Field(sa_column=Column(Text, info={"length_override": "MAX"})) author_guid: str created: dt.datetime From a51f7376bc447a84c013a80fa70f0073a066ea29 Mon Sep 17 00:00:00 2001 From: boonhapus Date: Tue, 21 May 2024 22:19:42 -0500 Subject: [PATCH 2/5] fix for missing object data --- cs_tools/api/middlewares/logical_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cs_tools/api/middlewares/logical_table.py b/cs_tools/api/middlewares/logical_table.py index 8c963789..74eb6fd0 100644 --- a/cs_tools/api/middlewares/logical_table.py +++ b/cs_tools/api/middlewares/logical_table.py @@ -106,7 +106,7 @@ def all( # noqa: A003 info = self.ts.metadata.fetch_header_and_extras(metadata_type="DATA_SOURCE", guids=[connection_guid]) # type: ignore # BUGFIX: SCAL-199984 .. eta 10.0.0.cl - if info[0]["type"] == "Default": + if not info or info[0]["type"] == "Default": continue table["data_source"] = info[0]["header"] From 5f927e9207d22478e1a554e4037cb656a6a4dabb Mon Sep 17 00:00:00 2001 From: boonhapus Date: Tue, 21 May 2024 22:20:24 -0500 Subject: [PATCH 3/5] fix for [ex/in]cluding system generated content (#142) --- cs_tools/cli/tools/scriptability/_export.py | 7 +++++-- cs_tools/cli/tools/scriptability/app.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cs_tools/cli/tools/scriptability/_export.py b/cs_tools/cli/tools/scriptability/_export.py index 93879936..24ede901 100644 --- a/cs_tools/cli/tools/scriptability/_export.py +++ b/cs_tools/cli/tools/scriptability/_export.py @@ -14,7 +14,7 @@ from thoughtspot_tml.utils import determine_tml_type, disambiguate from cs_tools.cli.ux import rich_console -from cs_tools.errors import CSToolsError, CSToolsCLIError +from cs_tools.errors import CSToolsCLIError from cs_tools.types import GUID, TMLSupportedContent, TMLSupportedContentSubtype from .tmlfs import ExportTMLFS @@ -44,7 +44,9 @@ def is_success(self) -> bool: return self.status_code == "OK" -def export(ts, path, guids, tags, author, include_types, exclude_types, pattern, export_associated, org): +def export( + ts, path, guids, tags, author, include_types, exclude_types, exclude_system_content, pattern, export_associated, org +): if guids and (tags or author or include_types or exclude_types or pattern): raise CSToolsCLIError( title="GUID cannot be used with other filters.", @@ -110,6 +112,7 @@ def export(ts, path, guids, tags, author, include_types, exclude_types, pattern, pattern=pattern, include_types=include_types, include_subtypes=include_subtypes, + exclude_system_content=exclude_system_content, exclude_types=exclude_types, exclude_subtypes=exclude_subtypes, ) diff --git a/cs_tools/cli/tools/scriptability/app.py b/cs_tools/cli/tools/scriptability/app.py index 041da247..1f80a36b 100644 --- a/cs_tools/cli/tools/scriptability/app.py +++ b/cs_tools/cli/tools/scriptability/app.py @@ -232,6 +232,7 @@ def scriptability_export( "worksheet" ), ), + include_system_content: bool = typer.Option(False, help="include System User content in export"), export_associated: bool = typer.Option( False, "--export-associated", help="if specified, also export related content (does not export connections)" ), @@ -261,6 +262,7 @@ def scriptability_export( pattern=pattern, include_types=include_types, exclude_types=exclude_types, + exclude_system_content=not include_system_content, export_associated=export_associated, org=org_override, ) From 9579a6a72b71480cacdb921eca123474efd5587d Mon Sep 17 00:00:00 2001 From: boonhapus Date: Tue, 21 May 2024 22:20:41 -0500 Subject: [PATCH 4/5] fix the offline install generator --- cs_tools/cli/commands/self.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/cs_tools/cli/commands/self.py b/cs_tools/cli/commands/self.py index 402ca6c7..4b26cddb 100644 --- a/cs_tools/cli/commands/self.py +++ b/cs_tools/cli/commands/self.py @@ -14,6 +14,7 @@ from cs_tools.cli.types import Directory from cs_tools.cli.ux import CSToolsCommand, CSToolsGroup, rich_console from cs_tools.settings import _meta_config as meta +from cs_tools.sync import base from cs_tools.updater import cs_tools_venv from cs_tools.updater._bootstrapper import get_latest_cs_tools_release import rich @@ -140,10 +141,11 @@ def analytics(): def download( directory: pathlib.Path = typer.Option(help="location to download the python binaries to", click_type=Directory()), platform: str = typer.Option(help="tag describing the OS and CPU architecture of the target environment"), - python_version: str = typer.Option( + python_version: AwesomeVersion = typer.Option( metavar="X.Y", help="major and minor version of your python install", parser=AwesomeVersion ), beta: bool = typer.Option(False, "--beta", help="if included, download the latest pre-release binary"), + syncer_dialect: str = typer.Option(None, help="the name of the dialect to fetch dependencies for"), ): """ Generate an offline binary. @@ -156,14 +158,35 @@ def download( [b yellow]python -m sysconfig[/] """ + # DEV NOTE: @boonhapus, 2024/05/21 + # + # The idea behind the offline installer is to simulate `pip install` by downloading + # all the necessary packages to build our environment. + # requirements = directory.joinpath("requirements") + + log.info("Fetching latest CS Tools release from GitHub") release_info = get_latest_cs_tools_release(allow_beta=beta) release_tag = release_info["tag_name"] - venv = cs_tools_venv + if syncer_dialect is not None: + syncer_dir = utils.get_package_directory("cs_tools") / "sync" / syncer_dialect + + if not syncer_dir.exists(): + log.error(f"Syncer dialect {syncer_dialect} not found") + raise typer.Exit(1) + + log.info(f"Fetching {syncer_dialect} syncer dependencies..") + manifest = base.SyncerManifest.model_validate_json(syncer_dir.joinpath("MANIFEST.json").read_text()) + manifest.__ensure_pip_requirements__() # freeze our own environment, which has all the dependencies needed to build - frozen = {req for req in venv.pip("freeze", "--quiet").stdout.decode().split("\n") if "cs_tools" not in req} + log.info("Freezing existing virtual environment..") + frozen = { + r + for r in cs_tools_venv.pip("freeze", "--quiet", visible_output=False).stdout.decode().split("\n") + if "cs_tools" not in r + } # add in the latest release frozen.add(f"cs_tools @ https://github.com/thoughtspot/cs_tools/archive/{release_tag}.zip") @@ -183,7 +206,8 @@ def download( if "win" in platform: frozen.add("pyreadline3 == 3.4.1") # from cs_tools - venv.pip( + log.info("Downloading dependent packages..") + cs_tools_venv.pip( "download", *frozen, "--no-deps", # we shouldn't need transitive dependencies, since we've build all the dependencies above "--dest", requirements.as_posix(), @@ -199,6 +223,8 @@ def download( from cs_tools.updater import _bootstrapper, _updater zip_fp = directory.joinpath(f"cs-tools_{__version__}_{platform}_{python_version}") + + log.info(f"Preparing your download at {zip_fp}..") shutil.copy(_bootstrapper.__file__, requirements.joinpath("_bootstrapper.py")) shutil.copy(_updater.__file__, requirements.joinpath("_updater.py")) shutil.make_archive(zip_fp.as_posix(), "zip", requirements) From 20fd1878a39c06bc24b34fc677b441a1a4cfea3a Mon Sep 17 00:00:00 2001 From: boonhapus Date: Tue, 21 May 2024 22:21:25 -0500 Subject: [PATCH 5/5] bump version --> (1.5.7) --- cs_tools/__project__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cs_tools/__project__.py b/cs_tools/__project__.py index 8f653272..89e9b2f3 100644 --- a/cs_tools/__project__.py +++ b/cs_tools/__project__.py @@ -1,4 +1,4 @@ -__version__ = "1.5.6" +__version__ = "1.5.7" __docs__ = "https://thoughtspot.github.io/cs_tools/" __repo__ = "https://github.com/thoughtspot/cs_tools" __help__ = f"{__repo__}/discussions/"