Extract TML with CS Tools. #7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: | |
| Extract TML with CS Tools. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| cs_tools_version: | |
| description: "The CS Tools version to target for a manual run." | |
| required: false | |
| type: string | |
| schedule: | |
| # Runs every day at 5:20 AM UTC | |
| - cron: "20 5 * * *" | |
| jobs: | |
| extract_data_from_thoughtspot: | |
| runs-on: ubuntu-latest | |
| env: | |
| # CS TOOLS IS COMMAND LINE LIBRARY WRAPPING TS APIS | |
| # https://thoughtspot.github.io/cs_tools/ | |
| CS_TOOLS_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.cs_tools_version || 'v1.6.0' }} | |
| CS_TOOLS_THOUGHTSPOT__URL: ${{ secrets.THOUGHTSPOT_URL }} | |
| CS_TOOLS_THOUGHTSPOT__USERNAME: ${{ secrets.THOUGHTSPOT_USERNAME }} | |
| CS_TOOLS_THOUGHTSPOT__SECRET_KEY: ${{ secrets.THOUGHTSPOT_SECRET_KEY }} | |
| # COMMON PARAMETERS FOR THE SNOWFLAKE SYNCER | |
| # https://thoughtspot.github.io/cs_tools/syncer/snowflake/ | |
| DECLARATIVE_SYNCER_SYNTAX: | |
| "\ | |
| account_name=${{ secrets.SNOWFLAKE_ACCOUNT }}\ | |
| &username=${{ secrets.SNOWFLAKE_USERNAME }}\ | |
| &secret=${{ secrets.SNOWFLAKE_PASSWORD }}\ | |
| &warehouse=${{ secrets.SNOWFLAKE_WAREHOUSE }}\ | |
| &role=${{ secrets.SNOWFLAKE_ROLE }}\ | |
| &database=${{ secrets.SNOWFLAKE_DATABASE }}\ | |
| &schema=${{ secrets.SNOWFLAKE_SCHEMA }}\ | |
| &authentication=basic\ | |
| " | |
| steps: | |
| # SETUP PYTHON. | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| # UPDATE PIP. | |
| - name: Ensure pip is up to date. | |
| run: python -m pip install --upgrade pip | |
| # INSTALL A SPECIFIC VERSION OF cs_tools. | |
| - name: Install a pinned version of CS Tools | |
| run: python -m pip install "cs_tools[cli] @ https://github.com/thoughtspot/cs_tools/archive/${{ env.CS_TOOLS_VERSION }}.zip" | |
| # ENSURE SYNCER DEPENDENCIES ARE INSTALLED. | |
| # found in: https://github.com/thoughtspot/cs_tools/blob/master/sync/<dialect>/MANIFEST.json | |
| - name: Install a pinned version of CS Tools | |
| run: python -m pip install "snowflake-sqlalchemy >= 1.6.1" | |
| # RUNS THE searchable tml COMMAND. | |
| # https://thoughtspot.github.io/cs_tools/tools/searchable | |
| # | |
| # THE CLI OPTION --config ENV: TELLS CS TOOLS TO PULL THE INFORMATION FROM ENVIRONMENT VARIABLES. | |
| - name: Snapshot TML from ThoughtSpot | |
| run: >- | |
| cs_tools tools | |
| searchable tml | |
| --metadata-type "MODEL,LIVEBOARD" | |
| --strategy DELTA | |
| --tml-format JSON | |
| --syncer 'snowflake://${{ env.DECLARATIVE_SYNCER_SYNTAX }}&load_strategy=UPSERT' | |
| --config ENV: |