Repository navigation
generate-homeval #40
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: generate-homeval | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| year: | |
| type: string | |
| description: > | |
| Assessment year: Use values and comps from the final model for this year. | |
| Leave blank if you are supplying a run ID instead | |
| required: false | |
| default: "" | |
| run_id: | |
| type: string | |
| description: > | |
| Run ID: The model run whose comps should be used. Leave blank if you are supplying | |
| an assessment year instead | |
| required: false | |
| default: "" | |
| pins: | |
| type: string | |
| description: > | |
| PINs: One or more space‑separated Cook County PINs. Leave blank to | |
| generate reports for all PINs in the County, or if you are supplying | |
| township codes | |
| required: false | |
| default: "" | |
| townships: | |
| type: string | |
| description: > | |
| Township codes: One or more space-separated township codes. Leave blank to | |
| generate reports for all PINs in the County, or if you are supplying PINs | |
| required: false | |
| default: "" | |
| environment: | |
| type: choice | |
| description: > | |
| Environment: Choose where to upload the reports | |
| options: | |
| - Development bucket | |
| - Production bucket | |
| default: Development bucket | |
| required: true | |
| deploy-assets: | |
| type: boolean | |
| description: > | |
| Deploy static assets like favicons and error pages in addition | |
| to reports | |
| required: false | |
| default: false | |
| hugo-build-by-nbhd: | |
| type: boolean | |
| description: > | |
| Build reports by neighborhood: Slower, but uses less memory | |
| required: false | |
| default: false | |
| eligible-only: | |
| type: boolean | |
| description: > | |
| Only generate reports for PINs that are eligible for reports (i.e. no error pages) | |
| required: false | |
| default: false | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| AWS_REGION: us-east-1 | |
| AWS_ATHENA_S3_STAGING_DIR: s3://ccao-athena-results-us-east-1/ | |
| HUGO_VERSION: "0.159.2" | |
| defaults: | |
| run: | |
| # Always use builtin bash shell to take advantage of automatic -eo pipefail. | |
| # See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#defaultsrunshell | |
| shell: bash | |
| jobs: | |
| ############################################################################### | |
| # Setup | |
| ############################################################################### | |
| setup: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| outputs: | |
| matrix: ${{ steps.resolve-metadata.outputs.matrix }} | |
| count: ${{ steps.resolve-metadata.outputs.count }} | |
| assessment-year: ${{ steps.resolve-metadata.outputs.assessment-year }} | |
| run-id: ${{ steps.resolve-metadata.outputs.run-id }} | |
| triad: ${{ steps.resolve-metadata.outputs.triad }} | |
| bucket: ${{ steps.resolve-aws.outputs.bucket }} | |
| distribution-id: ${{ steps.resolve-aws.outputs.distribution-id }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd | |
| - name: Mask sensitive information | |
| run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.1.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: scripts/generate_homeval/uv.lock | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 | |
| with: | |
| python-version-file: scripts/generate_homeval/.python-version | |
| - name: Install Python requirements | |
| run: uv sync --frozen | |
| working-directory: scripts/generate_homeval | |
| - name: Configure AWS credentials (OIDC) | |
| uses: aws-actions/configure-aws-credentials@v6.1.1 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }} | |
| aws-region: us-east-1 | |
| - name: Resolve model metadata | |
| id: resolve-metadata | |
| run: | | |
| # Parse PIN and township lists to array so we can safely pass them | |
| # into this script, which accepts multiple space-delimited values for | |
| # both arguments | |
| read -ra PINS <<< "${{ inputs.pins }}" | |
| read -ra TOWNSHIPS <<< "${{ inputs.townships }}" | |
| ELIG_FLAG="" | |
| if [ "${{ inputs.eligible-only }}" = "true" ]; then | |
| ELIG_FLAG="--eligible-only" | |
| fi | |
| uv run \ | |
| --frozen \ | |
| --project scripts/generate_homeval \ | |
| python3 scripts/generate_homeval/resolve_model_metadata.py \ | |
| --year "${{ inputs.year }}" \ | |
| --run-id "${{ inputs.run_id }}" \ | |
| --pin "${PINS[@]}" \ | |
| --township "${TOWNSHIPS[@]}" \ | |
| --write-github-output \ | |
| $ELIG_FLAG | |
| - name: Resolve AWS resources | |
| id: resolve-aws | |
| run: | | |
| if [[ "${{ inputs.environment }}" == 'Development bucket' ]]; then | |
| BUCKET="${{ vars.AWS_S3_BUCKET_PINVAL_STAGING }}" | |
| DISTRIBUTION_ID="${{ vars.AWS_CLOUDFRONT_DISTRIBUTION_ID_STAGING }}" | |
| else | |
| BUCKET="${{ vars.AWS_S3_BUCKET_PINVAL_PROD }}" | |
| DISTRIBUTION_ID="${{ vars.AWS_CLOUDFRONT_DISTRIBUTION_ID_PROD }}" | |
| fi | |
| BUCKET_STR="bucket=${BUCKET}" | |
| echo "$BUCKET_STR" | |
| echo "$BUCKET_STR" >> "$GITHUB_OUTPUT" | |
| DISTRIBUTION_ID_STR="distribution-id=${DISTRIBUTION_ID}" | |
| echo "$DISTRIBUTION_ID_STR" | |
| echo "$DISTRIBUTION_ID_STR" >> "$GITHUB_OUTPUT" | |
| ############################################################################### | |
| # Generate HomeVal reports for a triad or explicit PINs | |
| ############################################################################### | |
| generate-homeval: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{ fromJson(needs.setup.outputs.matrix) }} | |
| max-parallel: ${{ fromJson(needs.setup.outputs.count) }} | |
| env: | |
| TOWNSHIP: ${{ matrix.township }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd | |
| - name: Mask sensitive information | |
| run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.1.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: scripts/generate_homeval/uv.lock | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 | |
| with: | |
| python-version-file: scripts/generate_homeval/.python-version | |
| - name: Install Python requirements | |
| run: uv sync --frozen | |
| working-directory: scripts/generate_homeval | |
| - name: Configure AWS credentials (OIDC) | |
| uses: aws-actions/configure-aws-credentials@v6.1.1 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }} | |
| aws-region: us-east-1 | |
| role-duration-seconds: 7200 | |
| - name: Cache Hugo package | |
| id: cache-hugo | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae | |
| with: | |
| path: ${{ runner.temp }}/hugo.deb | |
| key: hugo-deb-${{ env.HUGO_VERSION }} | |
| - name: Download Hugo | |
| if: steps.cache-hugo.outputs.cache-hit != 'true' | |
| run: | | |
| wget -O ${{ runner.temp }}/hugo.deb \ | |
| https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb | |
| - name: Install Hugo | |
| run: sudo dpkg -i ${{ runner.temp }}/hugo.deb | |
| - name: Generate Hugo content files and build reports | |
| env: | |
| RUN_ID: ${{ needs.setup.outputs.run-id }} | |
| ENV_CODE: ${{ inputs.environment == 'Production bucket' && 'prod' || 'dev' }} | |
| run: | | |
| read -ra PINS <<< "${{ inputs.pins }}" | |
| NBHD_FLAG="" | |
| if [ "${{ inputs.hugo-build-by-nbhd }}" = "true" ]; then | |
| NBHD_FLAG="--build-by-neighborhood" | |
| fi | |
| ELIG_FLAG="" | |
| if [ "${{ inputs.eligible-only }}" = "true" ]; then | |
| ELIG_FLAG="--eligible-only" | |
| fi | |
| # Unquoted neighborhood flag is intentional in this call, since we | |
| # want this to expand to nothing (rather than an empty string) if | |
| # the variable is empty | |
| uv run \ | |
| --frozen \ | |
| --project scripts/generate_homeval \ | |
| python3 scripts/generate_homeval/generate_homeval.py \ | |
| --run-id "$RUN_ID" \ | |
| --township "$TOWNSHIP" \ | |
| --pin "${PINS[@]}" \ | |
| --environment "$ENV_CODE" \ | |
| $NBHD_FLAG \ | |
| $ELIG_FLAG | |
| - name: Sync reports to S3 | |
| env: | |
| TARGET_BUCKET: ${{ needs.setup.outputs.bucket }} | |
| YEAR: ${{ needs.setup.outputs.assessment-year }} | |
| run: | | |
| LOCAL="hugo/public" | |
| REMOTE="${TARGET_BUCKET%/}" | |
| if [[ "${{ inputs.deploy-assets }}" == 'true' ]]; then | |
| echo "Deploying static assets in addition to reports" | |
| else | |
| echo "Skipping static assets and deploying reports only" | |
| LOCAL="${LOCAL}/${YEAR}" | |
| REMOTE="${REMOTE}/${YEAR}" | |
| fi | |
| aws s3 sync "${LOCAL}/" "${REMOTE}/" --no-progress | |
| ############################################################################### | |
| # Invalidate cache and notify | |
| ############################################################################### | |
| notify: | |
| needs: | |
| - setup | |
| - generate-homeval | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| steps: | |
| - name: Configure AWS credentials (OIDC) | |
| uses: aws-actions/configure-aws-credentials@v6.1.1 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }} | |
| aws-region: us-east-1 | |
| - name: Mask AWS identifiers | |
| run: | | |
| echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" | |
| echo "::add-mask::${{ secrets.AWS_SNS_NOTIFICATION_TOPIC_ARN }}" | |
| - name: Invalidate cache | |
| env: | |
| YEAR: ${{ needs.setup.outputs.assessment-year }} | |
| TARGET_DISTRIBUTION_ID: ${{ needs.setup.outputs.distribution-id }} | |
| run: | | |
| # If the user passed in PINs, we want to only invalidate the cache | |
| # for those PINs, and preserve the cache for everything else to | |
| # improve performance for unrelated reports | |
| read -ra PINS <<< "${{ inputs.pins }}" | |
| INVALIDATION_PATHS=() | |
| if (( ${#PINS[@]} > 0 )); then | |
| for pin in "${PINS[@]:-}"; do | |
| INVALIDATION_PATHS+=("/${YEAR}/${pin}.html") | |
| done | |
| else | |
| # If the user didn't pass in PINs, we must have updated the whole | |
| # tri, so invalidate everything in this year | |
| INVALIDATION_PATHS+=("/${YEAR}/*.html") | |
| fi | |
| # If the user is deploying static assets, also invalidate those | |
| # assets | |
| if [[ "${{ inputs.deploy-assets }}" == 'true' ]]; then | |
| # This pattern will exclude robots.txt, since that file must live at | |
| # the root, but this is fine since we rarely if ever expect that file | |
| # to change | |
| INVALIDATION_PATHS+=("/static/*") | |
| fi | |
| aws cloudfront create-invalidation \ | |
| --distribution-id "${TARGET_DISTRIBUTION_ID}" \ | |
| --paths "${INVALIDATION_PATHS[@]}" | |
| - name: Publish SNS notification | |
| env: | |
| RUN_ID: ${{ needs.setup.outputs.run-id }} | |
| YEAR: ${{ needs.setup.outputs.assessment-year }} | |
| TRIAD: ${{ needs.setup.outputs.triad }} | |
| ENV_LABEL: ${{ inputs.environment }} | |
| BUCKET: ${{ needs.setup.outputs.bucket }} | |
| run: | | |
| read -ra PINS <<< "${{ inputs.pins }}" | |
| SUBJECT="HomeVal reports ready for run-id $RUN_ID" | |
| MESSAGE=$(cat <<EOF | |
| Your requested HomeVal reports have been generated. | |
| Assessment year: $YEAR | |
| Run-ID: $RUN_ID | |
| Triad: $TRIAD | |
| PIN list: ${PINS[@]} | |
| Environment: $ENV_LABEL | |
| You can browse or download the reports at the following S3 path: | |
| $BUCKET/$YEAR/ | |
| EOF | |
| ) | |
| aws sns publish \ | |
| --topic-arn "${{ secrets.AWS_SNS_NOTIFICATION_TOPIC_ARN }}" \ | |
| --subject "$SUBJECT" \ | |
| --message "$MESSAGE" |