diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97a8769..7611261 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,8 +122,6 @@ jobs: - name: Run presubmit script run: | python run_presubmit.py \ - --xlsynth-tools $(pwd)/dependencies \ - --xlsynth-driver-dir $(dirname "$(which xlsynth-driver)") \ --dslx-path dependencies/dslx_stdlib dslx_ci_py36: @@ -264,6 +262,4 @@ jobs: run: | source $HOME/.cargo/env python3.6 run_presubmit.py \ - --xlsynth-tools $(pwd)/dependencies \ - --xlsynth-driver-dir $(dirname "$(which xlsynth-driver)") \ --dslx-path dependencies/dslx_stdlib diff --git a/BUILD.bazel b/BUILD.bazel index b5c9047..03c2b29 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,3 +1,5 @@ +load("@rules_python//python:defs.bzl", "py_test") + py_test( name = "make_env_helpers_test", srcs = ["make_env_helpers_test.py"], @@ -7,3 +9,94 @@ py_test( "make_env_helpers.py", ], ) + +py_test( + name = "env_helpers_test", + srcs = [ + "env_helpers.py", + "env_helpers_test.py", + ], +) + +py_test( + name = "artifact_resolution_test", + srcs = [ + "artifact_resolution_test.py", + "materialize_xls_bundle.py", + ], +) + +py_test( + name = "download_release_test", + srcs = [ + "download_release.py", + "download_release_test.py", + ], +) + +genrule( + name = "stdlib_location_probe", + srcs = ["@rules_xlsynth_selftest_xls//:dslx_stdlib"], + outs = ["stdlib_location.txt"], + cmd = "printf '%s\\n' '$(location @rules_xlsynth_selftest_xls//:dslx_stdlib)' > $@", +) + +genrule( + name = "bundle_alias_probe", + srcs = ["@rules_xlsynth_selftest_xls"], + outs = ["bundle_alias_locations.txt"], + cmd = "printf '%s\\n' '$(locations @rules_xlsynth_selftest_xls//:rules_xlsynth_selftest_xls)' > $@", +) + +genrule( + name = "xlsynth_sys_runtime_probe", + srcs = ["@rules_xlsynth_selftest_xls//:xlsynth_sys_runtime_files"], + outs = ["xlsynth_sys_runtime_locations.txt"], + cmd = "printf '%s\\n' '$(locations @rules_xlsynth_selftest_xls//:xlsynth_sys_runtime_files)' > $@", +) + +genrule( + name = "xlsynth_sys_dep_probe", + srcs = ["@rules_xlsynth_selftest_xls//:xlsynth_sys_dep"], + outs = ["xlsynth_sys_dep_locations.txt"], + cmd = "printf '%s\\n' '$(locations @rules_xlsynth_selftest_xls//:xlsynth_sys_dep)' > $@", +) + +genrule( + name = "xlsynth_sys_artifact_config_probe", + srcs = ["@rules_xlsynth_selftest_xls//:xlsynth_sys_artifact_config"], + outs = ["xlsynth_sys_artifact_config_location.txt"], + cmd = "printf '%s\\n' '$(location @rules_xlsynth_selftest_xls//:xlsynth_sys_artifact_config)' > $@", +) + +genrule( + name = "xlsynth_sys_legacy_inputs_probe", + srcs = [ + "@rules_xlsynth_selftest_xls//:xlsynth_sys_legacy_dso", + "@rules_xlsynth_selftest_xls//:xlsynth_sys_legacy_stdlib", + ], + outs = ["xlsynth_sys_legacy_input_locations.txt"], + cmd = "printf '%s\\n' '$(locations @rules_xlsynth_selftest_xls//:xlsynth_sys_legacy_dso)' '$(locations @rules_xlsynth_selftest_xls//:xlsynth_sys_legacy_stdlib)' > $@", +) + +py_test( + name = "external_bundle_exports_test", + srcs = ["external_bundle_exports_test.py"], + data = [ + ":bundle_alias_probe", + ":stdlib_location_probe", + ":xlsynth_sys_artifact_config_probe", + ":xlsynth_sys_dep_probe", + ":xlsynth_sys_legacy_inputs_probe", + ":xlsynth_sys_runtime_probe", + "@rules_xlsynth_selftest_xls", + "@rules_xlsynth_selftest_xls//:dslx_stdlib", + "@rules_xlsynth_selftest_xls//:xlsynth_sys_artifact_config", + ], + deps = ["@rules_python//python/runfiles"], +) + +toolchain_type( + name = "toolchain_type", + visibility = ["//visibility:public"], +) diff --git a/DESIGN.md b/DESIGN.md index 0a87eac..5167838 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,38 +1,101 @@ -# Environment Helpers - -`env_helpers.py` hosts the Python entry point that Bazel actions use to talk to the xlsynth -toolchain. When the program starts it harvests a handful of `XLSYNTH_*` variables from the -action environment. The values drive two behaviours: building a temporary `toolchain.toml` file -for driver invocations and deriving additional command-line flags for individual tools. Regular -strings such as `XLSYNTH_GATE_FORMAT` become TOML string fields, while boolean variables like -`XLSYNTH_USE_SYSTEM_VERILOG` are validated and converted into `true` or `false`. For dslx tools -the helper injects the path to the bundled standard library and threads through optional warning -and type-inference settings. The module exposes two subcommands. `driver` shells out to the -`xlsynth-driver` binary, passing the generated TOML file and forwarding any extra user -arguments. `tool` directly executes a requested binary from the downloaded toolset after -pre-pending any extra flags determined from the environment. - -# Generating the Bazel Helper - -`make_env_helpers.py` keeps the Starlark side of the repository in sync with the Python runner. -The script reads `env_helpers.py`, wraps the source in a Starlark function called -`python_runner_source`, and writes the result to `env_helpers.bzl`. Embedding the literal Python -string this way lets each Bazel action materialise the runner directly as a declared tool rather -than depending on a separate source file target (we don't want our Python runner becoming a -source file dependency of all the rules written by users), which gives hermeticity guarantees -about the helper version used inside the sandbox. The docstring embedded into that function -documents the runner’s responsibilities so that Bazel authors do not need to open the Python -implementation to understand it. A unit test asserts that the generated file matches the -checked-in version, so running `python make_env_helpers.py` is the required regeneration step -when the runner changes. - -# Bazel Integration Path - -Many Starlark rules load `python_runner_source` and materialise the runner inside the action -sandbox. Each rule writes the helper script to a temporary output, adds it to the action’s tool -inputs, and then calls it with either the `driver` or `tool` subcommand depending on the -workflow. For example, `dslx_to_ir.bzl` composes the runner with `driver dslx2ir` to build -intermediate representations, then calls `driver ir2opt` for optimisation passes. Because every -action invokes the same runner binary, rule authors can rely on environment variables (for -custom DSLX search paths, enabling warnings, or toggling SystemVerilog emission) to behave -consistently across all tool stages. +# Workspace toolchain design + +`rules_xlsynth` now exposes one public XLS artifact-selection surface: the +`xls` module extension. A Bazel workspace chooses one or more named bundles in +`MODULE.bazel`, publishes them with `use_repo(...)`, and registers one default +bundle with `register_toolchains("@//:all")`. Public artifact selection +no longer lives in `.bazelrc` `@rules_xlsynth//config:{driver_path,tools_path,runtime_library_path,dslx_stdlib_path}` +flags. + +## Bundle repos and exported targets + +Each `xls.toolchain(...)` call materializes a repo that contains the selected +tool binaries, the DSLX stdlib tree, the matching `xlsynth-driver`, and the +matching `libxls` shared library. That repo exports: + +- `@//:all` +- `@//:bundle` +- `@//:libxls` +- `@//:libxls_link` +- `@//:dslx_stdlib` +- `@//:xlsynth_sys_artifact_config` +- `@//:xlsynth_sys_legacy_stdlib` +- `@//:xlsynth_sys_legacy_dso` +- `@//:xlsynth_sys_dep` +- `@//:xlsynth_sys_runtime_files` +- `@//:xlsynth_sys_link_dep` + +The `xlsynth_sys_*` exports are the intended downstream contract for +`rules_rust` `crate_extension.annotation(...)` wiring. The preferred modern +shape is `build_script_data` / `build_script_env` for the build-script +contract, plus `deps = ["@//:xlsynth_sys_dep"]` for the combined +runtime-plus-link contract. The compatibility exports +`xlsynth_sys_runtime_files` and `xlsynth_sys_link_dep` remain available for +callers that still spell those phases separately. This lets root `MODULE.bazel` +files choose only a bundle and a build-script mode instead of coupling to +generic bundle internals. + +`artifact_source` controls how those artifacts are resolved: + +- `auto` probes a consumer-owned installed layout and otherwise downloads the + release artifacts. +- `installed_only` requires the matching installed layout. +- `download_only` always downloads the release artifacts. +- `local_paths` uses explicit local paths and is the documented escape hatch + for `/tmp/xls-local-dev/` style setups. + +For the installed-layout modes, the provider derives the concrete paths from +the toolchain declaration instead of hard-coding a repository-global install +root: `/v` for the tools tree and +`//bin/xlsynth-driver` +for the driver binary. The provider owns the version-derived suffixes; the +consumer workspace owns the root prefixes. + +## Default bundles and explicit overrides + +Most rules use the registered default workspace bundle through normal Bazel +toolchain resolution. Supported DSLX rules can opt into a named bundle with +`xls_bundle = "@//:bundle"`. That override changes only the artifact +bundle. The existing behavior settings - for example `dslx_path`, warnings, +`gate_format`, `assert_format`, `use_system_verilog`, and +`add_invariant_assertions` - still come from the registered toolchain. + +## Runner and toolchain TOML + +`env_helpers.py` hosts the Python entry point that Bazel actions use to talk to +the xlsynth toolchain. Bazel rules materialize a per-action +`xlsynth-toolchain.toml` file from the selected bundle plus any rule-level +behavior overrides, then pass that declared input to the runner. The runner +exposes two subcommands: `driver` shells out to the configured +`xlsynth-driver` binary with `--toolchain=`, while `tool` reads the same +TOML file and derives the extra DSLX flags needed by direct tool invocations +such as `dslx_interpreter_main` or `typecheck_main`. + +The helper uses the selected `libxls` file path directly and derives the +runtime library directory from `dirname(libxls_path)`, so users no longer need +to configure a separate runtime-library path. The old artifact-path build +settings are deleted; artifact selection is bundle-only. + +## Generating the Bazel helper + +`make_env_helpers.py` keeps the Starlark side of the repository in sync with +the Python runner. The script reads `env_helpers.py`, wraps the source in a +Starlark function called `python_runner_source`, and writes the result to +`env_helpers.bzl`. Embedding the literal Python string this way lets each Bazel +action materialize the runner directly as a declared tool rather than depending +on a separate source file target, which gives hermeticity guarantees about the +helper version used inside the sandbox. A unit test asserts that the generated +file matches the checked-in version, so running `python make_env_helpers.py` is +the required regeneration step when the runner changes. + +## Bazel integration path + +Many Starlark rules load `python_runner_source` and materialize the runner +inside the action sandbox. Each rule writes the helper script to a temporary +output, writes a declared TOML file for the configured toolchain, and then +calls the helper with either the `driver` or `tool` subcommand depending on the +workflow. For example, `dslx_to_ir.bzl` composes the runner with +`driver dslx2ir` to build intermediate representations, then calls +`driver ir2opt` for optimization passes. Artifact selection now comes from the +module-extension bundle instead of `XLSYNTH_*` action environment variables or +artifact-path build settings. diff --git a/MODULE.bazel b/MODULE.bazel index 3f1bd02..69ad728 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,3 +1,20 @@ module(name = "rules_xlsynth") bazel_dep(name = "bazel_skylib", version = "1.6.1") +bazel_dep(name = "rules_cc", version = "0.2.11") +bazel_dep(name = "rules_python", version = "1.0.0") + +xls = use_extension("//:extensions.bzl", "xls", dev_dependency = True) + +xls.toolchain( + name = "rules_xlsynth_selftest_xls", + xls_version = "0.39.0", + xlsynth_driver_version = "0.36.0", + artifact_source = "download_only", +) + +use_repo(xls, "rules_xlsynth_selftest_xls") +register_toolchains( + "@rules_xlsynth_selftest_xls//:all", + dev_dependency = True, +) diff --git a/README.md b/README.md index b5ef83d..b9615ab 100644 --- a/README.md +++ b/README.md @@ -2,38 +2,130 @@ [![CI](https://github.com/xlsynth/rules_xlsynth/actions/workflows/ci.yml/badge.svg)](https://github.com/xlsynth/rules_xlsynth/actions/workflows/ci.yml) -### `.bazelrc`-settable configuration - -These environment variables can act as a repository-level configuration for the `rules_xlsynth` rules: - -- `XLSYNTH_DRIVER_DIR`: the path to the `xlsynth-driver` directory, i.e. containing the - `xlsynth-driver` binary (which can be installed via `cargo` from its Rust crate). Note that this - is named with a `_DIR` suffix because that differentiates it from a direct path to the binary. -- `XLSYNTH_TOOLS`: the path to the xlsynth tool directory, i.e. containing tools from releases - such as `dslx_interpreter_main`, `ir_converter_main`, `codegen_main`, etc. (This can be used - by the `xlsynth-driver` program instead of it directly calling `libxls` runtime APIs.) -- `XLSYNTH_DSLX_STDLIB_PATH`: the path to the DSLX stdlib to use. (Note that this refers to a - directory that holds all the standard library files.) -- `XLSYNTH_DSLX_PATH`: a colon-separated list of additional DSLX paths to search for imported files. -- `XLSYNTH_DSLX_ENABLE_WARNINGS`: a comma-separated list of warnings to enable. -- `XLSYNTH_DSLX_DISABLE_WARNINGS`: a comma-separated list of warnings to disable. -- `XLSYNTH_GATE_FORMAT`: format string used when emitting `gate!()` operations; `{input}` and `{output}` placeholders will be substituted with signal names. -- `XLSYNTH_ASSERT_FORMAT`: format string used when emitting `assert!()` operations; `{condition}` and `{label}` placeholders will be substituted with the assertion expression and label. -- `XLSYNTH_USE_SYSTEM_VERILOG`: `true|false`; when `true`, SystemVerilog constructs are emitted instead of plain Verilog. -- `XLSYNTH_ADD_INVARIANT_ASSERTIONS`: `true|false`; when `true`, extra runtime assertions (e.g. one-hot validation checks) are inserted in generated RTL. - -These can be set in your `.bazelrc` file like this: +### Workspace toolchains +`rules_xlsynth` now selects XLS artifacts from `MODULE.bazel` through the `xls` +module extension. A workspace instantiates one or more named bundles with +`xls.toolchain(...)`, exposes them with `use_repo(...)`, and registers one +default bundle with `register_toolchains("@//:all")`. + +```starlark +bazel_dep(name = "rules_xlsynth", version = "") + +xls = use_extension("@rules_xlsynth//:extensions.bzl", "xls") + +xls.toolchain( + name = "workspace_xls", + xls_version = "0.39.0", + xlsynth_driver_version = "0.36.0", + artifact_source = "auto", + installed_tools_root_prefix = "/opt/xlsynth", + installed_driver_root_prefix = "/opt/xlsynth-driver", +) + +xls.toolchain( + name = "legacy_xls", + xls_version = "0.37.0", + xlsynth_driver_version = "0.32.0", + artifact_source = "download_only", +) + +use_repo(xls, "workspace_xls", "legacy_xls") +register_toolchains("@workspace_xls//:all") ``` -... -build --action_env XLSYNTH_DSLX_PATH="path/to/additional/dslx/files:another/path" -build --action_env XLSYNTH_DSLX_ENABLE_WARNINGS="warning1,warning2" -build --action_env XLSYNTH_DSLX_DISABLE_WARNINGS="warning3,warning4" + +`artifact_source` chooses how each bundle repo is materialized: + +- `auto` probes the consumer-owned installed layout and otherwise downloads the + release artifacts. +- `installed_only` requires the matching installed layout. +- `download_only` always downloads the release artifacts. +- `local_paths` uses `local_tools_path`, `local_dslx_stdlib_path`, + `local_driver_path`, and `local_libxls_path`. + +For the installed-layout modes, `rules_xlsynth` derives exact-version paths as: + +- `/v` for the tools tree +- `/v/xls/dslx/stdlib` for the DSLX + stdlib +- `/v/libxls.{so,dylib}` for `libxls` +- `//bin/xlsynth-driver` + for the driver binary + +The attributes accepted by each mode are strict: + +- `local_paths` requires all four `local_*` attrs and does not accept + `xls_version` or `xlsynth_driver_version`. +- `auto` and `installed_only` require `xls_version`, + `xlsynth_driver_version`, `installed_tools_root_prefix`, and + `installed_driver_root_prefix`, and do not accept any `local_*` attrs. +- `download_only` requires `xls_version` and `xlsynth_driver_version`, and + does not accept any `local_*` or `installed_*` attrs. + +Download-backed modes also have one host prerequisite: when `auto` falls back +to downloading, or when `download_only` is selected, the repository rule +installs `xlsynth-driver` with `rustup run nightly cargo install`. The host +running module resolution must have `rustup` available. If the nightly +toolchain is missing, `rules_xlsynth` bootstraps a repo-local `rustup` home +before installing the driver. + +Each `xls.toolchain(...)` call exports a small repo surface: + +- `@//:all` for `register_toolchains(...)` +- `@//:bundle` for explicit `xls_bundle` overrides +- `@//:libxls` and `@//:libxls_link` for native consumers +- `@//:dslx_stdlib` for packages that need the standard library tree +- `@//:xlsynth_sys_artifact_config` for the modern single-file + `xlsynth-sys` build-script contract +- `@//:xlsynth_sys_legacy_stdlib` and + `@//:xlsynth_sys_legacy_dso` for frozen `xlsynth-sys` releases that + still use the paired `DSLX_STDLIB_PATH` / `XLS_DSO_PATH` contract +- `@//:xlsynth_sys_dep` for the preferred `xlsynth-sys` + runtime-plus-link contract +- `@//:xlsynth_sys_runtime_files` and `@//:xlsynth_sys_link_dep` + as compatibility exports for callers that still spell runtime and link + separately + +`xlsynth-sys` consumers should prefer `xlsynth_sys_artifact_config`, +`xlsynth_sys_dep`, and, for frozen releases, `xlsynth_sys_legacy_stdlib` plus +`xlsynth_sys_legacy_dso`, rather than spelling generic bundle internals like +`artifact_config`, `libxls_file`, `libxls`, or `dslx_stdlib` directly in +downstream `MODULE.bazel` files. + +Supported DSLX rules may opt out of the registered default bundle with +`xls_bundle = "@//:bundle"`. Today that escape hatch is available on +`dslx_library`, `dslx_test`, `dslx_to_ir`, `dslx_prove_quickcheck_test`, +`dslx_to_sv_types`, `dslx_to_pipeline`, `dslx_to_pipeline_eco`, and +`dslx_stitch_pipeline`. + +```starlark +dslx_to_pipeline( + name = "legacy_pipeline", + delay_model = "asap7", + pipeline_stages = 1, + top = "main", + deps = [":my_dslx_library"], + xls_bundle = "@legacy_xls//:bundle", +) ``` -Or by passing `--action_env=` on the bazel command line. +Artifact-path build settings such as +`--@rules_xlsynth//config:driver_path=...`, +`--@rules_xlsynth//config:tools_path=...`, +`--@rules_xlsynth//config:runtime_library_path=...`, and +`--@rules_xlsynth//config:dslx_stdlib_path=...` are no longer supported. +Artifact selection lives only in `MODULE.bazel`. The remaining +`@rules_xlsynth//config:*` settings are behavior knobs, such as extra DSLX +search paths or warning toggles. -### `dslx_library`, `dslx_test` — libraries/tests for DSLX files +Self-hosted examples in this repo: + +- `examples/workspace_toolchain_smoke/` shows one registered default bundle and + one explicit `xls_bundle` override without any `.bazelrc` artifact flags. +- `examples/workspace_toolchain_local_dev/` shows a `local_paths` workspace + rooted at `/tmp/xls-local-dev/`. + +### `dslx_library`, `dslx_test` - libraries/tests for DSLX files ```starlark load("@rules_xlsynth//:rules.bzl", "dslx_library", "dslx_test") @@ -50,7 +142,7 @@ dslx_test( ) ``` -### `dslx_fmt_test` — format DSLX files +### `dslx_fmt_test` - format DSLX files ```starlark load("@rules_xlsynth//:rules.bzl", "dslx_fmt_test") @@ -61,7 +153,7 @@ dslx_fmt_test( ) ``` -### `dslx_to_sv_types` — create `_pkg.sv` file +### `dslx_to_sv_types` - create `_pkg.sv` file ```starlark load("@rules_xlsynth//:rules.bzl", "dslx_to_sv_types") @@ -74,10 +166,23 @@ dslx_to_sv_types( ) ``` -`sv_enum_case_naming_policy` is required. Allowed values (matching `xlsynth-driver`) are `unqualified` and `enum_qualified`. -`sv_struct_field_ordering` is optional. Allowed values are `as_declared` (default) and `reversed`. +`sv_enum_case_naming_policy` is required. Allowed values (matching +`xlsynth-driver`) are `unqualified` and `enum_qualified`. + +`sv_struct_field_ordering` is optional. Allowed values are `as_declared` +(default) and `reversed`. + +The selected bundle records whether its `xlsynth-driver` supports the +`--sv_enum_case_naming_policy` CLI flag. Older bundles still work with +`sv_enum_case_naming_policy = "unqualified"`; `enum_qualified` only works when +the chosen workspace bundle or explicit `xls_bundle` advertises support. + +Older bundles also keep working with the default +`sv_struct_field_ordering = "as_declared"` behavior. The explicit `reversed` +mode only works when the chosen workspace bundle or explicit `xls_bundle` +advertises support for `--sv_struct_field_ordering`. -### `dslx_to_ir` — convert DSLX to optimized IR +### `dslx_to_ir` - convert DSLX to optimized IR Given a DSLX library target as a dependency, this rule will generate: @@ -96,7 +201,7 @@ dslx_to_ir( ) ``` -### `ir_to_delay_info` — convert IR to delay info +### `ir_to_delay_info` - convert IR to delay info ```starlark load("@rules_xlsynth//:rules.bzl", "ir_to_delay_info") @@ -109,7 +214,7 @@ ir_to_delay_info( ) ``` -### `dslx_prove_quickcheck_test` — prove quickcheck holds for entire input domain +### `dslx_prove_quickcheck_test` - prove quickcheck holds for entire input domain ```starlark load("@rules_xlsynth//:rules.bzl", "dslx_prove_quickcheck_test") @@ -122,7 +227,7 @@ dslx_prove_quickcheck_test( ) ``` -### `ir_to_gates` — convert IR to gate-level analysis +### `ir_to_gates` - convert IR to gate-level analysis Given an IR target (typically from `dslx_to_ir`) as input via `ir_src`, this rule runs the `ir2gates` tool to produce a text file containing gate-level analysis (e.g., gate counts, depth). @@ -147,7 +252,7 @@ ir_to_gates( ) ``` -### `dslx_stitch_pipeline` — stitch pipeline stage functions +### `dslx_stitch_pipeline` - stitch pipeline stage functions ```starlark load("@rules_xlsynth//:rules.bzl", "dslx_stitch_pipeline") @@ -161,16 +266,16 @@ dslx_stitch_pipeline( #### Attributes (non-exhaustive) -* `stages` — optional explicit list of stage function names to stitch when auto-discovery is not desired. -* `input_valid_signal` / `output_valid_signal` — when provided, additional `valid` handshaking logic is generated. -* `reset` — name of the reset signal to thread through the generated wrapper. Use together with `reset_active_low` to control polarity. -* `reset_active_low` — `True` when the reset signal is active low (defaults to `False`). -* `flop_inputs` — `True` to insert an input register stage in front of the first stitched stage (defaults to `True`). -* `flop_outputs` — `True` to insert an output register stage after the final stage (defaults to `True`). +* `stages` - optional explicit list of stage function names to stitch when auto-discovery is not desired. +* `input_valid_signal` / `output_valid_signal` - when provided, additional `valid` handshaking logic is generated. +* `reset` - name of the reset signal to thread through the generated wrapper. Use together with `reset_active_low` to control polarity. +* `reset_active_low` - `True` when the reset signal is active low (defaults to `False`). +* `flop_inputs` - `True` to insert an input register stage in front of the first stitched stage (defaults to `True`). +* `flop_outputs` - `True` to insert an output register stage after the final stage (defaults to `True`). The `flop_inputs` and `flop_outputs` flags give fine-grained control over where pipeline registers are placed. For example, the `sample/BUILD.bazel` file contains demonstrations that verify: -* `flop_inputs = True, flop_outputs = False` — only input side flops. -* `flop_inputs = False, flop_outputs = True` — only output side flops. +* `flop_inputs = True, flop_outputs = False` - only input side flops. +* `flop_inputs = False, flop_outputs = True` - only output side flops. Corresponding golden SystemVerilog files live next to the BUILD file so you can observe the emitted RTL. diff --git a/artifact_resolution_test.py b/artifact_resolution_test.py new file mode 100644 index 0000000..3314889 --- /dev/null +++ b/artifact_resolution_test.py @@ -0,0 +1,425 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +import materialize_xls_bundle + + +class ArtifactResolutionTest(unittest.TestCase): + def test_auto_prefers_exact_installed_layout(self): + plan = materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "auto", + xls_version = "0.38.0", + driver_version = "0.33.0", + installed_tools_root_prefix = "/tools/xlsynth", + installed_driver_root_prefix = "/tools/xlsynth-driver", + exists_fn = lambda path: True, + ) + self.assertEqual(plan["mode"], "installed") + self.assertEqual( + plan["dslx_stdlib_root"], + Path("/tools/xlsynth/v0.38.0/xls/dslx/stdlib"), + ) + self.assertEqual( + materialize_xls_bundle.derive_runtime_library_path(plan["libxls"]), + "/tools/xlsynth/v0.38.0", + ) + + def test_auto_falls_back_to_download(self): + plan = materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "auto", + xls_version = "0.38.0", + driver_version = "0.33.0", + installed_tools_root_prefix = "/tools/xlsynth", + installed_driver_root_prefix = "/tools/xlsynth-driver", + exists_fn = lambda path: False, + ) + self.assertEqual(plan["mode"], "download") + self.assertEqual(plan["xls_version"], "0.38.0") + self.assertEqual(plan["driver_version"], "0.33.0") + + def test_auto_requires_installed_prefixes(self): + with self.assertRaises(ValueError): + materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "auto", + xls_version = "0.38.0", + driver_version = "0.33.0", + exists_fn = lambda path: False, + ) + + def test_installed_only_requires_installed_paths(self): + with self.assertRaises(ValueError): + materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "installed_only", + xls_version = "0.38.0", + driver_version = "0.33.0", + installed_tools_root_prefix = "/tools/xlsynth", + installed_driver_root_prefix = "/tools/xlsynth-driver", + exists_fn = lambda path: False, + ) + + def test_download_only_skips_installed_probe(self): + observed_paths = [] + + def exists_fn(path): + observed_paths.append(path) + return True + + plan = materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "download_only", + xls_version = "0.38.0", + driver_version = "0.33.0", + exists_fn = exists_fn, + ) + self.assertEqual(plan["mode"], "download") + self.assertEqual(observed_paths, []) + + def test_local_paths_bypass_versioned_selection(self): + plan = materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "local_paths", + xls_version = "", + driver_version = "", + local_tools_path = "/tmp/xls-local-dev/tools", + local_dslx_stdlib_path = "/tmp/xls-local-dev/stdlib", + local_driver_path = "/tmp/xls-local-dev/xlsynth-driver", + local_libxls_path = "/tmp/xls-local-dev/libxls.so", + ) + self.assertEqual(plan["mode"], "local_paths") + self.assertEqual( + materialize_xls_bundle.derive_runtime_library_path(plan["libxls"]), + "/tmp/xls-local-dev", + ) + + def test_download_only_rejects_installed_prefixes(self): + with self.assertRaises(ValueError): + materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "download_only", + xls_version = "0.38.0", + driver_version = "0.33.0", + installed_tools_root_prefix = "/tools/xlsynth", + installed_driver_root_prefix = "/tools/xlsynth-driver", + ) + + def test_detect_host_platform_rejects_intel_macos(self): + with mock.patch.object(materialize_xls_bundle.sys, "platform", "darwin"): + with mock.patch.object(materialize_xls_bundle.os, "uname", return_value = mock.Mock(machine = "x86_64")): + with self.assertRaisesRegex(RuntimeError, "Intel macOS"): + materialize_xls_bundle.detect_host_platform() + + def test_installed_paths_use_live_version_pattern(self): + plan = materialize_xls_bundle.derive_installed_paths( + xls_version = "0.38.0", + driver_version = "0.33.0", + installed_tools_root_prefix = "/eda-tools/xlsynth", + installed_driver_root_prefix = "/eda-tools/xlsynth-driver", + ) + self.assertEqual(plan["tools_root"], Path("/eda-tools/xlsynth/v0.38.0")) + self.assertEqual(plan["driver"], Path("/eda-tools/xlsynth-driver/0.33.0/bin/xlsynth-driver")) + expected_name = "libxls.dylib" if sys.platform == "darwin" else "libxls.so" + self.assertEqual(plan["libxls"].name, expected_name) + + def test_build_driver_environment_sets_runtime_library_search_path(self): + libxls_path = "/tmp/xls-bundle/libxls.dylib" if sys.platform == "darwin" else "/tmp/xls-bundle/libxls.so" + runtime_var = "DYLD_LIBRARY_PATH" if sys.platform == "darwin" else "LD_LIBRARY_PATH" + env = materialize_xls_bundle.build_driver_environment( + libxls_path = libxls_path, + dslx_stdlib_path = "/tmp/xls-bundle", + environ = { + runtime_var: "/existing/runtime/path", + }, + sys_platform = sys.platform, + ) + self.assertEqual( + env[runtime_var], + "/tmp/xls-bundle{}{}".format(os.pathsep, "/existing/runtime/path"), + ) + self.assertEqual(env["XLS_DSO_PATH"], libxls_path) + self.assertEqual(env["DSLX_STDLIB_PATH"], "/tmp/xls-bundle") + + def test_detect_driver_capabilities_reads_help_flags(self): + with mock.patch.object( + materialize_xls_bundle.subprocess, + "run", + return_value = mock.Mock( + returncode = 0, + stdout = "Usage: xlsynth-driver dslx2sv-types --sv_struct_field_ordering \n", + stderr = "--sv_enum_case_naming_policy \n", + ), + ): + self.assertEqual( + materialize_xls_bundle.detect_driver_capabilities( + Path("/tmp/xls-bundle/xlsynth-driver"), + Path("/tmp/xls-bundle/libxls.so"), + Path("/tmp/xls-bundle"), + ), + { + "driver_supports_sv_enum_case_naming_policy": True, + "driver_supports_sv_struct_field_ordering": True, + }, + ) + + def test_build_driver_install_command_uses_rustup_nightly(self): + command = materialize_xls_bundle.build_driver_install_command( + "/usr/bin/rustup", + "/tmp/xls-driver-root", + "0.33.0", + ) + self.assertEqual( + command, + [ + "/usr/bin/rustup", + "run", + "nightly", + "cargo", + "install", + "--locked", + "--root", + "/tmp/xls-driver-root", + "--version", + "0.33.0", + "xlsynth-driver", + ], + ) + + def test_build_rustup_toolchain_install_command_uses_minimal_profile(self): + command = materialize_xls_bundle.build_rustup_toolchain_install_command("/usr/bin/rustup") + self.assertEqual( + command, + [ + "/usr/bin/rustup", + "toolchain", + "install", + "nightly", + "--profile", + "minimal", + ], + ) + + def test_build_driver_install_environment_uses_platform_scoped_cache_dirs(self): + libxls_path = "/tmp/xls-bundle/libxls.dylib" if sys.platform == "darwin" else "/tmp/xls-bundle/libxls.so" + env = materialize_xls_bundle.build_driver_install_environment( + Path("/tmp/xls-bundle-repo"), + libxls_path = libxls_path, + dslx_stdlib_path = "/tmp/xls-bundle", + host_platform = "arm64", + ) + self.assertEqual(env["RUSTUP_HOME"], "/tmp/xls-bundle-repo/_rustup_home/arm64") + self.assertEqual(env["CARGO_TARGET_DIR"], "/tmp/xls-bundle-repo/_cargo_target/arm64") + + def test_driver_install_root_is_version_and_platform_scoped(self): + self.assertEqual( + materialize_xls_bundle.driver_install_root( + Path("/tmp/xls-bundle-repo"), + "0.33.0", + "arm64", + ), + Path("/tmp/xls-bundle-repo/_cargo_driver/arm64/0.33.0"), + ) + + def test_download_versioned_artifacts_reuses_valid_cache(self): + with tempfile.TemporaryDirectory() as tempdir: + repo_root = Path(tempdir) + download_root = repo_root / "_downloaded_xls" / "arm64" / "0.38.0" + (download_root / "xls" / "dslx" / "stdlib").mkdir(parents = True) + (download_root / "xls" / "dslx" / "stdlib" / "std.x").write_text("// stdlib\n", encoding = "utf-8") + for binary in materialize_xls_bundle.TOOL_BINARIES: + (download_root / binary).write_text("", encoding = "utf-8") + (download_root / "libxls-v0.38.0-arm64.dylib").write_text("", encoding = "utf-8") + + with mock.patch.object(materialize_xls_bundle, "detect_host_platform", return_value = "arm64"): + with mock.patch.object(materialize_xls_bundle.subprocess, "run") as mock_run: + resolved = materialize_xls_bundle.download_versioned_artifacts(repo_root, "0.38.0") + + self.assertEqual(resolved["tools_root"], download_root) + self.assertEqual(resolved["dslx_stdlib_root"], download_root / "xls" / "dslx" / "stdlib") + self.assertEqual( + resolved["libxls"], + download_root / "libxls-v0.38.0-arm64.dylib", + ) + mock_run.assert_not_called() + + def test_install_driver_reuses_valid_cached_binary(self): + with tempfile.TemporaryDirectory() as tempdir: + repo_root = Path(tempdir) + driver_path = repo_root / "_cargo_driver" / "arm64" / "0.33.0" / "bin" / "xlsynth-driver" + driver_path.parent.mkdir(parents = True) + driver_path.write_text("", encoding = "utf-8") + + with mock.patch.object(materialize_xls_bundle, "detect_host_platform", return_value = "arm64"): + with mock.patch.object(materialize_xls_bundle.shutil, "which") as mock_which: + with mock.patch.object( + materialize_xls_bundle.subprocess, + "run", + return_value = mock.Mock( + returncode = 0, + stdout = "xlsynth-driver 0.33.0\n", + stderr = "", + ), + ) as mock_run: + resolved = materialize_xls_bundle.install_driver( + repo_root = repo_root, + driver_version = "0.33.0", + libxls_path = "/tmp/xls-bundle/libxls.dylib" if sys.platform == "darwin" else "/tmp/xls-bundle/libxls.so", + dslx_stdlib_path = "/tmp/xls-bundle", + ) + + self.assertEqual(resolved, driver_path) + mock_which.assert_not_called() + mock_run.assert_called_once_with( + [str(driver_path), "--version"], + check = False, + stdout = materialize_xls_bundle.subprocess.PIPE, + stderr = materialize_xls_bundle.subprocess.PIPE, + universal_newlines = True, + env = mock.ANY, + ) + + def test_parse_readelf_soname_finds_soname(self): + self.assertEqual( + materialize_xls_bundle.parse_readelf_soname( + """ +Tag Type Name/Value +0x000000000000000e (SONAME) Library soname: [libxls-v0.38.0.so] +""" + ), + "libxls-v0.38.0.so", + ) + + def test_read_linux_soname_reads_elf_metadata(self): + with mock.patch.object( + materialize_xls_bundle.subprocess, + "run", + return_value = mock.Mock( + returncode = 0, + stdout = "0x000000000000000e (SONAME) Library soname: [libxls-v0.38.0.so]\n", + stderr = "", + ), + ) as mock_run: + self.assertEqual( + materialize_xls_bundle.read_linux_soname(Path("/tmp/xls-bundle/libxls.so")), + "libxls-v0.38.0.so", + ) + mock_run.assert_called_once_with( + ["readelf", "-d", "/tmp/xls-bundle/libxls.so"], + check = False, + stdout = materialize_xls_bundle.subprocess.PIPE, + stderr = materialize_xls_bundle.subprocess.PIPE, + universal_newlines = True, + env = None, + ) + + def test_normalize_linux_soname_sets_expected_name(self): + with mock.patch.object( + materialize_xls_bundle, + "read_linux_soname", + return_value = "libxls-v0.38.0.so", + ): + with mock.patch.object(materialize_xls_bundle.shutil, "which", return_value = "/usr/bin/patchelf"): + with mock.patch.object(materialize_xls_bundle.subprocess, "run") as mock_run: + materialize_xls_bundle.normalize_linux_soname(Path("/tmp/xls-bundle/libxls.so")) + mock_run.assert_called_once_with( + [ + "/usr/bin/patchelf", + "--set-soname", + "libxls.so", + "/tmp/xls-bundle/libxls.so", + ], + check = True, + ) + + def test_normalize_linux_soname_is_noop_when_matching(self): + with mock.patch.object( + materialize_xls_bundle, + "read_linux_soname", + return_value = "libxls.so", + ): + with mock.patch.object(materialize_xls_bundle.shutil, "which") as mock_which: + with mock.patch.object(materialize_xls_bundle.subprocess, "run") as mock_run: + self.assertEqual( + materialize_xls_bundle.normalize_linux_soname(Path("/tmp/xls-bundle/libxls.so")), + [], + ) + mock_which.assert_not_called() + mock_run.assert_not_called() + + def test_normalize_linux_soname_stages_runtime_alias_when_patchelf_missing(self): + with tempfile.TemporaryDirectory() as tempdir: + libxls_path = Path(tempdir) / "libxls.so" + libxls_path.write_text("xls\n", encoding = "utf-8") + + with mock.patch.object( + materialize_xls_bundle, + "read_linux_soname", + return_value = "libxls-v0.38.0.so", + ): + with mock.patch.object(materialize_xls_bundle.shutil, "which", return_value = None): + with mock.patch.object(materialize_xls_bundle.subprocess, "run") as mock_run: + self.assertEqual( + materialize_xls_bundle.normalize_linux_soname(libxls_path), + ["libxls-v0.38.0.so"], + ) + + alias_path = Path(tempdir) / "libxls-v0.38.0.so" + self.assertTrue(alias_path.exists()) + if alias_path.is_symlink(): + self.assertEqual(os.readlink(alias_path), "libxls.so") + else: + self.assertEqual(alias_path.read_text(encoding = "utf-8"), "xls\n") + mock_run.assert_not_called() + + def test_materialize_bundle_records_runtime_aliases(self): + with tempfile.TemporaryDirectory() as tempdir: + repo_root = Path(tempdir) + input_root = repo_root / "_inputs" + tools_root = input_root / "tools" + stdlib_root = tools_root / "xls" / "dslx" / "stdlib" + stdlib_root.mkdir(parents = True) + (stdlib_root / "std.x").write_text("// stdlib\n", encoding = "utf-8") + for binary in materialize_xls_bundle.TOOL_BINARIES: + (tools_root / binary).write_text("", encoding = "utf-8") + + driver_path = input_root / "xlsynth-driver" + driver_path.write_text("", encoding = "utf-8") + libxls_path = input_root / "libxls-v0.38.0.so" + libxls_path.write_text("xls\n", encoding = "utf-8") + + with mock.patch.object( + materialize_xls_bundle, + "normalize_runtime_library_identity", + return_value = ["libxls-v0.38.0.so"], + ): + with mock.patch.object( + materialize_xls_bundle, + "detect_driver_capabilities", + return_value = { + "driver_supports_sv_enum_case_naming_policy": True, + "driver_supports_sv_struct_field_ordering": False, + }, + ): + materialize_xls_bundle.materialize_bundle( + repo_root, + { + "mode": "installed", + "tools_root": tools_root, + "dslx_stdlib_root": stdlib_root, + "driver": driver_path, + "libxls": libxls_path, + }, + ) + + metadata = dict( + line.split("=", 1) + for line in (repo_root / "bundle_metadata.txt").read_text(encoding = "utf-8").splitlines() + if line + ) + self.assertEqual(metadata["libxls_name"], "libxls.so") + self.assertEqual(metadata["libxls_runtime_aliases"], "libxls-v0.38.0.so") + + +if __name__ == "__main__": + unittest.main() diff --git a/config/BUILD.bazel b/config/BUILD.bazel new file mode 100644 index 0000000..1e0fd80 --- /dev/null +++ b/config/BUILD.bazel @@ -0,0 +1,38 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") + +package(default_visibility = ["//visibility:public"]) + +string_flag( + name = "dslx_path", + build_setting_default = "", +) + +string_flag( + name = "enable_warnings", + build_setting_default = "", +) + +string_flag( + name = "disable_warnings", + build_setting_default = "", +) + +string_flag( + name = "gate_format", + build_setting_default = "", +) + +string_flag( + name = "assert_format", + build_setting_default = "", +) + +string_flag( + name = "use_system_verilog", + build_setting_default = "", +) + +string_flag( + name = "add_invariant_assertions", + build_setting_default = "", +) diff --git a/download_release.py b/download_release.py index 9f42aa7..a19049c 100644 --- a/download_release.py +++ b/download_release.py @@ -1,17 +1,67 @@ # SPDX-License-Identifier: Apache-2.0 -import os +import gzip import hashlib +import http.client +import json +from optparse import OptionParser +import os import shutil import tempfile import time -import json -from optparse import OptionParser -from urllib import request as urlrequest from urllib import error as urlerror +from urllib import request as urlrequest GITHUB_API_URL = "https://api.github.com/repos/xlsynth/xlsynth/releases" -SUPPORTED_PLATFORMS = ["ubuntu2004", "ubuntu2204", "rocky8", "arm64", "x64"] +SUPPORTED_PLATFORMS = { + "ubuntu2004": ".so", + "ubuntu2204": ".so", + "rocky8": ".so", + "arm64": ".dylib", + "x64": ".so", +} + +TOOL_BINARIES = [ + "dslx_interpreter_main", + "ir_converter_main", + "block_to_verilog_main", + "codegen_main", + "opt_main", + "prove_quickcheck_main", + "typecheck_main", + "dslx_fmt", + "delay_info_main", + "check_ir_equivalence_main", +] + + +def parse_xlsynth_release_tag(tag): + assert tag.startswith("v"), "Version tags must start with 'v': {}".format(tag) + main_and_patch2 = tag[1:].split("-") + main_parts = main_and_patch2[0].split(".") + assert len(main_parts) == 3, "Expected semantic version tag, got: {}".format(tag) + major, minor, patch = main_parts + patch2 = int(main_and_patch2[1]) if len(main_and_patch2) > 1 else 0 + return (int(major), int(minor), int(patch), patch2) + + +def build_binary_release_filename(binary_name, platform): + return "{}-{}".format(binary_name, platform) + + +def build_dso_release_filename(platform, version_tuple): + filename = "libxls-{}{}".format(platform, SUPPORTED_PLATFORMS[platform]) + if version_tuple >= (0, 0, 219, 0): + filename += ".gz" + return filename + + +def build_release_artifacts(version, platform, include_dso): + version_tuple = parse_xlsynth_release_tag(version) + artifacts = [(build_binary_release_filename(binary_name, platform), True) for binary_name in TOOL_BINARIES] + if include_dso: + artifacts.append((build_dso_release_filename(platform, version_tuple), False)) + return artifacts def get_headers(): """ @@ -60,6 +110,35 @@ def get_latest_release(max_attempts): print(f"Latest version discovered: {latest_version}") return latest_version +def copy_url_to_path(url, destination_path, headers, max_attempts): + """ + Downloads one URL to destination_path with exponential-backoff retries. + + This retries both connection setup failures and mid-stream read failures so + large artifact downloads survive transient disconnects. + """ + attempt = 0 + delay = 1 + while attempt < max_attempts: + attempt += 1 + try: + with request_with_retry(url, stream=True, headers=headers, max_attempts=max_attempts) as r: + with open(destination_path, 'wb') as f: + shutil.copyfileobj(r, f) + return + except urlerror.HTTPError as e: + if e.code == 404 or attempt == max_attempts: + print(f"All {attempt} attempts failed for {url}") + raise + print(f"Attempt {attempt} failed for {url}. HTTP {e.code}. Retrying in {delay} seconds...") + except (ConnectionResetError, EOFError, OSError, TimeoutError, http.client.HTTPException, urlerror.URLError) as e: + if attempt == max_attempts: + print(f"All {attempt} attempts failed for {url}") + raise + print(f"Attempt {attempt} failed for {url}. Error: {e}. Retrying in {delay} seconds...") + time.sleep(delay) + delay *= 2 + def high_integrity_download(base_url, filename, target_dir, max_attempts, is_binary=False, platform=None): print(f"Starting download of {filename}...") start_time = time.time() @@ -73,15 +152,9 @@ def high_integrity_download(base_url, filename, target_dir, max_attempts, is_bin headers = get_headers() - # Download SHA256 file with retry support - with request_with_retry(sha256_url, stream=True, headers=headers, max_attempts=max_attempts) as r: - with open(sha256_path, 'wb') as f: - shutil.copyfileobj(r, f) + copy_url_to_path(sha256_url, sha256_path, headers, max_attempts) - # Download the artifact with retry support - with request_with_retry(artifact_url, stream=True, headers=headers, max_attempts=max_attempts) as r: - with open(artifact_path, 'wb') as f: - shutil.copyfileobj(r, f) + copy_url_to_path(artifact_url, artifact_path, headers, max_attempts) # Verify checksum with open(sha256_path, 'r') as f: @@ -100,10 +173,18 @@ def high_integrity_download(base_url, filename, target_dir, max_attempts, is_bin target_filename = filename if is_binary and platform and filename.endswith(f"-{platform}"): target_filename = filename[:-(len(platform) + 1)] # Remove '-platform' + is_gz_dso = target_filename.endswith(".so.gz") or target_filename.endswith(".dylib.gz") + if is_gz_dso: + target_filename = target_filename[:-3] # Move to target directory target_path = os.path.join(target_dir, target_filename) - shutil.move(artifact_path, target_path) + if is_gz_dso: + with gzip.open(artifact_path, "rb") as fin, open(target_path, "wb") as fout: + shutil.copyfileobj(fin, fout) + os.remove(artifact_path) + else: + shutil.move(artifact_path, target_path) # Make binary artifacts executable if is_binary: @@ -118,6 +199,14 @@ def main(): parser.add_option("-v", "--version", dest="version", help="Specify release version (e.g., v0.0.0)") parser.add_option("-o", "--output", dest="output_dir", help="Output directory for artifacts") parser.add_option("-p", "--platform", dest="platform", help="Target platform (e.g., ubuntu2004, rocky8)") + parser.add_option( + "-d", + "--dso", + dest="dso", + help="Download the libxls dynamic library", + action="store_true", + default=False, + ) parser.add_option('--max_attempts', dest='max_attempts', help='Maximum number of attempts to download', type='int', default=10) (options, args) = parser.parse_args() @@ -138,23 +227,12 @@ def main(): base_url = f"https://github.com/xlsynth/xlsynth/releases/download/{version}" - artifacts = [ - ("dslx_interpreter_main", True), - ("ir_converter_main", True), - ("codegen_main", True), - ("opt_main", True), - ("prove_quickcheck_main", True), - ("typecheck_main", True), - ("dslx_fmt", True), - ("delay_info_main", True), - ("check_ir_equivalence_main", True), - ] + artifacts = build_release_artifacts(version, options.platform, options.dso) os.makedirs(options.output_dir, exist_ok=True) for artifact, is_binary in artifacts: - filename = f"{artifact}-{options.platform}" - high_integrity_download(base_url, filename, options.output_dir, options.max_attempts, is_binary, options.platform) + high_integrity_download(base_url, artifact, options.output_dir, options.max_attempts, is_binary, options.platform) # Download and extract dslx_stdlib.tar.gz stdlib_filename = "dslx_stdlib.tar.gz" diff --git a/download_release_test.py b/download_release_test.py new file mode 100644 index 0000000..88d278a --- /dev/null +++ b/download_release_test.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 + +import io +import tempfile +import unittest +from unittest import mock +from urllib import error as urlerror + +import download_release + + +class _FakeResponse: + def __init__(self, payload): + self._buffer = io.BytesIO(payload) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self, size = -1): + return self._buffer.read(size) + + +class _FlakyResponse: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self, size = -1): + raise ConnectionResetError("stream reset") + + +class DownloadReleaseTest(unittest.TestCase): + def test_binary_release_filename_keeps_platform_suffix(self): + self.assertEqual( + download_release.build_binary_release_filename("dslx_fmt", "arm64"), + "dslx_fmt-arm64", + ) + + def test_dso_release_filename_is_not_double_suffixed(self): + self.assertEqual( + download_release.build_dso_release_filename("arm64", (0, 38, 0, 0)), + "libxls-arm64.dylib.gz", + ) + self.assertEqual( + download_release.build_dso_release_filename("ubuntu2004", (0, 38, 0, 0)), + "libxls-ubuntu2004.so.gz", + ) + + def test_old_release_dso_filename_is_not_gzipped(self): + self.assertEqual( + download_release.build_dso_release_filename("ubuntu2004", (0, 0, 218, 0)), + "libxls-ubuntu2004.so", + ) + + def test_release_artifacts_mix_binary_and_dso_filenames(self): + artifacts = download_release.build_release_artifacts("v0.38.0", "arm64", True) + self.assertIn(("dslx_fmt-arm64", True), artifacts) + self.assertIn(("libxls-arm64.dylib.gz", False), artifacts) + + def test_copy_url_to_path_retries_after_stream_reset(self): + with tempfile.TemporaryDirectory() as temp_dir: + destination_path = f"{temp_dir}/artifact" + with mock.patch.object( + download_release, + "request_with_retry", + side_effect = [_FlakyResponse(), _FakeResponse(b"ok")], + ): + with mock.patch.object(download_release.time, "sleep"): + download_release.copy_url_to_path( + "https://example.invalid/artifact", + destination_path, + headers = {}, + max_attempts = 2, + ) + with open(destination_path, "rb") as f: + self.assertEqual(f.read(), b"ok") + + def test_copy_url_to_path_still_raises_not_found(self): + with tempfile.TemporaryDirectory() as temp_dir: + destination_path = f"{temp_dir}/missing" + not_found = urlerror.HTTPError( + url = "https://example.invalid/missing", + code = 404, + msg = "not found", + hdrs = None, + fp = None, + ) + with mock.patch.object(download_release, "request_with_retry", side_effect = not_found): + with self.assertRaises(urlerror.HTTPError): + download_release.copy_url_to_path( + "https://example.invalid/missing", + destination_path, + headers = {}, + max_attempts = 2, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/dslx_fmt.bzl b/dslx_fmt.bzl index 65d9635..5bfec92 100644 --- a/dslx_fmt.bzl +++ b/dslx_fmt.bzl @@ -1,5 +1,6 @@ load(":helpers.bzl", "write_executable_shell_script") load(":env_helpers.bzl", "python_runner_source") +load(":xls_toolchain.bzl", "declare_xls_toolchain_toml", "get_tool_artifact_inputs", "require_tools_toolchain") def _dslx_format_impl(ctx): src_depset_files = ctx.attr.srcs @@ -8,7 +9,10 @@ def _dslx_format_impl(ctx): formatted_files = [] runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = require_tools_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "dslx_fmt") + toolchain_inputs = [toolchain_file] + get_tool_artifact_inputs(toolchain, "dslx_fmt") for src in src_depset_files: input_file = src[DefaultInfo].files.to_list()[0] @@ -16,17 +20,22 @@ def _dslx_format_impl(ctx): formatted_file = ctx.actions.declare_file(input_file.basename + ".fmt") formatted_files.append(formatted_file) - ctx.actions.run_shell( - inputs=[input_file], - tools=[runner], - outputs=[formatted_file], - command="\"$1\" tool dslx_fmt \"$2\" > \"$3\"", + ctx.actions.run( + inputs = [input_file] + toolchain_inputs, + executable = runner, + outputs = [formatted_file], arguments=[ - runner.path, - input_file.path, + "tool", + "--toolchain", + toolchain_file.path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--stdout_path", formatted_file.path, + "dslx_fmt", + input_file.path, ], - use_default_shell_env=True, + use_default_shell_env = False, ) diff_commands = [] @@ -41,9 +50,9 @@ def _dslx_format_impl(ctx): ) return DefaultInfo( - runfiles=ctx.runfiles(files=input_files + formatted_files + [diff_script_file]), - files=depset(direct=[diff_script_file] + formatted_files), - executable=diff_script_file, + runfiles = ctx.runfiles(files = input_files + formatted_files + [diff_script_file] + toolchain_inputs), + files = depset(direct = [diff_script_file] + formatted_files), + executable = diff_script_file, ) @@ -53,5 +62,6 @@ dslx_fmt_test = rule( "srcs": attr.label_list(allow_files=[".x"], allow_empty=False, doc="Source files to check formatting"), }, doc="A rule that checks if the given DSLX files are properly formatted.", - test=True, + test = True, + toolchains = ["//:toolchain_type"], ) diff --git a/dslx_prove_quickcheck_test.bzl b/dslx_prove_quickcheck_test.bzl index 8e658c8..0438ec1 100644 --- a/dslx_prove_quickcheck_test.bzl +++ b/dslx_prove_quickcheck_test.bzl @@ -3,6 +3,13 @@ load(":dslx_provider.bzl", "DslxInfo") load(":helpers.bzl", "write_executable_shell_script", "get_srcs_from_lib") load(":env_helpers.bzl", "python_runner_source") +load( + ":xls_toolchain.bzl", + "XlsArtifactBundleInfo", + "declare_xls_toolchain_toml", + "get_selected_tools_toolchain", + "get_tool_artifact_inputs", +) def _dslx_prove_quickcheck_test_impl(ctx): @@ -15,15 +22,25 @@ def _dslx_prove_quickcheck_test_impl(ctx): srcs = get_srcs_from_lib(ctx) runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) - cmd = "/usr/bin/env python3 {} tool prove_quickcheck_main {}".format( + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_tools_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "prove_quickcheck", toolchain = toolchain) + cmd_parts = [ + "/usr/bin/env", + "python3", runner.short_path, - lib_src.short_path, - ) + "tool", + "--toolchain", + toolchain_file.short_path, + ] + if toolchain.runtime_library_path: + cmd_parts.extend(["--runtime_library_path", toolchain.runtime_library_path]) + cmd_parts.extend(["prove_quickcheck_main", lib_src.short_path]) + cmd = " ".join(["\"{}\"".format(part) for part in cmd_parts]) if ctx.attr.top: cmd += " --test_filter=" + ctx.attr.top - runfiles = ctx.runfiles(srcs + [runner]) + runfiles = ctx.runfiles(srcs + [runner, toolchain_file] + get_tool_artifact_inputs(toolchain, "prove_quickcheck_main")) executable_file = write_executable_shell_script( ctx = ctx, filename = ctx.label.name + ".sh", @@ -48,6 +65,11 @@ dslx_prove_quickcheck_test = rule( "top": attr.string( doc = "The quickcheck function to be tested. If none is provided, all quickcheck functions in the library will be tested.", ), + "xls_bundle": attr.label( + doc = "Optional XLS bundle override.", + providers = [XlsArtifactBundleInfo], + ), }, test = True, + toolchains = ["//:toolchain_type"], ) diff --git a/dslx_provider.bzl b/dslx_provider.bzl index 475a580..a05579d 100644 --- a/dslx_provider.bzl +++ b/dslx_provider.bzl @@ -1,6 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 load(":env_helpers.bzl", "python_runner_source") +load( + ":xls_toolchain.bzl", + "XlsArtifactBundleInfo", + "declare_xls_toolchain_toml", + "get_selected_tools_toolchain", + "get_tool_artifact_inputs", +) DslxInfo = provider( doc = "Contains DAG info per node in a struct.", @@ -62,19 +69,26 @@ def _dslx_library_impl(ctx): # Run typechecking via the embedded runner so env is read at action runtime. runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_tools_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "typecheck", toolchain = toolchain) + action_inputs = srcs + [toolchain_file] + get_tool_artifact_inputs(toolchain, "typecheck_main") ctx.actions.run( - inputs = srcs, + inputs = action_inputs, outputs = [typecheck_output], executable = runner, arguments = [ "tool", + "--toolchain", + toolchain_file.path, + "--runtime_library_path", + toolchain.runtime_library_path, "typecheck_main", srcs[-1].path, "--output_path", typecheck_output.path, ], - use_default_shell_env = True, + use_default_shell_env = False, ) return [ @@ -97,5 +111,10 @@ dslx_library = rule( doc = "DSLX sources.", allow_files = [".x"], ), + "xls_bundle": attr.label( + doc = "Optional XLS bundle override.", + providers = [XlsArtifactBundleInfo], + ), }, + toolchains = ["//:toolchain_type"], ) diff --git a/dslx_stitch_pipeline.bzl b/dslx_stitch_pipeline.bzl index 0ecb656..c97dff7 100644 --- a/dslx_stitch_pipeline.bzl +++ b/dslx_stitch_pipeline.bzl @@ -3,6 +3,7 @@ load(":dslx_provider.bzl", "DslxInfo") load(":helpers.bzl", "get_srcs_from_lib") load(":env_helpers.bzl", "python_runner_source") +load(":xls_toolchain.bzl", "XlsArtifactBundleInfo", "declare_xls_toolchain_toml", "get_driver_artifact_inputs", "get_selected_driver_toolchain") def _dslx_stitch_pipeline_impl(ctx): @@ -17,16 +18,16 @@ def _dslx_stitch_pipeline_impl(ctx): srcs = get_srcs_from_lib(ctx) - flags_str = " --use_system_verilog={}".format( - str(ctx.attr.use_system_verilog).lower() - ) + passthrough = [ + "--use_system_verilog={}".format(str(ctx.attr.use_system_verilog).lower()), + ] use_explicit_stages = len(ctx.attr.stages) > 0 if use_explicit_stages: - flags_str += " --stages=" + ",".join(ctx.attr.stages) + passthrough.append("--stages=" + ",".join(ctx.attr.stages)) # xlsynth-driver v0.33.0+ requires output_module_name when --stages is used # and rejects --dslx_top in that mode. Reuse the existing `top` attr as the # wrapper module name to preserve rule callsites. - flags_str += " --output_module_name=" + ctx.attr.top + passthrough.append("--output_module_name=" + ctx.attr.top) string_flags = [ "input_valid_signal", @@ -36,7 +37,7 @@ def _dslx_stitch_pipeline_impl(ctx): for flag in string_flags: value = getattr(ctx.attr, flag) if value: - flags_str += " --{}={}".format(flag, value) + passthrough.append("--{}={}".format(flag, value)) bool_flags = [ "reset_active_low", @@ -45,29 +46,40 @@ def _dslx_stitch_pipeline_impl(ctx): ] for flag in bool_flags: value = getattr(ctx.attr, flag) - flags_str += " --{}={}".format(flag, str(value).lower()) + passthrough.append("--{}={}".format(flag, str(value).lower())) runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_driver_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "dslx_stitch_pipeline", toolchain = toolchain) - dslx_top_flag = "" dslx_top_arg = "" if not use_explicit_stages: - dslx_top_flag = " --dslx_top=\"$3\"" dslx_top_arg = ctx.attr.top - ctx.actions.run_shell( - inputs = srcs, - tools = [runner], + arguments = [ + "driver", + "--driver_path", + toolchain.driver_path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--toolchain", + toolchain_file.path, + "--stdout_path", + ctx.outputs.sv_file.path, + "dslx-stitch-pipeline", + "--dslx_input_file=" + main_src.path, + ] + if dslx_top_arg: + arguments.append("--dslx_top=" + dslx_top_arg) + arguments.extend(passthrough) + + ctx.actions.run( + inputs = srcs + [toolchain_file] + get_driver_artifact_inputs(toolchain), + executable = runner, outputs = [ctx.outputs.sv_file], - command = "\"$1\" driver dslx-stitch-pipeline --dslx_input_file=\"$2\"" + dslx_top_flag + flags_str + " > \"$4\"", - arguments = [ - runner.path, - main_src.path, - dslx_top_arg, - ctx.outputs.sv_file.path, - ], - use_default_shell_env = True, + arguments = arguments, + use_default_shell_env = False, ) return DefaultInfo( @@ -116,8 +128,13 @@ dslx_stitch_pipeline = rule( doc = "Whether to insert flops on outputs (true) or not (false).", default = True, ), + "xls_bundle": attr.label( + doc = "Optional override bundle repo label, for example @legacy_xls//:bundle.", + providers = [XlsArtifactBundleInfo], + ), }, outputs = { "sv_file": "%{name}.sv", }, + toolchains = ["//:toolchain_type"], ) diff --git a/dslx_test.bzl b/dslx_test.bzl index 6dff439..04b27af 100644 --- a/dslx_test.bzl +++ b/dslx_test.bzl @@ -3,6 +3,13 @@ load(":dslx_provider.bzl", "DslxInfo") load(":helpers.bzl", "get_srcs_from_deps", "get_srcs_from_lib", "write_executable_shell_script") load(":env_helpers.bzl", "python_runner_source") +load( + ":xls_toolchain.bzl", + "XlsArtifactBundleInfo", + "declare_xls_toolchain_toml", + "get_selected_tools_toolchain", + "get_tool_artifact_inputs", +) def _dslx_test_impl(ctx): @@ -35,13 +42,24 @@ def _dslx_test_impl(ctx): srcs = test_src + srcs_from_deps runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) - cmd = "/usr/bin/env python3 {} tool dslx_interpreter_main {}".format( + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_tools_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "dslx_test", toolchain = toolchain) + cmd_parts = [ + "/usr/bin/env", + "python3", runner.short_path, - " ".join([src.short_path for src in srcs]), - ) + "tool", + "--toolchain", + toolchain_file.short_path, + ] + if toolchain.runtime_library_path: + cmd_parts.extend(["--runtime_library_path", toolchain.runtime_library_path]) + cmd_parts.append("dslx_interpreter_main") + cmd_parts.extend([src.short_path for src in srcs]) + cmd = " ".join(["\"{}\"".format(part) for part in cmd_parts]) - runfiles = ctx.runfiles(srcs + [runner]) + runfiles = ctx.runfiles(srcs + [runner, toolchain_file] + get_tool_artifact_inputs(toolchain, "dslx_interpreter_main")) executable_file = write_executable_shell_script( ctx = ctx, filename = ctx.label.name + ".sh", @@ -70,6 +88,11 @@ dslx_test = rule( doc = "The DSLX library dependencies for the test.", providers = [DslxInfo], ), + "xls_bundle": attr.label( + doc = "Optional XLS bundle override.", + providers = [XlsArtifactBundleInfo], + ), }, test = True, + toolchains = ["//:toolchain_type"], ) diff --git a/dslx_to_ir.bzl b/dslx_to_ir.bzl index 8229e89..dc24af2 100644 --- a/dslx_to_ir.bzl +++ b/dslx_to_ir.bzl @@ -4,6 +4,13 @@ load(":dslx_provider.bzl", "DslxInfo") load(":helpers.bzl", "get_srcs_from_lib", "mangle_dslx_name") load(":ir_provider.bzl", "IrInfo") load(":env_helpers.bzl", "python_runner_source") +load( + ":xls_toolchain.bzl", + "XlsArtifactBundleInfo", + "declare_xls_toolchain_toml", + "get_driver_artifact_inputs", + "get_selected_driver_toolchain", +) def _dslx_to_ir_impl(ctx): # Get the DslxInfo from the direct library target @@ -20,21 +27,34 @@ def _dslx_to_ir_impl(ctx): all_transitive_srcs = get_srcs_from_lib(ctx) runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_driver_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "dslx_to_ir", toolchain = toolchain) + dslx2ir_inputs = [toolchain_file] + get_driver_artifact_inputs(toolchain, ["ir_converter_main"]) + ir2opt_inputs = [toolchain_file] + get_driver_artifact_inputs(toolchain, ["opt_main"]) # Stage 1: dslx2ir - ctx.actions.run_shell( - inputs = all_transitive_srcs, - tools = [runner], + ctx.actions.run( + inputs = all_transitive_srcs + dslx2ir_inputs, + executable = runner, outputs = [ctx.outputs.ir_file], - command = "\"$1\" driver dslx2ir --dslx_input_file=\"$2\" --dslx_top=\"$3\" > \"$4\"", arguments = [ - runner.path, + "driver", + "--driver_path", + toolchain.driver_path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--toolchain", + toolchain_file.path, + "--stdout_path", + ctx.outputs.ir_file.path, + "dslx2ir", + "--dslx_input_file", main_src.path, + "--dslx_top", ctx.attr.top, - ctx.outputs.ir_file.path, ], - use_default_shell_env = True, + use_default_shell_env = False, progress_message = "Generating IR for DSLX", mnemonic = "DSLX2IR", ) @@ -42,18 +62,26 @@ def _dslx_to_ir_impl(ctx): ir_top = mangle_dslx_name(main_src.basename, ctx.attr.top) # Stage 2: ir2opt - ctx.actions.run_shell( - inputs = [ctx.outputs.ir_file], - tools = [runner], + ctx.actions.run( + inputs = [ctx.outputs.ir_file] + ir2opt_inputs, + executable = runner, outputs = [ctx.outputs.opt_ir_file], - command = "\"$1\" driver ir2opt \"$2\" --top \"$3\" > \"$4\"", arguments = [ - runner.path, + "driver", + "--driver_path", + toolchain.driver_path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--toolchain", + toolchain_file.path, + "--stdout_path", + ctx.outputs.opt_ir_file.path, + "ir2opt", ctx.outputs.ir_file.path, + "--top", ir_top, - ctx.outputs.opt_ir_file.path, ], - use_default_shell_env = True, + use_default_shell_env = False, progress_message = "Optimizing IR", mnemonic = "IR2OPT", ) @@ -76,9 +104,14 @@ dslx_to_ir = rule( doc = "The top-level DSLX module to be converted to IR.", mandatory = True, ), + "xls_bundle": attr.label( + doc = "Optional XLS bundle override.", + providers = [XlsArtifactBundleInfo], + ), }, outputs = { "ir_file": "%{name}.ir", "opt_ir_file": "%{name}.opt.ir", }, + toolchains = ["//:toolchain_type"], ) diff --git a/dslx_to_pipeline.bzl b/dslx_to_pipeline.bzl index 0299392..643efe6 100644 --- a/dslx_to_pipeline.bzl +++ b/dslx_to_pipeline.bzl @@ -3,27 +3,27 @@ load(":dslx_provider.bzl", "DslxInfo") load(":helpers.bzl", "get_srcs_from_deps") load(":env_helpers.bzl", "python_runner_source") +load(":xls_toolchain.bzl", "XlsArtifactBundleInfo", "declare_xls_toolchain_toml", "get_driver_artifact_inputs", "get_selected_driver_toolchain") def _dslx_to_pipeline_impl(ctx): srcs = get_srcs_from_deps(ctx) - # Flags for stdlib path - flags_str = "" + passthrough = [] # Delay model flag (required) - flags_str += " --delay_model=" + ctx.attr.delay_model + passthrough.append("--delay_model=" + ctx.attr.delay_model) # Forward string-valued flags when a non-empty value is provided. for flag in ["input_valid_signal", "output_valid_signal", "module_name"]: value = getattr(ctx.attr, flag) if value: - flags_str += " --{}={}".format(flag, value) + passthrough.append("--{}={}".format(flag, value)) # Forward integer-valued timing flags when >0. if ctx.attr.pipeline_stages > 0: - flags_str += " --pipeline_stages={}".format(ctx.attr.pipeline_stages) + passthrough.append("--pipeline_stages={}".format(ctx.attr.pipeline_stages)) if ctx.attr.clock_period_ps > 0: - flags_str += " --clock_period_ps={}".format(ctx.attr.clock_period_ps) + passthrough.append("--clock_period_ps={}".format(ctx.attr.clock_period_ps)) # Validate that either pipeline_stages or clock_period_ps is specified (>0). if ctx.attr.pipeline_stages == 0 and ctx.attr.clock_period_ps == 0: @@ -40,12 +40,12 @@ def _dslx_to_pipeline_impl(ctx): ] for flag in bool_flags: value = getattr(ctx.attr, flag) - flags_str += " --{}={}".format(flag, str(value).lower()) + passthrough.append("--{}={}".format(flag, str(value).lower())) # If the attribute is explicitly set ("true" or "false") forward it to # override whatever value may be present in the toolchain config. if ctx.attr.add_invariant_assertions != "": - flags_str += " --add_invariant_assertions={}".format(ctx.attr.add_invariant_assertions) + passthrough.append("--add_invariant_assertions={}".format(ctx.attr.add_invariant_assertions)) # Top entry function flag if ctx.attr.top: @@ -54,29 +54,46 @@ def _dslx_to_pipeline_impl(ctx): fail("Please specify the 'top' entry function to use") if ctx.attr.reset: - flags_str += " --reset={}".format(ctx.attr.reset) + passthrough.append("--reset={}".format(ctx.attr.reset)) output_sv_file = ctx.outputs.sv_file output_unopt_ir_file = ctx.outputs.unopt_ir_file output_opt_ir_file = ctx.outputs.opt_ir_file runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_driver_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml( + ctx, + name = "dslx_to_pipeline", + toolchain = toolchain, + add_invariant_assertions = ctx.attr.add_invariant_assertions, + ) - ctx.actions.run_shell( - inputs = srcs, - tools = [runner], + ctx.actions.run( + inputs = srcs + [toolchain_file] + get_driver_artifact_inputs( + toolchain, + ["ir_converter_main", "opt_main", "codegen_main"], + ), + executable = runner, outputs = [output_sv_file, output_unopt_ir_file, output_opt_ir_file], - command = "\"$1\" driver dslx2pipeline --dslx_input_file=\"$2\" --dslx_top=\"$3\" --output_unopt_ir=\"$4\" --output_opt_ir=\"$5\"" + flags_str + " > \"$6\"", arguments = [ - runner.path, - srcs[0].path, - top_entry, - output_unopt_ir_file.path, - output_opt_ir_file.path, + "driver", + "--driver_path", + toolchain.driver_path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--toolchain", + toolchain_file.path, + "--stdout_path", output_sv_file.path, - ], - use_default_shell_env = True, + "dslx2pipeline", + "--dslx_input_file=" + srcs[0].path, + "--dslx_top=" + top_entry, + "--output_unopt_ir=" + output_unopt_ir_file.path, + "--output_opt_ir=" + output_opt_ir_file.path, + ] + passthrough, + use_default_shell_env = False, ) return DefaultInfo( @@ -119,9 +136,9 @@ DslxToPipelineAttrs = { default = True, ), # Tri-state: "true" / "false" / "" (unspecified). When non-empty the - # provided value overrides the setting in the TOML (which may come from - # XLSYNTH_ADD_INVARIANT_ASSERTIONS). Using a string lets us detect the - # unspecified case, which is not possible with attr.bool. + # provided value overrides the setting in the generated TOML. Using a + # string lets us detect the unspecified case, which is not possible with + # attr.bool. "add_invariant_assertions": attr.string( doc = "Override for invariant assertions generation: 'true', 'false', or leave empty to use toolchain default.", default = "", @@ -139,6 +156,10 @@ DslxToPipelineAttrs = { doc = "The top entry function within the dependency module.", mandatory = True, ), + "xls_bundle": attr.label( + doc = "Optional override bundle repo label, for example @legacy_xls//:bundle.", + providers = [XlsArtifactBundleInfo], + ), } # Keep the public rule signature stable @@ -154,4 +175,5 @@ dslx_to_pipeline = rule( implementation = _dslx_to_pipeline_impl, attrs = DslxToPipelineAttrs, outputs = DslxToPipelineOutputs, + toolchains = ["//:toolchain_type"], ) diff --git a/dslx_to_pipeline_eco.bzl b/dslx_to_pipeline_eco.bzl index 2a495fe..d51885e 100644 --- a/dslx_to_pipeline_eco.bzl +++ b/dslx_to_pipeline_eco.bzl @@ -3,27 +3,27 @@ load(":dslx_provider.bzl", "DslxInfo") load(":env_helpers.bzl", "python_runner_source") load(":helpers.bzl", "get_srcs_from_deps") +load(":xls_toolchain.bzl", "XlsArtifactBundleInfo", "declare_xls_toolchain_toml", "get_driver_artifact_inputs", "get_selected_driver_toolchain") def _dslx_to_pipeline_eco_impl(ctx): srcs = get_srcs_from_deps(ctx) - # Flags for stdlib path - flags_str = "" + passthrough = [] # Delay model flag (required) - flags_str += " --delay_model=" + ctx.attr.delay_model + passthrough.append("--delay_model=" + ctx.attr.delay_model) # Forward string-valued flags when a non-empty value is provided. for flag in ["input_valid_signal", "output_valid_signal", "module_name"]: value = getattr(ctx.attr, flag) if value: - flags_str += " --{}={}".format(flag, value) + passthrough.append("--{}={}".format(flag, value)) # Forward integer-valued timing flags when >0. if ctx.attr.pipeline_stages > 0: - flags_str += " --pipeline_stages={}".format(ctx.attr.pipeline_stages) + passthrough.append("--pipeline_stages={}".format(ctx.attr.pipeline_stages)) if ctx.attr.clock_period_ps > 0: - flags_str += " --clock_period_ps={}".format(ctx.attr.clock_period_ps) + passthrough.append("--clock_period_ps={}".format(ctx.attr.clock_period_ps)) # Validate that either pipeline_stages or clock_period_ps is specified (>0). if ctx.attr.pipeline_stages == 0 and ctx.attr.clock_period_ps == 0: @@ -40,12 +40,12 @@ def _dslx_to_pipeline_eco_impl(ctx): ] for flag in bool_flags: value = getattr(ctx.attr, flag) - flags_str += " --{}={}".format(flag, str(value).lower()) + passthrough.append("--{}={}".format(flag, str(value).lower())) # If the attribute is explicitly set ("true" or "false") forward it to # override whatever value may be present in the toolchain config. if ctx.attr.add_invariant_assertions != "": - flags_str += " --add_invariant_assertions={}".format(ctx.attr.add_invariant_assertions) + passthrough.append("--add_invariant_assertions={}".format(ctx.attr.add_invariant_assertions)) # Top entry function flag if ctx.attr.top: @@ -58,7 +58,7 @@ def _dslx_to_pipeline_eco_impl(ctx): srcs.append(ctx.file.baseline_unopt_ir) if ctx.attr.reset: - flags_str += " --reset={}".format(ctx.attr.reset) + passthrough.append("--reset={}".format(ctx.attr.reset)) output_sv_file = ctx.outputs.sv_file output_unopt_ir_file = ctx.outputs.unopt_ir_file @@ -68,25 +68,42 @@ def _dslx_to_pipeline_eco_impl(ctx): output_eco_edit_file = ctx.outputs.eco_edit_file runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_driver_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml( + ctx, + name = "dslx_to_pipeline_eco", + toolchain = toolchain, + add_invariant_assertions = ctx.attr.add_invariant_assertions, + ) - ctx.actions.run_shell( - inputs = srcs, - tools = [runner], + ctx.actions.run( + inputs = srcs + [toolchain_file] + get_driver_artifact_inputs( + toolchain, + ["ir_converter_main", "opt_main", "codegen_main", "block_to_verilog_main"], + ), + executable = runner, outputs = [output_sv_file, output_unopt_ir_file, output_opt_ir_file, output_baseline_verilog_file, output_eco_edit_file], - command = "\"$1\" driver dslx2pipeline-eco --dslx_input_file=\"$2\" --dslx_top=\"$3\" --baseline_unopt_ir=\"$4\" --output_unopt_ir=\"$5\" --output_opt_ir=\"$6\" --output_baseline_verilog_path=\"$7\" --edits_debug_out=\"$9\"" + flags_str + " > \"$8\"", arguments = [ - runner.path, - srcs[0].path, - top_entry, - baseline_unopt_ir_file.path, - output_unopt_ir_file.path, - output_opt_ir_file.path, - output_baseline_verilog_file.path, + "driver", + "--driver_path", + toolchain.driver_path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--toolchain", + toolchain_file.path, + "--stdout_path", output_sv_file.path, - output_eco_edit_file.path, - ], - use_default_shell_env = True, + "dslx2pipeline-eco", + "--dslx_input_file=" + srcs[0].path, + "--dslx_top=" + top_entry, + "--baseline_unopt_ir=" + baseline_unopt_ir_file.path, + "--output_unopt_ir=" + output_unopt_ir_file.path, + "--output_opt_ir=" + output_opt_ir_file.path, + "--output_baseline_verilog_path=" + output_baseline_verilog_file.path, + "--edits_debug_out=" + output_eco_edit_file.path, + ] + passthrough, + use_default_shell_env = False, ) return DefaultInfo( @@ -129,9 +146,9 @@ DslxToPipelineEcoAttrs = { default = True, ), # Tri-state: "true" / "false" / "" (unspecified). When non-empty the - # provided value overrides the setting in the TOML (which may come from - # XLSYNTH_ADD_INVARIANT_ASSERTIONS). Using a string lets us detect the - # unspecified case, which is not possible with attr.bool. + # provided value overrides the setting in the generated TOML. Using a + # string lets us detect the unspecified case, which is not possible with + # attr.bool. "add_invariant_assertions": attr.string( doc = "Override for invariant assertions generation: 'true', 'false', or leave empty to use toolchain default.", default = "", @@ -154,6 +171,10 @@ DslxToPipelineEcoAttrs = { mandatory = True, allow_single_file = [".ir"], ), + "xls_bundle": attr.label( + doc = "Optional override bundle repo label, for example @legacy_xls//:bundle.", + providers = [XlsArtifactBundleInfo], + ), } # Keep the public rule signature stable @@ -170,4 +191,5 @@ dslx_to_pipeline_eco = rule( implementation = _dslx_to_pipeline_eco_impl, attrs = DslxToPipelineEcoAttrs, outputs = DslxToPipelineEcoOutputs, + toolchains = ["//:toolchain_type"], ) diff --git a/dslx_to_sv_types.bzl b/dslx_to_sv_types.bzl index 0f9e7d2..717641e 100644 --- a/dslx_to_sv_types.bzl +++ b/dslx_to_sv_types.bzl @@ -3,6 +3,7 @@ load(":dslx_provider.bzl", "DslxInfo") load(":env_helpers.bzl", "python_runner_source") load(":helpers.bzl", "get_srcs_from_deps") +load(":xls_toolchain.bzl", "XlsArtifactBundleInfo", "declare_xls_toolchain_toml", "get_driver_artifact_inputs", "get_selected_driver_toolchain") _SV_ENUM_CASE_NAMING_POLICIES = [ "unqualified", @@ -14,31 +15,52 @@ _SV_STRUCT_FIELD_ORDERING_POLICIES = [ "reversed", ] +_DEFAULT_SV_STRUCT_FIELD_ORDERING_POLICY = "as_declared" + def _dslx_to_sv_types_impl(ctx): srcs = get_srcs_from_deps(ctx) output_sv_file = ctx.outputs.sv_file runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_driver_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "dslx_to_sv_types", toolchain = toolchain) + arguments = [ + "driver", + "--driver_path", + toolchain.driver_path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--toolchain", + toolchain_file.path, + "--stdout_path", + output_sv_file.path, + "dslx2sv-types", + "--dslx_input_file", + srcs[0].path, + ] - struct_field_ordering_arg = ( - "--sv_struct_field_ordering=" + ctx.attr.sv_struct_field_ordering - ) + if toolchain.driver_supports_sv_struct_field_ordering: + arguments.append( + "--sv_struct_field_ordering=" + ctx.attr.sv_struct_field_ordering, + ) + else: + if ctx.attr.sv_struct_field_ordering != _DEFAULT_SV_STRUCT_FIELD_ORDERING_POLICY: + fail("sv_struct_field_ordering={} requires a selected XLS bundle whose xlsynth-driver supports that flag".format(ctx.attr.sv_struct_field_ordering)) - ctx.actions.run_shell( - inputs = srcs, - tools = [runner], + if toolchain.driver_supports_sv_enum_case_naming_policy: + arguments.append("--sv_enum_case_naming_policy=" + ctx.attr.sv_enum_case_naming_policy) + else: + if ctx.attr.sv_enum_case_naming_policy != "unqualified": + fail("sv_enum_case_naming_policy={} requires a selected XLS bundle whose xlsynth-driver supports that flag".format(ctx.attr.sv_enum_case_naming_policy)) + + ctx.actions.run( + inputs = srcs + [toolchain_file] + get_driver_artifact_inputs(toolchain), + executable = runner, outputs = [output_sv_file], - command = "\"$1\" driver dslx2sv-types --dslx_input_file=\"$2\" \"$3\" $4 > \"$5\"", - arguments = [ - runner.path, - srcs[0].path, - "--sv_enum_case_naming_policy=" + ctx.attr.sv_enum_case_naming_policy, - struct_field_ordering_arg, - output_sv_file.path, - ], - use_default_shell_env = True, + arguments = arguments, + use_default_shell_env = False, ) return DefaultInfo( @@ -60,11 +82,16 @@ dslx_to_sv_types = rule( ), "sv_struct_field_ordering": attr.string( doc = "Struct field ordering policy passed to xlsynth-driver (`as_declared` or `reversed`).", - default = "as_declared", + default = _DEFAULT_SV_STRUCT_FIELD_ORDERING_POLICY, values = _SV_STRUCT_FIELD_ORDERING_POLICIES, ), + "xls_bundle": attr.label( + doc = "Optional override bundle repo label, for example @legacy_xls//:bundle.", + providers = [XlsArtifactBundleInfo], + ), }, outputs = { "sv_file": "%{name}.sv", }, + toolchains = ["//:toolchain_type"], ) diff --git a/env_helpers.bzl b/env_helpers.bzl index 25f3b6c..237b585 100644 --- a/env_helpers.bzl +++ b/env_helpers.bzl @@ -1,94 +1,47 @@ def python_runner_source(): """Returns the embedded xlsynth_runner.py source. - The returned program reads XLSYNTH_* from the action execution environment - and invokes either the driver (via the 'driver' subcommand) or a tool - (via the 'tool' subcommand), forwarding passthrough flags accordingly. + The returned program reads a declared toolchain TOML input and invokes either + the driver (via the 'driver' subcommand) or a tool (via the 'tool' + subcommand), forwarding passthrough flags accordingly. """ return """#!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 import argparse +import ast import os import subprocess import sys -import tempfile from enum import Enum -from typing import List, Optional, Tuple, Dict, NamedTuple +from typing import Any, Dict, List, NamedTuple, Optional -def _bool_env_to_toml(name: str, val: str) -> Optional[str]: - v = val.strip() - if v == "": - return None - if v not in ("true", "false"): - raise ValueError(f"Invalid value for {name}: {val}") - return v - - -def _get_env(name: str) -> str: - return os.environ.get(name, "").strip() - - -def _require_env(name: str) -> str: - v = _get_env(name) - if not v: - raise RuntimeError(f"Please set {name} environment variable") - return v - - -def _get_env_list(name: str, separator: str) -> List[str]: - v = _get_env(name) - return v.split(separator) if v else [] - - -def _get_bool_toml_from_env(name: str) -> Optional[str]: - v = _get_env(name) - return _bool_env_to_toml(name, v) if v != "" else None - - -def _toml_lines_from_regular_envs(pairs: List[Tuple[str, str]]) -> List[str]: - lines: List[str] = [] - for env_name, toml_key in pairs: - v = _get_env(env_name) - if v: - lines.append(f"{toml_key} = {repr(v)}") - return lines - - -def _toml_lines_from_bool_envs(pairs: List[Tuple[str, str]]) -> List[str]: - lines: List[str] = [] - for env_name, toml_key in pairs: - b = _get_bool_toml_from_env(env_name) - if b is not None: - lines.append(f"{toml_key} = {b}") - return lines - - -# Declarative per-tool configuration for extra flags _TOOL_CONFIG = { "dslx_interpreter_main": { "base_flags": ["--compare=jit", "--alsologtostderr"], - "env_flags": [ - "XLSYNTH_DSLX_PATH", - "XLSYNTH_DSLX_ENABLE_WARNINGS", - "XLSYNTH_DSLX_DISABLE_WARNINGS", - ], + "dslx_config": True, + "dslx_scalar_settings": [], + "needs_dslx_stdlib_flag": True, }, "prove_quickcheck_main": { "base_flags": ["--alsologtostderr"], - "env_flags": [ - "XLSYNTH_DSLX_PATH", - ], + "dslx_config": True, + "dslx_scalar_settings": [], + "needs_dslx_stdlib_flag": True, }, "typecheck_main": { "base_flags": [], - "env_flags": [ - "XLSYNTH_DSLX_PATH", - "XLSYNTH_DSLX_ENABLE_WARNINGS", - "XLSYNTH_DSLX_DISABLE_WARNINGS", - ], + "dslx_config": True, + "dslx_scalar_settings": [], + "needs_dslx_stdlib_flag": True, + }, + "dslx_fmt": { + "base_flags": [], + "dslx_config": False, + "dslx_scalar_settings": [], + "needs_dslx_stdlib_flag": False, }, } @@ -102,19 +55,18 @@ class EnvFlagSpec(NamedTuple): mode: EnvFlagMode -# Declarative mapping from env var to flag-building behavior -_ENV_FLAG_SPECS: Dict[str, EnvFlagSpec] = { - "XLSYNTH_DSLX_PATH": +_DSLX_FLAG_SPECS: Dict[str, EnvFlagSpec] = { + "dslx_path": EnvFlagSpec("dslx_path", EnvFlagMode.PASSTHROUGH_IF_NONEMPTY), - "XLSYNTH_DSLX_ENABLE_WARNINGS": + "enable_warnings": EnvFlagSpec("enable_warnings", EnvFlagMode.PASSTHROUGH_IF_NONEMPTY), - "XLSYNTH_DSLX_DISABLE_WARNINGS": + "disable_warnings": EnvFlagSpec("disable_warnings", EnvFlagMode.PASSTHROUGH_IF_NONEMPTY), } -def _env_flag_builder(env_name: str, value: str) -> List[str]: - spec = _ENV_FLAG_SPECS.get(env_name) +def _setting_flag_builder(setting_name: str, value: str) -> List[str]: + spec = _DSLX_FLAG_SPECS.get(setting_name) if not spec: return [] @@ -124,90 +76,196 @@ def _env_flag_builder(env_name: str, value: str) -> List[str]: return [] -def _build_extra_args_for_tool(tool: str, tools_dir: str) -> List[str]: +def _parse_scalar(value_text: str) -> Any: + if value_text == "true": + return True + if value_text == "false": + return False + return ast.literal_eval(value_text) + + +def _parse_toolchain_toml(path: str) -> Dict[str, Any]: + parsed: Dict[str, Any] = {} + section_stack: List[str] = [] + with open(path, "r", encoding = "utf-8") as f: + for raw_line in f: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("[") and line.endswith("]"): + section_stack = line[1:-1].split(".") + continue + key, value_text = line.split("=", 1) + key = key.strip() + value_text = value_text.strip() + target = parsed + for section_name in section_stack: + target = target.setdefault(section_name, {}) + target[key] = _parse_scalar(value_text) + return parsed + + +def _toolchain_dslx_config(toolchain_data: Dict[str, Any]) -> Dict[str, Any]: + toolchain_section = toolchain_data.get("toolchain", {}) + return toolchain_section.get("dslx", {}) + + +def _toolchain_tool_path(toolchain_data: Dict[str, Any]) -> str: + toolchain_section = toolchain_data.get("toolchain", {}) + tool_path = toolchain_section.get("tool_path", "") + return _resolve_runtime_path(tool_path) + + +def _runfiles_roots() -> List[str]: + roots: List[str] = [] + for env_var in ["RUNFILES_DIR", "TEST_SRCDIR"]: + value = os.environ.get(env_var) + if value and value not in roots: + roots.append(value) + return roots + + +def _runfiles_candidates(path: str) -> List[str]: + candidates = [path] + for marker in ["external/", "_main/"]: + marker_index = path.find(marker) + if marker_index != -1: + candidate = path[marker_index + len(marker):] + if candidate not in candidates: + candidates.append(candidate) + prefixed = "_main/" + candidate + if prefixed not in candidates: + candidates.append(prefixed) + return candidates + + +def _resolve_runtime_path(path: str) -> str: + if not path or os.path.isabs(path): + return path + + for root in _runfiles_roots(): + for candidate in _runfiles_candidates(path): + resolved = os.path.join(root, candidate) + if os.path.exists(resolved): + return resolved + return path + + +def _resolve_executable_path(path: str) -> str: + return _resolve_runtime_path(path) + + +def _runtime_library_env_var(sys_platform: str) -> str: + if sys_platform == "darwin": + return "DYLD_LIBRARY_PATH" + return "LD_LIBRARY_PATH" + + +def _build_extra_args_for_tool(tool: str, toolchain_data: Dict[str, Any]) -> List[str]: cfg = _TOOL_CONFIG.get(tool) if not cfg: return [] - stdlib = os.path.join(tools_dir, "xls", "dslx", "stdlib") - extra: List[str] = [f"--dslx_stdlib_path={stdlib}"] + dslx_cfg = _toolchain_dslx_config(toolchain_data) + extra: List[str] = [] + if cfg.get("needs_dslx_stdlib_flag", True): + stdlib = _resolve_runtime_path(dslx_cfg.get("dslx_stdlib_path", "")) + if not stdlib: + raise RuntimeError("Toolchain TOML is missing toolchain.dslx.dslx_stdlib_path") + extra.append(f"--dslx_stdlib_path={stdlib}") extra.extend(cfg.get("base_flags", [])) - for env_name in cfg.get("env_flags", []): - v = _get_env(env_name) or "" - extra.extend(_env_flag_builder(env_name, v)) + if cfg.get("dslx_config"): + list_settings = [ + ("dslx_path", ":"), + ("enable_warnings", ","), + ("disable_warnings", ","), + ] + for setting_name, separator in list_settings: + values = dslx_cfg.get(setting_name, []) + joined = separator.join(values) + extra.extend(_setting_flag_builder(setting_name, joined)) + for setting_name in cfg.get("dslx_scalar_settings", []): + setting_value = dslx_cfg.get(setting_name) + if setting_value is not None: + extra.extend(_setting_flag_builder( + setting_name, + "true" if setting_value else "false", + )) return extra -def _build_toolchain_toml(tool_dir: str) -> str: - dslx_stdlib_path = os.path.join(tool_dir, "xls", "dslx", "stdlib") - - additional_dslx_paths_list = _get_env_list("XLSYNTH_DSLX_PATH", ":") - enable_warnings_list = _get_env_list("XLSYNTH_DSLX_ENABLE_WARNINGS", ",") - disable_warnings_list = _get_env_list("XLSYNTH_DSLX_DISABLE_WARNINGS", ",") - - codegen_regular_envs: List[Tuple[str, str]] = [ - ("XLSYNTH_GATE_FORMAT", "gate_format"), - ("XLSYNTH_ASSERT_FORMAT", "assert_format"), - ] - codegen_bool_envs: List[Tuple[str, str]] = [ - ("XLSYNTH_USE_SYSTEM_VERILOG", "use_system_verilog"), - ("XLSYNTH_ADD_INVARIANT_ASSERTIONS", "add_invariant_assertions"), - ] - - lines: List[str] = [] - lines.append("[toolchain]") - lines.append(f'tool_path = "{tool_dir}"') - lines.append("") - lines.append("[toolchain.dslx]") - lines.append(f'dslx_stdlib_path = "{dslx_stdlib_path}"') - lines.append(f"dslx_path = {repr(additional_dslx_paths_list)}") - lines.append(f"enable_warnings = {repr(enable_warnings_list)}") - lines.append(f"disable_warnings = {repr(disable_warnings_list)}") - lines.append("") - lines.append("[toolchain.codegen]") - lines.extend(_toml_lines_from_regular_envs(codegen_regular_envs)) - lines.extend(_toml_lines_from_bool_envs(codegen_bool_envs)) - - # Use escaped newlines so the generated Python remains single-line literals. - return "\\n".join(lines) + "\\n" - - -def _driver(args: argparse.Namespace) -> int: - tools_dir = _require_env("XLSYNTH_TOOLS") - driver_dir = _require_env("XLSYNTH_DRIVER_DIR") - driver_path = os.path.join(driver_dir, "xlsynth-driver") - passthrough = list(args.passthrough) - toml = _build_toolchain_toml(tools_dir) - with tempfile.NamedTemporaryFile("w", - delete=False, - prefix="xlsynth_toolchain_", - suffix=".toml") as tf: - tf.write(toml) - toolchain_path = tf.name +def _run_subprocess( + cmd: List[str], + *, + extra_env: Optional[Dict[str, str]] = None, + runtime_library_path: str, + stdout_path: str, + sys_platform: str = sys.platform) -> int: + env = os.environ.copy() + resolved_runtime_library_path = _resolve_runtime_path(runtime_library_path) + if resolved_runtime_library_path: + runtime_env_var = _runtime_library_env_var(sys_platform) + existing = env.get(runtime_env_var, "") + env[runtime_env_var] = ( + resolved_runtime_library_path + if not existing + else resolved_runtime_library_path + os.pathsep + existing + ) + if extra_env is not None: + for key, value in extra_env.items(): + if value: + env[key] = value + stdout_handle = None + stdout_stream = None + if stdout_path: + stdout_handle = open(stdout_path, "wb") + stdout_stream = stdout_handle try: - cmd = [ - driver_path, f"--toolchain={toolchain_path}", args.subcommand, - *passthrough - ] - proc = subprocess.run(cmd, check=False) + proc = subprocess.run(cmd, check = False, env = env, stdout = stdout_stream) return proc.returncode finally: - try: - os.unlink(toolchain_path) - except OSError: - pass + if stdout_handle is not None: + stdout_handle.close() + + +def _driver(args: argparse.Namespace) -> int: + toolchain_data = _parse_toolchain_toml(args.toolchain) + extra_env = {} + tool_path = _toolchain_tool_path(toolchain_data) + if tool_path: + # Older driver releases still discover external prover tools through + # XLSYNTH_TOOLS even when --toolchain is provided. + extra_env["XLSYNTH_TOOLS"] = tool_path + cmd = [ + _resolve_executable_path(args.driver_path), + f"--toolchain={args.toolchain}", + args.subcommand, + *list(args.passthrough), + ] + return _run_subprocess( + cmd, + extra_env = extra_env, + runtime_library_path = args.runtime_library_path, + stdout_path = args.stdout_path, + ) def _tool(args: argparse.Namespace) -> int: - tools_dir = _require_env("XLSYNTH_TOOLS") - tool_path = os.path.join(tools_dir, args.tool) + toolchain_data = _parse_toolchain_toml(args.toolchain) + tool_path_root = _toolchain_tool_path(toolchain_data) + if not tool_path_root: + raise RuntimeError("Toolchain TOML is missing toolchain.tool_path") + tool_path = _resolve_runtime_path(os.path.join(tool_path_root, args.tool)) passthrough = list(args.passthrough) - extra = _build_extra_args_for_tool(args.tool, tools_dir) + extra = _build_extra_args_for_tool(args.tool, toolchain_data) if extra: passthrough = extra + passthrough cmd = [tool_path, *passthrough] - proc = subprocess.run(cmd, check=False) - return proc.returncode + return _run_subprocess( + cmd, + runtime_library_path = args.runtime_library_path, + stdout_path = args.stdout_path, + ) def main(argv: List[str]) -> int: @@ -217,10 +275,17 @@ def main(argv: List[str]) -> int: # No global arguments; subcommands define their own. p_driver = sub.add_parser("driver") + p_driver.add_argument("--driver_path", required=True) + p_driver.add_argument("--runtime_library_path", default="") + p_driver.add_argument("--stdout_path", default="") + p_driver.add_argument("--toolchain", required=True) p_driver.add_argument("subcommand") p_driver.set_defaults(func=_driver) p_tool = sub.add_parser("tool") + p_tool.add_argument("--runtime_library_path", default="") + p_tool.add_argument("--stdout_path", default="") + p_tool.add_argument("--toolchain", required=True) p_tool.add_argument("tool") p_tool.set_defaults(func=_tool) diff --git a/env_helpers.py b/env_helpers.py index 3a60d1c..66b1b28 100644 --- a/env_helpers.py +++ b/env_helpers.py @@ -2,85 +2,38 @@ # SPDX-License-Identifier: Apache-2.0 import argparse +import ast import os import subprocess import sys -import tempfile from enum import Enum -from typing import List, Optional, Tuple, Dict, NamedTuple +from typing import Any, Dict, List, NamedTuple, Optional -def _bool_env_to_toml(name: str, val: str) -> Optional[str]: - v = val.strip() - if v == "": - return None - if v not in ("true", "false"): - raise ValueError(f"Invalid value for {name}: {val}") - return v - - -def _get_env(name: str) -> str: - return os.environ.get(name, "").strip() - - -def _require_env(name: str) -> str: - v = _get_env(name) - if not v: - raise RuntimeError(f"Please set {name} environment variable") - return v - - -def _get_env_list(name: str, separator: str) -> List[str]: - v = _get_env(name) - return v.split(separator) if v else [] - - -def _get_bool_toml_from_env(name: str) -> Optional[str]: - v = _get_env(name) - return _bool_env_to_toml(name, v) if v != "" else None - - -def _toml_lines_from_regular_envs(pairs: List[Tuple[str, str]]) -> List[str]: - lines: List[str] = [] - for env_name, toml_key in pairs: - v = _get_env(env_name) - if v: - lines.append(f"{toml_key} = {repr(v)}") - return lines - - -def _toml_lines_from_bool_envs(pairs: List[Tuple[str, str]]) -> List[str]: - lines: List[str] = [] - for env_name, toml_key in pairs: - b = _get_bool_toml_from_env(env_name) - if b is not None: - lines.append(f"{toml_key} = {b}") - return lines - - -# Declarative per-tool configuration for extra flags _TOOL_CONFIG = { "dslx_interpreter_main": { "base_flags": ["--compare=jit", "--alsologtostderr"], - "env_flags": [ - "XLSYNTH_DSLX_PATH", - "XLSYNTH_DSLX_ENABLE_WARNINGS", - "XLSYNTH_DSLX_DISABLE_WARNINGS", - ], + "dslx_config": True, + "dslx_scalar_settings": [], + "needs_dslx_stdlib_flag": True, }, "prove_quickcheck_main": { "base_flags": ["--alsologtostderr"], - "env_flags": [ - "XLSYNTH_DSLX_PATH", - ], + "dslx_config": True, + "dslx_scalar_settings": [], + "needs_dslx_stdlib_flag": True, }, "typecheck_main": { "base_flags": [], - "env_flags": [ - "XLSYNTH_DSLX_PATH", - "XLSYNTH_DSLX_ENABLE_WARNINGS", - "XLSYNTH_DSLX_DISABLE_WARNINGS", - ], + "dslx_config": True, + "dslx_scalar_settings": [], + "needs_dslx_stdlib_flag": True, + }, + "dslx_fmt": { + "base_flags": [], + "dslx_config": False, + "dslx_scalar_settings": [], + "needs_dslx_stdlib_flag": False, }, } @@ -94,19 +47,18 @@ class EnvFlagSpec(NamedTuple): mode: EnvFlagMode -# Declarative mapping from env var to flag-building behavior -_ENV_FLAG_SPECS: Dict[str, EnvFlagSpec] = { - "XLSYNTH_DSLX_PATH": +_DSLX_FLAG_SPECS: Dict[str, EnvFlagSpec] = { + "dslx_path": EnvFlagSpec("dslx_path", EnvFlagMode.PASSTHROUGH_IF_NONEMPTY), - "XLSYNTH_DSLX_ENABLE_WARNINGS": + "enable_warnings": EnvFlagSpec("enable_warnings", EnvFlagMode.PASSTHROUGH_IF_NONEMPTY), - "XLSYNTH_DSLX_DISABLE_WARNINGS": + "disable_warnings": EnvFlagSpec("disable_warnings", EnvFlagMode.PASSTHROUGH_IF_NONEMPTY), } -def _env_flag_builder(env_name: str, value: str) -> List[str]: - spec = _ENV_FLAG_SPECS.get(env_name) +def _setting_flag_builder(setting_name: str, value: str) -> List[str]: + spec = _DSLX_FLAG_SPECS.get(setting_name) if not spec: return [] @@ -116,90 +68,196 @@ def _env_flag_builder(env_name: str, value: str) -> List[str]: return [] -def _build_extra_args_for_tool(tool: str, tools_dir: str) -> List[str]: +def _parse_scalar(value_text: str) -> Any: + if value_text == "true": + return True + if value_text == "false": + return False + return ast.literal_eval(value_text) + + +def _parse_toolchain_toml(path: str) -> Dict[str, Any]: + parsed: Dict[str, Any] = {} + section_stack: List[str] = [] + with open(path, "r", encoding = "utf-8") as f: + for raw_line in f: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("[") and line.endswith("]"): + section_stack = line[1:-1].split(".") + continue + key, value_text = line.split("=", 1) + key = key.strip() + value_text = value_text.strip() + target = parsed + for section_name in section_stack: + target = target.setdefault(section_name, {}) + target[key] = _parse_scalar(value_text) + return parsed + + +def _toolchain_dslx_config(toolchain_data: Dict[str, Any]) -> Dict[str, Any]: + toolchain_section = toolchain_data.get("toolchain", {}) + return toolchain_section.get("dslx", {}) + + +def _toolchain_tool_path(toolchain_data: Dict[str, Any]) -> str: + toolchain_section = toolchain_data.get("toolchain", {}) + tool_path = toolchain_section.get("tool_path", "") + return _resolve_runtime_path(tool_path) + + +def _runfiles_roots() -> List[str]: + roots: List[str] = [] + for env_var in ["RUNFILES_DIR", "TEST_SRCDIR"]: + value = os.environ.get(env_var) + if value and value not in roots: + roots.append(value) + return roots + + +def _runfiles_candidates(path: str) -> List[str]: + candidates = [path] + for marker in ["external/", "_main/"]: + marker_index = path.find(marker) + if marker_index != -1: + candidate = path[marker_index + len(marker):] + if candidate not in candidates: + candidates.append(candidate) + prefixed = "_main/" + candidate + if prefixed not in candidates: + candidates.append(prefixed) + return candidates + + +def _resolve_runtime_path(path: str) -> str: + if not path or os.path.isabs(path): + return path + + for root in _runfiles_roots(): + for candidate in _runfiles_candidates(path): + resolved = os.path.join(root, candidate) + if os.path.exists(resolved): + return resolved + return path + + +def _resolve_executable_path(path: str) -> str: + return _resolve_runtime_path(path) + + +def _runtime_library_env_var(sys_platform: str) -> str: + if sys_platform == "darwin": + return "DYLD_LIBRARY_PATH" + return "LD_LIBRARY_PATH" + + +def _build_extra_args_for_tool(tool: str, toolchain_data: Dict[str, Any]) -> List[str]: cfg = _TOOL_CONFIG.get(tool) if not cfg: return [] - stdlib = os.path.join(tools_dir, "xls", "dslx", "stdlib") - extra: List[str] = [f"--dslx_stdlib_path={stdlib}"] + dslx_cfg = _toolchain_dslx_config(toolchain_data) + extra: List[str] = [] + if cfg.get("needs_dslx_stdlib_flag", True): + stdlib = _resolve_runtime_path(dslx_cfg.get("dslx_stdlib_path", "")) + if not stdlib: + raise RuntimeError("Toolchain TOML is missing toolchain.dslx.dslx_stdlib_path") + extra.append(f"--dslx_stdlib_path={stdlib}") extra.extend(cfg.get("base_flags", [])) - for env_name in cfg.get("env_flags", []): - v = _get_env(env_name) or "" - extra.extend(_env_flag_builder(env_name, v)) + if cfg.get("dslx_config"): + list_settings = [ + ("dslx_path", ":"), + ("enable_warnings", ","), + ("disable_warnings", ","), + ] + for setting_name, separator in list_settings: + values = dslx_cfg.get(setting_name, []) + joined = separator.join(values) + extra.extend(_setting_flag_builder(setting_name, joined)) + for setting_name in cfg.get("dslx_scalar_settings", []): + setting_value = dslx_cfg.get(setting_name) + if setting_value is not None: + extra.extend(_setting_flag_builder( + setting_name, + "true" if setting_value else "false", + )) return extra -def _build_toolchain_toml(tool_dir: str) -> str: - dslx_stdlib_path = os.path.join(tool_dir, "xls", "dslx", "stdlib") - - additional_dslx_paths_list = _get_env_list("XLSYNTH_DSLX_PATH", ":") - enable_warnings_list = _get_env_list("XLSYNTH_DSLX_ENABLE_WARNINGS", ",") - disable_warnings_list = _get_env_list("XLSYNTH_DSLX_DISABLE_WARNINGS", ",") - - codegen_regular_envs: List[Tuple[str, str]] = [ - ("XLSYNTH_GATE_FORMAT", "gate_format"), - ("XLSYNTH_ASSERT_FORMAT", "assert_format"), - ] - codegen_bool_envs: List[Tuple[str, str]] = [ - ("XLSYNTH_USE_SYSTEM_VERILOG", "use_system_verilog"), - ("XLSYNTH_ADD_INVARIANT_ASSERTIONS", "add_invariant_assertions"), - ] - - lines: List[str] = [] - lines.append("[toolchain]") - lines.append(f'tool_path = "{tool_dir}"') - lines.append("") - lines.append("[toolchain.dslx]") - lines.append(f'dslx_stdlib_path = "{dslx_stdlib_path}"') - lines.append(f"dslx_path = {repr(additional_dslx_paths_list)}") - lines.append(f"enable_warnings = {repr(enable_warnings_list)}") - lines.append(f"disable_warnings = {repr(disable_warnings_list)}") - lines.append("") - lines.append("[toolchain.codegen]") - lines.extend(_toml_lines_from_regular_envs(codegen_regular_envs)) - lines.extend(_toml_lines_from_bool_envs(codegen_bool_envs)) - - # Use escaped newlines so the generated Python remains single-line literals. - return "\\n".join(lines) + "\\n" - - -def _driver(args: argparse.Namespace) -> int: - tools_dir = _require_env("XLSYNTH_TOOLS") - driver_dir = _require_env("XLSYNTH_DRIVER_DIR") - driver_path = os.path.join(driver_dir, "xlsynth-driver") - passthrough = list(args.passthrough) - toml = _build_toolchain_toml(tools_dir) - with tempfile.NamedTemporaryFile("w", - delete=False, - prefix="xlsynth_toolchain_", - suffix=".toml") as tf: - tf.write(toml) - toolchain_path = tf.name +def _run_subprocess( + cmd: List[str], + *, + extra_env: Optional[Dict[str, str]] = None, + runtime_library_path: str, + stdout_path: str, + sys_platform: str = sys.platform) -> int: + env = os.environ.copy() + resolved_runtime_library_path = _resolve_runtime_path(runtime_library_path) + if resolved_runtime_library_path: + runtime_env_var = _runtime_library_env_var(sys_platform) + existing = env.get(runtime_env_var, "") + env[runtime_env_var] = ( + resolved_runtime_library_path + if not existing + else resolved_runtime_library_path + os.pathsep + existing + ) + if extra_env is not None: + for key, value in extra_env.items(): + if value: + env[key] = value + stdout_handle = None + stdout_stream = None + if stdout_path: + stdout_handle = open(stdout_path, "wb") + stdout_stream = stdout_handle try: - cmd = [ - driver_path, f"--toolchain={toolchain_path}", args.subcommand, - *passthrough - ] - proc = subprocess.run(cmd, check=False) + proc = subprocess.run(cmd, check = False, env = env, stdout = stdout_stream) return proc.returncode finally: - try: - os.unlink(toolchain_path) - except OSError: - pass + if stdout_handle is not None: + stdout_handle.close() + + +def _driver(args: argparse.Namespace) -> int: + toolchain_data = _parse_toolchain_toml(args.toolchain) + extra_env = {} + tool_path = _toolchain_tool_path(toolchain_data) + if tool_path: + # Older driver releases still discover external prover tools through + # XLSYNTH_TOOLS even when --toolchain is provided. + extra_env["XLSYNTH_TOOLS"] = tool_path + cmd = [ + _resolve_executable_path(args.driver_path), + f"--toolchain={args.toolchain}", + args.subcommand, + *list(args.passthrough), + ] + return _run_subprocess( + cmd, + extra_env = extra_env, + runtime_library_path = args.runtime_library_path, + stdout_path = args.stdout_path, + ) def _tool(args: argparse.Namespace) -> int: - tools_dir = _require_env("XLSYNTH_TOOLS") - tool_path = os.path.join(tools_dir, args.tool) + toolchain_data = _parse_toolchain_toml(args.toolchain) + tool_path_root = _toolchain_tool_path(toolchain_data) + if not tool_path_root: + raise RuntimeError("Toolchain TOML is missing toolchain.tool_path") + tool_path = _resolve_runtime_path(os.path.join(tool_path_root, args.tool)) passthrough = list(args.passthrough) - extra = _build_extra_args_for_tool(args.tool, tools_dir) + extra = _build_extra_args_for_tool(args.tool, toolchain_data) if extra: passthrough = extra + passthrough cmd = [tool_path, *passthrough] - proc = subprocess.run(cmd, check=False) - return proc.returncode + return _run_subprocess( + cmd, + runtime_library_path = args.runtime_library_path, + stdout_path = args.stdout_path, + ) def main(argv: List[str]) -> int: @@ -209,10 +267,17 @@ def main(argv: List[str]) -> int: # No global arguments; subcommands define their own. p_driver = sub.add_parser("driver") + p_driver.add_argument("--driver_path", required=True) + p_driver.add_argument("--runtime_library_path", default="") + p_driver.add_argument("--stdout_path", default="") + p_driver.add_argument("--toolchain", required=True) p_driver.add_argument("subcommand") p_driver.set_defaults(func=_driver) p_tool = sub.add_parser("tool") + p_tool.add_argument("--runtime_library_path", default="") + p_tool.add_argument("--stdout_path", default="") + p_tool.add_argument("--toolchain", required=True) p_tool.add_argument("tool") p_tool.set_defaults(func=_tool) diff --git a/env_helpers_test.py b/env_helpers_test.py new file mode 100644 index 0000000..cb02f9b --- /dev/null +++ b/env_helpers_test.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +import env_helpers + + +class EnvHelpersTest(unittest.TestCase): + + def test_driver_resolves_runfiles_relative_path(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + runfiles_root = tmp_path / "runfiles" + driver_path = runfiles_root / "+xls+rules_xlsynth_selftest_xls" / "xlsynth-driver" + driver_path.parent.mkdir(parents = True) + driver_path.write_text("#!/bin/sh\n", encoding = "utf-8") + toolchain_path = tmp_path / "toolchain.toml" + toolchain_path.write_text("[toolchain]\n", encoding = "utf-8") + + argv = mock.Mock( + driver_path = "external/+xls+rules_xlsynth_selftest_xls/xlsynth-driver", + toolchain = str(toolchain_path), + subcommand = "--version", + passthrough = [], + runtime_library_path = "", + stdout_path = "", + ) + + captured = {} + + def fake_run(cmd, check = False, env = None, stdout = None): + captured["cmd"] = list(cmd) + + class Result: + returncode = 0 + + return Result() + + with mock.patch.dict(os.environ, {"RUNFILES_DIR": str(runfiles_root)}, clear = False): + with mock.patch.object(env_helpers.subprocess, "run", side_effect = fake_run): + self.assertEqual(env_helpers._driver(argv), 0) + + self.assertEqual(captured["cmd"][0], str(driver_path)) + + def test_driver_exports_xlsynth_tools_from_toolchain(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + runfiles_root = tmp_path / "runfiles" + tools_path = runfiles_root / "+xls+rules_xlsynth_selftest_xls" / "tools" + tools_path.mkdir(parents = True) + toolchain_path = tmp_path / "toolchain.toml" + toolchain_path.write_text( + "[toolchain]\n" + "tool_path = \"external/+xls+rules_xlsynth_selftest_xls/tools\"\n", + encoding = "utf-8", + ) + + argv = mock.Mock( + driver_path = "/tmp/xlsynth-driver", + toolchain = str(toolchain_path), + subcommand = "ir-equiv", + passthrough = [], + runtime_library_path = "", + stdout_path = "", + ) + + captured = {} + + def fake_run(cmd, check = False, env = None, stdout = None): + captured["env"] = dict(env) + + class Result: + returncode = 0 + + return Result() + + with mock.patch.dict(os.environ, {"RUNFILES_DIR": str(runfiles_root)}, clear = False): + with mock.patch.object(env_helpers.subprocess, "run", side_effect = fake_run): + self.assertEqual(env_helpers._driver(argv), 0) + + self.assertEqual(captured["env"]["XLSYNTH_TOOLS"], str(tools_path)) + + def test_run_subprocess_uses_darwin_runtime_library_env_var(self) -> None: + captured = {} + + def fake_run(cmd, check = False, env = None, stdout = None): + captured["env"] = dict(env) + + class Result: + returncode = 0 + + return Result() + + with mock.patch.object(env_helpers.subprocess, "run", side_effect = fake_run): + self.assertEqual( + env_helpers._run_subprocess( + ["dummy-tool"], + runtime_library_path = "/tmp/runtime", + stdout_path = "", + sys_platform = "darwin", + ), + 0, + ) + + self.assertEqual(captured["env"]["DYLD_LIBRARY_PATH"], "/tmp/runtime") + self.assertNotIn("LD_LIBRARY_PATH", captured["env"]) + + def test_dslx_fmt_does_not_receive_stdlib_flag(self) -> None: + toolchain_data = { + "toolchain": { + "dslx": { + "dslx_stdlib_path": "/tmp/stdlib", + }, + }, + } + + self.assertEqual( + env_helpers._build_extra_args_for_tool("dslx_fmt", toolchain_data), + [], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/workspace_toolchain_local_dev/BUILD.bazel b/examples/workspace_toolchain_local_dev/BUILD.bazel new file mode 100644 index 0000000..8a0f3e2 --- /dev/null +++ b/examples/workspace_toolchain_local_dev/BUILD.bazel @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_xlsynth//:rules.bzl", "dslx_library", "dslx_to_pipeline", "dslx_to_sv_types") + +dslx_library( + name = "smoke", + srcs = ["smoke.x"], +) + +dslx_to_sv_types( + name = "smoke_sv_types", + sv_enum_case_naming_policy = "unqualified", + deps = [":smoke"], +) + +dslx_to_pipeline( + name = "smoke_pipeline", + delay_model = "asap7", + pipeline_stages = 1, + top = "main", + deps = [":smoke"], +) diff --git a/examples/workspace_toolchain_local_dev/MODULE.bazel b/examples/workspace_toolchain_local_dev/MODULE.bazel new file mode 100644 index 0000000..5a9d723 --- /dev/null +++ b/examples/workspace_toolchain_local_dev/MODULE.bazel @@ -0,0 +1,20 @@ +module(name = "workspace_toolchain_local_dev") + +bazel_dep(name = "rules_xlsynth", version = "0.0.0") +local_path_override( + module_name = "rules_xlsynth", + path = "../..", +) + +xls = use_extension("@rules_xlsynth//:extensions.bzl", "xls") +xls.toolchain( + name = "xls_local_dev", + artifact_source = "local_paths", + local_driver_path = "/tmp/xls-local-dev/bin/xlsynth-driver", + local_dslx_stdlib_path = "/tmp/xls-local-dev/xls/dslx/stdlib", + local_libxls_path = "/tmp/xls-local-dev/libxls.so", + local_tools_path = "/tmp/xls-local-dev/tools", +) +use_repo(xls, "xls_local_dev") + +register_toolchains("@xls_local_dev//:all") diff --git a/examples/workspace_toolchain_local_dev/README.md b/examples/workspace_toolchain_local_dev/README.md new file mode 100644 index 0000000..e8c41eb --- /dev/null +++ b/examples/workspace_toolchain_local_dev/README.md @@ -0,0 +1,55 @@ +# Local XLS development workspace + +This workspace documents the `artifact_source = "local_paths"` flow. It points +`rules_xlsynth` at a fixed `/tmp/xls-local-dev/` staging tree instead of using +artifact-path `.bazelrc` flags. + +Expected layout: + +```text +/tmp/xls-local-dev/ +|-- bin/ +| `-- xlsynth-driver +|-- libxls.so +|-- tools/ +| |-- check_ir_equivalence_main +| |-- block_to_verilog_main +| |-- codegen_main +| |-- delay_info_main +| |-- dslx_fmt +| |-- dslx_interpreter_main +| |-- ir_converter_main +| |-- opt_main +| |-- prove_quickcheck_main +| `-- typecheck_main +`-- xls/dslx/stdlib/ + `-- std.x +``` + +`local_tools_path` must be the directory that directly contains the XLS tool +binaries, and `local_dslx_stdlib_path` must be the directory that directly +contains `std.x`. + +That `tools/` list is the full bundle tool set expected by +`materialize_xls_bundle.py`, even though the smoke targets below only exercise +a subset of those binaries. + +Populate that tree by copying or symlinking outputs from local XLS and +`xlsynth-driver` builds. A typical setup stages the compiled tool binaries into +`/tmp/xls-local-dev/tools`, points `xlsynth-driver` at +`/tmp/xls-local-dev/bin/xlsynth-driver`, and places the matching shared library +at `/tmp/xls-local-dev/libxls.so`. + +On macOS, update `MODULE.bazel` to use `/tmp/xls-local-dev/libxls.dylib` +instead of `/tmp/xls-local-dev/libxls.so`. + +After the staging tree exists, run: + +```bash +bazel build //:smoke_sv_types //:smoke_pipeline +``` + +`examples/workspace_toolchain_smoke/` shows the same rule surface with one +registered default bundle and one explicit `xls_bundle` override. This local +development variant keeps the BUILD file minimal and pushes all artifact +selection into `MODULE.bazel`. diff --git a/examples/workspace_toolchain_local_dev/smoke.x b/examples/workspace_toolchain_local_dev/smoke.x new file mode 100644 index 0000000..a5adac1 --- /dev/null +++ b/examples/workspace_toolchain_local_dev/smoke.x @@ -0,0 +1,7 @@ +struct SmokeWord { + value: u32, +} + +fn main(x: u32) -> u32 { + x + u32:1 +} diff --git a/examples/workspace_toolchain_smoke/BUILD.bazel b/examples/workspace_toolchain_smoke/BUILD.bazel new file mode 100644 index 0000000..e3364a2 --- /dev/null +++ b/examples/workspace_toolchain_smoke/BUILD.bazel @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_xlsynth//:rules.bzl", "dslx_library", "dslx_to_pipeline", "dslx_to_sv_types") + +dslx_library( + name = "smoke", + srcs = ["smoke.x"], +) + +# Uses the registered default workspace bundle. +dslx_to_sv_types( + name = "smoke_sv_types", + sv_enum_case_naming_policy = "unqualified", + deps = [":smoke"], +) + +# Uses an explicit bundle override on a supported leaf rule. +dslx_to_pipeline( + name = "smoke_pipeline", + delay_model = "asap7", + pipeline_stages = 1, + top = "main", + xls_bundle = "@workspace_override_xls//:bundle", + deps = [":smoke"], +) diff --git a/examples/workspace_toolchain_smoke/MODULE.bazel b/examples/workspace_toolchain_smoke/MODULE.bazel new file mode 100644 index 0000000..9898bc1 --- /dev/null +++ b/examples/workspace_toolchain_smoke/MODULE.bazel @@ -0,0 +1,27 @@ +module(name = "workspace_toolchain_smoke") + +bazel_dep(name = "rules_xlsynth", version = "0.0.0") +local_path_override( + module_name = "rules_xlsynth", + path = "../..", +) + +xls = use_extension("@rules_xlsynth//:extensions.bzl", "xls") +xls.toolchain( + name = "workspace_default_xls", + artifact_source = "download_only", + xls_version = "0.39.0", + xlsynth_driver_version = "0.36.0", +) + +# Keep the override bundle on the same versions so this smoke workspace proves +# the explicit override path without adding version-skew maintenance. +xls.toolchain( + name = "workspace_override_xls", + artifact_source = "download_only", + xls_version = "0.39.0", + xlsynth_driver_version = "0.36.0", +) +use_repo(xls, "workspace_default_xls", "workspace_override_xls") + +register_toolchains("@workspace_default_xls//:all") diff --git a/examples/workspace_toolchain_smoke/smoke.x b/examples/workspace_toolchain_smoke/smoke.x new file mode 100644 index 0000000..a5adac1 --- /dev/null +++ b/examples/workspace_toolchain_smoke/smoke.x @@ -0,0 +1,7 @@ +struct SmokeWord { + value: u32, +} + +fn main(x: u32) -> u32 { + x + u32:1 +} diff --git a/extensions.bzl b/extensions.bzl new file mode 100644 index 0000000..224e8cd --- /dev/null +++ b/extensions.bzl @@ -0,0 +1,278 @@ +_TOOL_BINARIES = [ + "dslx_interpreter_main", + "ir_converter_main", + "block_to_verilog_main", + "codegen_main", + "opt_main", + "prove_quickcheck_main", + "typecheck_main", + "dslx_fmt", + "delay_info_main", + "check_ir_equivalence_main", +] + +def _metadata_dict(repo_ctx): + metadata = {} + metadata_path = repo_ctx.path("bundle_metadata.txt") + if not metadata_path.exists: + fail("Missing bundle metadata at {}".format(metadata_path)) + for line in repo_ctx.read("bundle_metadata.txt").splitlines(): + if not line: + continue + key, value = line.split("=", 1) + metadata[key] = value + return metadata + +def _bundle_build_file( + repo_alias, + libxls_name, + runtime_aliases, + driver_supports_sv_enum_case_naming_policy, + driver_supports_sv_struct_field_ordering): + tool_list = ",\n ".join(['"{}"'.format(name) for name in _TOOL_BINARIES]) + exported_files = ",\n ".join( + ['"{}"'.format(name) for name in _TOOL_BINARIES + ["xlsynth-driver", libxls_name] + runtime_aliases], + ) + runtime_alias_srcs = ",\n ".join(['"{}"'.format(name) for name in runtime_aliases]) + lib_target = libxls_name + lib_file_rule = """ +filegroup( + name = "libxls_file", + srcs = ["{lib_target}"], + visibility = ["//visibility:public"], +) +filegroup( + name = "libxls_runtime_files", + srcs = [":libxls_file"{runtime_alias_srcs}], + visibility = ["//visibility:public"], +) +cc_import( + name = "libxls", + shared_library = ":libxls_file", + visibility = ["//visibility:public"], +) +xls_shared_library_link( + name = "libxls_link", + runtime_files = ":libxls_runtime_files", + shared_library = ":libxls_file", + visibility = ["//visibility:public"], +) +""".format( + lib_target = lib_target, + runtime_alias_srcs = "" if not runtime_aliases else ",\n {}".format(runtime_alias_srcs), + ) + return """# SPDX-License-Identifier: Apache-2.0 + +load("@rules_cc//cc:defs.bzl", "cc_import") +load("@rules_xlsynth//:xls_toolchain.bzl", "copy_flat_files_to_directory", "xls_bundle", "xls_shared_library_link", "xls_toolchain", "xlsynth_artifact_config") + +exports_files([ + {exported_files}, + "xlsynth_artifact_config.toml", +]) + +filegroup( + name = "tools_root_files", + srcs = [ + {tool_list}, + ], +) + +copy_flat_files_to_directory( + name = "dslx_stdlib", + srcs = glob(["*.x"]), + visibility = ["//visibility:public"], +) + +{lib_file_rule} + +xlsynth_artifact_config( + name = "artifact_config", + dso_name = "{libxls_name}", + dslx_stdlib = ":dslx_stdlib", + shared_library = ":libxls_file", + visibility = ["//visibility:public"], +) + +alias( + name = "xlsynth_sys_artifact_config", + actual = ":artifact_config", + visibility = ["//visibility:public"], +) + +alias( + name = "xlsynth_sys_legacy_stdlib", + actual = ":dslx_stdlib", + visibility = ["//visibility:public"], +) + +alias( + name = "xlsynth_sys_legacy_dso", + actual = ":libxls_file", + visibility = ["//visibility:public"], +) + +filegroup( + name = "xlsynth_sys_runtime_files", + srcs = [ + ":dslx_stdlib", + ":libxls_runtime_files", + ], + visibility = ["//visibility:public"], +) + +xls_shared_library_link( + name = "xlsynth_sys_dep", + runtime_files = ":xlsynth_sys_runtime_files", + shared_library = ":libxls_file", + visibility = ["//visibility:public"], +) + +alias( + name = "xlsynth_sys_link_dep", + actual = ":libxls_link", + visibility = ["//visibility:public"], +) + +xls_bundle( + name = "bundle", + driver = ":xlsynth-driver", + driver_supports_sv_enum_case_naming_policy = {driver_supports_sv_enum_case_naming_policy}, + driver_supports_sv_struct_field_ordering = {driver_supports_sv_struct_field_ordering}, + dslx_stdlib = ":dslx_stdlib", + libxls = ":libxls_file", + tools_root = ":tools_root_files", + visibility = ["//visibility:public"], +) + +alias( + name = "{repo_alias}", + actual = ":bundle", + visibility = ["//visibility:public"], +) + +xls_toolchain( + name = "toolchain_impl", + bundle = ":bundle", +) + +toolchain( + name = "toolchain", + toolchain = ":toolchain_impl", + toolchain_type = "@rules_xlsynth//:toolchain_type", + visibility = ["//visibility:public"], +) +""".format( + exported_files = exported_files, + tool_list = tool_list, + libxls_name = libxls_name, + lib_file_rule = lib_file_rule.strip(), + repo_alias = repo_alias, + driver_supports_sv_enum_case_naming_policy = "True" if driver_supports_sv_enum_case_naming_policy else "False", + driver_supports_sv_struct_field_ordering = "True" if driver_supports_sv_struct_field_ordering else "False", + ) + +def _bundle_repo_impl(repo_ctx): + python3 = repo_ctx.which("python3") + if python3 == None: + fail("python3 is required to materialize XLS bundles") + args = [ + str(python3), + str(repo_ctx.path(Label("//:materialize_xls_bundle.py"))), + "--repo-root", + str(repo_ctx.path(".")), + "--artifact-source", + repo_ctx.attr.artifact_source, + ] + if repo_ctx.attr.xls_version: + args.extend(["--xls-version", repo_ctx.attr.xls_version]) + if repo_ctx.attr.xlsynth_driver_version: + args.extend(["--xlsynth-driver-version", repo_ctx.attr.xlsynth_driver_version]) + if repo_ctx.attr.installed_tools_root_prefix: + args.extend(["--installed-tools-root-prefix", repo_ctx.attr.installed_tools_root_prefix]) + if repo_ctx.attr.installed_driver_root_prefix: + args.extend(["--installed-driver-root-prefix", repo_ctx.attr.installed_driver_root_prefix]) + if repo_ctx.attr.local_tools_path: + args.extend(["--local-tools-path", repo_ctx.attr.local_tools_path]) + if repo_ctx.attr.local_dslx_stdlib_path: + args.extend(["--local-dslx-stdlib-path", repo_ctx.attr.local_dslx_stdlib_path]) + if repo_ctx.attr.local_driver_path: + args.extend(["--local-driver-path", repo_ctx.attr.local_driver_path]) + if repo_ctx.attr.local_libxls_path: + args.extend(["--local-libxls-path", repo_ctx.attr.local_libxls_path]) + result = repo_ctx.execute(args, quiet = False) + if result.return_code != 0: + fail("Failed to materialize XLS bundle {}:\nstdout:\n{}\nstderr:\n{}".format( + repo_ctx.name, + result.stdout, + result.stderr, + )) + metadata = _metadata_dict(repo_ctx) + runtime_aliases = [ + alias + for alias in metadata.get("libxls_runtime_aliases", "").split(",") + if alias + ] + repo_ctx.file( + "BUILD.bazel", + _bundle_build_file( + repo_alias = repo_ctx.attr.repo_alias, + libxls_name = metadata["libxls_name"], + runtime_aliases = runtime_aliases, + driver_supports_sv_enum_case_naming_policy = metadata["driver_supports_sv_enum_case_naming_policy"] == "true", + driver_supports_sv_struct_field_ordering = metadata["driver_supports_sv_struct_field_ordering"] == "true", + ), + ) + +_xls_bundle_repo = repository_rule( + implementation = _bundle_repo_impl, + attrs = { + "artifact_source": attr.string(mandatory = True), + "installed_driver_root_prefix": attr.string(), + "installed_tools_root_prefix": attr.string(), + "local_driver_path": attr.string(), + "local_dslx_stdlib_path": attr.string(), + "local_libxls_path": attr.string(), + "local_tools_path": attr.string(), + "repo_alias": attr.string(mandatory = True), + "xls_version": attr.string(), + "xlsynth_driver_version": attr.string(), + }, +) + +_toolchain_tag = tag_class(attrs = { + "artifact_source": attr.string(mandatory = True), + "installed_driver_root_prefix": attr.string(), + "installed_tools_root_prefix": attr.string(), + "local_driver_path": attr.string(), + "local_dslx_stdlib_path": attr.string(), + "local_libxls_path": attr.string(), + "local_tools_path": attr.string(), + "name": attr.string(mandatory = True), + "xls_version": attr.string(), + "xlsynth_driver_version": attr.string(), +}) + +def _xls_extension_impl(module_ctx): + for module in module_ctx.modules: + for toolchain in module.tags.toolchain: + _xls_bundle_repo( + name = toolchain.name, + artifact_source = toolchain.artifact_source, + installed_driver_root_prefix = toolchain.installed_driver_root_prefix, + installed_tools_root_prefix = toolchain.installed_tools_root_prefix, + local_driver_path = toolchain.local_driver_path, + local_dslx_stdlib_path = toolchain.local_dslx_stdlib_path, + local_libxls_path = toolchain.local_libxls_path, + local_tools_path = toolchain.local_tools_path, + repo_alias = toolchain.name, + xls_version = toolchain.xls_version, + xlsynth_driver_version = toolchain.xlsynth_driver_version, + ) + +xls = module_extension( + implementation = _xls_extension_impl, + tag_classes = { + "toolchain": _toolchain_tag, + }, +) diff --git a/external_bundle_exports_test.py b/external_bundle_exports_test.py new file mode 100644 index 0000000..5ad1ff2 --- /dev/null +++ b/external_bundle_exports_test.py @@ -0,0 +1,116 @@ +import os +from pathlib import Path +import unittest + +from python.runfiles import runfiles + + +def expected_libxls_name(): + return "libxls.dylib" if os.uname().sysname == "Darwin" else "libxls.so" + + +class ExternalBundleExportsTest(unittest.TestCase): + def test_stdlib_target_is_a_single_directory_runfile(self): + runfiles_lookup = runfiles.Create() + workspace = os.environ["TEST_WORKSPACE"] + + location_file = Path(runfiles_lookup.Rlocation("{}/stdlib_location.txt".format(workspace))) + self.assertTrue(location_file.is_file()) + self.assertEqual(Path(location_file.read_text(encoding = "utf-8").strip()).name, "dslx_stdlib") + + stdlib_dir = Path(runfiles_lookup.Rlocation("rules_xlsynth_selftest_xls/dslx_stdlib")) + self.assertTrue(stdlib_dir.is_dir()) + self.assertTrue(any(stdlib_dir.glob("*.x"))) + + def test_repo_named_alias_exposes_bundle_runfiles(self): + runfiles_lookup = runfiles.Create() + workspace = os.environ["TEST_WORKSPACE"] + + locations_file = Path(runfiles_lookup.Rlocation("{}/bundle_alias_locations.txt".format(workspace))) + self.assertTrue(locations_file.is_file()) + + locations = locations_file.read_text(encoding = "utf-8").split() + self.assertTrue(any(Path(path).name == "xlsynth-driver" for path in locations)) + self.assertTrue(any(Path(path).name == expected_libxls_name() for path in locations)) + self.assertTrue(any(Path(path).name == "block_to_verilog_main" for path in locations)) + + def test_xlsynth_sys_runtime_export_is_narrow(self): + runfiles_lookup = runfiles.Create() + workspace = os.environ["TEST_WORKSPACE"] + + locations_file = Path( + runfiles_lookup.Rlocation("{}/xlsynth_sys_runtime_locations.txt".format(workspace)), + ) + self.assertTrue(locations_file.is_file()) + + locations = locations_file.read_text(encoding = "utf-8").split() + basenames = [Path(path).name for path in locations] + + self.assertIn("dslx_stdlib", basenames) + self.assertTrue(any(name.startswith("libxls") for name in basenames)) + self.assertFalse(any(name == "xlsynth-driver" for name in basenames)) + self.assertFalse(any(name == "block_to_verilog_main" for name in basenames)) + + def test_xlsynth_sys_artifact_config_export_points_at_config_file(self): + runfiles_lookup = runfiles.Create() + workspace = os.environ["TEST_WORKSPACE"] + + location_file = Path( + runfiles_lookup.Rlocation("{}/xlsynth_sys_artifact_config_location.txt".format(workspace)), + ) + self.assertTrue(location_file.is_file()) + self.assertEqual( + Path(location_file.read_text(encoding = "utf-8").strip()).name, + "xlsynth_artifact_config.toml", + ) + + def test_xlsynth_sys_artifact_config_export_is_self_contained(self): + runfiles_lookup = runfiles.Create() + config_path = Path( + runfiles_lookup.Rlocation( + "rules_xlsynth_selftest_xls/artifact_config/xlsynth_artifact_config.toml", + ), + ) + self.assertTrue(config_path.is_file()) + + config_lines = dict( + line.split(" = ", 1) + for line in config_path.read_text(encoding = "utf-8").splitlines() + if line + ) + expected_dso_name = expected_libxls_name() + self.assertEqual(config_lines["dso_path"].strip('"'), expected_dso_name) + self.assertEqual(config_lines["dslx_stdlib_path"].strip('"'), "dslx_stdlib") + self.assertTrue((config_path.parent / expected_dso_name).is_file()) + self.assertTrue((config_path.parent / "dslx_stdlib").is_dir()) + + def test_xlsynth_sys_dep_export_packages_runtime_payload(self): + runfiles_lookup = runfiles.Create() + workspace = os.environ["TEST_WORKSPACE"] + + locations_file = Path( + runfiles_lookup.Rlocation("{}/xlsynth_sys_dep_locations.txt".format(workspace)), + ) + self.assertTrue(locations_file.is_file()) + + basenames = [Path(path).name for path in locations_file.read_text(encoding = "utf-8").split()] + self.assertIn("dslx_stdlib", basenames) + self.assertTrue(any(name.startswith("libxls") for name in basenames)) + self.assertFalse(any(name == "xlsynth-driver" for name in basenames)) + + def test_xlsynth_sys_legacy_exports_are_stdlib_plus_dso(self): + runfiles_lookup = runfiles.Create() + workspace = os.environ["TEST_WORKSPACE"] + + locations_file = Path( + runfiles_lookup.Rlocation("{}/xlsynth_sys_legacy_input_locations.txt".format(workspace)), + ) + self.assertTrue(locations_file.is_file()) + + basenames = [Path(path).name for path in locations_file.read_text(encoding = "utf-8").split()] + self.assertIn("dslx_stdlib", basenames) + self.assertTrue(any(name.startswith("libxls") for name in basenames)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ir_prove_equiv_test.bzl b/ir_prove_equiv_test.bzl index 9ec4df6..6336d93 100644 --- a/ir_prove_equiv_test.bzl +++ b/ir_prove_equiv_test.bzl @@ -3,6 +3,7 @@ load(":helpers.bzl", "write_executable_shell_script") load(":ir_provider.bzl", "IrInfo") load(":env_helpers.bzl", "python_runner_source") +load(":xls_toolchain.bzl", "declare_xls_toolchain_toml", "get_driver_artifact_inputs", "require_driver_toolchain") def _ir_prove_equiv_test_impl(ctx): @@ -16,13 +17,28 @@ def _ir_prove_equiv_test_impl(ctx): rhs_file = list(rhs_files)[0] runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) - cmd = "/usr/bin/env python3 {} driver ir-equiv --top={} {} {}".format( + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = require_driver_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "ir_equiv") + cmd_parts = [ + "/usr/bin/env", + "python3", runner.short_path, - ctx.attr.top, + "driver", + "--driver_path", + toolchain.driver_path, + "--toolchain", + toolchain_file.short_path, + ] + if toolchain.runtime_library_path: + cmd_parts.extend(["--runtime_library_path", toolchain.runtime_library_path]) + cmd_parts.extend([ + "ir-equiv", + "--top={}".format(ctx.attr.top), lhs_file.short_path, rhs_file.short_path, - ) + ]) + cmd = " ".join(["\"{}\"".format(part) for part in cmd_parts]) run_script = write_executable_shell_script( ctx = ctx, filename = ctx.label.name + ".sh", @@ -31,7 +47,7 @@ def _ir_prove_equiv_test_impl(ctx): return DefaultInfo( files = depset(direct = [run_script]), runfiles = ctx.runfiles( - files = [lhs_file, rhs_file, runner], + files = [lhs_file, rhs_file, runner, toolchain_file] + get_driver_artifact_inputs(toolchain, ["check_ir_equivalence_main"]), ), executable = run_script, ) @@ -58,4 +74,5 @@ ir_prove_equiv_test = rule( }, executable = True, test = True, + toolchains = ["//:toolchain_type"], ) diff --git a/ir_to_delay_info.bzl b/ir_to_delay_info.bzl index ae650c9..ccd3441 100644 --- a/ir_to_delay_info.bzl +++ b/ir_to_delay_info.bzl @@ -1,36 +1,51 @@ # SPDX-License-Identifier: Apache-2.0 -load(":ir_provider.bzl", "IrInfo") load(":env_helpers.bzl", "python_runner_source") - +load(":ir_provider.bzl", "IrInfo") +load( + ":xls_toolchain.bzl", + "XlsArtifactBundleInfo", + "declare_xls_toolchain_toml", + "get_driver_artifact_inputs", + "get_selected_driver_toolchain", +) def _ir_to_delay_info_impl(ctx): opt_ir_file = ctx.attr.ir[IrInfo].ir_file if ctx.attr.use_unopt_ir else ctx.attr.ir[IrInfo].opt_ir_file output_file = ctx.outputs.delay_info runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_driver_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "ir_to_delay_info", toolchain = toolchain) - ctx.actions.run_shell( - inputs = [opt_ir_file], - tools = [runner], + ctx.actions.run( + inputs = [opt_ir_file, toolchain_file] + get_driver_artifact_inputs(toolchain, ["delay_info_main"]), + executable = runner, outputs = [output_file], - command = "\"$1\" driver ir2delayinfo --delay_model=\"$2\" \"$3\" \"$4\" > \"$5\"", arguments = [ - runner.path, + "driver", + "--driver_path", + toolchain.driver_path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--toolchain", + toolchain_file.path, + "--stdout_path", + output_file.path, + "ir2delayinfo", + "--delay_model", ctx.attr.delay_model, opt_ir_file.path, ctx.attr.top, - output_file.path, ], - use_default_shell_env = True, + use_default_shell_env = False, ) return DefaultInfo( files = depset(direct = [output_file]), ) - ir_to_delay_info = rule( doc = "Convert an IR file to delay info", implementation = _ir_to_delay_info_impl, @@ -51,8 +66,13 @@ ir_to_delay_info = rule( doc = "Whether to use the unoptimized IR file instead of optimized IR file.", default = False, ), + "xls_bundle": attr.label( + doc = "Optional XLS bundle override.", + providers = [XlsArtifactBundleInfo], + ), }, outputs = { "delay_info": "%{name}.txt", }, + toolchains = ["//:toolchain_type"], ) diff --git a/ir_to_gates.bzl b/ir_to_gates.bzl index 1afc69e..f29cff9 100644 --- a/ir_to_gates.bzl +++ b/ir_to_gates.bzl @@ -1,8 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 -load(":ir_provider.bzl", "IrInfo") load(":env_helpers.bzl", "python_runner_source") - +load(":ir_provider.bzl", "IrInfo") +load( + ":xls_toolchain.bzl", + "XlsArtifactBundleInfo", + "declare_xls_toolchain_toml", + "get_driver_artifact_inputs", + "get_selected_driver_toolchain", +) def _ir_to_gates_impl(ctx): ir_info = ctx.attr.ir_src[IrInfo] @@ -11,21 +17,30 @@ def _ir_to_gates_impl(ctx): metrics_file = ctx.outputs.metrics_json runner = ctx.actions.declare_file(ctx.label.name + "_runner.py") - ctx.actions.write(output = runner, content = python_runner_source()) + ctx.actions.write(output = runner, content = python_runner_source(), is_executable = True) + toolchain = get_selected_driver_toolchain(ctx) + toolchain_file = declare_xls_toolchain_toml(ctx, name = "ir_to_gates", toolchain = toolchain) - ctx.actions.run_shell( - inputs = [ir_file_to_use], - tools = [runner], + ctx.actions.run( + inputs = [ir_file_to_use, toolchain_file] + get_driver_artifact_inputs(toolchain), + executable = runner, outputs = [gates_file, metrics_file], - command = "\"$1\" driver ir2gates --fraig=\"$2\" --output_json=\"$3\" \"$4\" > \"$5\"", arguments = [ - runner.path, - ("true" if ctx.attr.fraig else "false"), - metrics_file.path, - ir_file_to_use.path, + "driver", + "--driver_path", + toolchain.driver_path, + "--runtime_library_path", + toolchain.runtime_library_path, + "--toolchain", + toolchain_file.path, + "--stdout_path", gates_file.path, + "ir2gates", + "--fraig={}".format("true" if ctx.attr.fraig else "false"), + "--output_json={}".format(metrics_file.path), + ir_file_to_use.path, ], - use_default_shell_env = True, + use_default_shell_env = False, progress_message = "Generating gate-level analysis for IR", mnemonic = "IR2GATES", ) @@ -34,7 +49,6 @@ def _ir_to_gates_impl(ctx): files = depset(direct = [gates_file, metrics_file]), ) - ir_to_gates = rule( doc = "Convert an IR file to gate-level analysis", implementation = _ir_to_gates_impl, @@ -48,9 +62,14 @@ ir_to_gates = rule( doc = "If true, perform \"fraig\" optimization; can be slow when gate graph is large.", default = True, ), + "xls_bundle": attr.label( + doc = "Optional XLS bundle override.", + providers = [XlsArtifactBundleInfo], + ), }, outputs = { "gates_file": "%{name}.txt", "metrics_json": "%{name}.json", }, + toolchains = ["//:toolchain_type"], ) diff --git a/make_env_helpers.py b/make_env_helpers.py index 7035eb3..e91fc6e 100644 --- a/make_env_helpers.py +++ b/make_env_helpers.py @@ -5,12 +5,13 @@ import sys import textwrap from pathlib import Path +from typing import List _DOCSTRING = """Returns the embedded xlsynth_runner.py source. -The returned program reads XLSYNTH_* from the action execution environment -and invokes either the driver (via the 'driver' subcommand) or a tool -(via the 'tool' subcommand), forwarding passthrough flags accordingly. +The returned program reads a declared toolchain TOML input and invokes either +the driver (via the 'driver' subcommand) or a tool (via the 'tool' +subcommand), forwarding passthrough flags accordingly. """ @@ -31,7 +32,7 @@ def _generate_bzl(py_source: str) -> str: return "\n".join([*header_lines, docstring, *return_lines, ""]) -def main(argv: list[str]) -> int: +def main(argv: List[str]) -> int: parser = argparse.ArgumentParser(description="Regenerate env_helpers.bzl") parser.add_argument("--output", type=Path, default=Path("env_helpers.bzl")) parser.add_argument("--source", type=Path, default=Path("env_helpers.py")) diff --git a/materialize_xls_bundle.py b/materialize_xls_bundle.py new file mode 100644 index 0000000..822f96a --- /dev/null +++ b/materialize_xls_bundle.py @@ -0,0 +1,677 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Materializes an XLS bundle repository for the rules_xlsynth module extension.""" + +import argparse +import os +from pathlib import Path +import shutil +import subprocess +import sys + +TOOL_BINARIES = [ + "dslx_interpreter_main", + "ir_converter_main", + "block_to_verilog_main", + "codegen_main", + "opt_main", + "prove_quickcheck_main", + "typecheck_main", + "dslx_fmt", + "delay_info_main", + "check_ir_equivalence_main", +] + +_DRIVER_CAPABILITY_FLAGS = { + "driver_supports_sv_enum_case_naming_policy": "--sv_enum_case_naming_policy", + "driver_supports_sv_struct_field_ordering": "--sv_struct_field_ordering", +} + + +def run_captured_text_command(args, check, env = None): + return subprocess.run( + args, + check = check, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + universal_newlines = True, + env = env, + ) + + +def normalize_version(version): + if not version: + return version + if version.startswith("v"): + return version[1:] + return version + + +def version_tag(version): + return "v{}".format(normalize_version(version)) + + +def libxls_name_for_platform(sys_platform): + if sys_platform == "darwin": + return "libxls.dylib" + elif sys_platform == "linux": + return "libxls.so" + else: + raise RuntimeError("Unsupported host platform: {}".format(sys_platform)) + + +def derive_installed_paths( + xls_version, + driver_version, + installed_tools_root_prefix, + installed_driver_root_prefix, + sys_platform = sys.platform): + normalized_xls_version = normalize_version(xls_version) + normalized_driver_version = normalize_version(driver_version) + tools_root = Path(installed_tools_root_prefix) / "v{}".format(normalized_xls_version) + return { + "tools_root": tools_root, + "dslx_stdlib_root": tools_root / "xls" / "dslx" / "stdlib", + "driver": Path(installed_driver_root_prefix) / normalized_driver_version / "bin" / "xlsynth-driver", + "libxls": tools_root / libxls_name_for_platform(sys_platform), + } + + +def validate_stdlib_root(stdlib_root): + if not stdlib_root.exists(): + raise ValueError("DSLX stdlib root does not exist: {}".format(stdlib_root)) + if not stdlib_root.is_dir(): + raise ValueError("DSLX stdlib root is not a directory: {}".format(stdlib_root)) + std_x = stdlib_root / "std.x" + if not std_x.exists(): + raise ValueError("DSLX stdlib root must contain std.x directly: {}".format(stdlib_root)) + + +def resolve_artifact_plan( + artifact_source, + xls_version, + driver_version, + installed_tools_root_prefix = "", + installed_driver_root_prefix = "", + local_tools_path = "", + local_dslx_stdlib_path = "", + local_driver_path = "", + local_libxls_path = "", + exists_fn = os.path.exists, +): + if artifact_source == "local_paths": + if xls_version or driver_version: + raise ValueError("local_paths does not accept xls_version or xlsynth_driver_version") + required = { + "local_tools_path": local_tools_path, + "local_dslx_stdlib_path": local_dslx_stdlib_path, + "local_driver_path": local_driver_path, + "local_libxls_path": local_libxls_path, + } + missing = [name for name, value in required.items() if not value] + if missing: + raise ValueError("local_paths requires {}".format(", ".join(sorted(missing)))) + return { + "mode": "local_paths", + "tools_root": Path(local_tools_path), + "dslx_stdlib_root": Path(local_dslx_stdlib_path), + "driver": Path(local_driver_path), + "libxls": Path(local_libxls_path), + } + + if artifact_source not in ("auto", "installed_only", "download_only"): + raise ValueError("Unknown artifact_source: {}".format(artifact_source)) + if not xls_version or not driver_version: + raise ValueError("{} requires xls_version and xlsynth_driver_version".format(artifact_source)) + if local_tools_path or local_dslx_stdlib_path or local_driver_path or local_libxls_path: + raise ValueError("{} does not accept local_paths attrs".format(artifact_source)) + if artifact_source == "download_only": + if installed_tools_root_prefix or installed_driver_root_prefix: + raise ValueError("download_only does not accept installed_* attrs") + else: + missing_installed = [ + name + for name, value in { + "installed_tools_root_prefix": installed_tools_root_prefix, + "installed_driver_root_prefix": installed_driver_root_prefix, + }.items() + if not value + ] + if missing_installed: + raise ValueError("{} requires {}".format(artifact_source, ", ".join(sorted(missing_installed)))) + + installed_paths = derive_installed_paths( + xls_version = xls_version, + driver_version = driver_version, + installed_tools_root_prefix = installed_tools_root_prefix, + installed_driver_root_prefix = installed_driver_root_prefix, + ) + if artifact_source == "download_only": + return { + "mode": "download", + "xls_version": normalize_version(xls_version), + "driver_version": normalize_version(driver_version), + } + + installed_paths_present = all(exists_fn(str(path)) for path in installed_paths.values()) + if artifact_source == "auto" and installed_paths_present: + return { + "mode": "installed", + "tools_root": installed_paths["tools_root"], + "dslx_stdlib_root": installed_paths["dslx_stdlib_root"], + "driver": installed_paths["driver"], + "libxls": installed_paths["libxls"], + } + if artifact_source == "installed_only": + if not installed_paths_present: + raise ValueError( + "installed_only requires exact-version installed paths for XLS {} and driver {}".format( + normalize_version(xls_version), + normalize_version(driver_version), + ) + ) + return { + "mode": "installed", + "tools_root": installed_paths["tools_root"], + "dslx_stdlib_root": installed_paths["dslx_stdlib_root"], + "driver": installed_paths["driver"], + "libxls": installed_paths["libxls"], + } + return { + "mode": "download", + "xls_version": normalize_version(xls_version), + "driver_version": normalize_version(driver_version), + } + + +def derive_runtime_library_path(libxls_path): + return str(Path(libxls_path).parent) + + +def detect_host_platform(): + if sys.platform == "darwin": + machine = os.uname().machine + if machine == "arm64": + return "arm64" + if machine == "x86_64": + raise RuntimeError( + "download-backed XLS bundles do not support Intel macOS because the xlsynth " + "releases do not publish matching x86_64 Darwin artifacts" + ) + raise RuntimeError("Unsupported macOS architecture: {}".format(machine)) + + if sys.platform != "linux": + raise RuntimeError("Unsupported host platform: {}".format(sys.platform)) + + machine = os.uname().machine + if machine != "x86_64": + raise RuntimeError("Unsupported Linux architecture: {}".format(machine)) + + os_release_path = Path("/etc/os-release") + if os_release_path.exists(): + os_release_data = os_release_path.read_text(encoding = "utf-8") + lower = os_release_data.lower() + if any(marker in lower for marker in ["rocky", "rhel", "almalinux", "centos"]): + return "rocky8" + return "ubuntu2004" + + +def driver_install_root(repo_root, driver_version, host_platform): + return repo_root / "_cargo_driver" / host_platform / normalize_version(driver_version) + + +def rustup_home_root(repo_root, host_platform): + return repo_root / "_rustup_home" / host_platform + + +def cargo_target_root(repo_root, host_platform): + return repo_root / "_cargo_target" / host_platform + + +def downloaded_xls_root(repo_root, xls_version, host_platform): + return repo_root / "_downloaded_xls" / host_platform / normalize_version(xls_version) + + +def ensure_clean_path(path): + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) + + +def symlink_or_copy(src, dest): + ensure_clean_path(dest) + try: + os.symlink(str(src), str(dest)) + except OSError: + if src.is_dir(): + shutil.copytree(str(src), str(dest)) + else: + shutil.copy2(str(src), str(dest)) + + +def copy_path(src, dest): + ensure_clean_path(dest) + if src.is_dir(): + shutil.copytree(str(src), str(dest)) + else: + shutil.copy2(str(src), str(dest)) + + +def link_stdlib_sources(stdlib_root, repo_root): + validate_stdlib_root(stdlib_root) + for child in stdlib_root.iterdir(): + if child.name.endswith(".x"): + symlink_or_copy(child, repo_root / child.name) + + +def link_tool_binaries(tools_root, repo_root): + for binary in TOOL_BINARIES: + source = tools_root / binary + if not source.exists(): + raise ValueError("Expected tool binary at {}".format(source)) + symlink_or_copy(source, repo_root / binary) + + +def normalized_libxls_name(libxls_path): + suffix = Path(libxls_path).suffix + if suffix == ".dylib": + return "libxls.dylib" + return "libxls.so" + + +def runtime_library_env_var(sys_platform): + if sys_platform == "darwin": + return "DYLD_LIBRARY_PATH" + elif sys_platform == "linux": + return "LD_LIBRARY_PATH" + else: + raise RuntimeError("Unsupported host platform: {}".format(sys_platform)) + + +def prepend_search_path(path_value, existing_value): + if existing_value: + return "{}{}{}".format(path_value, os.pathsep, existing_value) + return path_value + + +def build_driver_environment( + libxls_path, + dslx_stdlib_path, + environ = None, + sys_platform = sys.platform): + env = dict(os.environ if environ == None else environ) + runtime_library_path = str(Path(libxls_path).parent) + runtime_var = runtime_library_env_var(sys_platform) + env[runtime_var] = prepend_search_path(runtime_library_path, env.get(runtime_var, "")) + env["XLS_DSO_PATH"] = str(libxls_path) + env["DSLX_STDLIB_PATH"] = str(dslx_stdlib_path) + return env + + +def parse_readelf_soname(stdout): + marker = "Library soname: [" + for line in stdout.splitlines(): + if marker not in line: + continue + return line.split(marker, 1)[1].split("]", 1)[0] + return "" + + +def read_linux_soname(libxls_path): + result = run_captured_text_command( + ["readelf", "-d", str(libxls_path)], + check = False, + ) + if result.returncode != 0: + raise RuntimeError( + "Failed to inspect ELF SONAME at {}\nstdout:\n{}\nstderr:\n{}".format( + libxls_path, + result.stdout, + result.stderr, + ) + ) + return parse_readelf_soname(result.stdout) + + +def materialize_runtime_library_aliases(libxls_path, runtime_aliases): + staged_aliases = [] + for alias in runtime_aliases: + if not alias or alias == Path(libxls_path).name: + continue + alias_path = Path(libxls_path).parent / alias + ensure_clean_path(alias_path) + try: + os.symlink(Path(libxls_path).name, str(alias_path)) + except OSError: + shutil.copy2(str(libxls_path), str(alias_path)) + staged_aliases.append(alias) + return staged_aliases + + +def normalize_linux_soname(libxls_path): + soname = read_linux_soname(libxls_path) + expected = Path(libxls_path).name + if not soname or soname == expected: + return [] + + patchelf = shutil.which("patchelf") + if patchelf == None: + return materialize_runtime_library_aliases(libxls_path, [soname]) + + subprocess.run( + [patchelf, "--set-soname", expected, str(libxls_path)], + check = True, + ) + return [] + + +def normalize_runtime_library_identity(libxls_path, sys_platform = sys.platform): + if sys_platform == "darwin": + subprocess.run( + [ + "install_name_tool", + "-id", + "@rpath/{}".format(Path(libxls_path).name), + str(libxls_path), + ], + check = True, + ) + return [] + elif sys_platform == "linux": + return normalize_linux_soname(libxls_path) + else: + raise RuntimeError("Unsupported host platform: {}".format(sys_platform)) + + +def detect_driver_capabilities(driver_path, libxls_path, dslx_stdlib_path): + env = build_driver_environment(libxls_path, dslx_stdlib_path) + result = run_captured_text_command( + [str(driver_path), "dslx2sv-types", "--help"], + check = False, + env = env, + ) + if result.returncode != 0: + raise RuntimeError( + "Failed to inspect xlsynth-driver capability at {}\nstdout:\n{}\nstderr:\n{}".format( + driver_path, + result.stdout, + result.stderr, + ) + ) + help_text = "{}\n{}".format(result.stdout, result.stderr) + return { + capability_name: capability_flag in help_text + for capability_name, capability_flag in _DRIVER_CAPABILITY_FLAGS.items() + } + + +def build_driver_install_command(rustup_path, install_root, driver_version): + return [ + rustup_path, + "run", + "nightly", + "cargo", + "install", + "--locked", + "--root", + str(install_root), + "--version", + normalize_version(driver_version), + "xlsynth-driver", + ] + + +def build_rustup_toolchain_install_command(rustup_path): + return [ + rustup_path, + "toolchain", + "install", + "nightly", + "--profile", + "minimal", + ] + + +def build_driver_install_environment( + repo_root, + libxls_path, + dslx_stdlib_path, + environ = None, + sys_platform = sys.platform, + host_platform = ""): + env = build_driver_environment( + libxls_path = libxls_path, + dslx_stdlib_path = dslx_stdlib_path, + environ = environ, + sys_platform = sys_platform, + ) + resolved_host_platform = host_platform or detect_host_platform() + env["RUSTUP_HOME"] = str(rustup_home_root(repo_root, resolved_host_platform)) + env["CARGO_TARGET_DIR"] = str(cargo_target_root(repo_root, resolved_host_platform)) + return env + + +def ensure_rustup_nightly_toolchain(rustup_path, env): + probe = run_captured_text_command( + [rustup_path, "run", "nightly", "cargo", "--version"], + check = False, + env = env, + ) + if probe.returncode == 0: + return + subprocess.run( + build_rustup_toolchain_install_command(rustup_path), + check = True, + env = env, + ) + + +def validate_installed_driver(driver_path, env, driver_version): + result = run_captured_text_command( + [str(driver_path), "--version"], + check = False, + env = env, + ) + if result.returncode != 0: + raise RuntimeError( + "Installed xlsynth-driver is not runnable at {}\nstdout:\n{}\nstderr:\n{}".format( + driver_path, + result.stdout, + result.stderr, + ) + ) + version_text = "{}\n{}".format(result.stdout, result.stderr) + expected_version = normalize_version(driver_version) + if expected_version not in version_text: + raise RuntimeError( + "Installed xlsynth-driver at {} reported an unexpected version.\nexpected substring: {}\nstdout:\n{}\nstderr:\n{}".format( + driver_path, + expected_version, + result.stdout, + result.stderr, + ) + ) + + +def install_driver(repo_root, driver_version, libxls_path, dslx_stdlib_path): + host_platform = detect_host_platform() + install_root = driver_install_root(repo_root, driver_version, host_platform) + rustup_home = rustup_home_root(repo_root, host_platform) + target_root = cargo_target_root(repo_root, host_platform) + env = build_driver_install_environment( + repo_root, + libxls_path, + dslx_stdlib_path, + host_platform = host_platform, + ) + for path in [install_root, rustup_home, target_root]: + path.mkdir(parents = True, exist_ok = True) + driver_path = install_root / "bin" / "xlsynth-driver" + if driver_path.exists(): + try: + validate_installed_driver(driver_path, env, driver_version) + return driver_path + except RuntimeError: + ensure_clean_path(install_root) + install_root.mkdir(parents = True, exist_ok = True) + + rustup = shutil.which("rustup") + if rustup is None: + raise RuntimeError( + "rules_xlsynth download fallback requires rustup to install xlsynth-driver {}".format( + driver_version + ) + ) + ensure_rustup_nightly_toolchain(rustup, env) + subprocess.run( + build_driver_install_command(rustup, install_root, driver_version), + check = True, + env = env, + ) + validate_installed_driver(driver_path, env, driver_version) + return driver_path + + +def resolve_downloaded_artifacts(download_root): + stdlib_root = download_root / "xls" / "dslx" / "stdlib" + validate_stdlib_root(stdlib_root) + for binary in TOOL_BINARIES: + tool_path = download_root / binary + if not tool_path.exists(): + raise RuntimeError("Expected tool binary at {}".format(tool_path)) + libxls_candidates = sorted(download_root.glob("libxls-*.so")) + sorted(download_root.glob("libxls-*.dylib")) + if len(libxls_candidates) != 1: + raise RuntimeError("Expected exactly one libxls artifact in {}, found {}".format(download_root, libxls_candidates)) + return { + "tools_root": download_root, + "dslx_stdlib_root": stdlib_root, + "libxls": libxls_candidates[0], + } + + +def download_versioned_artifacts(repo_root, xls_version): + script_path = Path(__file__).with_name("download_release.py") + host_platform = detect_host_platform() + download_root = downloaded_xls_root(repo_root, xls_version, host_platform) + if download_root.exists(): + try: + return resolve_downloaded_artifacts(download_root) + except (RuntimeError, ValueError): + ensure_clean_path(download_root) + download_root.mkdir(parents = True, exist_ok = True) + subprocess.run( + [ + sys.executable, + str(script_path), + "--output", + str(download_root), + "--platform", + host_platform, + "--version", + version_tag(xls_version), + "--dso", + ], + check = True, + ) + return resolve_downloaded_artifacts(download_root) + + +def materialize_bundle(repo_root, plan): + if plan["mode"] == "download": + resolved = download_versioned_artifacts(repo_root, plan["xls_version"]) + link_tool_binaries(resolved["tools_root"], repo_root) + link_stdlib_sources(resolved["dslx_stdlib_root"], repo_root) + + libxls_dest = repo_root / normalized_libxls_name(resolved["libxls"]) + copy_path(resolved["libxls"], libxls_dest) + runtime_aliases = normalize_runtime_library_identity(libxls_dest) + + driver_path = install_driver( + repo_root, + plan["driver_version"], + libxls_dest, + repo_root, + ) + driver_dest = repo_root / "xlsynth-driver" + symlink_or_copy(driver_path, driver_dest) + else: + resolved = plan + validate_stdlib_root(resolved["dslx_stdlib_root"]) + link_tool_binaries(resolved["tools_root"], repo_root) + link_stdlib_sources(resolved["dslx_stdlib_root"], repo_root) + + driver_dest = repo_root / "xlsynth-driver" + symlink_or_copy(resolved["driver"], driver_dest) + + libxls_dest = repo_root / normalized_libxls_name(resolved["libxls"]) + copy_path(resolved["libxls"], libxls_dest) + runtime_aliases = normalize_runtime_library_identity(libxls_dest) + + artifact_config_path = repo_root / "xlsynth_artifact_config.toml" + artifact_config_path.write_text( + "".join([ + "dso_path = \"{}\"\n".format(libxls_dest.name), + "dslx_stdlib_path = \".\"\n", + ]), + encoding = "utf-8", + ) + + driver_capabilities = detect_driver_capabilities(driver_dest, libxls_dest, repo_root) + metadata_path = repo_root / "bundle_metadata.txt" + metadata_path.write_text( + "".join([ + "".join([ + "{}={}\n".format( + capability_name, + "true" if capability_enabled else "false", + ) + for capability_name, capability_enabled in driver_capabilities.items() + ]), + "libxls_name={}\n".format(libxls_dest.name), + "libxls_runtime_aliases={}\n".format(",".join(runtime_aliases)), + ]), + encoding = "utf-8", + ) + return { + **driver_capabilities, + "runtime_library_path": derive_runtime_library_path(libxls_dest), + "libxls_name": libxls_dest.name, + } + + +def parse_args(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--repo-root", required = True) + parser.add_argument("--artifact-source", required = True) + parser.add_argument("--xls-version", default = "") + parser.add_argument("--xlsynth-driver-version", default = "") + parser.add_argument("--installed-tools-root-prefix", default = "") + parser.add_argument("--installed-driver-root-prefix", default = "") + parser.add_argument("--local-tools-path", default = "") + parser.add_argument("--local-dslx-stdlib-path", default = "") + parser.add_argument("--local-driver-path", default = "") + parser.add_argument("--local-libxls-path", default = "") + return parser.parse_args(argv) + + +def main(argv): + args = parse_args(argv) + repo_root = Path(args.repo_root) + repo_root.mkdir(parents = True, exist_ok = True) + plan = resolve_artifact_plan( + artifact_source = args.artifact_source, + xls_version = args.xls_version, + driver_version = args.xlsynth_driver_version, + installed_tools_root_prefix = args.installed_tools_root_prefix, + installed_driver_root_prefix = args.installed_driver_root_prefix, + local_tools_path = args.local_tools_path, + local_dslx_stdlib_path = args.local_dslx_stdlib_path, + local_driver_path = args.local_driver_path, + local_libxls_path = args.local_libxls_path, + ) + materialize_bundle(repo_root, plan) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/run_presubmit.py b/run_presubmit.py index b258475..ab951cf 100644 --- a/run_presubmit.py +++ b/run_presubmit.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -from typing import Optional, Tuple, List, Callable, Dict, Set +from typing import Optional, Tuple, List, Callable, Dict, NamedTuple, Set import subprocess import optparse import os @@ -9,27 +9,93 @@ import tempfile import shutil import sys -import hashlib -import urllib.request -Runnable = Callable[['PathData'], None] +import materialize_xls_bundle + +Runnable = Callable[['PresubmitConfig'], None] TO_RUN: List[Runnable] = [] def register(f: Runnable): TO_RUN.append(f) return f -class PathData: - xlsynth_tools: str - xlsynth_driver_dir: str + +class PresubmitConfig(NamedTuple): + repo_root: Path dslx_path: Optional[Tuple[str, ...]] + xlsynth_driver_version: str + xls_version: str + + +def _build_setting_override_flags(more_action_env: Optional[Dict[str, str]]) -> List[str]: + if not more_action_env: + return [] + mapping = { + 'XLSYNTH_DSLX_PATH': '@rules_xlsynth//config:dslx_path', + 'XLSYNTH_DSLX_ENABLE_WARNINGS': '@rules_xlsynth//config:enable_warnings', + 'XLSYNTH_DSLX_DISABLE_WARNINGS': '@rules_xlsynth//config:disable_warnings', + 'XLSYNTH_GATE_FORMAT': '@rules_xlsynth//config:gate_format', + 'XLSYNTH_ASSERT_FORMAT': '@rules_xlsynth//config:assert_format', + 'XLSYNTH_USE_SYSTEM_VERILOG': '@rules_xlsynth//config:use_system_verilog', + 'XLSYNTH_ADD_INVARIANT_ASSERTIONS': '@rules_xlsynth//config:add_invariant_assertions', + } + flags: List[str] = [] + for key, value in more_action_env.items(): + if key not in mapping: + raise ValueError('Unsupported override key: {}'.format(key)) + flags.append('--{}={}'.format(mapping[key], value)) + return flags + + +def _dslx_path_flags(dslx_path: Optional[Tuple[str, ...]]) -> List[str]: + if dslx_path is None: + return [] + return ['--@rules_xlsynth//config:dslx_path=' + ':'.join(dslx_path)] + + +def _presubmit_bazel_flags( + config: PresubmitConfig, + *, + dslx_path: Optional[Tuple[str, ...]] = None, + more_action_env: Optional[Dict[str, str]] = None) -> List[str]: + resolved_dslx_path = config.dslx_path if dslx_path is None else dslx_path + flags = _dslx_path_flags(resolved_dslx_path) + flags.extend(_build_setting_override_flags(more_action_env)) + return flags + + +def _run_bazel( + workspace_dir: Path, + subcommand: str, + targets: Tuple[str, ...], + flags: List[str], + *, + capture_output: bool) -> subprocess.CompletedProcess: + cmdline = [ + 'bazel', + '--bazelrc=/dev/null', + subcommand, + '--subcommands', + ] + flags + ['--', *targets] + print('Running command: ' + subprocess.list2cmdline(cmdline)) + return subprocess.run( + cmdline, + check = True, + cwd = str(workspace_dir), + stderr = subprocess.PIPE if capture_output else None, + stdout = subprocess.PIPE if capture_output else None, + encoding = 'utf-8' if capture_output else None, + ) - def __init__(self, xlsynth_tools: str, xlsynth_driver_dir: str, dslx_path: Optional[Tuple[str, ...]]): - self.xlsynth_tools = xlsynth_tools - self.xlsynth_driver_dir = xlsynth_driver_dir - self.dslx_path = dslx_path -def bazel_test_opt(targets: Tuple[str, ...], path_data: PathData, *, capture_output: bool = False, more_action_env: Optional[Dict[str, str]] = None): +def bazel_test_opt( + targets: Tuple[str, ...], + config: PresubmitConfig, + *, + workspace_dir: Optional[Path] = None, + capture_output: bool = False, + dslx_path: Optional[Tuple[str, ...]] = None, + more_action_env: Optional[Dict[str, str]] = None): assert isinstance(targets, tuple), targets flags = [] # Force Bazel to rebuild rather than reusing the local shared disk cache so that @@ -37,70 +103,54 @@ def bazel_test_opt(targets: Tuple[str, ...], path_data: PathData, *, capture_out # * --disk_cache= : overrides any ~/.bazelrc --disk_cache setting with an empty value # * --nocache_test_results : still avoid caching test results inside the build tree flags += ['--nocache_test_results', '--disk_cache=', '-c', 'opt', '--test_output=errors'] - flags += [ - '--action_env=XLSYNTH_TOOLS=' + path_data.xlsynth_tools, - '--action_env=XLSYNTH_DRIVER_DIR=' + path_data.xlsynth_driver_dir, - ] - if path_data.dslx_path is not None: - flags += [ - '--action_env=XLSYNTH_DSLX_PATH=' + ':'.join(path_data.dslx_path), - ] - if more_action_env: - for k, v in more_action_env.items(): - flags += ['--action_env=' + k + '=' + v] - # Use the caller's default .bazelrc files so presubmit and one-off runs behave consistently. - cmdline = [ - 'bazel', '--bazelrc=/dev/null', 'test', - '--test_output=errors', - '--subcommands', - ] + flags + ['--', *targets] - print('Running command: ' + subprocess.list2cmdline(cmdline)) - if capture_output: - subprocess.run(cmdline, check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8') - else: - subprocess.run(cmdline, check=True) - -def bazel_build_opt(targets: Tuple[str, ...], path_data: PathData, *, capture_output: bool = False, more_action_env: Optional[Dict[str, str]] = None): + flags += _presubmit_bazel_flags( + config, + dslx_path = dslx_path, + more_action_env = more_action_env, + ) + resolved_workspace_dir = config.repo_root if workspace_dir is None else workspace_dir + _run_bazel(resolved_workspace_dir, 'test', targets, flags, capture_output = capture_output) + + +def bazel_build_opt( + targets: Tuple[str, ...], + config: PresubmitConfig, + *, + workspace_dir: Optional[Path] = None, + capture_output: bool = False, + dslx_path: Optional[Tuple[str, ...]] = None, + more_action_env: Optional[Dict[str, str]] = None): """Run a `bazel build` over the given targets with the standard flags.""" assert isinstance(targets, tuple), targets flags = [] # Disable the shared disk cache for builds as well so we always rebuild fresh. flags += ['--disk_cache=', '-c', 'opt'] - flags += [ - '--action_env=XLSYNTH_TOOLS=' + path_data.xlsynth_tools, - '--action_env=XLSYNTH_DRIVER_DIR=' + path_data.xlsynth_driver_dir, - ] - if path_data.dslx_path is not None: - flags += [ - '--action_env=XLSYNTH_DSLX_PATH=' + ':'.join(path_data.dslx_path), - ] - if more_action_env: - for k, v in more_action_env.items(): - flags += ['--action_env=' + k + '=' + v] - # Use the caller's default .bazelrc so build behaviour matches regular developer invocations. - cmdline = [ - 'bazel', '--bazelrc=/dev/null', 'build', - '--subcommands', - ] + flags + ['--', *targets] - print('Running command: ' + subprocess.list2cmdline(cmdline)) - if capture_output: - subprocess.run(cmdline, check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8') - else: - subprocess.run(cmdline, check=True) + flags += _presubmit_bazel_flags( + config, + dslx_path = dslx_path, + more_action_env = more_action_env, + ) + resolved_workspace_dir = config.repo_root if workspace_dir is None else workspace_dir + _run_bazel(resolved_workspace_dir, 'build', targets, flags, capture_output = capture_output) @register -def run_sample(path_data: PathData): - bazel_test_opt(('//sample/...',), path_data) +def run_sample(config: PresubmitConfig): + bazel_test_opt(('//sample/...',), config) + @register -def run_sample_expecting_dslx_path(path_data: PathData): - path_data.dslx_path = ('sample_expecting_dslx_path', 'sample_expecting_dslx_path/subdir') - bazel_test_opt(('//sample_expecting_dslx_path:main_test', '//sample_expecting_dslx_path:add_mol_pipeline_sv_test'), path_data) +def run_sample_expecting_dslx_path(config: PresubmitConfig): + bazel_test_opt( + ('//sample_expecting_dslx_path:main_test', '//sample_expecting_dslx_path:add_mol_pipeline_sv_test'), + config, + dslx_path = ('sample_expecting_dslx_path', 'sample_expecting_dslx_path/subdir'), + ) + @register -def run_sample_failing_quickcheck(path_data: PathData): +def run_sample_failing_quickcheck(config: PresubmitConfig): try: - bazel_test_opt(('//sample_failing_quickcheck:failing_quickcheck_test',), path_data, capture_output=True) + bazel_test_opt(('//sample_failing_quickcheck:failing_quickcheck_test',), config, capture_output = True) except subprocess.CalledProcessError as e: if 'Found falsifying example after 1 tests' in e.stdout: pass @@ -110,7 +160,7 @@ def run_sample_failing_quickcheck(path_data: PathData): raise ValueError('Expected quickcheck to fail') try: - bazel_test_opt(('//sample_failing_quickcheck:failing_quickcheck_proof_test',), path_data, capture_output=True) + bazel_test_opt(('//sample_failing_quickcheck:failing_quickcheck_proof_test',), config, capture_output = True) except subprocess.CalledProcessError as e: m = re.search( r'ProofError: Failed to prove the property! counterexample: bits\[1\]:\d+, bits\[2\]:\d+', @@ -123,10 +173,10 @@ def run_sample_failing_quickcheck(path_data: PathData): raise ValueError('Unexpected error proving quickcheck: stdout: ' + e.stdout + ' stderr: ' + e.stderr) @register -def run_sample_disabling_warning(path_data: PathData): +def run_sample_disabling_warning(config: PresubmitConfig): try: print('Running with warnings as default...') - bazel_test_opt(('//sample_disabling_warning/...',), path_data, capture_output=True) + bazel_test_opt(('//sample_disabling_warning/...',), config, capture_output = True) except subprocess.CalledProcessError as e: want_warnings = [ 'is an empty range', @@ -142,17 +192,22 @@ def run_sample_disabling_warning(path_data: PathData): # Now we disable the warning and it should be ok. print('== Now running with warnings disabled...') - bazel_test_opt(('//sample_disabling_warning/...',), path_data, more_action_env={ - 'XLSYNTH_DSLX_DISABLE_WARNINGS': 'unused_definition,empty_range_literal' - }) + bazel_test_opt( + ('//sample_disabling_warning/...',), + config, + more_action_env = { + 'XLSYNTH_DSLX_DISABLE_WARNINGS': 'unused_definition,empty_range_literal', + }, + ) + @register -def run_sample_nonequiv_ir(path_data: PathData): +def run_sample_nonequiv_ir(config: PresubmitConfig): print('== Running yes-equivalent IR test...') - bazel_test_opt(('//sample_nonequiv_ir:add_one_ir_prove_equiv_test',), path_data, capture_output=True) + bazel_test_opt(('//sample_nonequiv_ir:add_one_ir_prove_equiv_test',), config, capture_output = True) print('== Running no-not-equivalent IR test...') try: - bazel_test_opt(('//sample_nonequiv_ir:add_one_ir_prove_equiv_expect_failure_test',), path_data, capture_output=True) + bazel_test_opt(('//sample_nonequiv_ir:add_one_ir_prove_equiv_expect_failure_test',), config, capture_output = True) except subprocess.CalledProcessError as e: if 'Verified NOT equivalent' in e.stdout: print('IRs are not equivalent as expected; bazel stdout: ' + repr(e.stdout) + ' bazel stderr: ' + repr(e.stderr)) @@ -162,20 +217,22 @@ def run_sample_nonequiv_ir(path_data: PathData): else: raise ValueError('Expected nonequiv IR to fail') + @register -def run_sample_with_formats(path_data: PathData): +def run_sample_with_formats(config: PresubmitConfig): bazel_test_opt( ('//sample_with_formats:gate_assert_minimal_sv_test',), - path_data, - more_action_env={ + config, + more_action_env = { 'XLSYNTH_GATE_FORMAT': 'br_gate_buf gated_{output}(.in({input}), .out({output}))', 'XLSYNTH_ASSERT_FORMAT': '`BR_ASSERT({label}, {condition})', 'XLSYNTH_USE_SYSTEM_VERILOG': 'true', }, ) + @register -def run_readme_sample_snippets(path_data: PathData): +def run_readme_sample_snippets(config: PresubmitConfig): """Ensures that the Starlark BUILD snippets in the README can be loaded by Bazel. The strategy is: @@ -189,13 +246,13 @@ def run_readme_sample_snippets(path_data: PathData): names used in the snippets are valid. """ - repo_root = os.path.dirname(__file__) - readme_path = os.path.join(repo_root, "README.md") + repo_root = config.repo_root + readme_path = repo_root / 'README.md' - if not os.path.exists(readme_path): + if not readme_path.exists(): raise RuntimeError(f"README.md not found at {readme_path}") - with open(readme_path, "r", encoding="utf-8") as f: + with open(readme_path, "r", encoding = "utf-8") as f: readme_text = f.read() # Extract ```starlark``` blocks. @@ -203,14 +260,34 @@ def run_readme_sample_snippets(path_data: PathData): if not snippet_blocks: raise RuntimeError("No starlark code blocks found in README.md") + + def is_module_snippet(block: str) -> bool: + module_markers = ( + 'bazel_dep(', + 'local_path_override(', + 'use_extension(', + 'use_repo(', + 'register_toolchains(', + 'xls.toolchain(', + ) + return any(marker in block for marker in module_markers) + # Flatten snippets into a list of lines, stripping trailing whitespace. snippet_lines = [] for idx, block in enumerate(snippet_blocks, 1): + if is_module_snippet(block): + print("--- Skipping README module snippet {} ---".format(idx)) + print(block.strip()) + print("---------------------------------------") + continue print("--- README snippet {} ---".format(idx)) print(block.strip()) print("----------------------") snippet_lines.extend([ln.rstrip() for ln in block.splitlines() if ln.strip()]) + if not snippet_lines: + raise RuntimeError('No BUILD-compatible starlark snippets found in README.md') + # Determine which rule symbols are used so we can create a single load(...). rule_name_pattern = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\(") used_rule_names: Set[str] = set() @@ -221,10 +298,10 @@ def run_readme_sample_snippets(path_data: PathData): # Inspect rules.bzl to know what symbols it actually exports so we only # attempt to load valid ones (e.g. we do NOT load `glob`). - rules_bzl_path = os.path.join(repo_root, "rules.bzl") + rules_bzl_path = repo_root / 'rules.bzl' exported_rule_names: Set[str] = set() - if os.path.exists(rules_bzl_path): - with open(rules_bzl_path, "r", encoding="utf-8") as rbzl: + if rules_bzl_path.exists(): + with open(rules_bzl_path, "r", encoding = "utf-8") as rbzl: for line in rbzl: m = re.match(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=", line) if m: @@ -236,9 +313,9 @@ def run_readme_sample_snippets(path_data: PathData): xfile_names = set(re.findall(r"['\"]([^'\"]+\.x)['\"]", "\n".join(snippet_lines))) # Create a temporary package under the repository root. - temp_pkg_dir = tempfile.mkdtemp(prefix="readme_snippets_", dir=repo_root) + temp_pkg_dir = tempfile.mkdtemp(prefix = 'readme_snippets_', dir = str(repo_root)) try: - build_path = os.path.join(temp_pkg_dir, "BUILD.bazel") + build_path = os.path.join(temp_pkg_dir, 'BUILD.bazel') with open(build_path, "w", encoding="utf-8") as bf: bf.write("# Auto-generated test BUILD file for README snippets.\n") @@ -285,20 +362,15 @@ def run_readme_sample_snippets(path_data: PathData): rel_pkg = os.path.relpath(temp_pkg_dir, repo_root) query_target = f"//{rel_pkg}:all" - env = os.environ.copy() - env["XLSYNTH_TOOLS"] = path_data.xlsynth_tools - env["XLSYNTH_DRIVER_DIR"] = path_data.xlsynth_driver_dir - if path_data.dslx_path is not None: - env["XLSYNTH_DSLX_PATH"] = ":".join(path_data.dslx_path) - print(f"Running bazel query on README snippets: {query_target}\nBUILD file used: {build_path}") try: result = subprocess.run([ - "bazel", - "query", - "--noshow_progress", + 'bazel', + 'query', + *_presubmit_bazel_flags(config), + '--noshow_progress', query_target, - ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8", env=env) + ], check = True, cwd = str(repo_root), stdout = subprocess.PIPE, stderr = subprocess.PIPE, encoding = 'utf-8') except subprocess.CalledProcessError as e: print("=== Bazel query failed ===") print("STDOUT:\n" + (e.stdout or "")) @@ -313,38 +385,124 @@ def run_readme_sample_snippets(path_data: PathData): else: shutil.rmtree(temp_pkg_dir) + @register -def run_sample_stitch_pipeline_expecting_dslx_path(path_data: PathData): - """Runs the pipeline stitching sample that relies on XLSYNTH_DSLX_PATH search paths.""" - path_data.dslx_path = ( - 'sample_stitch_expecting_dslx_path', - 'sample_stitch_expecting_dslx_path/subdir', +def run_workspace_toolchain_smoke_example(config: PresubmitConfig): + bazel_build_opt( + ('//:smoke_sv_types', '//:smoke_pipeline'), + config, + workspace_dir = config.repo_root / 'examples' / 'workspace_toolchain_smoke', ) + + +def _resolve_example_artifacts(config: PresubmitConfig, temp_root: Path) -> Dict[str, Path]: + plan = materialize_xls_bundle.resolve_artifact_plan( + artifact_source = 'download_only', + xls_version = config.xls_version, + driver_version = config.xlsynth_driver_version, + ) + if plan['mode'] != 'download': + raise ValueError('Expected download-backed example artifacts, got {}'.format(plan['mode'])) + resolved = materialize_xls_bundle.download_versioned_artifacts(temp_root, plan['xls_version']) + normalized_libxls = temp_root / materialize_xls_bundle.normalized_libxls_name(resolved['libxls']) + materialize_xls_bundle.copy_path(resolved['libxls'], normalized_libxls) + driver_path = materialize_xls_bundle.install_driver( + temp_root, + plan['driver_version'], + normalized_libxls, + resolved['dslx_stdlib_root'], + ) + return { + 'tools_root': resolved['tools_root'], + 'dslx_stdlib_root': resolved['dslx_stdlib_root'], + 'driver': driver_path, + 'libxls': normalized_libxls, + } + + +@register +def run_toolchain_helper_tests(config: PresubmitConfig): + bazel_test_opt( + ( + '//:make_env_helpers_test', + '//:env_helpers_test', + '//:artifact_resolution_test', + '//:download_release_test', + '//:external_bundle_exports_test', + ), + config, + ) + + +def _stage_local_dev_example_tree(config: PresubmitConfig) -> Path: + stage_root = Path('/tmp/xls-local-dev') + with tempfile.TemporaryDirectory(prefix = 'xls_local_dev_stage_', dir = str(config.repo_root)) as temp_dir: + temp_root = Path(temp_dir) + resolved = _resolve_example_artifacts(config, temp_root) + materialize_xls_bundle.ensure_clean_path(stage_root) + stage_root.mkdir(parents = True, exist_ok = True) + (stage_root / 'bin').mkdir(parents = True, exist_ok = True) + (stage_root / 'xls' / 'dslx').mkdir(parents = True, exist_ok = True) + materialize_xls_bundle.copy_path(resolved['driver'], stage_root / 'bin' / 'xlsynth-driver') + materialize_xls_bundle.copy_path(resolved['tools_root'], stage_root / 'tools') + materialize_xls_bundle.copy_path( + resolved['dslx_stdlib_root'], + stage_root / 'xls' / 'dslx' / 'stdlib', + ) + materialize_xls_bundle.copy_path( + resolved['libxls'], + stage_root / materialize_xls_bundle.normalized_libxls_name(resolved['libxls']), + ) + return stage_root + + +@register +def run_workspace_toolchain_local_dev_example(config: PresubmitConfig): + if sys.platform != 'linux': + print('Skipping local_paths example on non-Linux hosts; MODULE.bazel is pinned to libxls.so') + return + staged_root = _stage_local_dev_example_tree(config) + print('Staged local_paths example inputs at {}'.format(staged_root)) + bazel_build_opt( + ('//:smoke_sv_types', '//:smoke_pipeline'), + config, + workspace_dir = config.repo_root / 'examples' / 'workspace_toolchain_local_dev', + ) + + +@register +def run_sample_stitch_pipeline_expecting_dslx_path(config: PresubmitConfig): + """Runs the pipeline stitching sample that relies on the configured DSLX import paths.""" bazel_test_opt( ( '//sample_stitch_expecting_dslx_path:pipeline_stages_pipeline_build_test', ), - path_data, + config, + dslx_path = ( + 'sample_stitch_expecting_dslx_path', + 'sample_stitch_expecting_dslx_path/subdir', + ), ) + @register -def run_sample_invariant_assertions(path_data: PathData): +def run_sample_invariant_assertions(config: PresubmitConfig): """Builds a tiny design twice – with and without invariant assertions – and checks that the flag actually toggles assertion emission in the generated SystemVerilog. """ target = "//sample_invariant_assertions:array_match_sv" - repo_root = os.path.dirname(__file__) + repo_root = str(config.repo_root) # First, build with the flag *disabled* (explicit "false") and record the # produced Verilog so we know what the baseline looks like. - bazel_build_opt((target,), path_data, more_action_env={"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "false"}) + bazel_build_opt((target,), config, more_action_env = {"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "false"}) sv_path = os.path.join(repo_root, "bazel-bin", "sample_invariant_assertions", "array_match_sv.sv") with open(sv_path, "r", encoding="utf-8") as f: sv_without = f.read() # Now, build again but with the flag enabled. - bazel_build_opt((target,), path_data, more_action_env={"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "true"}) + bazel_build_opt((target,), config, more_action_env = {"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "true"}) with open(sv_path, "r", encoding="utf-8") as f: sv_with = f.read() @@ -358,14 +516,13 @@ def count_asserts(text: str) -> int: count_with_env = count_asserts(sv_with) if count_with_env <= count_without: raise ValueError( - "Expected more assertion machinery when enabling env-var; got {} vs {}".format(count_with_env, count_without) + "Expected more assertion machinery when enabling the build setting; got {} vs {}".format(count_with_env, count_without) ) - # Avoid non-ASCII arrow to ensure output is safe under ASCII-only stdout (e.g. Python 3.6 CI) - print(f"Env-var toggling works: {count_without} -> {count_with_env} assertions.") + print(f"Build-setting toggling works: {count_without} -> {count_with_env} assertions.") # -- Now verify rule-level override behaviour. tgt_attr_false = "//sample_invariant_assertions:array_match_sv_attr_false" - bazel_build_opt((tgt_attr_false,), path_data, more_action_env={"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "true"}) + bazel_build_opt((tgt_attr_false,), config, more_action_env = {"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "true"}) sv_path_attr_false = os.path.join(repo_root, "bazel-bin", "sample_invariant_assertions", "array_match_sv_attr_false.sv") with open(sv_path_attr_false, "r", encoding="utf-8") as f: sv_attr_false = f.read() @@ -373,12 +530,12 @@ def count_asserts(text: str) -> int: count_attr_false = count_asserts(sv_attr_false) if count_attr_false != count_without: raise ValueError( - "Rule attribute 'false' did not override env-var 'true'; expected {} asserts but saw {}".format(count_without, count_attr_false) + "Rule attribute 'false' did not override build-setting 'true'; expected {} asserts but saw {}".format(count_without, count_attr_false) ) - print("Rule override to 'false' correctly suppressed extra assertions despite env-var=true.") + print("Rule override to 'false' correctly suppressed extra assertions despite build-setting=true.") tgt_attr_true = "//sample_invariant_assertions:array_match_sv_attr_true" - bazel_build_opt((tgt_attr_true,), path_data, more_action_env={"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "false"}) + bazel_build_opt((tgt_attr_true,), config, more_action_env = {"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "false"}) sv_path_attr_true = os.path.join(repo_root, "bazel-bin", "sample_invariant_assertions", "array_match_sv_attr_true.sv") with open(sv_path_attr_true, "r", encoding="utf-8") as f: sv_attr_true = f.read() @@ -386,27 +543,27 @@ def count_asserts(text: str) -> int: count_attr_true = count_asserts(sv_attr_true) if count_attr_true <= count_without: raise ValueError( - "Rule attribute 'true' did not override env-var 'false'; counts {} vs baseline {}".format(count_attr_true, count_without) + "Rule attribute 'true' did not override build-setting 'false'; counts {} vs baseline {}".format(count_attr_true, count_without) ) - print("Rule override to 'true' correctly enabled assertions despite env-var=false.") + print("Rule override to 'true' correctly enabled assertions despite build-setting=false.") # ----------------------------------------------------------------------------- @register -def run_stitch_invariant_assertions(path_data: PathData): +def run_stitch_invariant_assertions(config: PresubmitConfig): """Analogous checks for dslx_stitch_pipeline rule overrides.""" - repo_root = os.path.dirname(__file__) + repo_root = str(config.repo_root) base_tgt = "//sample_stitch_invariant_assertions:stages_pipeline" - # Build with env-var off, record baseline. - bazel_build_opt((base_tgt,), path_data, more_action_env={"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "false"}) + # Build with the build setting off, record baseline. + bazel_build_opt((base_tgt,), config, more_action_env = {"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "false"}) sv_base = os.path.join(repo_root, "bazel-bin", "sample_stitch_invariant_assertions", "stages_pipeline.sv") with open(sv_base, "r", encoding="utf-8") as f: sv_without = f.read() - # Build with env-var on, capture. - bazel_build_opt((base_tgt,), path_data, more_action_env={"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "true"}) + # Build with the build setting on, capture. + bazel_build_opt((base_tgt,), config, more_action_env = {"XLSYNTH_ADD_INVARIANT_ASSERTIONS": "true"}) with open(sv_base, "r", encoding="utf-8") as f: sv_with_env = f.read() @@ -417,13 +574,14 @@ def cnt(txt: str) -> int: env_cnt = cnt(sv_with_env) if base_cnt != 0: - raise ValueError(f"Expected zero assertions with env-var=false; got {base_cnt}") + raise ValueError(f"Expected zero assertions with build-setting=false; got {base_cnt}") if env_cnt == 0: - raise ValueError("Expected assertions to be present with env-var=true but count was zero") + raise ValueError("Expected assertions to be present with build-setting=true but count was zero") print(f"Stitch pipeline assertion counts: disabled={base_cnt}, enabled={env_cnt} (ok)") + def parse_versions_toml(path): crate_version = None dso_version = None @@ -440,139 +598,28 @@ def parse_versions_toml(path): raise RuntimeError(f'Could not parse crate or dso version from {path}') return crate_version, dso_version -def find_dso(dso_filename, search_dirs): - for d in search_dirs: - candidate = os.path.join(d, dso_filename) - if os.path.exists(candidate): - return candidate - # Use ldconfig to find shared library paths - try: - output = subprocess.check_output(['ldconfig', '-p'], encoding='utf-8', stderr=subprocess.DEVNULL) - for line in output.splitlines(): - line = line.strip() - if dso_filename in line: - parts = line.split('=>') - if len(parts) == 2: - candidate = parts[1].strip() - if os.path.exists(candidate): - return candidate - except Exception: - pass # ldconfig may not be available (e.g. on macOS) - return None - -def _fetch_remote_sha256(url: str) -> str: - """Fetches the expected SHA-256 (first token) from a .sha256 URL.""" - try: - with urllib.request.urlopen(url, timeout=15) as response: - text = response.read().decode('utf-8') - except Exception as e: - raise RuntimeError(f'Failed to fetch SHA256 from {url}: {e}') - first_token = text.strip().split()[0] - if not re.fullmatch(r'[0-9a-fA-F]{64}', first_token): - raise RuntimeError(f'Unexpected SHA256 file contents from {url}: {text}') - return first_token - - -def _sha256_of_file(path: str) -> str: - """Computes the SHA-256 digest of the given file and returns it as hex.""" - h = hashlib.sha256() - with open(path, 'rb') as f: - for chunk in iter(lambda: f.read(8192), b''): - h.update(chunk) - return h.hexdigest() - -def verify_tool_binaries(tools_dir: str, version: str, *, platform: str = 'ubuntu2004') -> None: - """Verifies that each tool binary's SHA-256 matches the released checksum. +def build_presubmit_config(repo_root: Path, dslx_path: Optional[Tuple[str, ...]]) -> PresubmitConfig: + versions_path = repo_root / 'xlsynth-versions.toml' + xlsynth_driver_version, xls_version = parse_versions_toml(versions_path) + return PresubmitConfig( + repo_root = repo_root, + dslx_path = dslx_path, + xlsynth_driver_version = xlsynth_driver_version, + xls_version = xls_version, + ) - Raises RuntimeError on mismatch. - """ - base_url = f"https://github.com/xlsynth/xlsynth/releases/download/v{version}" - artifacts = [ - 'dslx_interpreter_main', - 'ir_converter_main', - 'codegen_main', - 'opt_main', - 'prove_quickcheck_main', - 'typecheck_main', - 'dslx_fmt', - 'delay_info_main', - 'check_ir_equivalence_main', - ] - - for art in artifacts: - local_path = os.path.join(tools_dir, art) - if not os.path.exists(local_path): - raise RuntimeError(f'Expected tool binary not found at {local_path}') - remote_asset = f"{art}-{platform}" - remote_sha_url = f"{base_url}/{remote_asset}.sha256" - expected = _fetch_remote_sha256(remote_sha_url) - actual = _sha256_of_file(local_path) - if actual != expected: - raise RuntimeError( - f'SHA256 mismatch for {art}: local {actual} != expected {expected} (from {remote_sha_url})' - ) - else: - print(f"Verified SHA256 for {art} matches release v{version}.") - - # Verify the libxls DSO if present next to the repository root. - dso_name = f"libxls-{platform}.so.gz" - dso_path = find_dso(dso_name, [os.getcwd(), '/usr/lib', '/usr/local/lib']) - if dso_path: - remote_sha_url = f"{base_url}/libxls-{platform}.so.gz.sha256" - expected = _fetch_remote_sha256(remote_sha_url) - actual = _sha256_of_file(dso_path) - if actual != expected: - raise RuntimeError( - f'SHA256 mismatch for {dso_name}: local {actual} != expected {expected} (from {remote_sha_url})' - ) - else: - print(f"Verified SHA256 for {dso_name} matches release v{version}.") - else: - print(f"WARNING: Could not find local libxls DSO ({dso_name}); skipping checksum verification.") def main(): parser = optparse.OptionParser() - parser.add_option('--xlsynth-tools', type='string', help='Path to xlsynth tools') - parser.add_option('--xlsynth-driver-dir', type='string', help='Path to xlsynth driver directory') - parser.add_option('--dslx-path', type='string', help='Path to DSLX standard library') + parser.add_option('--dslx-path', type = 'string', help = 'Colon-separated extra DSLX import roots for steps that need them') parser.add_option('-k', '--keyword', type='string', help='Only run tests whose function name contains this keyword') (options, args) = parser.parse_args() - if options.xlsynth_tools is None or options.xlsynth_driver_dir is None: - parser.error('Missing required argument(s): --xlsynth-tools, --xlsynth-driver-dir, --dslx-path') + if args: + parser.error('Unexpected positional arguments: {}'.format(' '.join(args))) return dslx_path = options.dslx_path.split(':') if options.dslx_path else None - # Canonicalize directories (remove redundant slashes, resolve '..') so that - # downstream path concatenations do not introduce double-slash artefacts. - tools_dir = str(Path(options.xlsynth_tools).expanduser().resolve()) - driver_dir = str(Path(options.xlsynth_driver_dir).expanduser().resolve()) - - path_data = PathData( - xlsynth_tools=tools_dir, - xlsynth_driver_dir=driver_dir, - dslx_path=dslx_path, - ) - - # Version check for xlsynth-driver and DSO - versions_path = os.path.join(os.path.dirname(__file__), 'xlsynth-versions.toml') - crate_version, dso_version = parse_versions_toml(versions_path) - driver_path = os.path.join(path_data.xlsynth_driver_dir, 'xlsynth-driver') - try: - version_out = subprocess.check_output([driver_path, '--version'], encoding='utf-8').strip() - except Exception as e: - raise RuntimeError(f'Could not run xlsynth-driver at {driver_path}: {e}') - m = re.search(r'(\d+\.\d+\.\d+)', version_out) - if not m: - raise RuntimeError(f'Could not parse version from xlsynth-driver --version output: {version_out}') - actual_version = m.group(1) - if actual_version != crate_version: - raise RuntimeError(f'xlsynth-driver version {actual_version} does not match required {crate_version}. Please update your xlsynth-driver.') - # DSO existence check removed; assume xlsynth-driver can run if version matches - - verify_tool_binaries(path_data.xlsynth_tools, dso_version) - - assert os.path.exists(os.path.join(path_data.xlsynth_tools, 'dslx_interpreter_main')), 'dslx_interpreter_main not found in XLSYNTH_TOOLS=' + path_data.xlsynth_tools - assert os.path.exists(os.path.join(path_data.xlsynth_driver_dir, 'xlsynth-driver')), 'xlsynth-driver not found in XLSYNTH_DRIVER_DIR=' + path_data.xlsynth_driver_dir + config = build_presubmit_config(Path(__file__).resolve().parent, tuple(dslx_path) if dslx_path else None) to_run = TO_RUN if options.keyword: @@ -584,10 +631,8 @@ def main(): print('Executing', f.__name__) print('-' * 80) - print(f"xlsynth-driver version: {version_out}") - try: - f(path_data) + f(config) except Exception as e: err_msg = str(e) failures.append((f.__name__, err_msg)) diff --git a/sample/BUILD.bazel b/sample/BUILD.bazel index 6f48923..80be411 100644 --- a/sample/BUILD.bazel +++ b/sample/BUILD.bazel @@ -37,12 +37,26 @@ dslx_to_sv_types( deps = [":imported"], ) +# Prove the explicit bundle override path on a supported leaf rule. +dslx_to_sv_types( + name = "imported_pkg_explicit_bundle", + sv_enum_case_naming_policy = "unqualified", + xls_bundle = "@rules_xlsynth_selftest_xls//:bundle", + deps = [":imported"], +) + diff_test( name = "imported_pkg_test", file1 = ":imported_pkg", file2 = ":imported_pkg.golden.sv", ) +diff_test( + name = "imported_pkg_explicit_bundle_test", + file1 = ":imported_pkg_explicit_bundle", + file2 = ":imported_pkg.golden.sv", +) + dslx_library( name = "reversed_struct_ordering", srcs = ["reversed_struct_ordering.x"], @@ -67,17 +81,41 @@ dslx_library( deps = [":imported"], ) +dslx_library( + name = "sample_explicit_bundle", + srcs = ["sample.x"], + deps = [":imported"], + xls_bundle = "@rules_xlsynth_selftest_xls//:bundle", +) + +build_test( + name = "sample_explicit_bundle_typecheck_test", + targets = [":sample_explicit_bundle"], +) + dslx_to_ir( name = "sample_ir", lib = ":sample", top = "main", ) +dslx_to_ir( + name = "sample_ir_explicit_bundle", + lib = ":sample_explicit_bundle", + top = "main", + xls_bundle = "@rules_xlsynth_selftest_xls//:bundle", +) + build_test( name = "sample_ir_test", targets = [":sample_ir"], ) +build_test( + name = "sample_ir_explicit_bundle_test", + targets = [":sample_ir_explicit_bundle"], +) + # Prove the unoptimized and optimized IRs are equivalent. ir_prove_equiv_test( name = "sample_ir_prove_equiv_test", @@ -138,6 +176,12 @@ dslx_test( deps = [":sample"], ) +dslx_test( + name = "sample_explicit_bundle_test", + deps = [":sample_explicit_bundle"], + xls_bundle = "@rules_xlsynth_selftest_xls//:bundle", +) + dslx_test( name = "sample_src_test", src = "sample_src_test.x", @@ -240,6 +284,13 @@ dslx_prove_quickcheck_test( top = "quickcheck_main", ) +dslx_prove_quickcheck_test( + name = "sample_prove_quickcheck_explicit_bundle_test", + lib = ":sample_explicit_bundle", + top = "quickcheck_main", + xls_bundle = "@rules_xlsynth_selftest_xls//:bundle", +) + # Pipeline stitching example dslx_library( diff --git a/sample_expecting_dslx_path/BUILD.bazel b/sample_expecting_dslx_path/BUILD.bazel index df7cc5a..6e79f12 100644 --- a/sample_expecting_dslx_path/BUILD.bazel +++ b/sample_expecting_dslx_path/BUILD.bazel @@ -3,8 +3,8 @@ # Sample BUILD file # This example shows how to use the dslx_library and dslx_test rules # -# This particular sample shows how things work when we give a XLSYNTH_DSLX_PATH -# environment variable. +# This particular sample shows how things work when +# `@rules_xlsynth//config:dslx_path` adds an import root with bare module names. load("//:rules.bzl", "dslx_library", "dslx_test", "dslx_to_pipeline") load("@bazel_skylib//rules:diff_test.bzl", "diff_test") diff --git a/sample_invariant_assertions/BUILD.bazel b/sample_invariant_assertions/BUILD.bazel index 5810b84..b11490e 100644 --- a/sample_invariant_assertions/BUILD.bazel +++ b/sample_invariant_assertions/BUILD.bazel @@ -3,8 +3,8 @@ load("//:rules.bzl", "dslx_library", "dslx_to_pipeline") # DSLX library containing the example function that will exercise the -# invariant-assertion code-generation path when -# XLSYNTH_ADD_INVARIANT_ASSERTIONS=true. +# invariant-assertion code-generation path when the toolchain or build-setting +# default enables invariant assertions. dslx_library( name = "array_match", srcs = ["array_match.x"], @@ -21,8 +21,8 @@ dslx_to_pipeline( ) # -- Targets that explicitly override the invariant-assertion setting via rule -# attribute so we can test that it supersedes any value coming from the -# environment / toolchain TOML. +# attribute so we can test that it supersedes any repo-wide toolchain TOML +# default. dslx_to_pipeline( name = "array_match_sv_attr_true", diff --git a/sample_stitch_expecting_dslx_path/BUILD.bazel b/sample_stitch_expecting_dslx_path/BUILD.bazel index cbeb61f..4ff3af8 100644 --- a/sample_stitch_expecting_dslx_path/BUILD.bazel +++ b/sample_stitch_expecting_dslx_path/BUILD.bazel @@ -19,8 +19,8 @@ dslx_library( ":imported", "//sample_stitch_expecting_dslx_path/subdir:another", ], - # Note: we still demonstrate XLSYNTH_DSLX_PATH because the DSLX imports use - # bare module names (no explicit path prefixes). + # Note: we still demonstrate @rules_xlsynth//config:dslx_path because the + # DSLX imports use bare module names (no explicit path prefixes). ) # Stitch the stage functions into a wrapper module. diff --git a/xls_toolchain.bzl b/xls_toolchain.bzl new file mode 100644 index 0000000..b0c8f4c --- /dev/null +++ b/xls_toolchain.bzl @@ -0,0 +1,458 @@ +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain", "use_cpp_toolchain") +load("@rules_cc//cc:defs.bzl", "CcInfo", "cc_common") + +_XLS_TOOLCHAIN_TYPE = "//:toolchain_type" +_TRI_STATE_VALUES = ["", "true", "false"] + +XlsArtifactBundleInfo = provider( + doc = "Versioned or local XLS bundle artifacts materialized by the module extension.", + fields = { + "artifact_inputs": "Declared files that must be present for actions using the bundle.", + "driver": "The xlsynth-driver binary.", + "driver_supports_sv_enum_case_naming_policy": "Whether the driver accepts --sv_enum_case_naming_policy.", + "driver_supports_sv_struct_field_ordering": "Whether the driver accepts --sv_struct_field_ordering.", + "dslx_stdlib": "The DSLX stdlib root tree artifact.", + "dslx_stdlib_path": "Directory path containing the DSLX stdlib sources.", + "libxls": "The libxls shared library file.", + "runtime_library_path": "Directory containing libxls for runtime loading.", + "tools_root": "The XLS tool root tree artifact.", + "tools_path": "Directory path containing the XLS tool binaries.", + }, +) + +def _split_nonempty(value, separator): + if not value: + return [] + items = [] + for item in value.split(separator): + stripped = item.strip() + if stripped: + items.append(stripped) + return items + +def _validate_tri_state(value, label): + if value not in _TRI_STATE_VALUES: + fail("{} must be one of {}".format(label, _TRI_STATE_VALUES)) + return value + +def _single_artifact(target, label): + files = target[DefaultInfo].files.to_list() + if len(files) != 1: + fail("{} must provide exactly one artifact, got {}".format(label, len(files))) + return files[0] + +def _single_directory_artifact(target, label): + files = target[DefaultInfo].files.to_list() + if len(files) != 1: + fail("{} must provide exactly one directory artifact, got {}".format(label, len(files))) + artifact = files[0] + if not artifact.is_directory: + fail("{} must provide a directory artifact".format(label)) + return artifact + +def _artifact_directory(files, label): + if not files: + fail("{} must provide at least one artifact".format(label)) + dirname = files[0].dirname + for file in files[1:]: + if file.dirname != dirname: + fail("{} artifacts must share one directory; got {} and {}".format(label, dirname, file.dirname)) + return dirname + +def _bundle_struct_from_provider(bundle): + return struct( + artifact_inputs = bundle.artifact_inputs, + driver = bundle.driver, + driver_path = bundle.driver.path, + driver_supports_sv_enum_case_naming_policy = bundle.driver_supports_sv_enum_case_naming_policy, + driver_supports_sv_struct_field_ordering = bundle.driver_supports_sv_struct_field_ordering, + dslx_stdlib = bundle.dslx_stdlib, + dslx_stdlib_path = bundle.dslx_stdlib_path, + libxls = bundle.libxls, + runtime_library_path = bundle.runtime_library_path, + tools_root = bundle.tools_root, + tools_path = bundle.tools_path, + ) + +def _toolchain_with_semantics(artifact_selection, ctx): + use_system_verilog = _validate_tri_state( + ctx.attr._use_system_verilog_flag[BuildSettingInfo].value, + "@rules_xlsynth//config:use_system_verilog", + ) + add_invariant_assertions = _validate_tri_state( + ctx.attr._add_invariant_assertions_flag[BuildSettingInfo].value, + "@rules_xlsynth//config:add_invariant_assertions", + ) + return platform_common.ToolchainInfo( + artifact_inputs = artifact_selection.artifact_inputs, + driver = artifact_selection.driver, + driver_path = artifact_selection.driver_path, + tools_root = artifact_selection.tools_root, + tools_path = artifact_selection.tools_path, + dslx_stdlib = artifact_selection.dslx_stdlib, + dslx_stdlib_path = artifact_selection.dslx_stdlib_path, + libxls = artifact_selection.libxls, + runtime_library_path = artifact_selection.runtime_library_path, + driver_supports_sv_enum_case_naming_policy = artifact_selection.driver_supports_sv_enum_case_naming_policy, + driver_supports_sv_struct_field_ordering = artifact_selection.driver_supports_sv_struct_field_ordering, + dslx_path = _split_nonempty(ctx.attr._dslx_path_flag[BuildSettingInfo].value, ":"), + enable_warnings = _split_nonempty(ctx.attr._enable_warnings_flag[BuildSettingInfo].value, ","), + disable_warnings = _split_nonempty(ctx.attr._disable_warnings_flag[BuildSettingInfo].value, ","), + gate_format = ctx.attr._gate_format_flag[BuildSettingInfo].value, + assert_format = ctx.attr._assert_format_flag[BuildSettingInfo].value, + use_system_verilog = use_system_verilog, + add_invariant_assertions = add_invariant_assertions, + ) + +def _xls_toolchain_impl(ctx): + artifact_selection = _bundle_struct_from_provider(ctx.attr.bundle[XlsArtifactBundleInfo]) + return [_toolchain_with_semantics(artifact_selection, ctx)] + +xls_toolchain = rule( + implementation = _xls_toolchain_impl, + attrs = { + "bundle": attr.label(mandatory = True, providers = [XlsArtifactBundleInfo]), + "_dslx_path_flag": attr.label(default = "//config:dslx_path"), + "_enable_warnings_flag": attr.label(default = "//config:enable_warnings"), + "_disable_warnings_flag": attr.label(default = "//config:disable_warnings"), + "_gate_format_flag": attr.label(default = "//config:gate_format"), + "_assert_format_flag": attr.label(default = "//config:assert_format"), + "_use_system_verilog_flag": attr.label(default = "//config:use_system_verilog"), + "_add_invariant_assertions_flag": attr.label(default = "//config:add_invariant_assertions"), + }, +) + +def _xls_bundle_impl(ctx): + tool_files = ctx.attr.tools_root[DefaultInfo].files.to_list() + dslx_stdlib = _single_directory_artifact(ctx.attr.dslx_stdlib, "dslx_stdlib") + driver = _single_artifact(ctx.attr.driver, "driver") + libxls = _single_artifact(ctx.attr.libxls, "libxls") + artifact_inputs = tool_files + [dslx_stdlib, driver, libxls] + tools_path = _artifact_directory(tool_files, "tools_root") + return [ + XlsArtifactBundleInfo( + artifact_inputs = artifact_inputs, + driver = driver, + driver_supports_sv_enum_case_naming_policy = ctx.attr.driver_supports_sv_enum_case_naming_policy, + driver_supports_sv_struct_field_ordering = ctx.attr.driver_supports_sv_struct_field_ordering, + dslx_stdlib = dslx_stdlib, + dslx_stdlib_path = dslx_stdlib.path, + libxls = libxls, + runtime_library_path = libxls.dirname, + tools_root = tool_files[0], + tools_path = tools_path, + ), + DefaultInfo(files = depset(direct = artifact_inputs)), + ] + +xls_bundle = rule( + implementation = _xls_bundle_impl, + attrs = { + "driver": attr.label(allow_files = True, mandatory = True), + "driver_supports_sv_enum_case_naming_policy": attr.bool(default = False), + "driver_supports_sv_struct_field_ordering": attr.bool(default = False), + "dslx_stdlib": attr.label(mandatory = True), + "libxls": attr.label(allow_files = True, mandatory = True), + "tools_root": attr.label(mandatory = True), + }, +) + +def get_xls_toolchain(ctx): + return ctx.toolchains[_XLS_TOOLCHAIN_TYPE] + +def _require_common_toolchain(toolchain): + if not toolchain.tools_path: + fail("rules_xlsynth requires a configured XLS tools root") + if not toolchain.dslx_stdlib_path: + fail("rules_xlsynth requires a configured DSLX stdlib root") + +def _merge_toolchain_with_bundle(toolchain, bundle): + artifact_selection = _bundle_struct_from_provider(bundle) + return struct( + artifact_inputs = artifact_selection.artifact_inputs, + driver = artifact_selection.driver, + driver_path = artifact_selection.driver_path, + driver_supports_sv_enum_case_naming_policy = artifact_selection.driver_supports_sv_enum_case_naming_policy, + driver_supports_sv_struct_field_ordering = artifact_selection.driver_supports_sv_struct_field_ordering, + dslx_stdlib = artifact_selection.dslx_stdlib, + dslx_stdlib_path = artifact_selection.dslx_stdlib_path, + libxls = artifact_selection.libxls, + runtime_library_path = artifact_selection.runtime_library_path, + tools_root = artifact_selection.tools_root, + tools_path = artifact_selection.tools_path, + dslx_path = toolchain.dslx_path, + enable_warnings = toolchain.enable_warnings, + disable_warnings = toolchain.disable_warnings, + gate_format = toolchain.gate_format, + assert_format = toolchain.assert_format, + use_system_verilog = toolchain.use_system_verilog, + add_invariant_assertions = toolchain.add_invariant_assertions, + ) + +def require_driver_toolchain(ctx): + toolchain = get_xls_toolchain(ctx) + if not toolchain.driver_path: + fail("rules_xlsynth requires a configured xlsynth-driver") + _require_common_toolchain(toolchain) + return toolchain + +def require_tools_toolchain(ctx): + toolchain = get_xls_toolchain(ctx) + _require_common_toolchain(toolchain) + return toolchain + +def get_selected_tools_toolchain(ctx): + toolchain = require_tools_toolchain(ctx) + if hasattr(ctx.attr, "xls_bundle") and ctx.attr.xls_bundle: + return _merge_toolchain_with_bundle(toolchain, ctx.attr.xls_bundle[XlsArtifactBundleInfo]) + return toolchain + +def get_selected_driver_toolchain(ctx): + toolchain = get_selected_tools_toolchain(ctx) + if not toolchain.driver_path: + fail("rules_xlsynth requires a configured xlsynth-driver") + return toolchain + +def _toml_quote(value): + return "\"{}\"".format(value.replace("\\", "\\\\").replace("\"", "\\\"")) + +def _toml_array(values): + return "[{}]".format(", ".join([_toml_quote(value) for value in values])) + +def _resolve_tri_state(default_value, override_value): + if override_value == "": + return default_value + return override_value + +def declare_xls_toolchain_toml( + ctx, + *, + name, + toolchain = None, + gate_format = None, + assert_format = None, + use_system_verilog = "", + add_invariant_assertions = "", + array_index_bounds_checking = ""): + resolved_toolchain = require_tools_toolchain(ctx) if toolchain == None else toolchain + + resolved_use_system_verilog = _resolve_tri_state(resolved_toolchain.use_system_verilog, use_system_verilog) + resolved_add_invariant_assertions = _resolve_tri_state(resolved_toolchain.add_invariant_assertions, add_invariant_assertions) + resolved_gate_format = resolved_toolchain.gate_format if gate_format == None else gate_format + resolved_assert_format = resolved_toolchain.assert_format if assert_format == None else assert_format + + lines = [ + "[toolchain]", + "tool_path = {}".format(_toml_quote(resolved_toolchain.tools_path)), + "", + "[toolchain.dslx]", + "dslx_stdlib_path = {}".format(_toml_quote(resolved_toolchain.dslx_stdlib_path)), + "dslx_path = {}".format(_toml_array(resolved_toolchain.dslx_path)), + "enable_warnings = {}".format(_toml_array(resolved_toolchain.enable_warnings)), + "disable_warnings = {}".format(_toml_array(resolved_toolchain.disable_warnings)), + ] + lines.extend([ + "", + "[toolchain.codegen]", + ]) + if resolved_gate_format: + lines.append("gate_format = {}".format(_toml_quote(resolved_gate_format))) + if resolved_assert_format: + lines.append("assert_format = {}".format(_toml_quote(resolved_assert_format))) + if resolved_use_system_verilog: + lines.append("use_system_verilog = {}".format(resolved_use_system_verilog)) + if resolved_add_invariant_assertions: + lines.append("add_invariant_assertions = {}".format(resolved_add_invariant_assertions)) + if array_index_bounds_checking: + lines.append("array_index_bounds_checking = {}".format(array_index_bounds_checking)) + + toolchain_toml = ctx.actions.declare_file("{}_{}.toml".format(ctx.label.name, name)) + ctx.actions.write( + output = toolchain_toml, + content = "\n".join(lines) + "\n", + ) + return toolchain_toml + +def get_toolchain_artifact_inputs(toolchain): + return getattr(toolchain, "artifact_inputs", []) + +def _bundle_runtime_inputs(toolchain): + inputs = [] + for field_name in ["dslx_stdlib", "libxls"]: + artifact = getattr(toolchain, field_name, None) + if artifact != None: + inputs.append(artifact) + return inputs + +def _bundle_tool_input(toolchain, tool_name): + artifact_inputs = get_toolchain_artifact_inputs(toolchain) + if not artifact_inputs: + return None + matches = [artifact for artifact in artifact_inputs if artifact.basename == tool_name] + if len(matches) != 1: + fail("rules_xlsynth toolchain is missing tool artifact {}".format(tool_name)) + return matches[0] + +def _bundle_tool_inputs(toolchain, tool_names): + inputs = [] + for tool_name in tool_names: + tool_input = _bundle_tool_input(toolchain, tool_name) + if tool_input not in inputs: + inputs.append(tool_input) + return inputs + +def get_driver_artifact_inputs(toolchain, tool_names = []): + driver = getattr(toolchain, "driver", None) + if driver == None: + return get_toolchain_artifact_inputs(toolchain) + return [driver] + _bundle_tool_inputs(toolchain, tool_names) + _bundle_runtime_inputs(toolchain) + +def get_tool_artifact_inputs(toolchain, tool_name): + tool_input = _bundle_tool_input(toolchain, tool_name) + if tool_input == None: + return get_toolchain_artifact_inputs(toolchain) + return [tool_input] + _bundle_runtime_inputs(toolchain) + +def _patch_dylib_impl(ctx): + ctx.actions.run_shell( + inputs = [ctx.file.src], + outputs = [ctx.outputs.out], + command = """ + cp {infile} {outfile} + install_name_tool -id @rpath/{libname} {outfile} + """.format( + infile = ctx.file.src.path, + outfile = ctx.outputs.out.path, + libname = ctx.outputs.out.basename, + ), + progress_message = "Patching dylib install name", + ) + return DefaultInfo(files = depset([ctx.outputs.out])) + +patch_dylib = rule( + implementation = _patch_dylib_impl, + attrs = { + "src": attr.label(mandatory = True, allow_single_file = True), + "out": attr.output(), + }, +) + +def _copy_flat_files_to_directory_impl(ctx): + output = ctx.actions.declare_directory(ctx.label.name) + input_paths = [src.path for src in ctx.files.srcs] + ctx.actions.run_shell( + inputs = ctx.files.srcs, + outputs = [output], + arguments = [output.path] + input_paths, + command = """ + set -euo pipefail + out="$1" + shift + mkdir -p "$out" + for src in "$@"; do + cp "$src" "$out/$(basename "$src")" + done + """, + progress_message = "Copying files into {}".format(ctx.label), + ) + return DefaultInfo(files = depset([output])) + +copy_flat_files_to_directory = rule( + implementation = _copy_flat_files_to_directory_impl, + attrs = { + "srcs": attr.label_list(allow_files = True, mandatory = True), + }, +) + +def _xlsynth_artifact_config_impl(ctx): + bundle_root = ctx.label.name + config_output = ctx.actions.declare_file("{}/xlsynth_artifact_config.toml".format(bundle_root)) + dso_output = ctx.actions.declare_file("{}/{}".format(bundle_root, ctx.attr.dso_name)) + stdlib_output = ctx.actions.declare_directory("{}/dslx_stdlib".format(bundle_root)) + dslx_stdlib = _single_directory_artifact(ctx.attr.dslx_stdlib, "dslx_stdlib") + shared_library = ctx.file.shared_library + ctx.actions.run_shell( + inputs = [dslx_stdlib, shared_library], + outputs = [config_output, dso_output, stdlib_output], + arguments = [ + shared_library.path, + dso_output.path, + dslx_stdlib.path, + stdlib_output.path, + config_output.path, + ctx.attr.dso_name, + ], + command = """ + set -euo pipefail + shared_library="$1" + dso_output="$2" + dslx_stdlib="$3" + stdlib_output="$4" + config_output="$5" + dso_name="$6" + + mkdir -p "$(dirname "$dso_output")" "$(dirname "$config_output")" "$stdlib_output" + cp "$shared_library" "$dso_output" + cp -R "$dslx_stdlib"/. "$stdlib_output" + printf 'dso_path = "%s"\\ndslx_stdlib_path = "dslx_stdlib"\\n' \ + "$dso_name" > "$config_output" + """, + progress_message = "Packaging xlsynth artifact config for {}".format(ctx.label), + ) + packaged_files = [config_output, dso_output, stdlib_output] + return DefaultInfo( + files = depset([config_output]), + runfiles = ctx.runfiles(files = packaged_files), + ) + +xlsynth_artifact_config = rule( + implementation = _xlsynth_artifact_config_impl, + attrs = { + "dso_name": attr.string(mandatory = True), + "dslx_stdlib": attr.label(mandatory = True), + "shared_library": attr.label(allow_single_file = True, mandatory = True), + }, +) + +def _xls_shared_library_link_impl(ctx): + cc_toolchain = find_cpp_toolchain(ctx) + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + shared_library = ctx.file.shared_library + library_to_link = cc_common.create_library_to_link( + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + dynamic_library = shared_library, + ) + linker_input = cc_common.create_linker_input( + owner = ctx.label, + libraries = depset([library_to_link]), + ) + linking_context = cc_common.create_linking_context( + linker_inputs = depset([linker_input]), + ) + runtime_files = [shared_library] + ctx.files.runtime_files + return [ + DefaultInfo( + files = depset(runtime_files), + runfiles = ctx.runfiles(files = runtime_files), + ), + CcInfo(linking_context = linking_context), + ] + +xls_shared_library_link = rule( + implementation = _xls_shared_library_link_impl, + attrs = { + "runtime_files": attr.label(allow_files = True), + "shared_library": attr.label(mandatory = True, allow_single_file = True), + "_cc_toolchain": attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")), + }, + fragments = ["cpp"], + toolchains = use_cpp_toolchain(), +)