WIP: Split crates requiring nightly Rust into separate unstable workspace

.github/workflows/ci.yml

@@ -65,11 +65,13 @@ jobs:
           # Don't build with `--all-features` as `--all-features` includes `--features llvm-static`,
           # which we don't want to test here (see https://github.com/immunant/c2rust/issues/500).
           cargo build --release
+          (cd unstable && cargo build --release)
       - name: cargo test --release --workspace
         run: |
           export RUSTFLAGS="-D warnings"
           export RUSTDOCFLAGS="-D warnings"
           cargo test --release --workspace
+          (cd unstable && cargo test --release --workspace)
       - name: Test translator
         run: |
           # `test_translator.py` compiles translated code,

.github/workflows/internal-testsuite.yml

@@ -104,12 +104,13 @@ jobs:
       run: |
         export LLVM_CONFIG_PATH=/usr/bin/llvm-config-15
         cargo build --release
+        (cd unstable && cargo build --release)
 
     # TODO(pl): figure out why compile_commands.json may cause json-c to fail
     - name: Run c2rust testsuite
       run: |
         find testsuite -type f -name compile_commands.json -delete
-        export PATH=$PWD/target/release:$HOME/.local/bin:$PATH
+        export PATH=$PWD/target/release:$PWD/unstable/target/release:$HOME/.local/bin:$PATH
         echo "PATH=$PATH"
         python3 testsuite/test.py curl json-c lua nginx zstd
 

.gitignore

@@ -14,6 +14,7 @@
 **/compile_commands.json
 **/Cargo.lock
 !/Cargo.lock
+!/unstable/Cargo.lock
 .python-version
 .clangd
 # ignore Clion build dirs

Cargo.toml

@@ -1,9 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
-    "analysis/runtime",
     "c2rust",
-    "c2rust-analyze",
     "c2rust-asm-casts",
     "c2rust-ast-builder",
     "c2rust-ast-exporter",
@@ -12,15 +10,11 @@ members = [
     "c2rust-bitfields-derive",
     "c2rust-build-paths",
     "c2rust-transpile",
-    "dynamic_instrumentation",
-    "pdg",
 ]
 exclude = [
-    "analysis/tests",
-    "c2rust-macros",
-    "c2rust-refactor",
     "examples",
     "tests",
+    "unstable",
 ]
 
 [workspace.package]

README.md

@@ -23,7 +23,9 @@ required to lift unsafe Rust into safe Rust types.
 This work is still in the early stages; please get in touch if you're interested!
 We previously maintained a scriptable refactoring tool, [`c2rust refactor`](./c2rust-refactor/),
 that reduces the tedium of refactoring, but this tool is now deprecated
-so that we can move forward with a recent Rust toolchain.
+so that we can move forward with a recent Rust toolchain. The `unstable`
+subdirectory contains some additional tools for working with Rust code, which
+depend on a previous nightly Rust toolchain and its internal rustc libraries.
 
 Here's the big picture:
 

docker/Dockerfile

@@ -1,8 +1,8 @@
 # Docker image for c2rust CI and containerized development.
 # Multi-distro via optional BASE_IMAGE argument.
-# RUST_VER argument should be set to the channel of the rust-toolchain.toml file in
-# the repo root (because dockerfiles can't access files above the location where
-# `docker build` is run).
+# `NIGHTLY_RUST_VER` argument should be set to the channel of the
+# unstable/rust-toolchain.toml file (because dockerfiles can't access files
+# above the location where `docker build` is run).
 
 ARG BASE_IMAGE=ubuntu:bionic
 FROM ${BASE_IMAGE}
@@ -29,8 +29,8 @@ RUN chmod +x /tmp/provision*.sh && /tmp/provision.sh
 
 USER $USER
 # Provision rust language and packages (should not run as root)
-ARG RUST_VER
-RUN RUST_VER=$RUST_VER /tmp/provision_rust.sh
+ARG NIGHTLY_RUST_VER
+RUN NIGHTLY_RUST_VER=$NIGHTLY_RUST_VER /tmp/provision_rust.sh
 
 ENV PATH="/home/${USER}/.cargo/bin:${PATH}"
 ENV CARGO_HOME="/tmp/.cargo"

examples/json-c/translate.py

@@ -63,10 +63,10 @@ def run_refactor(args, mode='inplace'):
             '--', 'src/lib.rs', '--crate-type=dylib',
             '--crate-name=json_c',
             '-L{rust_libdir}/rustlib/{triple}/lib/'.format(
-                rust_libdir=get_rust_toolchain_libpath(),
+                rust_libdir=get_nightly_rust_toolchain_libpath(),
                 triple=get_host_triplet())]
 
-    ld_lib_path = get_rust_toolchain_libpath()
+    ld_lib_path = get_nightly_rust_toolchain_libpath()
 
     # don't overwrite existing ld lib path if any...
     if 'LD_LIBRARY_PATH' in local.env:

manual/c2rust-refactor/README.md

@@ -1 +1 @@
-../../c2rust-refactor/README.md
+../../unstable/c2rust-refactor/README.md

manual/c2rust-refactor/doc/ldoc.css

@@ -1 +1 @@
-../../../c2rust-refactor/doc/ldoc.css
+../../../unstable/c2rust-refactor/doc/ldoc.css

manual/c2rust-refactor/doc/scripting_api.html

@@ -1 +1 @@
-../../../c2rust-refactor/doc/scripting_api.html
+../../../unstable/c2rust-refactor/doc/scripting_api.html

manual/c2rust-refactor/src/analysis/ownership/README.md

@@ -1 +1 @@
-../../../../../c2rust-refactor/src/analysis/ownership/README.md
+../../../../../unstable/c2rust-refactor/src/analysis/ownership/README.md

manual/preprocessors/generator_dispatch.py

@@ -18,7 +18,7 @@ def quote(args):
 
 def refactor_commands(args):
     assert len(args) == 0
-    sys.path.append(os.path.join(ROOT_DIR, 'c2rust-refactor/doc'))
+    sys.path.append(os.path.join(ROOT_DIR, 'unstable/c2rust-refactor/doc'))
     import gen_command_docs
     return gen_command_docs.generate_commands()
 
@@ -44,7 +44,7 @@ def literate(args):
     if 'C2RUST_MANUAL_LITERATE_ARGS' in os.environ:
         args = shlex.split(os.environ['C2RUST_MANUAL_LITERATE_ARGS']) + args
 
-    sys.path.append(os.path.join(ROOT_DIR, 'c2rust-refactor/doc'))
+    sys.path.append(os.path.join(ROOT_DIR, 'unstable/c2rust-refactor/doc'))
     with tempfile.TemporaryDirectory() as td:
         cmd_args = ['render'] + args + [os.path.join(td, 'out.md')]
         import literate

scripts/common.py

@@ -39,7 +39,7 @@ class Config:
     ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
     ROOT_DIR = os.path.abspath(os.path.join(ROOT_DIR, os.pardir))
     BUILD_DIR = os.path.join(ROOT_DIR, 'build' + BUILD_SUFFIX)
-    RREF_DIR = os.path.join(ROOT_DIR, 'c2rust-refactor')
+    RREF_DIR = os.path.join(ROOT_DIR, 'unstable', 'c2rust-refactor')
     C2RUST_DIR = os.path.join(ROOT_DIR, 'c2rust')
     CROSS_CHECKS_DIR = os.path.join(ROOT_DIR, "cross-checks")
     REMON_SUBMOD_DIR = os.path.join(CROSS_CHECKS_DIR, 'ReMon')
@@ -52,7 +52,7 @@ class Config:
     RUST_CHECKS_DIR = os.path.join(CROSS_CHECKS_DIR, 'rust-checks')
 
     TRANSPILE_CRATE_DIR = os.path.join(ROOT_DIR, 'c2rust-transpile')
-    REFACTOR_CRATE_DIR = os.path.join(ROOT_DIR, 'c2rust-refactor')
+    REFACTOR_CRATE_DIR = os.path.join(ROOT_DIR, 'unstable', 'c2rust-refactor')
     AST_BUILDER_CRATE_DIR = os.path.join(ROOT_DIR, 'c2rust-ast-builder')
     AST_EXPORTER_CRATE_DIR = os.path.join(ROOT_DIR, 'c2rust-ast-exporter')
     BITFIELDS_CRATE_DIR = os.path.join(ROOT_DIR, 'c2rust-bitfields')
@@ -61,7 +61,7 @@ class Config:
     XCHECK_DERIVE_CRATE_DIR = os.path.join(RUST_CHECKS_DIR, 'derive-macros')
     XCHECK_BACKEND_DYNAMIC_DLSYM_CRATE_DIR = os.path.join(RUST_CHECKS_DIR, 'backends', 'dynamic-dlsym')
     XCHECK_CONFIG_CRATE_DIR = os.path.join(RUST_CHECKS_DIR, 'config')
-    MACROS_CRATE_DIR = os.path.join(ROOT_DIR, 'c2rust-macros')
+    MACROS_CRATE_DIR = os.path.join(ROOT_DIR, 'unstable', 'c2rust-macros')
     AST_PRINTER_CRATE_DIR = os.path.join(ROOT_DIR, 'c2rust-ast-printer')
 
     CBOR_PREFIX = os.path.join(BUILD_DIR, "tinycbor")
@@ -99,7 +99,7 @@ class Config:
 
     CC_DB_JSON = "compile_commands.json"
 
-    CUSTOM_RUST_NAME = query_toml(path=Path(ROOT_DIR).joinpath("rust-toolchain.toml"), query=("toolchain", "channel"))
+    CUSTOM_RUST_NAME = query_toml(path=Path(ROOT_DIR).joinpath("unstable/rust-toolchain.toml"), query=("toolchain", "channel"))
 
     LLVM_SKIP_SIGNATURE_CHECKS  = False
 
@@ -213,15 +213,15 @@ def update_or_init_submodule(submodule_path: str) -> None:
     logging.debug("updated submodule %s", submodule_path)
 
 
-def get_rust_toolchain_libpath() -> str:
-    return _get_rust_toolchain_path("lib")
+def get_nightly_rust_toolchain_libpath() -> str:
+    return _get_nightly_rust_toolchain_path("lib")
 
 
-def get_rust_toolchain_binpath() -> str:
-    return _get_rust_toolchain_path("bin")
+def get_nightly_rust_toolchain_binpath() -> str:
+    return _get_nightly_rust_toolchain_path("bin")
 
 
-def _get_rust_toolchain_path(dirtype: str) -> str:
+def _get_nightly_rust_toolchain_path(dirtype: str) -> str:
     """
     Ask rustc for the correct path to its {lib,bin} directory.
     """

scripts/docker_build.sh

@@ -27,16 +27,16 @@ build_image() {
     BASE_IMAGE=${1}
     IMAGE_TAG=$(echo ${BASE_IMAGE} | tr -s :/ - ) # replace colons and slashes with hyphens
 
-    # pull the rust version out of ../rust-toolchain.toml to keep things synced
-    RUST_TOOLCHAIN_FILE="$SCRIPT_DIR/../rust-toolchain.toml"
-    RUST_VER=$($SCRIPT_DIR/query_toml.py toolchain.channel $RUST_TOOLCHAIN_FILE)
+    # pull the nightly rust version out of the rust-toolchain.toml file to keep things synced
+    RUST_TOOLCHAIN_FILE="$SCRIPT_DIR/../unstable/rust-toolchain.toml"
+    NIGHTLY_RUST_VER=$($SCRIPT_DIR/query_toml.py toolchain.channel $RUST_TOOLCHAIN_FILE)
 
     docker pull "$BASE_IMAGE"
     docker build -f $SCRIPT_DIR/../docker/Dockerfile \
            --build-arg BASE_IMAGE=$BASE_IMAGE \
            --build-arg UID=$(id -u $(logname)) \
            --build-arg GID=$(id -g $(logname)) \
-           --build-arg RUST_VER=$RUST_VER \
+           --build-arg NIGHTLY_RUST_VER=$NIGHTLY_RUST_VER \
            --tag "$REPO_NAME:$IMAGE_TAG-$DATE_TAG" \
            --tag "$REPO_NAME:$IMAGE_TAG-latest" \
            $SCRIPT_DIR

scripts/integration_test_translator.py

@@ -15,7 +15,6 @@
     pb,
     get_cmd_or_die,
     invoke,
-    get_rust_toolchain_libpath,
     download_archive,
     invoke_quietly,
     die,
@@ -88,19 +87,12 @@ def _test_minimal(code_snippet: str) -> bool:
     with open(cc_json, 'w') as fh:
         fh.write(minimal_cc_db)
 
-    ld_lib_path = get_rust_toolchain_libpath()
-
-    # don't overwrite existing ld lib path if any...
-    if 'LD_LIBRARY_PATH' in pb.local.env:
-        ld_lib_path += ':' + pb.local.env['LD_LIBRARY_PATH']
-
     args = []
     args += ['--ddump-untyped-clang-ast']
     args += [cfile]
 
     # import ast
-    with pb.local.env(RUST_BACKTRACE='1',
-                      LD_LIBRARY_PATH=ld_lib_path):
+    with pb.local.env(RUST_BACKTRACE='1'):
         invoke(transpiler, args)
 
     return True  # if we get this far, test passed

scripts/pdg_setup.sh

@@ -38,7 +38,7 @@ _c2rust_prepare_vars() {
     metadata="${test_dir}/metadata.bc"
     pdg="${test_dir}/pdg.bc"
     event_log="${test_dir}/log.bc"
-    runtime="${C2RUST_DIR}/analysis/runtime"
+    runtime="${C2RUST_DIR}/unstable/analysis/runtime"
 }
 
 c2rust-set-instrument-vars() {
@@ -60,7 +60,7 @@ c2rust-instrument() (
     unset RUSTFLAGS # transpiled code has tons of warnings; don't allow `-D warnings`
     export RUST_BACKTRACE=full
 
-    cd "${C2RUST_DIR}"
+    cd "${C2RUST_DIR}/unstable"
     cargo run \
         --bin c2rust-instrument \
         "${profile_args[@]}" \
@@ -80,7 +80,7 @@ c2rust-pdg() (
     export RUST_BACKTRACE=full # print sources w/ color-eyre
     export RUST_LOG=error
 
-    cd "${C2RUST_DIR}"
+    cd "${C2RUST_DIR}/unstable"
     cargo run \
         --bin c2rust-pdg \
         "${profile_args[@]}" \
@@ -104,7 +104,7 @@ c2rust-analyze-with-pdg() (
         export PDG_FILE="${pdg}"
     fi
 
-    cd "${C2RUST_DIR}"
+    cd "${C2RUST_DIR}/unstable"
     cargo run \
         --bin c2rust-analyze \
         "${profile_args[@]}" \

scripts/provision_mac.sh

@@ -27,6 +27,6 @@ python3 -m pip install --user --upgrade pip
 python3 -m pip install --user -r "$SCRIPT_DIR/requirements.txt"
 
 # Rust and dependencies
-RUST_TOOLCHAIN_FILE="$SCRIPT_DIR/../rust-toolchain.toml"
-export RUST_VER=$($SCRIPT_DIR/query_toml.py toolchain.channel $RUST_TOOLCHAIN_FILE)
+RUST_TOOLCHAIN_FILE="$SCRIPT_DIR/../unstable/rust-toolchain.toml"
+export NIGHTLY_RUST_VER=$($SCRIPT_DIR/query_toml.py toolchain.channel $RUST_TOOLCHAIN_FILE)
 "$SCRIPT_DIR/provision_rust.sh"

scripts/provision_rust.sh

@@ -9,18 +9,15 @@ if [[ "$EUID" -eq 0 ]]
   exit
 fi
 
-# $RUST_VER must be set.
-if [[ -z "$RUST_VER" ]]; then
-  echo "RUST_VER must be set to the desired rust version"
+# $NIGHTLY_RUST_VER must be set.
+if [[ -z "$NIGHTLY_RUST_VER" ]]; then
+  echo "NIGHTLY_RUST_VER must be set to the desired nightly rust version"
   exit
 fi
 
-if hash rustup 2>/dev/null; then # rustup is installed
-  rustup toolchain install $RUST_VER
-  rustup default $RUST_VER
-else # rustup is not installed  
-  curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_VER
-fi 
+if ! hash rustup 2>/dev/null; then # rustup is not installed
+  curl https://sh.rustup.rs -sSf | sh -s -- -y
+fi
 
 # make rust environment available on next login 
 if ! grep "source ~/.cargo/env" ~/.bashrc >/dev/null; then 
@@ -29,9 +26,12 @@ fi
 # make rust environment available for commands below 
 source ~/.cargo/env
 
+rustup toolchain install $NIGHTLY_RUST_VER
+rustup toolchain install stable
+
 # rustfmt is required for c2rust-refactor tests
 # rustc-dev was added make sure it is installed on Azure/macOS-10.15
-rustup component add rustfmt-preview rustc-dev rust-src
+rustup component add rustfmt rustc-dev rust-src
 
 # Make rustup directory world-writable so other test users can install new rust
 # versions

scripts/run_ci_checks.sh

@@ -12,16 +12,19 @@ fi
 
 fmt() {
     cargo fmt --check
+    (cd unstable && cargo fmt --check)
 }
 
 # Differs from `cargo build` since we can use `--all-features` here.
 # We plan to eventually replace `cargo check` with `cargo clippy`.
 check() {
     cargo check --tests --all-features
+    (cd unstable && cargo check --tests --all-features)
 }
 
 doc() {
     cargo doc --all-features --document-private-items --no-deps
+    (cd unstable && cargo doc --all-features --document-private-items --no-deps)
 }
 
 # At this point, we could unset `RUSTFLAGS` and `RUSTDOCFLAGS`,
@@ -34,10 +37,12 @@ doc() {
 # see https://github.com/immunant/c2rust/issues/500).
 build() {
     cargo build --release
+    (cd unstable && cargo build --release)
 }
 
 test() {
     cargo test --release --workspace
+    (cd unstable && cargo test --release --workspace)
 }
 
 # `test_translator.py` compiles translated code,