[reduce] Add CIRCT-specific test case runner (#2129)

Setup a CIRCT-specific runner for reduction test cases. This
implementation is more elaborate than what's in MLIR at the moment, and
allows for test cases to be carefully validated, written to disk and
checked for their size, and then run through the tester, in distinct
steps. The MLIR version does all of these things at the same time, which
can be prohibitively expensive if the test case is a two-hour formal
equivalence check -- just to find out that the module size increased and
we have to throw away the reduction regardless.

Eventually we'll want to upstream this back to MLIR.

tools/circt-reduce/CMakeLists.txt

@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
 add_llvm_tool(circt-reduce
  circt-reduce.cpp
  Reduction.cpp
+ Tester.cpp
 )
 llvm_update_compile_flags(circt-reduce)
 target_link_libraries(circt-reduce

tools/circt-reduce/Reduction.cpp

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Reduction.h"
 #include "circt/Dialect/FIRRTL/FIRRTLOps.h"
 #include "circt/Dialect/FIRRTL/Passes.h"
 #include "circt/InitAllDialects.h"
@@ -25,8 +26,6 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/Debug.h"
 
-#include "Reduction.h"
-
 #define DEBUG_TYPE "circt-reduce"
 
 using namespace llvm;
@@ -235,7 +234,6 @@ void circt::createAllReductions(
   // sorted by decreasing reduction potential/benefit. For example, things that
   // can knock out entire modules while being cheap should be tried first,
   // before trying to tweak operands of individual arithmetic ops.
-  add(std::make_unique<ModuleExternalizer>());
   add(std::make_unique<PassReduction>(context, firrtl::createInlinerPass()));
   add(std::make_unique<PassReduction>(context,
                                       createSimpleCanonicalizerPass()));
@@ -249,6 +247,7 @@ void circt::createAllReductions(
       context, firrtl::createLowerFIRRTLTypesPass(), true, true));
   add(std::make_unique<PassReduction>(context, firrtl::createExpandWhensPass(),
                                       true, true));
+  add(std::make_unique<ModuleExternalizer>());
   add(std::make_unique<PassReduction>(context, createCSEPass()));
   add(std::make_unique<ConnectInvalidator>());
   add(std::make_unique<OperationPruner>());

tools/circt-reduce/Reduction.h

@@ -16,8 +16,9 @@
 #include <memory>
 #include <string>
 
+#include "llvm/ADT/StringRef.h"
+
 namespace llvm {
-class StringRef;
 template <typename T>
 class function_ref;
 } // namespace llvm

tools/circt-reduce/Tester.cpp

@@ -0,0 +1,151 @@
+//===- Tester.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the Tester class used in the CIRCT reduce tool.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Tester.h"
+#include "mlir/IR/Verifier.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+using namespace llvm;
+using namespace mlir;
+using namespace circt;
+
+//===----------------------------------------------------------------------===//
+// Tester
+//===----------------------------------------------------------------------===//
+
+Tester::Tester(StringRef scriptName, ArrayRef<std::string> scriptArgs)
+    : testScript(scriptName), testScriptArgs(scriptArgs) {}
+
+std::pair<bool, size_t> Tester::isInteresting(ModuleOp module) const {
+  auto test = get(module);
+  return std::make_pair(test.isInteresting(), test.getSize());
+}
+
+/// Runs the interestingness testing script on a MLIR test case file. Returns
+/// true if the interesting behavior is present in the test case or false
+/// otherwise.
+bool Tester::isInteresting(StringRef testCase) const {
+  // Assemble the arguments to the tester. Note that the first one has to be the
+  // name of the program.
+  SmallVector<StringRef> testerArgs;
+  testerArgs.push_back(testScript);
+  testerArgs.append(testScriptArgs.begin(), testScriptArgs.end());
+  testerArgs.push_back(testCase);
+
+  // Run the tester.
+  std::string errMsg;
+  int result = llvm::sys::ExecuteAndWait(
+      testScript, testerArgs, /*Env=*/None, /*Redirects=*/None,
+      /*SecondsToWait=*/0, /*MemoryLimit=*/0, &errMsg);
+  if (result < 0)
+    llvm::report_fatal_error(
+        Twine("Error running interestingness test: ") + errMsg, false);
+
+  return result > 0;
+}
+
+/// Create a new test case for the given `module`.
+TestCase Tester::get(mlir::ModuleOp module) const {
+  return TestCase(*this, module);
+}
+
+/// Create a new test case for the given file already on disk.
+TestCase Tester::get(llvm::Twine filepath) const {
+  return TestCase(*this, filepath);
+}
+
+//===----------------------------------------------------------------------===//
+// Test Case
+//===----------------------------------------------------------------------===//
+
+/// Check whether the MLIR module is valid. Actual validation is only
+/// performed on the first call; subsequent calls return the cached result.
+bool TestCase::isValid() {
+  // Assume already-provided test cases on disk are valid.
+  if (!module)
+    return true;
+  if (!valid)
+    valid = succeeded(verify(module));
+  return *valid;
+}
+
+/// Determine the path to the MLIR module on disk. Actual writing to disk is
+/// only performed on the first call; subsequent calls return the cached result.
+StringRef TestCase::getFilepath() {
+  if (!isValid())
+    return "";
+  ensureFileOnDisk();
+  return filepath;
+}
+
+/// Determine the size of the MLIR module on disk. Actual writing to disk is
+/// only performed on the first call; subsequent calls return the cached result.
+size_t TestCase::getSize() {
+  if (!isValid())
+    return 0;
+  ensureFileOnDisk();
+  return *size;
+}
+
+/// Run the tester on the MLIR module and return whether it is deemed
+/// interesting. Actual testing is only performed on the first call; subsequent
+/// calls return the cached result.
+bool TestCase::isInteresting() {
+  if (!isValid())
+    return false;
+  ensureFileOnDisk();
+  if (!interesting)
+    interesting = tester.isInteresting(filepath);
+  return *interesting;
+}
+
+/// Ensure `filepath` and `size` are populated, and that the test case is in a
+/// file on disk.
+void TestCase::ensureFileOnDisk() {
+  // Write the module to a temporary file if no already-prepared file path has
+  // been provided to the test.
+  if (filepath.empty()) {
+    assert(module);
+
+    // Pick a temporary output file path.
+    int fd;
+    std::error_code ec =
+        llvm::sys::fs::createTemporaryFile("mlir-reduce", "mlir", fd, filepath);
+    if (ec)
+      llvm::report_fatal_error(
+          Twine("Error making unique filename: ") + ec.message(), false);
+
+    // Write to the output.
+    file = std::make_unique<llvm::ToolOutputFile>(filepath, fd);
+    module.print(file->os());
+    file->os().close();
+    if (file->os().has_error())
+      llvm::report_fatal_error(llvm::Twine("Error emitting the IR to file `") +
+                                   filepath + "`",
+                               false);
+
+    // Update the file size.
+    size = file->os().tell();
+    return;
+  }
+
+  // Otherwise just determine the size of the already-prepared file on disk.
+  if (!size) {
+    uint64_t fileSize;
+    std::error_code ec = llvm::sys::fs::file_size(filepath, fileSize);
+    if (ec)
+      llvm::report_fatal_error(Twine("Error determining size of file `") +
+                                   filepath + "`: " + ec.message(),
+                               false);
+    size = fileSize;
+  }
+}

tools/circt-reduce/Tester.h

@@ -0,0 +1,135 @@
+//===- Tester.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the Tester class used in the CIRCT reduce tool.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CIRCT_REDUCE_TESTER_H
+#define CIRCT_REDUCE_TESTER_H
+
+#include <memory>
+#include <vector>
+
+#include "mlir/IR/BuiltinOps.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Program.h"
+
+namespace llvm {
+class ToolOutputFile;
+} // namespace llvm
+
+namespace circt {
+
+class TestCase;
+
+/// A testing environment for reduction attempts.
+///
+/// This class tracks the program used to check reduction attempts for
+/// interestingness and additional arguments to pass to that tool. Use `get()`
+/// to obtain a new test case that can be queried for information on an
+/// individual MLIR module.
+class Tester {
+public:
+  Tester(llvm::StringRef testScript,
+         llvm::ArrayRef<std::string> testScriptArgs);
+
+  /// Runs the interestingness testing script on a MLIR test case file. Returns
+  /// true if the interesting behavior is present in the test case or false
+  /// otherwise.
+  std::pair<bool, size_t> isInteresting(mlir::ModuleOp module) const;
+
+  /// Return whether the file in the given path is interesting.
+  bool isInteresting(llvm::StringRef testCase) const;
+
+  /// Create a new test case for the given `module`.
+  TestCase get(mlir::ModuleOp module) const;
+
+  /// Create a new test case for the given file already on disk.
+  TestCase get(llvm::Twine filepath) const;
+
+private:
+  /// The binary to execute in order to check a reduction attempt for
+  /// interestingness.
+  llvm::StringRef testScript;
+
+  /// Additional arguments to pass to `testScript`.
+  llvm::ArrayRef<std::string> testScriptArgs;
+};
+
+/// A single test case to be run by a tester.
+///
+/// This is a helper object that wraps a `ModuleOp` and can be used to query
+/// initial information about the test, such as validity of the module and size
+/// on disk, before the test is actually executed.
+class TestCase {
+public:
+  /// Create a test case with an MLIR module that will be written to a temporary
+  /// file on disk. The `TestCase` will clean up the temporary file after use.
+  TestCase(const Tester &tester, mlir::ModuleOp module)
+      : tester(tester), module(module) {}
+
+  /// Create a test case for an already-prepared file on disk. The caller
+  /// remains responsible for cleaning up the file on disk.
+  TestCase(const Tester &tester, llvm::Twine filepath) : tester(tester) {
+    filepath.toVector(this->filepath);
+  }
+
+  /// Check whether the MLIR module is valid. Actual validation is only
+  /// performed on the first call; subsequent calls return the cached result.
+  bool isValid();
+
+  /// Determine the path to the MLIR module on disk. Actual writing to disk is
+  /// only performed on the first call; subsequent calls return the cached
+  /// result.
+  llvm::StringRef getFilepath();
+
+  /// Determine the size of the MLIR module on disk. Actual writing to disk is
+  /// only performed on the first call; subsequent calls return the cached
+  /// result.
+  size_t getSize();
+
+  /// Run the tester on the MLIR module and return whether it is deemed
+  /// interesting. Actual testing is only performed on the first call;
+  /// subsequent calls return the cached result.
+  bool isInteresting();
+
+private:
+  friend class Tester;
+
+  /// Ensure `filepath` and `size` are populated, and that the test case is in a
+  /// file on disk.
+  void ensureFileOnDisk();
+
+  /// The tester that is used to run this test case.
+  const Tester &tester;
+  /// The module to be tested.
+  mlir::ModuleOp module;
+  /// The path on disk where the test case is located.
+  llvm::SmallString<32> filepath;
+
+  /// In case this test case has created a temporary file on disk, this is the
+  /// `ToolOutputFile` that did the writing. Keeping this class around ensures
+  /// that the file will be cleaned up properly afterwards. This field remains
+  /// null if the user already has provided a filepath in the constructor.
+  std::unique_ptr<llvm::ToolOutputFile> file;
+
+  /// Whether the MLIR module validation has run, and its result.
+  llvm::Optional<bool> valid;
+  /// Whether the size of the test case on disk has already been determined, and
+  /// if yes, that size.
+  llvm::Optional<size_t> size;
+  /// Whether the tester has run on this test case, and its result.
+  llvm::Optional<bool> interesting;
+};
+
+} // namespace circt
+
+#endif // CIRCT_REDUCE_TESTER_H