Point release adding the uuid, time, and walk builtins. (#163)

In order to add support for the `time` built-ins that worked cross-platform and with the widest range of C++ compilers, we had to introduce a dependency on the [`date`](https://github.com/HowardHinnant/date) library. This contains a full implementations of the `std::chrono` functionality which will eventually be supported across most c++ compilers as part of the STL. In particular, it provides the crucial Time Zone lookup functionalities required by the Rego `time` built-ins. The Time Zone lookup requires a valid [`tzdata`](https://www.iana.org/time-zones) database from the IANA. This can either be configured to use the local system TZ data (default for Linux variants) or a manual database (default for Windows). If the `REGOCPP_USE_MANUAL_TZDATA` flag is set, then the project will download the latest database and install it as part of its build process.

**New Features**
- Added the `uuid` built-ins
- Added the `time` built-ins
- Added the `walk` built-in
- It is now possible for `BuiltInDef` implementations to cache values or otherwise maintain state. To facilitate this, a new virtual `clear()` method has been added which will be called at the start of each query evaluation.

**Bug Fixes**
- Fixed a bug with adding zero to negative integers.

Signed-off-by: Matthew Johnson <[email protected]>

Author: matajoh

.github/workflows/build_wheels.yml

@@ -19,7 +19,7 @@ jobs:
           git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/*
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.18.1
+        uses: pypa/cibuildwheel@v2.21.1
         env:
           CIBW_BUILD: ${{ matrix.python }}-win*
           CIBW_ENVIRONMENT: >
@@ -61,7 +61,7 @@ jobs:
           export PATH=$(python3 -m site --user-base)/bin:$PATH
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.18.1
+        uses: pypa/cibuildwheel@v2.21.1
         env:
           CIBW_BUILD: ${{ matrix.python }}-macosx*
           CIBW_ARCHS_MACOS: arm64
@@ -90,7 +90,7 @@ jobs:
           git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/*
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.18.1
+        uses: pypa/cibuildwheel@v2.21.1
         env:
           CIBW_BUILD: ${{ matrix.python }}-manylinux*
           CIBW_BEFORE_ALL: yum makecache && yum -y install devtoolset-10-libatomic-devel
@@ -119,7 +119,7 @@ jobs:
           git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/*
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.18.1
+        uses: pypa/cibuildwheel@v2.21.1
         env:
           CIBW_BUILD: ${{ matrix.python }}-musllinux*
           CIBW_ENVIRONMENT: >

.github/workflows/pr_gate.yml

@@ -209,7 +209,7 @@ jobs:
 
       - name: CMake config
         run: |
-          cmake -B ${{github.workspace}}/build --preset release-opa
+          cmake -B ${{github.workspace}}/build --preset release-opa -DREGOCPP_TZDATA_PATH=${{github.workspace}}/build
       
       - name: CMake build
         working-directory: ${{github.workspace}}/build

CHANGELOG

@@ -1,5 +1,28 @@
 # Changelog
 
+## 2024-09-23 - Version 0.4.4
+Point release adding the `uuid`, `time`, and `walk` builtins.
+
+In order to add support for the `time` built-ins that worked cross-platform and with the widest
+range of C++ compilers, we had to introduce a dependency on the [`date`](https://github.com/HowardHinnant/date)
+library. This contains a full implementations of the `std::chrono` functionality which will eventually be
+supported across most c++ compilers as part of the STL. In particular, it provides the crucial Time Zone
+lookup functionalities required by the Rego `time` built-ins. The Time Zone lookup requires a valid
+[`tzdata`](https://www.iana.org/time-zones) database from the IANA. This can either be configured to use the local
+system TZ data (default for Linux variants) or a manual database (default for Windows). If the `REGOCPP_USE_MANUAL_TZDATA` flag
+is set, then the project will download the latest database and install it as part of its build process.
+
+**New Features**
+- Added the `uuid` built-ins
+- Added the `time` built-ins
+- Added the `walk` built-in
+- It is now possible for `BuiltInDef` implementations to cache values or otherwise maintain state. To facilitate this,
+  a new virtual `clear()` method has been added which will be called at the start of each query evaluation.
+
+**Bug Fixes**
+- Fixed a bug with adding zero to negative integers.
+
+
 ## 2024-09-10 - Version 0.4.3
 Point release updating to Rego v0.68.0
 

CMakeLists.txt

@@ -69,6 +69,7 @@ option(REGOCPP_COPY_EXAMPLES "Specifies whether to copy the examples to the inst
 option(REGOCPP_USE_CXX17 "Specifies whether to use C++17" OFF)
 option(REGOCPP_ACTION_METRICS "Specifies whether to metricate Trieste Actions" OFF)
 option(REGOCPP_CLEAN_INSTALL "Whether the install directory should be cleaned before install" OFF)
+option(REGOCPP_USE_MANUAL_TZDATA "Whether to use the manual tzdata" OFF)
 set(REGOCPP_SANITIZE "" CACHE STRING "Argument to pass to sanitize (disabled by default)")
 
 if(REGOCPP_USE_CXX17)
@@ -90,14 +91,58 @@ FetchContent_Declare(
     GIT_SHALLOW FALSE
 )
 
+FetchContent_Declare(
+  date
+  GIT_REPOSITORY https://github.com/HowardHinnant/date.git
+  GIT_TAG 51ce7e131079c061533d741be5fe7cca57f2faac
+)
+
 FetchContent_Declare(
   trieste
   GIT_REPOSITORY https://github.com/microsoft/trieste
-  GIT_TAG        467157a167d2e929095498089f29fc9419f4ef11
+  GIT_TAG        70a4d5f79cb574070ff69b2517b422c5126e2024
 )
 
 FetchContent_MakeAvailable(cmake_utils)
 
+if(MSVC)
+  message("MSVC: must use manual TZ database")
+  set(REGOCPP_USE_MANUAL_TZDATA ON)
+endif()
+
+set(REGOCPP_TZDATA_PATH "${CMAKE_INSTALL_PREFIX}" CACHE STRING "Path to the tzdata directory")
+file(TO_CMAKE_PATH ${REGOCPP_TZDATA_PATH} REGOCPP_TZDATA_PATH)
+
+if(REGOCPP_USE_MANUAL_TZDATA)
+  set(MANUAL_TZ_DB ON)
+  set(USE_SYSTEM_TZ_DB OFF)
+  set(TZDATA_GZ_PATH "${CMAKE_CURRENT_BINARY_DIR}/tzdata-latest.tar.gz")
+  set(TZDATA_PATH "${CMAKE_CURRENT_BINARY_DIR}/tzdata")
+  if(NOT EXISTS ${TZDATA_PATH})
+    message("Downloading tzdata to ${TZDATA_GZ_PATH}")
+    # we need to download the TZ database and make it available
+    FILE(DOWNLOAD https://www.iana.org/time-zones/repository/tzdata-latest.tar.gz ${TZDATA_GZ_PATH} SHOW_PROGRESS)
+    message("Extracting ${TZDATA_GZ_PATH} to ${TZDATA_PATH}")
+    FILE(ARCHIVE_EXTRACT INPUT ${TZDATA_GZ_PATH} DESTINATION ${TZDATA_PATH})
+
+    # Copy the tzdata directory to the specified path.
+    message("Copying ${TZDATA_PATH} to ${REGOCPP_TZDATA_PATH}")
+    file(COPY ${TZDATA_PATH} DESTINATION ${REGOCPP_TZDATA_PATH})
+    if(MSVC)
+      message("MSVC: Copying windowsZones.xml to ${REGOCPP_TZDATA_PATH}/tzdata")
+      file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/templates/tzdata/windowsZones.xml DESTINATION ${REGOCPP_TZDATA_PATH}/tzdata)
+    endif()
+  endif()
+else()
+  message("Attempting to use system TZ database")
+  set(USE_SYSTEM_TZ_DB ON)
+  set(MANUAL_TZ_DB OFF)
+endif()
+
+set(BUILD_TZ_LIB ON)
+set(COMPILE_WITH_C_LOCALE ON)
+FetchContent_MakeAvailable_ExcludeFromAll(date)
+
 set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) 
 
 set(TRIESTE_USE_CXX17 ${REGOCPP_USE_CXX17})
@@ -195,11 +240,18 @@ if(REGOCPP_CLEAN_INSTALL)
     ]])
 endif()
 
+if(REGOCPP_USE_MANUAL_TZDATA)
+  # If the install depends on this, we want to include it in the install package.
+  INSTALL(DIRECTORY ${TZDATA_PATH} DESTINATION .)
+  if(MSVC)
+    INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/templates/tzdata/windowsZones.xml DESTINATION tzdata)
+  endif()
+endif()
 
 set(INSTALL_CONFIGDIR cmake)
 set(INSTALL_LIBDIR lib)
 set(INSTALL_INCLUDEDIR include)
-set(REGOCPP_INSTALL_TARGETS rego trieste json yaml snmalloc re2)
+set(REGOCPP_INSTALL_TARGETS rego trieste json yaml snmalloc re2 date-tz)
 
 if(TRIESTE_USE_SNMALLOC)
   list(APPEND REGOCPP_INSTALL_TARGETS snmalloc-new-override)
@@ -257,6 +309,6 @@ export(PACKAGE ${PROJECT_NAME})
 
 install(DIRECTORY include/ DESTINATION ${INSTALL_INCLUDEDIR})
 install(FILES 
-  ${CMAKE_CURRENT_BINARY_DIR}/src/version.h
+  ${CMAKE_CURRENT_BINARY_DIR}/src/version.h  
   DESTINATION ${INSTALL_INCLUDEDIR}/rego
 )

README.md

@@ -76,7 +76,7 @@ library from different langauages.
 
 ## Language Support
 
-At present we support v0.65.0 of Rego as defined by OPA, with the following grammar:
+We support v0.68.0 of Rego as defined by OPA, with the following grammar:
 
 ```ebnf
 module          = package { import } policy
@@ -150,8 +150,7 @@ LF     Line Feed
 
 ### Builtins
 
-At the moment only support a few builtins, but are actively working on adding
-all the standard builtins. The following builtins are currently supported:
+At the moment support the following builtins are available:
 
 - `aggregates`
 - `arrays`
@@ -165,12 +164,13 @@ all the standard builtins. The following builtins are currently supported:
 - `semver`
 - `sets`
 - `strings`
+- `time`
 - `types`
 - `units`
+- `uuid`
 - miscellaneous
     * `opa.runtime`
     * `print`
-    * `time.now_ns`
 
 ### Compatibility with the OPA Rego Go implementation
 
@@ -193,8 +193,6 @@ At present, we are **NOT** passing the following test suites in full:
 - `net*`
 - `planner-ir`
 - `providers-aws`
-- `time`
-- `walkbuiltin`
 
 ## Contributing
 

VERSION

@@ -1 +1 @@
-0.4.3
+0.4.4

examples/rust/Cargo.toml

@@ -6,5 +6,5 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-regorust = "0.4.0"
+regorust = "0.4.4"
 clap = { version = "4.0", features = ["derive"] }

include/rego/rego.hh

@@ -206,7 +206,7 @@ namespace rego
   using BuiltIn = std::shared_ptr<BuiltInDef>;
 
   /** The function pointer to the behavior of the built-in. */
-  using BuiltInBehavior = Node (*)(const Nodes&);
+  using BuiltInBehavior = std::function<Node(const Nodes&)>;
 
   /**
    * This is a basic, non-optimized implementation of a big integer using
@@ -622,6 +622,18 @@ namespace rego
     /** The function which will be called when the built-in is evaluated. */
     BuiltInBehavior behavior;
 
+    /**
+     * Constructor.
+     */
+    BuiltInDef(Location name_, std::size_t arity_, BuiltInBehavior behavior_);
+
+    virtual ~BuiltInDef() = default;
+
+    /**
+     * Called to clear any persistent state or caching.
+     */
+    virtual void clear();
+
     /**
      * Creates a new built-in.
      *
@@ -672,8 +684,12 @@ namespace rego
      * @param version The Rego version.
      * @return The result of the built-in call.
      */
-    Node call(
-      const Location& name, const Location& version, const Nodes& args) const;
+    Node call(const Location& name, const Location& version, const Nodes& args);
+
+    /**
+     * Called to clear any persistent state or caching.
+     */
+    void clear();
 
     /**
      * Registers a built-in.

src/CMakeLists.txt

@@ -1,4 +1,5 @@
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/version.h.in" "${CMAKE_CURRENT_BINARY_DIR}/version.h" @ONLY)
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/builtins/tzdata.h.in" "${CMAKE_CURRENT_BINARY_DIR}/tzdata.h" @ONLY)
 
 set( SOURCES
 bigint.cc
@@ -59,6 +60,7 @@ builtins/strings.cc
 builtins/time.cc
 builtins/types.cc
 builtins/units.cc
+builtins/uuid.cc
 # third-party
 builtins/base64/base64.cpp
 )
@@ -71,11 +73,16 @@ if (REGOCPP_SPECULATIVE_LOAD_HARDENING)
   target_compile_options(rego PRIVATE -mllvm -x86-speculative-load-hardening)
 endif()
 
+if (MANUAL_TZ_DB)
+  target_compile_definitions(rego PUBLIC "MANUAL_TZ_DB")
+endif()
+
 target_link_libraries(rego
   PUBLIC
     trieste::trieste
     trieste::json
     trieste::yaml
+    date-tz
 )
 
 if(MSVC)

src/bigint.cc

@@ -132,6 +132,11 @@ namespace rego
 
   BigInt operator-(const BigInt& lhs, const BigInt& rhs)
   {
+    if (rhs.is_zero())
+    {
+      return lhs;
+    }
+
     bool negate = false;
     if (lhs.is_negative())
     {
@@ -173,11 +178,6 @@ namespace rego
       }
     }
 
-    if (rhs.is_zero())
-    {
-      return lhs;
-    }
-
     std::string difference =
       BigInt::subtract(lhs.digits(), rhs.digits(), negate);
     return BigInt(difference);

src/builtins.cc

@@ -38,6 +38,13 @@ namespace
 
 namespace rego
 {
+  BuiltInDef::BuiltInDef(
+    Location name_, std::size_t arity_, BuiltInBehavior behavior_) :
+    name(name_), arity(arity_), behavior(behavior_)
+  {}
+
+  void BuiltInDef::clear() {}
+
   BuiltIn BuiltInDef::create(
     const Location& name, std::size_t arity, BuiltInBehavior behavior)
   {
@@ -46,6 +53,14 @@ namespace rego
 
   BuiltInsDef::BuiltInsDef() noexcept : m_strict_errors(false) {}
 
+  void BuiltInsDef::clear()
+  {
+    for (auto& builtin : m_builtins)
+    {
+      builtin.second->clear();
+    }
+  }
+
   bool BuiltInsDef::strict_errors() const
   {
     return m_strict_errors;
@@ -91,7 +106,7 @@ namespace rego
   }
 
   Node BuiltInsDef::call(
-    const Location& name, const Location& version, const Nodes& args) const
+    const Location& name, const Location& version, const Nodes& args)
   {
     if (!is_builtin(name))
     {
@@ -164,6 +179,7 @@ namespace rego
     register_builtins(builtins::time());
     register_builtins(builtins::types());
     register_builtins(builtins::units());
+    register_builtins(builtins::uuid());
 
     return *this;
   }

src/builtins/builtins.h

@@ -21,5 +21,6 @@ namespace rego
     std::vector<BuiltIn> time();
     std::vector<BuiltIn> types();
     std::vector<BuiltIn> units();
+    std::vector<BuiltIn> uuid();
   }
 }