test

Author: robertbastian

Cargo.lock

@@ -38,8 +38,6 @@ zerovec = { version = "0.9.4", path = "../../utils/zerovec", features = ["yoke"]
 databake = { version = "0.1.3", path = "../../utils/databake", optional = true, features = ["derive"] }
 serde = { version = "1.0", default-features = false, features = ["derive", "alloc"], optional = true }
 
-num-traits = { version = "0.2", default-features = false, features = ["libm"], optional = true }
-
 [dev-dependencies]
 criterion = "0.4"
 icu_testdata = { path = "../../provider/testdata", default-features = false, features = ["buffer", "icu_segmenter", "icu_properties"] }
@@ -56,7 +54,7 @@ default = ["auto"]
 std = ["icu_collections/std", "icu_locid/std", "icu_provider/std"]
 serde = ["dep:serde", "zerovec/serde", "icu_collections/serde", "icu_provider/serde"]
 datagen = ["serde", "dep:databake", "zerovec/databake", "icu_collections/databake"]
-lstm = ["dep:num-traits"]
+lstm = []
 auto = ["lstm"] # Enabled try_new_auto_unstable constructors
 
 [lib]

components/segmenter/Cargo.toml

@@ -8,20 +8,28 @@ use core::ops::Range;
 use zerovec::ule::AsULE;
 use zerovec::ZeroSlice;
 
-// Polyfill float operations with libm in case we're no_std.
-#[allow(unused_imports)]
-use num_traits::Float;
-
-/// `tanh` computes the tanh function for a scalar value.
-#[inline]
-fn tanh(x: f32) -> f32 {
-    x.tanh()
+// This will be used in #[no_std] as f32::exp/f32::tanh are not in core.
+trait CoreFloat {
+    fn exp(self) -> Self;
+    fn tanh(self) -> Self;
 }
 
-/// `sigmoid` computes the sigmoid function for a scalar value.
-#[inline]
-fn sigmoid(x: f32) -> f32 {
-    1.0 / (1.0 + (-x).exp())
+impl CoreFloat for f32 {
+    fn exp(self) -> Self {
+        #[link(name = "m")]
+        extern "C" {
+            fn expf(val: f32) -> f32;
+        }
+        unsafe { expf(self) }
+    }
+
+    fn tanh(self) -> Self {
+        #[link(name = "m")]
+        extern "C" {
+            fn tanhf(val: f32) -> f32;
+        }
+        unsafe { tanhf(self) }
+    }
 }
 
 /// A `D`-dimensional, heap-allocated matrix.
@@ -217,13 +225,13 @@ impl<'a, const D: usize> MatrixBorrowedMut<'a, D> {
 
     pub(super) fn sigmoid_transform(&mut self) {
         for x in &mut self.data.iter_mut() {
-            *x = sigmoid(*x);
+            *x = 1.0 / (1.0 + (-*x).exp());
         }
     }
 
     pub(super) fn tanh_transform(&mut self) {
         for x in &mut self.data.iter_mut() {
-            *x = tanh(*x);
+            *x = x.tanh();
         }
     }
 
@@ -262,7 +270,7 @@ impl<'a, const D: usize> MatrixBorrowedMut<'a, D> {
             // Safety: The lengths are all the same (checked above)
             unsafe {
                 *self.data.get_unchecked_mut(idx) =
-                    o.get_unchecked(idx) * tanh(*c.get_unchecked(idx));
+                    o.get_unchecked(idx) * c.get_unchecked(idx).tanh();
             }
         }
     }

components/segmenter/README.md

@@ -106,6 +106,19 @@
 //! ```
 //!
 //! See [`SentenceSegmenter`] for more examples.
+//!
+//! # `#![no_std]` support
+//!
+//! Like all of ICU4X, this module is `#![no_std]` by default. However, the LSTM segmenters (enabled
+//! with features `default`, `auto` or `lstm`) require `f32::exp` and `f32::tanh`, which are currently
+//! not included in `core`[^1]. Building without `std` will instead use `math.h`'s `expf` and `tanhf`
+//! functions, which need to be linked. There are two ways to do this:
+//! * If your target has a `libm` implementation, you can link with `-lm` to use native implementations.
+//! * If your target does not have a `libm` implementation, you can use the [`compiler_builtins`](https://github.com/rust-lang/compiler-builtins/tree/master)
+//!   crate to provide the implementations. Keep in mind that these will usually perform worse than a
+//!   native `libm`.
+//!
+//! [^1]: See [rust-lang/rfcs#2505](https://github.com/rust-lang/rfcs/issues/2505)
 
 // https://github.com/unicode-org/icu4x/blob/main/docs/process/boilerplate.md#library-annotations
 #![cfg_attr(not(any(test, feature = "std")), no_std)]

components/segmenter/src/complex/lstm/matrix.rs

@@ -12,4 +12,4 @@
 //! respective library types are needed.
 
 // Necessary for symbols to be linked in
-extern crate icu_capi;
+extern crate icu_capi;

components/segmenter/src/lib.rs

@@ -36,12 +36,6 @@ rustflags = []
 [gn.package.syn."1.0.109"]
 rustflags = ["--cfg=syn_disable_nightly_tests"]
 
-[gn.package.num-traits."0.2.15"]
-rustflags = ["--cfg=has_i128"]
-
-[gn.package.libm."0.2.6"]
-rustflags = []
-
 # Build cargo-gnaw with the checked-in lockfile
 [workspace]
 members = ["third_party_tools/fuchsia/tools/cargo-gnaw"]

ffi/capi_cdylib/src/lib.rs

@@ -628,7 +628,6 @@ rust_library("icu_segmenter-v1_2_1") {
   deps += [ ":icu_collections-v1_2_0" ]
   deps += [ ":icu_locid-v1_2_0" ]
   deps += [ ":icu_provider-v1_2_0" ]
-  deps += [ ":num-traits-v0_2_15" ]
   deps += [ ":utf8_iter-v1_0_3" ]
   deps += [ ":zerovec-v0_9_4" ]
 
@@ -738,26 +737,6 @@ rust_library("lazy_static-v1_4_0") {
   visibility = [ ":*" ]
 }
 
-rust_library("libm-v0_2_6") {
-  crate_name = "libm"
-  crate_root = "//ffi/gn/vendor/libm/src/lib.rs"
-  output_name = "libm-6b5214f2031c6f9e"
-
-  deps = []
-
-  rustenv = []
-
-  rustflags = [
-    "--cap-lints=allow",
-    "--edition=2018",
-    "-Cmetadata=6b5214f2031c6f9e",
-    "-Cextra-filename=-6b5214f2031c6f9e",
-    "--cfg=feature=\"default\"",
-  ]
-
-  visibility = [ ":*" ]
-}
-
 rust_library("litemap-v0_7_0") {
   crate_name = "litemap"
   crate_root = "//utils/litemap/src/lib.rs"
@@ -820,28 +799,6 @@ rust_library("memchr-v2_5_0") {
   visibility = [ ":*" ]
 }
 
-rust_library("num-traits-v0_2_15") {
-  crate_name = "num_traits"
-  crate_root = "//ffi/gn/vendor/num-traits/src/lib.rs"
-  output_name = "num_traits-64c0e09f0f13aa66"
-
-  deps = []
-  deps += [ ":libm-v0_2_6" ]
-
-  rustenv = []
-
-  rustflags = [
-    "--cap-lints=allow",
-    "--edition=2015",
-    "-Cmetadata=64c0e09f0f13aa66",
-    "-Cextra-filename=-64c0e09f0f13aa66",
-    "--cfg=feature=\"libm\"",
-    "--cfg=has_i128",
-  ]
-
-  visibility = [ ":*" ]
-}
-
 rust_library("proc-macro2-v1_0_56") {
   crate_name = "proc_macro2"
   crate_root = "//ffi/gn/vendor/proc-macro2/src/lib.rs"

ffi/gn/Cargo.lock

@@ -71,9 +71,6 @@ pub const EXTRA_EXPERIMENTAL_DEPS: &[&str] = &[
     "icu_compactdecimal",
 ];
 
-/// Dependencies allowed when opting in to LSTM segmenter
-pub const EXTRA_LSTM_DEPS: &[&str] = &["libm", "num-traits"];
-
 /// Dependencies allowed when opting in to fixed_decimal's `ryu` backend
 /// This should never change
 pub const EXTRA_RYU_DEPS: &[&str] = &["ryu"];
@@ -122,6 +119,7 @@ pub const EXTRA_DATAGEN_DEPS: &[&str] = &[
     "ndarray",
     "num-complex",
     "num-integer",
+    "num-traits",
     "rawpointer",
     "regex-syntax",
     "rust-format",

ffi/gn/Cargo.toml

@@ -135,7 +135,6 @@ fn main() {
     let basic: BTreeSet<_> = basic_runtime.union(&basic_build).copied().collect();
     let serde: BTreeSet<_> = EXTRA_SERDE_DEPS.iter().copied().collect();
     let experimental: BTreeSet<_> = EXTRA_EXPERIMENTAL_DEPS.iter().copied().collect();
-    let lstm: BTreeSet<_> = EXTRA_LSTM_DEPS.iter().copied().collect();
     let ryu: BTreeSet<_> = EXTRA_RYU_DEPS.iter().copied().collect();
     let capi_runtime: BTreeSet<_> = EXTRA_CAPI_DEPS.iter().copied().collect();
     let capi_build: BTreeSet<_> = EXTRA_CAPI_BUILD_DEPS.iter().copied().collect();
@@ -177,15 +176,15 @@ fn main() {
         "icu",
         "normal",
         "--features serde,experimental,icu_segmenter/lstm",
-        &[&basic, &serde, &experimental, &lstm],
-        "`EXTRA_LSTM_DEPS`",
+        &[&basic, &serde, &experimental],
+        "`EXTRA_EXPERIMENTAL_DEPS`",
     );
     test_dep_list(
         "icu_segmenter",
         "normal",
         "--features lstm",
-        &[&basic, &lstm],
-        "`EXTRA_LSTM_DEPS`",
+        &[&basic],
+        "`BASIC_RUNTIME_DEPS",
     );
     test_dep_list(
         "fixed_decimal",
@@ -200,53 +199,44 @@ fn main() {
         "normal,no-proc-macro",
         "",
         // capi should NOT pull in serde or capi-build when the proc macro is disabled
-        &[&basic, &experimental, &lstm, &ryu, &capi_runtime],
+        &[&basic, &experimental, &ryu, &capi_runtime],
         "`EXTRA_CAPI_DEPS`",
     );
 
     test_dep_list(
         "icu_capi",
         "normal",
         "",
-        &[&basic, &serde, &experimental, &lstm, &ryu, &capi],
+        &[&basic, &serde, &experimental, &ryu, &capi],
         "`EXTRA_CAPI_BUILD_DEPS`",
     );
 
     test_dep_list(
         "icu_capi",
         "normal",
         "--features buffer_provider",
-        &[&basic, &serde, &experimental, &lstm, &ryu, &capi, &blob],
+        &[&basic, &serde, &experimental, &ryu, &capi, &blob],
         "`EXTRA_BLOB_DEPS`",
     );
     test_dep_list(
         "icu_capi",
         "normal",
         "--features provider_fs",
-        &[
-            &basic,
-            &serde,
-            &experimental,
-            &lstm,
-            &ryu,
-            &capi,
-            &blob,
-            &fs,
-        ],
+        &[&basic, &serde, &experimental, &ryu, &capi, &blob, &fs],
         "`EXTRA_FS_DEPS`",
     );
     test_dep_list(
         "icu_capi",
         "normal",
         "--features provider_test",
-        &[&basic, &serde, &experimental, &lstm, &ryu, &capi, &test],
+        &[&basic, &serde, &experimental, &ryu, &capi, &test],
         "`EXTRA_TEST_DEPS`",
     );
     test_dep_list(
         "icu_capi",
         "normal",
         "--features logging",
-        &[&basic, &serde, &experimental, &lstm, &ryu, &capi, &logging],
+        &[&basic, &serde, &experimental, &ryu, &capi, &logging],
         "`EXTRA_CAPI_LOGGING_DEPS`",
     );
 
@@ -258,7 +248,6 @@ fn main() {
             &basic,
             &serde,
             &experimental,
-            &lstm,
             &blob,
             &fs,
             &zip,