diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1beff6a..0952617 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,7 @@ Ensure the following are installed: - Java 8+ (JDK) - [Rust](https://www.rust-lang.org/tools/install) - [sbt](https://www.scala-sbt.org/) +- [just](https://github.com/casey/just) --- @@ -53,31 +54,31 @@ compiler are installed, then follow the commands below as per the need. ### Full project (Rust + Scala/Java) ```bash -sbt compile +just compile ``` ### Native Rust library only ```bash -sbt generateNativeLibrary +just build-native ``` ### Native Rust library only (Release profile) ```bash -NATIVE_RELEASE=true sbt generateNativeLibrary +NATIVE_RELEASE=true just build-native ``` ### Locally publish ```bash -sbt publishLocal +just release-local ``` ### Fat JAR ```bash -sbt assembly +just assembly ``` --- @@ -87,7 +88,7 @@ sbt assembly Run unit tests via: ```bash -sbt test +just test ``` --- @@ -107,7 +108,7 @@ If you're a maintainer: ```bash # Publish to Sonatype snapshots -sbt +publish +just release ``` --- @@ -118,4 +119,3 @@ sbt +publish - Join the [Polars Discord](https://discord.gg/4UfP5cfBE7) We appreciate every contribution ❤️ - diff --git a/README.md b/README.md index fa01c82..d959caf 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,9 @@ repositories { } implementation("com.github.chitralverma:scala-polars_2.12:SOME-VERSION-SNAPSHOT") ``` + > Note: Use `scala-polars_2.13` for Scala 2.13.x projects or `scala-polars_3` for Scala 3.x projects as the artifact ID + --- ## 🧱 Modules @@ -95,7 +97,7 @@ implementation("com.github.chitralverma:scala-polars_2.12:SOME-VERSION-SNAPSHOT" ## 🧪 Getting Started -### Scala +### Scala ```scala import com.github.chitralverma.polars.api.{DataFrame, Series} @@ -116,6 +118,7 @@ result.show() ``` ### Java + ```java import com.github.chitralverma.polars.api.DataFrame; import com.github.chitralverma.polars.api.Series; @@ -161,24 +164,25 @@ df.show(); - JDK 8+ - [Rust](https://www.rust-lang.org/tools/install) - [sbt](https://www.scala-sbt.org/) +- [just](https://github.com/casey/just) ### Commands ```bash # Compile Rust + Scala + Java -sbt compile +just compile # Publish locally -sbt publishLocal +just release-local # Fat JAR (default Scala version) -sbt assembly +just assembly # Rust native only -sbt generateNativeLibrary +just build-native # Rust native only (Release profile) -NATIVE_RELEASE=true sbt generateNativeLibrary +NATIVE_RELEASE=true just build-native ``` --- @@ -192,4 +196,4 @@ Apache 2.0 — see [LICENSE](LICENSE) ## 🤝 Community - Discuss Polars on [Polars Discord](https://discord.gg/4UfP5cfBE7) -- To contribute, see [CONTRIBUTING.md](CONTRIBUTING.md) \ No newline at end of file +- To contribute, see [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/build.sbt b/build.sbt index 5169f76..214a9e8 100644 --- a/build.sbt +++ b/build.sbt @@ -28,8 +28,10 @@ lazy val core = project publishMavenStyle := true ) .settings( + nativeRoot := baseDirectory.value.toPath.resolveSibling("native").toFile, inConfig(Compile)(NativeBuildSettings.settings) ) + .settings(ExtraCommands.commands) .settings(ExtraCommands.commandAliases) // .configureUnidoc("scala-polars API Reference") diff --git a/justfile b/justfile index 059bd78..d6e611b 100644 --- a/justfile +++ b/justfile @@ -2,8 +2,8 @@ set shell := ["bash", "-c"] set ignore-comments := true root := justfile_directory() -native_root := "native" -native_manifest := "native/Cargo.toml" +native_root := root / 'native' +native_manifest := native_root / 'Cargo.toml' cargo_flags := env("CARGO_FLAGS", "--locked") # Default recipe to 'help' to display this help screen @@ -22,9 +22,9 @@ echo-command args: # Format all code (Scala, Java, Rust, sbt) [group('lint')] fmt: - @just echo-command 'Formatting Scala, Java & Sbt' - @sbt -error --batch scalafmtAll scalafmtSbt javafmtAll - @just echo-command 'Formatting Rust' + @just echo-command 'Formatting core module' + @sbt -error scalafmtAll scalafmtSbt javafmtAll reload + @just echo-command 'Formatting native module' @cargo clippy -q {{ cargo_flags }} --no-deps --fix --allow-dirty --allow-staged --manifest-path {{ native_manifest }} @cargo sort {{ native_root }} @cargo fmt --quiet --manifest-path {{ native_manifest }} @@ -33,9 +33,9 @@ fmt: # Check formatting and linting [group('lint')] lint: - @just echo-command 'Checking Scala, Java & Sbt' - @sbt -error --batch scalafmtCheckAll scalafmtSbtCheck javafmtCheckAll - @just echo-command 'Checking Rust' + @just echo-command 'Checking core module' + @sbt -error scalafmtCheckAll scalafmtSbtCheck javafmtCheckAll + @just echo-command 'Checking native module' @cargo clippy -q {{ cargo_flags }} --no-deps --manifest-path {{ native_manifest }} -- -D warnings @cargo sort {{ native_root }} --check @cargo fmt --check --manifest-path {{ native_manifest }} @@ -47,60 +47,35 @@ pre-commit: fmt lint # Generate JNI headers [group('dev')] -gen-headers: clean-headers - @sbt -error --batch genHeaders - -# Remove generated JNI headers -[group('dev')] -clean-headers: - @rm -rf core/target/native - @just echo-command 'Removed JNI headers directory' +gen-headers: + @just echo-command 'Generating JNI headers' + @sbt genHeaders # Build native library TARGET_TRIPLE, NATIVE_RELEASE, NATIVE_LIB_LOCATION env vars are supported [group('dev')] build-native: - #!/usr/bin/env bash - set -euo pipefail - if [ "${SKIP_NATIVE_GENERATION:-false}" = "false" ]; then - TRIPLE="${TARGET_TRIPLE:-$(rustc -vV | grep host | cut -d' ' -f2)}" - ARCH=$(echo "$TRIPLE" | cut -d'-' -f1) - RELEASE_FLAG="" - if [ "${NATIVE_RELEASE:-false}" = "true" ]; then - RELEASE_FLAG="--release" - fi - - # Generate native library artifacts in a predictable output directory - NATIVE_OUTPUT_DIR="core/target/native-libs/$ARCH" - mkdir -p "$NATIVE_OUTPUT_DIR" - cargo build {{ cargo_flags }} --manifest-path {{ native_manifest }} -Z unstable-options $RELEASE_FLAG --lib --target "$TRIPLE" --artifact-dir "$NATIVE_OUTPUT_DIR" - - if [ -n "${NATIVE_LIB_LOCATION:-}" ]; then - # Remove trailing slash if present - CLEAN_NATIVE_LIB_LOCATION="${NATIVE_LIB_LOCATION%/}" - DEST="$CLEAN_NATIVE_LIB_LOCATION/$ARCH" - echo "Environment variable NATIVE_LIB_LOCATION is set, copying built native library from location '$NATIVE_OUTPUT_DIR' to '$DEST'." - mkdir -p "$DEST" - cp -rf "$NATIVE_OUTPUT_DIR"/* "$DEST/" - fi - else - @just echo-command 'Environment variable SKIP_NATIVE_GENERATION is set, skipping cargo build.' - fi + @just echo-command 'Building native library' + @sbt generateNativeLibrary # Build assembly jars [group('dev')] -assembly: build-native - sbt +assembly +assembly: + @sbt +assembly # Compile [group('dev')] -compile: build-native - sbt compile +compile: + @sbt compile # Clean build artifacts [group('dev')] -clean: clean-headers - @sbt clean cleanFiles - @cargo clean --manifest-path {{ native_manifest }} +clean: + @just echo-command 'Cleaning native artifacts' + @cargo clean --manifest-path {{ native_manifest }} --quiet + @just echo-command 'Cleaning JNI headers' + @sbt -error cleanHeaders + @just echo-command 'Cleaning core artifacts' + @sbt -error clean cleanFiles reload # Run tests [group('dev')] @@ -121,3 +96,8 @@ publish-site: [group('release')] release: @sbt ci-release + +# Release/publish artifacts locally +[group('release')] +release-local: + @sbt publishLocal diff --git a/project/ExtraCommands.scala b/project/ExtraCommands.scala index 60aaf3c..1000b7b 100644 --- a/project/ExtraCommands.scala +++ b/project/ExtraCommands.scala @@ -5,8 +5,23 @@ import com.github.sbt.jni.plugins.JniJavah.autoImport.javah object ExtraCommands { + lazy val cleanHeaders = + taskKey[Unit]("Removes all previously generated headers") + lazy val commandAliases: Seq[Setting[_]] = Seq( - addCommandAlias("genHeaders", "javah") + addCommandAlias("genHeaders", ";cleanHeaders; javah") ).flatten + lazy val commands: Seq[Setting[_]] = Seq( + cleanHeaders := { + import scala.reflect.io.Directory + + val headerDir = (javah / target).value + val directory = new Directory(headerDir) + + directory.deleteRecursively() + sLog.value.info(s"Removed headers directory $headerDir") + } + ) + } diff --git a/project/NativeBuildSettings.scala b/project/NativeBuildSettings.scala index e5413f9..1005dec 100644 --- a/project/NativeBuildSettings.scala +++ b/project/NativeBuildSettings.scala @@ -4,74 +4,181 @@ import sbt.* import sbt.Keys.* import scala.collection.JavaConverters.* +import scala.sys.process.* + +import Utils.* object NativeBuildSettings { + lazy val generateNativeLibrary = taskKey[Unit]( + "Generates native library using Cargo which can be added as managed resource to classpath. " + + "The following environment variables are supported: " + + "SKIP_NATIVE_GENERATION: If set, skips cargo build. " + + "TARGET_TRIPLE: The target triple for cargo build. " + + "NATIVE_RELEASE: If set to 'true', cargo build will use the '--release' flag. " + + "NATIVE_LIB_LOCATION: If set, copies the built native library to this location." + ) + lazy val managedNativeLibraries = taskKey[Seq[Path]]( "Maps locally built, platform-dependent libraries to their locations on the classpath." ) lazy val settings: Seq[Setting[_]] = Seq( - managedNativeLibraries := Def.task { - val logger: Logger = sLog.value - val nativeLibsDir = target.value.toPath.resolve("native-libs") - - val managedLibs = if (Files.exists(nativeLibsDir)) { - Files - .find( - nativeLibsDir, - Int.MaxValue, - (filePath, _) => filePath.toFile.isFile - ) - .iterator() - .asScala - .toSeq - } else { - Seq.empty[Path] - } + generateNativeLibrary := Def + .taskDyn[Unit] { + Def.task { + val logger: Logger = sLog.value + + sys.env.get("SKIP_NATIVE_GENERATION") match { + case None => + + val targetTriple = sys.env.getOrElse( + "TARGET_TRIPLE", { + logger.warn( + "Environment variable TARGET_TRIPLE was not set, getting value from `rustc`." + ) + + s"rustc -vV".!!.split("\n") + .map(_.trim) + .find(_.startsWith("host")) + .map(_.split(" ")(1).trim) + .getOrElse(throw new IllegalStateException("No target triple found.")) + } + ) - val externalNativeLibs = sys.env.get("NATIVE_LIB_LOCATION") match { - case Some(path) => - val externalPath = Paths.get(path) - if (Files.exists(externalPath)) { - Files - .find( - externalPath, - Int.MaxValue, - (filePath, _) => filePath.toFile.isFile + val arch = targetTriple.toLowerCase(java.util.Locale.ROOT).split("-").head + + val nativeOutputDir = resourceManaged.value.toPath.resolve(s"native/$arch/") + + val releaseFlag = sys.env.get("NATIVE_RELEASE") match { + case Some("true") => Seq("--release") + case _ => Seq.empty + } + + val cargoFlags = sys.env.get("CARGO_FLAGS") match { + case Some(s) => s + case _ => "--locked" + } + + val extraFlags = + if (cargoFlags.nonEmpty) + cargoFlags.split("\\s+").filterNot(_ == "--release").toSeq + else Seq.empty + + val baseCmd = Seq( + "cargo", + "build", + "-Z", + "unstable-options", + "--lib", + "--target", + targetTriple, + "--artifact-dir", + nativeOutputDir ) - .iterator() - .asScala - .toSeq - } else { - Seq.empty[Path] - } - case None => Seq.empty[Path] - } + // Build the native project using cargo + val cmd = (baseCmd ++ releaseFlag ++ extraFlags).mkString(" ") + + executeProcess(cmd = cmd, cwd = Some(nativeRoot.value), sLog.value, infoOnly = true) + logger.success(s"Successfully built native library at location '$nativeOutputDir'") - val allLibs = (managedLibs ++ externalNativeLibs).distinct.map(_.toAbsolutePath) + sys.env.get("NATIVE_LIB_LOCATION") match { + case Some(path) => + val dest = Paths.get(path, arch).toAbsolutePath + logger.info( + "Environment variable NATIVE_LIB_LOCATION is set, " + + s"copying built native library from location '$nativeOutputDir' to '$dest'." + ) - if (allLibs.isEmpty) { - logger.warn( - s"Native libraries directory $nativeLibsDir does not exist and NATIVE_LIB_LOCATION is not set. " + - "Run 'just build-native' first." - ) + IO.copyDirectory(nativeOutputDir.toFile, dest.toFile) + + case None => () + } + + case Some(_) => + logger.info( + "Environment variable SKIP_NATIVE_GENERATION is set, skipping cargo build." + ) + } + } } + .value, + managedNativeLibraries := Def + .taskDyn[Seq[Path]] { + Def.task { + val managedLibs = sys.env.get("SKIP_NATIVE_GENERATION") match { + case None => + val nativeDir = resourceManaged.value.toPath.resolve("native/") + if (Files.exists(nativeDir)) { + Files + .find( + nativeDir, + Int.MaxValue, + (filePath, _) => filePath.toFile.isFile + ) + .iterator() + .asScala + .toSeq + } else { + Seq.empty[Path] + } + + case Some(_) => Seq.empty[Path] + } + + val externalNativeLibs = sys.env.get("NATIVE_LIB_LOCATION") match { + case Some(path) => + val externalPath = Paths.get(path) + if (Files.exists(externalPath)) { + Files + .find( + externalPath, + Int.MaxValue, + (filePath, _) => filePath.toFile.isFile + ) + .iterator() + .asScala + .toSeq + } else { + sLog.value.warn(s"NATIVE_LIB_LOCATION '$path' does not exist; skipping.") + Seq.empty[Path] + } + + case None => Seq.empty[Path] + } + + // Collect paths of built resources to later include in classpath + val allLibs = (managedLibs ++ externalNativeLibs).distinct.map(_.toAbsolutePath) - allLibs - }.value, + if (allLibs.isEmpty) { + sLog.value.warn( + "No native libraries were found. " + + "If you have not built them yet, run 'just build-native' first, " + + "or set the NATIVE_LIB_LOCATION environment variable to point to existing native libraries." + ) + } + + allLibs + } + } + .dependsOn(generateNativeLibrary) + .value, resourceGenerators += Def.task { + // Add all generated resources to manage resources' classpath managedNativeLibraries.value .map { path => val pathStr = path.toString val arch = path.getParent.getFileName.toString val libraryFile = path.toFile + + // native library as a managed resource file val resource = resourceManaged.value / "native" / arch / libraryFile.getName - if (libraryFile.isDirectory) IO.copyDirectory(libraryFile, resource) - else IO.copyFile(libraryFile, resource) + // copy native library to a managed resource, so that it is always available + // on the classpath, even when not packaged as a jar + IO.copyDirectory(libraryFile, resource) sLog.value.success( s"Added resource from location '$pathStr' " + diff --git a/project/Utils.scala b/project/Utils.scala index b0748e1..5463920 100644 --- a/project/Utils.scala +++ b/project/Utils.scala @@ -1,11 +1,38 @@ import sbt.* +import scala.sys.process.* + object Utils { + lazy val nativeRoot = taskKey[File]("Directory pointing to the native project root.") + + def executeProcess( + cmd: String, + cwd: Option[File] = None, + logger: Logger, + infoOnly: Boolean = false, + extraEnv: Seq[(String, String)] = Nil + ): Unit = { + val exitCode = + Process(cmd, cwd, extraEnv: _*).run(getProcessLogger(logger, infoOnly)).exitValue() + + if (exitCode != 0) { + sys.error(s"Failed to execute command `$cmd` with exit code $exitCode.") + } else { + logger.success(s"Successfully executed command `$cmd`.") + } + } + def priorTo213(scalaVersion: String): Boolean = CrossVersion.partialVersion(scalaVersion) match { case Some((2, minor)) if minor < 13 => true case _ => false } + def getProcessLogger(logger: Logger, infoOnly: Boolean = false): ProcessLogger = + ProcessLogger( + (o: String) => logger.info(o), + (e: String) => if (infoOnly) logger.info(e) else logger.error(e) + ) + }