Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade ScalaPB #1

Open
wants to merge 3 commits into
base: release/v0.17.1
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Minimal changes for sync
AlexisBRENON committed Oct 18, 2021
commit 7809cfe7ce58409cfc3024ede66ff0ba6bb90da2
9 changes: 4 additions & 5 deletions project/Common.scala
Original file line number Diff line number Diff line change
@@ -2,8 +2,8 @@ package ml.combust.mleap

import sbt._
import Keys._
import com.typesafe.sbt.SbtPgp.autoImportImpl._
import com.typesafe.sbt.pgp.PgpKeys._
import com.jsuereth.sbtpgp.SbtPgp.autoImport._
import com.jsuereth.sbtpgp.PgpKeys._
import sbtrelease.ReleasePlugin.autoImport._
import xerial.sbt.Sonatype.autoImport._

@@ -16,8 +16,7 @@ object Common {
lazy val defaultSettings = buildSettings ++ sonatypeSettings

lazy val buildSettings: Seq[Def.Setting[_]] = Seq(
scalaVersion := "2.11.12",
crossScalaVersions := Seq("2.11.12", "2.12.10"),
scalaVersion := "2.12.10",
scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature"),
fork in Test := true,
javaOptions in test += sys.env.getOrElse("JVM_OPTS", ""),
@@ -40,7 +39,7 @@ object Common {
lazy val bundleSettings: Seq[Def.Setting[_]] = Seq(organization := "ml.combust.bundle")

lazy val noPublishSettings: Seq[Def.Setting[_]] = Seq(
publishTo in publishSigned := None,
publishSigned / publishTo := None,
publishTo := None
)

8 changes: 4 additions & 4 deletions project/Dependencies.scala
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ import Keys._
object Dependencies {
import DependencyHelpers._

val sparkVersion = "2.4.5"
val sparkVersion = "2.4.8"
val scalaTestVersion = "3.0.3"
val akkaVersion = "2.5.12"
val akkaHttpVersion = "10.0.3"
@@ -58,7 +58,7 @@ object Dependencies {
val scalaPb = Seq(
"com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion % "protobuf",
"com.thesamet.scalapb" %% "scalapb-runtime-grpc" % scalapb.compiler.Version.scalapbVersion,
"com.thesamet.scalapb" %% "scalapb-json4s" % scalapb.compiler.Version.scalapbVersion
"com.thesamet.scalapb" %% "scalapb-json4s" % "0.10.3"
)

val awsS3 = "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion
@@ -141,7 +141,7 @@ object Dependencies {

val springBootServing = l ++= Seq(springBoot, springBootActuator, commonsLang, Test.scalaTest, Test.springBootTest) ++ scalaPb

val benchmark = l ++= Seq(scalameter, scopt) ++ Compile.spark
val benchmark = l ++= Compile.spark

val databricksRuntimeTestkit = l ++= Provided.spark

@@ -163,7 +163,7 @@ object Dependencies {
* dependent entries.
*/
def versionDependentDeps(modules: ScalaVersionDependentModuleID*): Def.Setting[Seq[ModuleID]] =
libraryDependencies <++= scalaVersion(version => modules.flatMap(m => m.modules(version)))
libraryDependencies ++= scalaVersion(version => modules.flatMap(m => m.modules(version))).value

val ScalaVersion = """\d\.\d+\.\d+(?:-(?:M|RC)\d+)?""".r
val nominalScalaVersion: String => String = {
8 changes: 1 addition & 7 deletions project/DockerConfig.scala
Original file line number Diff line number Diff line change
@@ -7,14 +7,8 @@ object DockerConfig {
dockerExposedPorts := Seq(65327, 65328),
dockerBaseImage := "openjdk:8-jre-slim",
dockerRepository := Some("combustml"),
dockerBuildOptions := Seq("-t", dockerAlias.value.versioned) ++ (
if (dockerUpdateLatest.value)
Seq("-t", dockerAlias.value.latest)
else
Seq()
),
dockerCommands := dockerCommands.value.filterNot {
case ExecCmd("RUN", args @ _*) => args.contains("chown")
case cmd => false
})
}
}
166 changes: 71 additions & 95 deletions project/MleapProject.scala
Original file line number Diff line number Diff line change
@@ -35,9 +35,9 @@ object MleapProject {

lazy val root = Project(
id = "mleap",
base = file("."),
aggregate = aggregatedProjects
base = file(".")
).settings(rootSettings)
.aggregate(aggregatedProjects: _*)

lazy val baseProject = Project(
id = "mleap-base",
@@ -46,173 +46,149 @@ object MleapProject {

lazy val tensor = Project(
id = "mleap-tensor",
base = file("mleap-tensor"),
dependencies = Seq(baseProject)
)
base = file("mleap-tensor")
).dependsOn(baseProject)

lazy val bundleMl = Project(
id = "bundle-ml",
base = file("bundle-ml"),
dependencies = Seq(baseProject, tensor)
)
base = file("bundle-ml")
).dependsOn(baseProject, tensor)

lazy val bundleHdfs = Project(
id = "bundle-hdfs",
base = file("bundle-hdfs"),
dependencies = Seq(bundleMl)
)
base = file("bundle-hdfs")
).dependsOn(bundleMl)

lazy val core = Project(
id = "mleap-core",
base = file("mleap-core"),
dependencies = Seq(baseProject, tensor)
)
base = file("mleap-core")
).dependsOn(baseProject, tensor)

lazy val runtime = Project(
id = "mleap-runtime",
base = file("mleap-runtime"),
dependencies = Seq(core, bundleMl)
)
base = file("mleap-runtime")
).dependsOn(core, bundleMl)

lazy val avro = Project(
id = "mleap-avro",
base = file("mleap-avro"),
dependencies = Seq(runtime)
)
base = file("mleap-avro")
).dependsOn(runtime)

lazy val sparkBase = Project(
id = "mleap-spark-base",
base = file("mleap-spark-base"),
dependencies = Seq(runtime, bundleHdfs)
)
base = file("mleap-spark-base")
).dependsOn(runtime, bundleHdfs)

lazy val sparkTestkit = Project(
id = "mleap-spark-testkit",
base = file("mleap-spark-testkit"),
dependencies = Seq(sparkBase)
)
base = file("mleap-spark-testkit")
).dependsOn(sparkBase)

lazy val spark = Project(
id = "mleap-spark",
base = file("mleap-spark"),
dependencies = Seq(sparkBase, sparkTestkit % "test")
)
base = file("mleap-spark")
).dependsOn(sparkBase, sparkTestkit % "test")

lazy val sparkExtension = Project(
id = "mleap-spark-extension",
base = file("mleap-spark-extension"),
dependencies = Seq(spark, sparkTestkit % "test")
)
base = file("mleap-spark-extension")
).dependsOn(spark, sparkTestkit % "test")

lazy val tensorflow = Project(
id = "mleap-tensorflow",
base = file("mleap-tensorflow"),
dependencies = Seq(runtime)
)
base = file("mleap-tensorflow")
).dependsOn(runtime)

lazy val xgboostRuntime = Project(
id = "mleap-xgboost-runtime",
base = file("mleap-xgboost-runtime"),
dependencies = Seq(
runtime,
sparkTestkit % "test")
)
base = file("mleap-xgboost-runtime")
).dependsOn(
runtime,
sparkTestkit % "test")

lazy val xgboostSpark = Project(
id = "mleap-xgboost-spark",
base = file("mleap-xgboost-spark"),
dependencies = Seq(sparkBase % "provided",
xgboostRuntime % "test",
spark % "test",
sparkTestkit % "test")
)
base = file("mleap-xgboost-spark")
).dependsOn(sparkBase % "provided",
xgboostRuntime % "test",
spark % "test",
sparkTestkit % "test")

lazy val serving = Project(
id = "mleap-serving",
base = file("mleap-serving"),
dependencies = Seq(springBootServing, grpcServer)
)
base = file("mleap-serving")
).dependsOn(springBootServing, grpcServer)

lazy val executor = Project(
id = "mleap-executor",
base = file("mleap-executor"),
dependencies = Seq(runtime)
)
base = file("mleap-executor")
).dependsOn(runtime)

lazy val executorTestKit = Project(
id = "mleap-executor-testkit",
base = file("mleap-executor-testkit"),
dependencies = Seq(executor)
)
base = file("mleap-executor-testkit")
).dependsOn(executor)

lazy val executorTests = Project(
id = "mleap-executor-tests",
base = file("mleap-executor-tests"),
dependencies = Seq(executor, executorTestKit % "test")
)
base = file("mleap-executor-tests")
).dependsOn(executor, executorTestKit % "test")

lazy val grpc = Project(
id = "mleap-grpc",
base = file("mleap-grpc"),
dependencies = Seq(`executor`)
)
base = file("mleap-grpc")
).dependsOn(`executor`)

lazy val grpcServer = Project(
id = "mleap-grpc-server",
base = file("mleap-grpc-server"),
dependencies = Seq(grpc, executorTestKit % "test")
)
base = file("mleap-grpc-server")
).dependsOn(grpc, executorTestKit % "test")

lazy val repositoryS3 = Project(
id = "mleap-repository-s3",
base = file("mleap-repository-s3"),
dependencies = Seq(executor)
)
base = file("mleap-repository-s3")
).dependsOn(executor)

lazy val springBootServing = Project(
id = "mleap-spring-boot",
base = file("mleap-spring-boot"),
dependencies = Seq(executor)
)
base = file("mleap-spring-boot")
).dependsOn(executor)

lazy val benchmark = Project(
id = "mleap-benchmark",
base = file("mleap-benchmark"),
dependencies = Seq(runtime, spark, avro)
)
base = file("mleap-benchmark")
).dependsOn(runtime, spark, avro)

// Create underlying fat jar project as per: https://github.com/sbt/sbt-assembly#q-despite-the-concerned-friends-i-still-want-publish-fat-jars-what-advice-do-you-have
lazy val databricksRuntimeFat = Project(
id = "mleap-databricks-runtime-fat",
base = file("mleap-databricks-runtime-fat"),
dependencies = Seq(baseProject,
tensor,
core,
runtime,
bundleMl,
spark,
sparkExtension,
tensorflow,
xgboostRuntime,
xgboostSpark)
base = file("mleap-databricks-runtime-fat")
).dependsOn(baseProject,
tensor,
core,
runtime,
bundleMl,
spark,
sparkExtension,
tensorflow,
xgboostRuntime,
xgboostSpark
).settings(excludeDependencies ++= Seq(
SbtExclusionRule("org.tensorflow"),
SbtExclusionRule("org.apache.spark"),
SbtExclusionRule("ml.dmlc")
ExclusionRule("org.tensorflow"),
ExclusionRule("org.apache.spark"),
ExclusionRule("ml.dmlc")
))

lazy val databricksRuntime = Project(
id = "mleap-databricks-runtime",
base = file("mleap-databricks-runtime"),
dependencies = Seq()
base = file("mleap-databricks-runtime")
)

lazy val databricksRuntimeTestkit = Project(
id = "mleap-databricks-runtime-testkit",
base = file("mleap-databricks-runtime-testkit"),
dependencies = Seq(spark % "provided",
sparkExtension % "provided",
xgboostSpark % "provided",
tensorflow % "provided")
)
base = file("mleap-databricks-runtime-testkit")
).dependsOn(spark % "provided",
sparkExtension % "provided",
xgboostSpark % "provided",
tensorflow % "provided")
}
6 changes: 2 additions & 4 deletions project/Release.scala
Original file line number Diff line number Diff line change
@@ -4,7 +4,6 @@ import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport
import sbt.Keys._
import sbtrelease.ReleasePlugin.autoImport.{ReleaseStep, _}
import ReleaseTransformations._
import xerial.sbt.Sonatype.SonatypeCommand

object Release {
lazy val settings = Seq(releaseVersionBump := sbtrelease.Version.Bump.Minor,
@@ -19,10 +18,9 @@ object Release {
commitReleaseVersion,
tagRelease,
publishArtifacts,
releaseStepCommand(SonatypeCommand.sonatypeRelease),
releaseStepTask(publish in autoImport.Docker in MleapProject.serving),
releaseStepCommand("sonatypeBundleRelease"),
setNextVersion,
commitNextVersion,
pushChanges
))
}
}
2 changes: 1 addition & 1 deletion project/build.properties
Original file line number Diff line number Diff line change
@@ -1 +1 @@
sbt.version = 0.13.18
sbt.version = 1.5.5
12 changes: 6 additions & 6 deletions project/plugins.sbt
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
logLevel := Level.Warn

addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.18")
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0-M8")
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.0")
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.10")
addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.6")
addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2")
addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.6")
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.6.1")
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.9.3")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2")


libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.7.1"
libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.10.11"