From 7486b5f35f8dc29019eeee8fe8c96ef7ecc7fee6 Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Thu, 2 Feb 2023 16:12:37 +0100 Subject: [PATCH 1/2] Add instance-wide PR throttle --- .../scalasteward/core/application/Cli.scala | 16 +++++- .../core/application/Config.scala | 8 ++- .../core/application/Context.scala | 13 ++++- .../core/application/StewardAlg.scala | 7 ++- .../core/nurture/NurtureAlg.scala | 2 + .../core/nurture/PullRequestThrottle.scala | 55 ++++++++++++++++++ .../scalasteward/core/util/SimpleTimer.scala | 56 ++++++++++++++++++ .../core/util/SimpleTimerTest.scala | 57 +++++++++++++++++++ 8 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 modules/core/src/main/scala/org/scalasteward/core/nurture/PullRequestThrottle.scala create mode 100644 modules/core/src/main/scala/org/scalasteward/core/util/SimpleTimer.scala create mode 100644 modules/core/src/test/scala/org/scalasteward/core/util/SimpleTimerTest.scala diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala index b11e43c431..a1afb5e89e 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala @@ -323,6 +323,19 @@ object Cli { .withDefault(default) } + private val prThrottleSkip = { + val help = "" // TODO + option[FiniteDuration]("pr-throttle-skip", help).orNone + } + + private val prThrottleWait = { + val help = "" // TODO + option[FiniteDuration]("pr-throttle-wait", help).orNone + } + + private val pullRequestThrottleCfg: Opts[PullRequestThrottleCfg] = + (prThrottleSkip, prThrottleWait).mapN(PullRequestThrottleCfg.apply) + private val configFile: Opts[File] = Opts.argument[File]() @@ -350,7 +363,8 @@ object Cli { gitHubApp, urlCheckerTestUrls, defaultMavenRepo, - refreshBackoffPeriod + refreshBackoffPeriod, + pullRequestThrottleCfg ).mapN(Config.apply) val command: Command[StewardUsage] = diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala index 2ced3f8a64..4fa59544cb 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala @@ -69,7 +69,8 @@ final case class Config( githubApp: Option[GitHubApp], urlCheckerTestUrls: Nel[Uri], defaultResolver: Resolver, - refreshBackoffPeriod: FiniteDuration + refreshBackoffPeriod: FiniteDuration, + pullRequestThrottleCfg: PullRequestThrottleCfg ) { def forgeUser[F[_]](implicit processAlg: ProcessAlg[F], @@ -139,6 +140,11 @@ object Config { disableDefaults: Boolean ) + final case class PullRequestThrottleCfg( + skipFor: Option[FiniteDuration], + waitFor: Option[FiniteDuration] + ) + sealed trait ForgeSpecificCfg extends Product with Serializable final case class AzureReposCfg( diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala index 53b654905b..ac70cc7042 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala @@ -42,7 +42,12 @@ import org.scalasteward.core.forge.github.{GitHubAppApiAlg, GitHubAuthAlg} import org.scalasteward.core.forge.{ForgeApiAlg, ForgeRepoAlg, ForgeSelection} import org.scalasteward.core.git.{GenGitAlg, GitAlg} import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg} -import org.scalasteward.core.nurture.{NurtureAlg, PullRequestRepository, UpdateInfoUrlFinder} +import org.scalasteward.core.nurture.{ + NurtureAlg, + PullRequestRepository, + PullRequestThrottle, + UpdateInfoUrlFinder +} import org.scalasteward.core.persistence.{CachingKeyValueStore, JsonKeyValueStore} import org.scalasteward.core.repocache._ import org.scalasteward.core.repoconfig.{RepoConfigAlg, RepoConfigLoader, ValidateRepoConfigAlg} @@ -193,13 +198,16 @@ object Context { .create[F, Repo, RepoCache]("repo_cache", "1", kvsPrefix) versionsStore <- JsonKeyValueStore .create[F, VersionsCache.Key, VersionsCache.Value]("versions", "2") + dateTimeAlg0 = DateTimeAlg.create[F] + pullRequestThrottle0 <- PullRequestThrottle + .create[F](config.pullRequestThrottleCfg)(dateTimeAlg0, logger, F) } yield { implicit val artifactMigrationsLoader: ArtifactMigrationsLoader[F] = artifactMigrationsLoader0 implicit val artifactMigrationsFinder: ArtifactMigrationsFinder = artifactMigrationsFinder0 implicit val scalafixMigrationsLoader: ScalafixMigrationsLoader[F] = scalafixMigrationsLoader0 implicit val scalafixMigrationsFinder: ScalafixMigrationsFinder = scalafixMigrationsFinder0 implicit val urlChecker: UrlChecker[F] = urlChecker0 - implicit val dateTimeAlg: DateTimeAlg[F] = DateTimeAlg.create[F] + implicit val dateTimeAlg: DateTimeAlg[F] = dateTimeAlg0 implicit val repoConfigAlg: RepoConfigAlg[F] = new RepoConfigAlg[F](maybeGlobalRepoConfig) implicit val filterAlg: FilterAlg[F] = new FilterAlg[F] implicit val gitAlg: GitAlg[F] = GenGitAlg.create[F](config.gitCfg) @@ -232,6 +240,7 @@ object Context { implicit val repoCacheAlg: RepoCacheAlg[F] = new RepoCacheAlg[F](config) implicit val scannerAlg: ScannerAlg[F] = new ScannerAlg[F] implicit val editAlg: EditAlg[F] = new EditAlg[F] + implicit val pullRequestThrottle: PullRequestThrottle[F] = pullRequestThrottle0 implicit val nurtureAlg: NurtureAlg[F] = new NurtureAlg[F](config.forgeCfg) implicit val pruningAlg: PruningAlg[F] = new PruningAlg[F] implicit val gitHubAppApiAlg: GitHubAppApiAlg[F] = diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/StewardAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/application/StewardAlg.scala index 415a9916e9..aad971a890 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/StewardAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/StewardAlg.scala @@ -24,7 +24,7 @@ import org.scalasteward.core.data.Repo import org.scalasteward.core.forge.github.{GitHubApp, GitHubAppApiAlg, GitHubAuthAlg} import org.scalasteward.core.git.GitAlg import org.scalasteward.core.io.{FileAlg, WorkspaceAlg} -import org.scalasteward.core.nurture.NurtureAlg +import org.scalasteward.core.nurture.{NurtureAlg, PullRequestThrottle} import org.scalasteward.core.repocache.RepoCacheAlg import org.scalasteward.core.update.PruningAlg import org.scalasteward.core.util @@ -42,6 +42,7 @@ final class StewardAlg[F[_]](config: Config)(implicit logger: Logger[F], nurtureAlg: NurtureAlg[F], pruningAlg: PruningAlg[F], + pullRequestThrottle: PullRequestThrottle[F], repoCacheAlg: RepoCacheAlg[F], selfCheckAlg: SelfCheckAlg[F], workspaceAlg: WorkspaceAlg[F], @@ -79,7 +80,9 @@ final class StewardAlg[F[_]](config: Config)(implicit F.guarantee( repoCacheAlg.checkCache(repo).flatMap { case (data, fork) => pruningAlg.needsAttention(data).flatMap { - _.traverse_(states => nurtureAlg.nurture(data, fork, states.map(_.update))) + _.traverse_ { states => + pullRequestThrottle.throttle(nurtureAlg.nurture(data, fork, states.map(_.update))) + } } }, gitAlg.removeClone(repo) diff --git a/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala index 408a61da85..90bc5c7b52 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala @@ -42,6 +42,7 @@ final class NurtureAlg[F[_]](config: ForgeCfg)(implicit gitAlg: GitAlg[F], logger: Logger[F], pullRequestRepository: PullRequestRepository[F], + pullRequestThrottle: PullRequestThrottle[F], updateInfoUrlFinder: UpdateInfoUrlFinder[F], urlChecker: UrlChecker[F], F: Concurrent[F] @@ -252,6 +253,7 @@ final class NurtureAlg[F[_]](config: ForgeCfg)(implicit _ <- forgeApiAlg .labelPullRequest(data.repo, pr.number, requestData.labels) .whenA(config.addLabels && requestData.labels.nonEmpty) + _ <- pullRequestThrottle.hit prData = PullRequestData[Id]( pr.html_url, data.baseSha1, diff --git a/modules/core/src/main/scala/org/scalasteward/core/nurture/PullRequestThrottle.scala b/modules/core/src/main/scala/org/scalasteward/core/nurture/PullRequestThrottle.scala new file mode 100644 index 0000000000..89d84d69b4 --- /dev/null +++ b/modules/core/src/main/scala/org/scalasteward/core/nurture/PullRequestThrottle.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2018-2023 Scala Steward contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.scalasteward.core.nurture + +import cats.Monad +import cats.effect.Async +import cats.syntax.all._ +import org.scalasteward.core.application.Config.PullRequestThrottleCfg +import org.scalasteward.core.util.{dateTime, DateTimeAlg, SimpleTimer} +import org.typelevel.log4cats.Logger +import scala.concurrent.duration.FiniteDuration + +final class PullRequestThrottle[F[_]](config: PullRequestThrottleCfg, timer: SimpleTimer[F])( + implicit + logger: Logger[F], + F: Monad[F] +) { + def hit: F[Unit] = + config.skipFor.orElse(config.waitFor).fold(F.unit)(timer.start) + + def throttle(f: F[Unit]): F[Unit] = + timer.remaining.flatMap { + case Some(remaining) if config.skipFor.isDefined => + log("skipping", remaining) + case Some(remaining) if config.waitFor.isDefined => + log("waiting", remaining) >> timer.await >> f + case _ => f + } + + private def log(action: String, remaining: FiniteDuration): F[Unit] = + logger.info(s"PR throttle is active: $action for ${dateTime.showDuration(remaining)}") +} + +object PullRequestThrottle { + def create[F[_]](config: PullRequestThrottleCfg)(implicit + dateTimeAlg: DateTimeAlg[F], + logger: Logger[F], + F: Async[F] + ): F[PullRequestThrottle[F]] = + SimpleTimer.create.map(new PullRequestThrottle(config, _)) +} diff --git a/modules/core/src/main/scala/org/scalasteward/core/util/SimpleTimer.scala b/modules/core/src/main/scala/org/scalasteward/core/util/SimpleTimer.scala new file mode 100644 index 0000000000..992d1d0d8a --- /dev/null +++ b/modules/core/src/main/scala/org/scalasteward/core/util/SimpleTimer.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2018-2023 Scala Steward contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.scalasteward.core.util + +import cats.data.OptionT +import cats.effect.{Async, Ref} +import cats.syntax.all._ +import scala.concurrent.duration.FiniteDuration + +final class SimpleTimer[F[_]]( + currentDuration: Ref[F, Option[FiniteDuration]], + lastStart: Ref[F, Option[Timestamp]] +)(implicit dateTimeAlg: DateTimeAlg[F], F: Async[F]) { + def start(duration: FiniteDuration): F[Unit] = + currentDuration.set(duration.some) >> + dateTimeAlg.currentTimestamp.map(Some.apply).flatMap(lastStart.set) + + def remaining: F[Option[FiniteDuration]] = + OptionT(currentDuration.get).flatMap { duration => + OptionT(lastStart.get).flatMapF { setTimestamp => + dateTimeAlg.currentTimestamp.flatMap { now => + val elapsed = setTimestamp.until(now) + if (elapsed < duration) F.pure((duration - elapsed).some) + else currentDuration.set(None).as(none[FiniteDuration]) + } + } + }.value + + def await: F[Unit] = + remaining.flatMap(_.fold(F.unit)(F.sleep)) + + def expired: F[Boolean] = + remaining.map(_.isEmpty) +} + +object SimpleTimer { + def create[F[_]](implicit dateTimeAlg: DateTimeAlg[F], F: Async[F]): F[SimpleTimer[F]] = + for { + currentDuration <- Ref[F].of(Option.empty[FiniteDuration]) + lastStart <- Ref[F].of(Option.empty[Timestamp]) + } yield new SimpleTimer(currentDuration, lastStart) +} diff --git a/modules/core/src/test/scala/org/scalasteward/core/util/SimpleTimerTest.scala b/modules/core/src/test/scala/org/scalasteward/core/util/SimpleTimerTest.scala new file mode 100644 index 0000000000..3a8aa87f42 --- /dev/null +++ b/modules/core/src/test/scala/org/scalasteward/core/util/SimpleTimerTest.scala @@ -0,0 +1,57 @@ +package org.scalasteward.core.util + +import cats.effect.IO +import munit.CatsEffectSuite +import scala.concurrent.duration._ + +class SimpleTimerTest extends CatsEffectSuite { + implicit private val dateTimeAlg: DateTimeAlg[IO] = DateTimeAlg.create[IO] + + test("'expired' is true without 'start'") { + for { + t <- SimpleTimer.create[IO] + expired <- t.expired + _ = assert(expired) + } yield () + } + + test("'expired' is true after 'start' if 'duration' is zero") { + for { + t <- SimpleTimer.create[IO] + _ <- t.start(0.seconds) + expired <- t.expired + _ = assert(expired) + } yield () + } + + test("'expired' is false after 'start' if 'duration' is non-zero") { + for { + t <- SimpleTimer.create[IO] + _ <- t.start(1.second) + expired <- t.expired + _ = assert(!expired) + } yield () + } + + test("'expired' is true after 'start' if 'duration' elapsed") { + for { + t <- SimpleTimer.create[IO] + d = 50.millis + _ <- t.start(d) + _ <- IO.sleep(d) + expired <- t.expired + _ = assert(expired) + } yield () + } + + test("'await' sleeps for 'duration' after 'start'") { + for { + t <- SimpleTimer.create[IO] + d = 50.millis + res <- dateTimeAlg.timed(t.start(d) >> t.await >> t.expired) + (expired, duration) = res + _ = assert(expired) + _ = assert(clue(duration) >= d) + } yield () + } +} From 1783dd9419cb18afd97639e0fda1a18ecd0886bc Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Sat, 11 Feb 2023 18:10:15 +0100 Subject: [PATCH 2/2] Add help text for CLI options --- docs/help.md | 6 +++++- .../main/scala/org/scalasteward/core/application/Cli.scala | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/help.md b/docs/help.md index c98155f69b..fa7529294a 100644 --- a/docs/help.md +++ b/docs/help.md @@ -5,7 +5,7 @@ All command line arguments for the `scala-steward` application. ``` Usage: scala-steward validate-repo-config - scala-steward --workspace --repos-file [--git-author-name ] --git-author-email [--git-author-signing-key ] --git-ask-pass [--sign-commits] [--forge-type ] [--forge-api-host ] --forge-login [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var ]... [--process-timeout ] [--whitelist ]... [--read-only ]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size ] [--repo-config ]... [--disable-default-repo-config] [--scalafix-migrations ]... [--disable-default-scalafix-migrations] [--artifact-migrations ]... [--disable-default-artifact-migrations] [--cache-ttl ] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers ] [--azure-repos-organization ] [--github-app-id --github-app-key-file ] [--url-checker-test-url ]... [--default-maven-repo ] [--refresh-backoff-period ] + scala-steward --workspace --repos-file [--git-author-name ] --git-author-email [--git-author-signing-key ] --git-ask-pass [--sign-commits] [--forge-type ] [--forge-api-host ] --forge-login [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var ]... [--process-timeout ] [--whitelist ]... [--read-only ]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size ] [--repo-config ]... [--disable-default-repo-config] [--scalafix-migrations ]... [--disable-default-scalafix-migrations] [--artifact-migrations ]... [--disable-default-artifact-migrations] [--cache-ttl ] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers ] [--azure-repos-organization ] [--github-app-id --github-app-key-file ] [--url-checker-test-url ]... [--default-maven-repo ] [--refresh-backoff-period ] [--pr-throttle-skip ] [--pr-throttle-wait ] @@ -92,6 +92,10 @@ Options and flags: default: https://repo1.maven.org/maven2/ --refresh-backoff-period Period of time a failed build won't be triggered again; default: 0days + --pr-throttle-skip + Skips creating PRs for the given duration after the last PR + --pr-throttle-wait + Waits for the given duration between creating two consequent PRs Subcommands: validate-repo-config diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala index a1afb5e89e..7a1ecbab36 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala @@ -324,12 +324,12 @@ object Cli { } private val prThrottleSkip = { - val help = "" // TODO + val help = "Skips creating PRs for the given duration after the last PR" option[FiniteDuration]("pr-throttle-skip", help).orNone } private val prThrottleWait = { - val help = "" // TODO + val help = "Waits for the given duration between creating two consequent PRs" option[FiniteDuration]("pr-throttle-wait", help).orNone }