Rename flags and rate limiters dedup

Author: phbnf

cmd/tesseract/README.md

@@ -119,7 +119,7 @@ This flag is on by default.
 
 #### Antispam
 
-The `pushback_max_antispam_lag`, `pushback_max_dedup_in_flight` and
+The `pushback_max_antispam_lag`, `rate_limit_dedup` and
 `inmemory_antispam_cache_size` flags control how [TesseraCT's Antispam feature](/docs/architecture.md#antispam)
 works, which itself is built on top of [Tessera's Antispam](https://github.com/transparency-dev/tessera?tab=readme-ov-file#antispam)
 capabilities. It is composed of three main steps:
@@ -146,14 +146,13 @@ calls faster, and provides optimistic coverage for entries submitted _very_
 recently and which have not yet been processed by the asynchronous process in
 `(1)`.
 
-The `pushback_max_dedup_in_flight` flag rate limits how many concurrent `add-*`
-requests identified as duplicates will be processed by the
-**synchronous** process in `(3)` wich fetches entries and extracts information
-required to build SCTs. When this value is exceeded, TesseraCT returns
-`429 -Too Many Requests` to subsequent **duplicate** `add-*` requests only.
-Non-duplicate `add-*` requests are not impacted, and can still be processed.
-This limits the amount of resources TesseraCT spends on servicing duplicate
-requests.
+The `rate_limit_dedup` flag rate limits how many concurrent `add-*` requests
+identified as duplicates will be processed by the **synchronous** process in
+`(3)` wich fetches entries and extracts information required to build SCTs.
+When this value is exceeded, TesseraCT returns `429 -Too Many Requests` to
+subsequent **duplicate** `add-*` requests only. Non-duplicate `add-*` requests
+are not impacted, and can still be processed. This limits the amount of
+resources TesseraCT spends on servicing duplicate requests.
 
 ### Setup
 

cmd/tesseract/aws/main.go

@@ -47,16 +47,16 @@ import (
 func init() {
 	flag.Var(&notAfterStart, "not_after_start", "Start of the range of acceptable NotAfter values, inclusive. Leaving this unset or empty implies no lower bound to the range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
 	flag.Var(&notAfterLimit, "not_after_limit", "Cut off point of notAfter dates - only notAfter dates strictly *before* notAfterLimit will be accepted. Leaving this unset or empty means no upper bound on the accepted range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
-	flag.Float64Var(&limitDedupInFlight, "limit_dedup_in_flight", -1, "Optionally rate limits the number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always rate limited. When negative, no rate limits apply.")
+	flag.Float64Var(&dedupRL, "rate_limit_dedup", -1, "Optionally rate limits the number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always rate limited. When negative, no rate limits apply.")
 	// DEPRECATED: will be removed shortly
-	flag.Float64Var(&limitDedupInFlight, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use pushback_max_dedup_in_flight. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Float64Var(&dedupRL, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use rate_limit_dedup. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
 }
 
 // Global flags that affect all log instances.
 var (
-	notAfterStart      timestampFlag
-	notAfterLimit      timestampFlag
-	limitDedupInFlight float64
+	notAfterStart timestampFlag
+	notAfterLimit timestampFlag
+	dedupRL       float64
 
 	// Functionality flags
 	httpEndpoint             = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port).")
@@ -141,7 +141,7 @@ eventually go away. See /internal/lax509/README.md for more information.`)
 
 	hOpts := tesseract.LogHandlerOpts{
 		OldSubmissionLimit: rateLimitFromFlags(),
-		DedupInFlightLimit: limitDedupInFlight,
+		DedupRL:            dedupRL,
 	}
 	logHandler, err := tesseract.NewLogHandler(ctx, *origin, signer, chainValidationConfig, newAWSStorage, *httpDeadline, *maskInternalErrors, *pathPrefix, hOpts)
 	if err != nil {

cmd/tesseract/gcp/main.go

@@ -46,17 +46,17 @@ func init() {
 	flag.Var(&notAfterStart, "not_after_start", "Start of the range of acceptable NotAfter values, inclusive. Leaving this unset or empty implies no lower bound to the range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
 	flag.Var(&notAfterLimit, "not_after_limit", "Cut off point of notAfter dates - only notAfter dates strictly *before* notAfterLimit will be accepted. Leaving this unset or empty means no upper bound on the accepted range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
 	flag.Var(&additionalSigners, "additional_signer_private_key_secret_name", "Private key secret name for additional Ed25519 checkpoint signatures, may be supplied multiple times. Format: projects/{projectId}/secrets/{secretName}/versions/{secretVersion}.")
-	flag.Float64Var(&limitDedupInFlight, "limit_dedup_in_flight", -1, "Optionally rate limits the number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always rate limited. When negative, no rate limits apply.")
+	flag.Float64Var(&dedupRL, "rate_limit_dedup", -1, "Optionally rate limits the number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always rate limited. When negative, no rate limits apply.")
 	// DEPRECATED: will be removed shortly
-	flag.Float64Var(&limitDedupInFlight, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use limit_dedup_in_flight. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Float64Var(&dedupRL, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use rate_limit_dedup. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
 }
 
 // Global flags that affect all log instances.
 var (
-	notAfterStart      timestampFlag
-	notAfterLimit      timestampFlag
-	additionalSigners  multiStringFlag
-	limitDedupInFlight float64
+	notAfterStart     timestampFlag
+	notAfterLimit     timestampFlag
+	additionalSigners multiStringFlag
+	dedupRL           float64
 
 	// Functionality flags
 	httpEndpoint             = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port).")
@@ -128,7 +128,7 @@ eventually go away. See /internal/lax509/README.md for more information.`)
 
 	hOpts := tesseract.LogHandlerOpts{
 		OldSubmissionLimit: rateLimitFromFlags(),
-		DedupInFlightLimit: limitDedupInFlight,
+		DedupRL:            dedupRL,
 	}
 	logHandler, err := tesseract.NewLogHandler(ctx, *origin, signer, chainValidationConfig, newGCPStorage, *httpDeadline, *maskInternalErrors, *pathPrefix, hOpts)
 	if err != nil {

cmd/tesseract/posix/main.go

@@ -49,17 +49,17 @@ func init() {
 	flag.Var(&notAfterStart, "not_after_start", "Start of the range of acceptable NotAfter values, inclusive. Leaving this unset implies no lower bound to the range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
 	flag.Var(&notAfterLimit, "not_after_limit", "Cut off point of notAfter dates - only notAfter dates strictly *before* notAfterLimit will be accepted. Leaving this unset means no upper bound on the accepted range. RFC3339 UTC format, e.g: 2024-01-02T15:04:05Z.")
 	flag.Var(&additionalSigners, "additional_signer", "Path to a file containing an additional note Signer formatted keys for checkpoints. May be specified multiple times.")
-	flag.Float64Var(&pushbackMaxDedupInFlight, "pushback_max_dedup_in_flight", 100, "Maximum number of number of in-flight duplicate add requests per second - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Float64Var(&dedupRL, "rate_limit_dedup", 100, "Maximum number of number of in-flight duplicate add requests per second - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
 	// DEPRECATED: will be removed shortly
-	flag.Float64Var(&pushbackMaxDedupInFlight, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use pushback_max_dedup_in_flight. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Float64Var(&dedupRL, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use rate_limit_dedup. Maximum number of number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
 }
 
 // Global flags that affect all log instances.
 var (
-	notAfterStart            timestampFlag
-	notAfterLimit            timestampFlag
-	additionalSigners        multiStringFlag
-	pushbackMaxDedupInFlight float64
+	notAfterStart     timestampFlag
+	notAfterLimit     timestampFlag
+	additionalSigners multiStringFlag
+	dedupRL           float64
 
 	// Functionality flags
 	httpEndpoint             = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port).")

ctlog.go

@@ -131,7 +131,7 @@ type OldSubmissionLimit struct {
 
 type LogHandlerOpts struct {
 	OldSubmissionLimit *OldSubmissionLimit
-	DedupInFlightLimit float64
+	DedupRL            float64
 }
 
 // NewLogHandler creates a Tessera based CT log pluged into HTTP handlers.
@@ -163,8 +163,8 @@ func NewLogHandler(ctx context.Context, origin string, signer crypto.Signer, cfg
 	if opts.OldSubmissionLimit != nil {
 		ctOpts.RateLimits.OldSubmission(opts.OldSubmissionLimit.AgeThreshold, opts.OldSubmissionLimit.RateLimit)
 	}
-	if opts.DedupInFlightLimit >= 0 {
-		ctOpts.RateLimits.DedupInFlight(opts.DedupInFlightLimit)
+	if opts.DedupRL >= 0 {
+		ctOpts.RateLimits.Dedup(opts.DedupRL)
 	}
 
 	handlers := ct.NewPathHandlers(ctx, ctOpts, log)

deployment/modules/gcp/gce/tesseract/main.tf

@@ -40,7 +40,8 @@ module "gce_container_tesseract" {
       "--batch_max_age=${var.batch_max_age}",
       "--enable_publication_awaiter=${var.enable_publication_awaiter}",
       "--accept_sha1_signing_algorithms=true",
-      "--limit_old_submissions=${var.limit_old_submissions}"
+      "--limit_old_submissions=${var.limit_old_submissions}",
+      "--rate_limit_dedup=${var.rate_limit_dedup}"
     ]
     tty : true # maybe remove this
   }

deployment/modules/gcp/gce/tesseract/variables.tf

@@ -106,3 +106,8 @@ variable "limit_old_submissions" {
   default     = ""
 }
 
+variable "rate_limit_dedup" {
+  description = "Set to rate limit duplicate submissions per second."
+  type        = number
+  default     = -1 
+}

deployment/modules/gcp/tesseract/gce/main.tf

@@ -42,6 +42,7 @@ module "gce" {
   batch_max_size                 = var.batch_max_size
   enable_publication_awaiter     = var.enable_publication_awaiter
   limit_old_submissions          = var.limit_old_submissions
+  rate_limit_dedup               = var.rate_limit_dedup
 
   depends_on = [
     module.secretmanager,

deployment/modules/gcp/tesseract/gce/variables.tf

@@ -104,3 +104,9 @@ variable "limit_old_submissions" {
   type        = string
   default     = ""
 }
+
+variable "rate_limit_dedup" {
+  description = "Set to rate limit duplicate submissions per second."
+  type        = number
+  default     = -1 
+}

internal/ct/handlers.go

@@ -192,12 +192,12 @@ func (a appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 // RateLimits knows how to apply configurable rate limits to submissions.
 type RateLimits struct {
-	oldAge       time.Duration
-	oldLimiter   *rate.Limiter
-	dedupLimiter *rate.Limiter
+	oldAge     time.Duration
+	oldLimiter *rate.Limiter
+	dedup      *rate.Limiter
 }
 
-// OldSubmission configures a rate limit on old certs.
+// NotBefore configures a rate limit on old certs.
 //
 // Submissions whose notBefore date is at least as old as age will be subject to the specified number of entries per second.
 func (r *RateLimits) OldSubmission(age time.Duration, limit float64) {
@@ -206,11 +206,11 @@ func (r *RateLimits) OldSubmission(age time.Duration, limit float64) {
 	klog.Infof("Configured OldSubmission limiter with %0.2f qps for certs aged >= %s", limit, age)
 }
 
-// DedupInFlight configures a rate limit on entries being deduplicated.
+// Dedup configures a rate limit on entries being deduplicated.
 //
 // Submissions will be subject to the specified number of entries per second.
-func (r *RateLimits) DedupInFlight(limit float64) {
-	r.dedupLimiter = rate.NewLimiter(rate.Limit(limit), int(math.Ceil(limit)))
+func (r *RateLimits) Dedup(limit float64) {
+	r.dedup = rate.NewLimiter(rate.Limit(limit), int(math.Ceil(limit)))
 	klog.Infof("Configured DedupInFlight limiter with %0.2f qps", limit)
 }
 
@@ -233,8 +233,8 @@ func (r *RateLimits) Accept(ctx context.Context, chain []*x509.Certificate) bool
 
 // AcceptDedup returns true if a duplicate entry is permitted to be resolved.
 func (r *RateLimits) AcceptDedup(ctx context.Context) bool {
-	if r.dedupLimiter != nil {
-		if r.dedupLimiter.Allow() {
+	if r.dedup != nil {
+		if r.dedup.Allow() {
 			return true
 		}
 		rateLimitedRequests.Add(ctx, 1, metric.WithAttributes(rateLimitReasonKey.String("dedup")))