From ad17e19e1bb2fbb347276c6ccb74c3fb93dd45d4 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Tue, 11 Nov 2025 10:33:31 -0800 Subject: [PATCH 1/8] VerifAI Dynamic Sampling initial changes --- src/scenic/core/external_params.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/scenic/core/external_params.py b/src/scenic/core/external_params.py index 88848db4b..92fe0b565 100644 --- a/src/scenic/core/external_params.py +++ b/src/scenic/core/external_params.py @@ -252,14 +252,15 @@ def __init__(self, params, globalParams): self.rejectionFeedback = 1 self.cachedSample = None - def nextSample(self, feedback): - return self.sampler.nextSample(feedback) + self._lastSample = None - def update(self, sample, info, rho): - self.sampler.update(sample, info, rho) + def nextSample(self, feedback): + if feedback is not None: + assert self._lastSample is not None + self._lastSample.update(feedback) - def getSample(self): - return self.sampler.getSample() + self._lastSample = self.sampler.getSample() + return self._lastSample.staticSample def valueFor(self, param): return getattr(self.cachedSample, self.nameForParam(param.index)) From a5fc5d260bf34b0e1b53e11f8f8db17a64f7ab03 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 12 Nov 2025 17:42:56 -0800 Subject: [PATCH 2/8] Mostly working dynamic sampling. --- src/scenic/core/external_params.py | 63 ++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/src/scenic/core/external_params.py b/src/scenic/core/external_params.py index 92fe0b565..fcd2a6fcb 100644 --- a/src/scenic/core/external_params.py +++ b/src/scenic/core/external_params.py @@ -96,6 +96,8 @@ """ +from abc import ABC, abstractmethod + from dotmap import DotMap import numpy @@ -182,6 +184,7 @@ def __init__(self, params, globalParams): import verifai.server # construct FeatureSpace + timeBound = globalParams.get("verifaiTimeBound", 0) usingProbs = False self.params = tuple(params) for index, param in enumerate(self.params): @@ -195,9 +198,14 @@ def __init__(self, params, globalParams): usingProbs = True space = verifai.features.FeatureSpace( { - self.nameForParam(index): verifai.features.Feature(param.domain) + self.nameForParam(index): ( + verifai.features.Feature(param.domain) + if not param.timeSeries + else verifai.features.TimeSeriesFeature(param.domain) + ) for index, param in enumerate(self.params) - } + }, + timeBound=timeBound, ) # set up VerifAI sampler @@ -260,10 +268,21 @@ def nextSample(self, feedback): self._lastSample.update(feedback) self._lastSample = self.sampler.getSample() - return self._lastSample.staticSample + return self._lastSample def valueFor(self, param): - return getattr(self.cachedSample, self.nameForParam(param.index)) + if not param.timeSeries: + return param.extractOutput( + getattr(self.cachedSample.staticSample, self.nameForParam(param.index)) + ) + else: + callback = lambda feedback: param.extractOutput( + getattr( + self.cachedSample.getDynamicSample(feedback), + self.nameForParam(param.index), + ) + ) + return TimeSeriesParameter(callback) @staticmethod def nameForParam(i): @@ -271,12 +290,13 @@ def nameForParam(i): return f"param{i}" -class ExternalParameter(Distribution): +class ExternalParameter(Distribution, ABC): """A value determined by external code rather than Scenic's internal sampler.""" - def __init__(self): + def __init__(self, timeSeries): super().__init__() self.sampler = None + self.timeSeries = timeSeries import scenic.syntax.veneer as veneer # TODO improve? veneer.registerExternalParameter(self) @@ -290,12 +310,25 @@ def sampleGiven(self, value): assert self.sampler is not None return self.sampler.valueFor(self) + @abstractmethod + def extractOutput(self, value): + pass + + +class TimeSeriesParameter: + def __init__(self, callback): + self._callback = callback + + def getSample(self): + scenic_context = None # TODO + return self._callback(scenic_context) + class VerifaiParameter(ExternalParameter): """An external parameter sampled using one of VerifAI's samplers.""" - def __init__(self, domain): - super().__init__() + def __init__(self, domain, timeSeries=False): + super().__init__(timeSeries=timeSeries) self.domain = domain @staticmethod @@ -323,10 +356,10 @@ class VerifaiRange(VerifaiParameter): _defaultValueType = float - def __init__(self, low, high, buckets=None, weights=None): + def __init__(self, low, high, buckets=None, weights=None, timeSeries=False): import verifai.features - super().__init__(verifai.features.Box([low, high])) + super().__init__(verifai.features.Box([low, high]), timeSeries=timeSeries) if weights is not None: weights = tuple(weights) if buckets is not None and len(weights) != buckets: @@ -342,8 +375,7 @@ def __init__(self, low, high, buckets=None, weights=None): total = sum(weights) self.probs = tuple(wt / total for wt in weights) - def sampleGiven(self, value): - value = super().sampleGiven(value) + def extractOutput(self, value): assert len(value) == 1 return value[0] @@ -353,10 +385,10 @@ class VerifaiDiscreteRange(VerifaiParameter): _defaultValueType = float - def __init__(self, low, high, weights=None): + def __init__(self, low, high, weights=None, timeSeries=False): import verifai.features - super().__init__(verifai.features.DiscreteBox([low, high])) + super().__init__(verifai.features.DiscreteBox([low, high]), timeSeries=timeSeries) if weights is not None: if len(weights) != (high - low + 1): raise RuntimeError( @@ -368,8 +400,7 @@ def __init__(self, low, high, weights=None): else: self.probs = None - def sampleGiven(self, value): - value = super().sampleGiven(value) + def extractOutput(self, value): assert len(value) == 1 return value[0] From 187345c8cc4209ce2d8eb1a4e1e32dd9ce33952e Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Tue, 25 Nov 2025 12:55:36 -0800 Subject: [PATCH 3/8] Proper handling of when to pull new dynamic sample --- src/scenic/core/external_params.py | 32 ++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/scenic/core/external_params.py b/src/scenic/core/external_params.py index fcd2a6fcb..02d15535b 100644 --- a/src/scenic/core/external_params.py +++ b/src/scenic/core/external_params.py @@ -261,6 +261,8 @@ def __init__(self, params, globalParams): self.cachedSample = None self._lastSample = None + self._lastDynamicSample = None + self._lastTime = -1 def nextSample(self, feedback): if feedback is not None: @@ -270,15 +272,27 @@ def nextSample(self, feedback): self._lastSample = self.sampler.getSample() return self._lastSample + def nextDynamicSample(self): + import scenic.syntax.veneer as veneer + + assert veneer.currentSimulation is not None + + if veneer.currentSimulation.currentTime > self._lastTime: + feedback = None # TODO + self._lastDynamicSample = self.cachedSample.getDynamicSample(feedback) + self._lastTime = veneer.currentSimulation.currentTime + + return self._lastDynamicSample + def valueFor(self, param): if not param.timeSeries: return param.extractOutput( getattr(self.cachedSample.staticSample, self.nameForParam(param.index)) ) else: - callback = lambda feedback: param.extractOutput( + callback = lambda: param.extractOutput( getattr( - self.cachedSample.getDynamicSample(feedback), + self.nextDynamicSample(), self.nameForParam(param.index), ) ) @@ -318,10 +332,20 @@ def extractOutput(self, value): class TimeSeriesParameter: def __init__(self, callback): self._callback = callback + self._lastTime = -1 def getSample(self): - scenic_context = None # TODO - return self._callback(scenic_context) + import scenic.syntax.veneer as veneer + + assert veneer.currentSimulation is not None + + if veneer.currentSimulation.currentTime <= self._lastTime: + raise RuntimeError( + "Attempted `getSample` for a timeSeries property twice in one timestep." + ) + + self._lastTime = veneer.currentSimulation.currentTime + return self._callback() class VerifaiParameter(ExternalParameter): From 829473982357ec7d64cffc8a28589c01b93f9df8 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 26 Nov 2025 21:23:11 -0800 Subject: [PATCH 4/8] Added Scenic external parameter feedback. --- src/scenic/core/external_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scenic/core/external_params.py b/src/scenic/core/external_params.py index 02d15535b..ed4a310f9 100644 --- a/src/scenic/core/external_params.py +++ b/src/scenic/core/external_params.py @@ -278,7 +278,7 @@ def nextDynamicSample(self): assert veneer.currentSimulation is not None if veneer.currentSimulation.currentTime > self._lastTime: - feedback = None # TODO + feedback = veneer.currentSimulation self._lastDynamicSample = self.cachedSample.getDynamicSample(feedback) self._lastTime = veneer.currentSimulation.currentTime From 3965d5e6fc17add704bead0db599f7e9efcd5666 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Tue, 2 Dec 2025 14:47:59 -0800 Subject: [PATCH 5/8] Dynamic sampling tweaks. --- src/scenic/core/external_params.py | 34 ++++++++++++++++++++++-------- src/scenic/syntax/veneer.py | 2 ++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/scenic/core/external_params.py b/src/scenic/core/external_params.py index ed4a310f9..fb1a445f9 100644 --- a/src/scenic/core/external_params.py +++ b/src/scenic/core/external_params.py @@ -97,6 +97,7 @@ """ from abc import ABC, abstractmethod +import warnings from dotmap import DotMap import numpy @@ -184,7 +185,7 @@ def __init__(self, params, globalParams): import verifai.server # construct FeatureSpace - timeBound = globalParams.get("verifaiTimeBound", 0) + timeBound = globalParams.get("timeBound", 0) usingProbs = False self.params = tuple(params) for index, param in enumerate(self.params): @@ -196,6 +197,13 @@ def __init__(self, params, globalParams): param.index = index if param.probs is not None: usingProbs = True + + if timeBound == 0 and any(param.timeSeries for param in self.params): + warnings.warn( + "TimeSeries external parameter used by no time bound specified " + "(Did you provide `maxSteps` when creating ScenicSampler?)." + ) + space = verifai.features.FeatureSpace( { self.nameForParam(index): ( @@ -307,10 +315,10 @@ def nameForParam(i): class ExternalParameter(Distribution, ABC): """A value determined by external code rather than Scenic's internal sampler.""" - def __init__(self, timeSeries): + def __init__(self): super().__init__() self.sampler = None - self.timeSeries = timeSeries + self.timeSeries = False import scenic.syntax.veneer as veneer # TODO improve? veneer.registerExternalParameter(self) @@ -348,11 +356,19 @@ def getSample(self): return self._callback() +def TimeSeries(param): + if not isinstance(param, ExternalParameter): + raise ValueError("Cannot turn a non `ExternalParameter` into a time series") + + param.timeSeries = True + return param + + class VerifaiParameter(ExternalParameter): """An external parameter sampled using one of VerifAI's samplers.""" - def __init__(self, domain, timeSeries=False): - super().__init__(timeSeries=timeSeries) + def __init__(self, domain): + super().__init__() self.domain = domain @staticmethod @@ -380,10 +396,10 @@ class VerifaiRange(VerifaiParameter): _defaultValueType = float - def __init__(self, low, high, buckets=None, weights=None, timeSeries=False): + def __init__(self, low, high, buckets=None, weights=None): import verifai.features - super().__init__(verifai.features.Box([low, high]), timeSeries=timeSeries) + super().__init__(verifai.features.Box([low, high])) if weights is not None: weights = tuple(weights) if buckets is not None and len(weights) != buckets: @@ -409,10 +425,10 @@ class VerifaiDiscreteRange(VerifaiParameter): _defaultValueType = float - def __init__(self, low, high, weights=None, timeSeries=False): + def __init__(self, low, high, weights=None): import verifai.features - super().__init__(verifai.features.DiscreteBox([low, high]), timeSeries=timeSeries) + super().__init__(verifai.features.DiscreteBox([low, high])) if weights is not None: if len(weights) != (high - low + 1): raise RuntimeError( diff --git a/src/scenic/syntax/veneer.py b/src/scenic/syntax/veneer.py index cce52162f..034ff61d8 100644 --- a/src/scenic/syntax/veneer.py +++ b/src/scenic/syntax/veneer.py @@ -120,6 +120,7 @@ "VerifaiRange", "VerifaiDiscreteRange", "VerifaiOptions", + "TimeSeries", "File", "Files", # Constructible types @@ -201,6 +202,7 @@ from scenic.core.dynamics.invocables import BlockConclusion, runTryInterrupt from scenic.core.dynamics.scenarios import DynamicScenario from scenic.core.external_params import ( + TimeSeries, VerifaiDiscreteRange, VerifaiOptions, VerifaiParameter, From 90697d94a0f74777cd2c111ebd3d11e4a3d16ebb Mon Sep 17 00:00:00 2001 From: Eric Vin <8935814+Eric-Vin@users.noreply.github.com> Date: Wed, 17 Dec 2025 17:08:10 -0800 Subject: [PATCH 6/8] Apply suggestions from code review Co-authored-by: Daniel Fremont --- src/scenic/core/external_params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scenic/core/external_params.py b/src/scenic/core/external_params.py index fb1a445f9..3354c361f 100644 --- a/src/scenic/core/external_params.py +++ b/src/scenic/core/external_params.py @@ -200,7 +200,7 @@ def __init__(self, params, globalParams): if timeBound == 0 and any(param.timeSeries for param in self.params): warnings.warn( - "TimeSeries external parameter used by no time bound specified " + "TimeSeries external parameter used but no time bound specified " "(Did you provide `maxSteps` when creating ScenicSampler?)." ) @@ -358,7 +358,7 @@ def getSample(self): def TimeSeries(param): if not isinstance(param, ExternalParameter): - raise ValueError("Cannot turn a non `ExternalParameter` into a time series") + raise TypeError("Cannot turn a non `ExternalParameter` into a time series") param.timeSeries = True return param From ebb67c520207a1a581867ddf781bc9810faf0af3 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 17 Dec 2025 18:38:31 -0800 Subject: [PATCH 7/8] Incorporated PR feedback. --- src/scenic/core/external_params.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/scenic/core/external_params.py b/src/scenic/core/external_params.py index 3354c361f..00c567e3f 100644 --- a/src/scenic/core/external_params.py +++ b/src/scenic/core/external_params.py @@ -198,17 +198,17 @@ def __init__(self, params, globalParams): if param.probs is not None: usingProbs = True - if timeBound == 0 and any(param.timeSeries for param in self.params): + if timeBound == 0 and any(param.isTimeSeries for param in self.params): warnings.warn( - "TimeSeries external parameter used but no time bound specified " - "(Did you provide `maxSteps` when creating ScenicSampler?)." + "TimeSeries external parameter used but no global parameter `timeBound` is specified. " + "(If using VerifAI’s ScenicSampler, set its maxSteps option)." ) space = verifai.features.FeatureSpace( { self.nameForParam(index): ( verifai.features.Feature(param.domain) - if not param.timeSeries + if not param.isTimeSeries else verifai.features.TimeSeriesFeature(param.domain) ) for index, param in enumerate(self.params) @@ -293,7 +293,7 @@ def nextDynamicSample(self): return self._lastDynamicSample def valueFor(self, param): - if not param.timeSeries: + if not param.isTimeSeries: return param.extractOutput( getattr(self.cachedSample.staticSample, self.nameForParam(param.index)) ) @@ -312,13 +312,13 @@ def nameForParam(i): return f"param{i}" -class ExternalParameter(Distribution, ABC): +class ExternalParameter(Distribution): """A value determined by external code rather than Scenic's internal sampler.""" def __init__(self): super().__init__() self.sampler = None - self.timeSeries = False + self.isTimeSeries = False import scenic.syntax.veneer as veneer # TODO improve? veneer.registerExternalParameter(self) @@ -332,9 +332,13 @@ def sampleGiven(self, value): assert self.sampler is not None return self.sampler.valueFor(self) - @abstractmethod def extractOutput(self, value): - pass + """ + Given a raw sampled value for a parameter, optionally extract the actual desired value. + + By default just passes the value through unchanged. + """ + return value class TimeSeriesParameter: @@ -349,7 +353,7 @@ def getSample(self): if veneer.currentSimulation.currentTime <= self._lastTime: raise RuntimeError( - "Attempted `getSample` for a timeSeries property twice in one timestep." + "Attempted `getSample` for a TimeSeries external parameter twice in one timestep." ) self._lastTime = veneer.currentSimulation.currentTime @@ -360,7 +364,7 @@ def TimeSeries(param): if not isinstance(param, ExternalParameter): raise TypeError("Cannot turn a non `ExternalParameter` into a time series") - param.timeSeries = True + param.isTimeSeries = True return param From 112ddc88fedfc31f801956fe4c6ce5fb5f6cb33c Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 24 Dec 2025 12:46:16 -0800 Subject: [PATCH 8/8] Fixed multiple simulation handling in external parameters. --- src/scenic/core/external_params.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/scenic/core/external_params.py b/src/scenic/core/external_params.py index 00c567e3f..24cae6acf 100644 --- a/src/scenic/core/external_params.py +++ b/src/scenic/core/external_params.py @@ -270,6 +270,7 @@ def __init__(self, params, globalParams): self._lastSample = None self._lastDynamicSample = None + self._lastSimulation = None self._lastTime = -1 def nextSample(self, feedback): @@ -285,6 +286,10 @@ def nextDynamicSample(self): assert veneer.currentSimulation is not None + if self._lastSimulation is not veneer.currentSimulation: + self._lastSimulation = veneer.currentSimulation + self._lastTime = -1 + if veneer.currentSimulation.currentTime > self._lastTime: feedback = veneer.currentSimulation self._lastDynamicSample = self.cachedSample.getDynamicSample(feedback)