diff --git a/doc/OnlineDocs/explanation/analysis/parmest/covariance.rst b/doc/OnlineDocs/explanation/analysis/parmest/covariance.rst index cc17feaae62..1d8e5591168 100644 --- a/doc/OnlineDocs/explanation/analysis/parmest/covariance.rst +++ b/doc/OnlineDocs/explanation/analysis/parmest/covariance.rst @@ -1,16 +1,156 @@ Covariance Matrix Estimation ================================= -If the optional argument ``calc_cov=True`` is specified for :class:`~pyomo.contrib.parmest.parmest.Estimator.theta_est`, -parmest will calculate the covariance matrix :math:`V_{\theta}` as follows: - -.. math:: - V_{\theta} = 2 \sigma^2 H^{-1} - -This formula assumes all measurement errors are independent and identically distributed with -variance :math:`\sigma^2`. :math:`H^{-1}` is the inverse of the Hessian matrix for an unweighted -sum of least squares problem. Currently, the covariance approximation is only valid if the -objective given to parmest is the sum of squared error. Moreover, parmest approximates the -variance of the measurement errors as :math:`\sigma^2 = \frac{1}{n-l} \sum e_i^2` where :math:`n` is -the number of data points, :math:`l` is the number of fitted parameters, and :math:`e_i` is the -residual for experiment :math:`i`. \ No newline at end of file +The uncertainty in the estimated parameters is quantified using the covariance matrix. +The diagonal of the covariance matrix contains the variance of the estimated parameters. +Assuming Gaussian independent and identically distributed measurement errors, the +covariance matrix of the estimated parameters can be computed using the following +methods which have been implemented in parmest. + +1. Reduced Hessian Method + + .. math:: + V_{\boldsymbol{\theta}} = 2 \sigma^2 \left(\frac{\partial^2 \text{SSE}} + {\partial \boldsymbol{\theta} \partial \boldsymbol{\theta}}\right)^{-1}_{\boldsymbol{\theta} + = \boldsymbol{\theta}^*} + + Where SSE is the sum of squared errors, WSSE is the weighted SSE, + :math:`\boldsymbol{\theta}` are the unknown parameters, :math:`\boldsymbol{\theta^*}` + are the estimate of the unknown parameters, and :math:`\sigma^2` is the variance of + the measurement error. When the standard deviation of the measurement error is not + supplied by the user, parmest approximates the variance of the measurement error as + :math:`\sigma^2 = \frac{1}{n-l} \sum e_i^2` where :math:`n` is the number of data + points, :math:`l` is the number of fitted parameters, and :math:`e_i` is the residual + for experiment :math:`i`. + +2. Finite Difference Method + + .. math:: + V_{\boldsymbol{\theta}} = \left( \sum_{r = 1}^n \mathbf{G}_{r}^{\mathrm{T}} \mathbf{W} + \mathbf{G}_{r} \right)^{-1} + + This method uses central finite difference to compute the Jacobian matrix, + :math:`\mathbf{G}_{r}`, which is the sensitivity of the measured variables with + respect to the parameters, `\boldsymbol{\theta}`. :math:`\mathbf{W}` is a diagonal + matrix containing the inverse of the variance of the measurement errors, + :math:`\sigma^2`. + +3. Automatic Differentiation Method + + .. math:: + V_{\boldsymbol{\theta}} = \left( \sum_{r = 1}^n \mathbf{G}_{\text{kaug},\, r}^{\mathrm{T}} + \mathbf{W} \mathbf{G}_{\text{kaug},\, r} \right)^{-1} + + This method uses the model optimality (KKT) condition to compute the Jacobian matrix, + :math:`\mathbf{G}_{\text{kaug},\, r}`. + +In parmest, the covariance matrix can be calculated after defining the +:class:`~pyomo.contrib.parmest.parmest.Estimator` object and estimating the unknown +parameters using :class:`~pyomo.contrib.parmest.parmest.Estimator.theta_est`. To +estimate the covariance matrix, call +:class:`~pyomo.contrib.parmest.parmest.Estimator.cov_est` and pass it the number +of data points, e.g., + +.. testsetup:: * + :skipif: not __import__('pyomo.contrib.parmest.parmest').contrib.parmest.parmest.parmest_available + + # Data + import pandas as pd + data = pd.DataFrame( + data=[[1, 8.3], [2, 10.3], [3, 19.0], + [4, 16.0], [5, 15.6], [7, 19.8]], + columns=['hour', 'y'], + ) + num_data = len(data) + + # Create the Rooney-Biegler model + def rooney_biegler_model(): + """ + Formulates the Pyomo model of the Rooney-Biegler example + + Returns: + m: Pyomo model + """ + m = pyo.ConcreteModel() + + m.asymptote = pyo.Var(within=pyo.NonNegativeReals, initialize=10) + m.rate_constant = pyo.Var(within=pyo.NonNegativeReals, initialize=0.2) + + m.hour = pyo.Var(within=pyo.PositiveReals, initialize=0.1) + m.y = pyo.Var(within=pyo.NonNegativeReals) + + @m.Constraint() + def response_rule(m): + return m.y == m.asymptote * (1 - pyo.exp(-m.rate_constant * m.hour)) + + return m + + # Create the Experiment class + from pyomo.contrib.parmest.experiment import Experiment + class RooneyBieglerExperiment(Experiment): + def __init__(self, hour, y): + self.y = y + self.hour = hour + self.model = None + + def get_labeled_model(self): + self.create_model() + self.finalize_model() + self.label_model() + + return self.model + + def create_model(self): + m = self.model = rooney_biegler_model() + + return m + + def finalize_model(self): + m = self.model + + # fix the input variable + m.hour.fix(self.hour) + + return m + + def label_model(self): + m = self.model + + # add experiment outputs + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, self.y)]) + + # add unknown parameters + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update( + (k, pyo.value(k)) for k in [m.asymptote, m.rate_constant] + ) + + # create the measurement error + m.measurement_error = pyo.Suffix(direction = pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + + # Create an experiment list + exp_list = [] + for i in range(data.shape[0]): + exp_list.append(RooneyBieglerExperiment(data["hour"][i], data["y"][i])) + +.. doctest:: + :skipif: not __import__('pyomo.contrib.parmest.parmest').contrib.parmest.parmest.parmest_available + + >>> import pyomo.contrib.parmest.parmest as parmest + >>> pest = parmest.Estimator(exp_list, obj_function="SSE") + >>> obj_val, theta_val = pest.theta_est() + >>> cov = pest.cov_est(cov_n=num_data) + +Optionally, one of the three methods; "reduced_hessian", "finite_difference", +and "automatic_differentiation_kaug" can be supplied for the covariance calculation, +e.g., + +.. doctest:: + :skipif: not __import__('pyomo.contrib.parmest.parmest').contrib.parmest.parmest.parmest_available + + >>> pest = parmest.Estimator(exp_list, obj_function="SSE") + >>> obj_val, theta_val = pest.theta_est() + >>> cov_method = "reduced_hessian" + >>> cov = pest.cov_est(cov_n=num_data, method=cov_method) \ No newline at end of file diff --git a/doc/OnlineDocs/explanation/analysis/parmest/datarec.rst b/doc/OnlineDocs/explanation/analysis/parmest/datarec.rst index 3c6e12196f7..16d4ce175e7 100644 --- a/doc/OnlineDocs/explanation/analysis/parmest/datarec.rst +++ b/doc/OnlineDocs/explanation/analysis/parmest/datarec.rst @@ -42,13 +42,6 @@ The following example returns model values from a Pyomo Expression. >>> for i in range(data.shape[0]): ... exp_list.append(RooneyBieglerExperiment(data.loc[i, :])) - >>> # Define objective - >>> def SSE(model): - ... expr = (model.experiment_outputs[model.y] - ... - model.response_function[model.experiment_outputs[model.hour]] - ... ) ** 2 - ... return expr - - >>> pest = parmest.Estimator(exp_list, obj_function=SSE, solver_options=None) + >>> pest = parmest.Estimator(exp_list, obj_function="SSE", solver_options=None) >>> obj, theta, var_values = pest.theta_est(return_values=['response_function']) >>> #print(var_values) diff --git a/doc/OnlineDocs/explanation/analysis/parmest/driver.rst b/doc/OnlineDocs/explanation/analysis/parmest/driver.rst index b3f212008ca..c4edcf03ea2 100644 --- a/doc/OnlineDocs/explanation/analysis/parmest/driver.rst +++ b/doc/OnlineDocs/explanation/analysis/parmest/driver.rst @@ -16,19 +16,21 @@ the model and the observations (typically defined as the sum of squared deviation between model values and observed values). If the Pyomo model is not formatted as a two-stage stochastic -programming problem in this format, the user can supply a custom -function to use as the second stage cost and the Pyomo model will be +programming problem in this format, the user can choose either the +built-in "SSE" or "SSE_weighted" objective functions, or supply a custom +objective function to use as the second stage cost. The Pyomo model will then be modified within parmest to match the required specifications. -The stochastic programming callback function is also defined within parmest. The callback -function returns a populated and initialized model for each scenario. +The stochastic programming callback function is also defined within parmest. +The callback function returns a populated and initialized model for each scenario. -To use parmest, the user creates a :class:`~pyomo.contrib.parmest.parmest.Estimator` object -which includes the following methods: +To use parmest, the user creates a :class:`~pyomo.contrib.parmest.parmest.Estimator` +object which includes the following methods: .. autosummary:: :nosignatures: ~pyomo.contrib.parmest.parmest.Estimator.theta_est + ~pyomo.contrib.parmest.parmest.Estimator.cov_est ~pyomo.contrib.parmest.parmest.Estimator.theta_est_bootstrap ~pyomo.contrib.parmest.parmest.Estimator.theta_est_leaveNout ~pyomo.contrib.parmest.parmest.Estimator.objective_at_theta @@ -65,25 +67,36 @@ Section. columns=['hour', 'y'], ) - # Sum of squared error function - def SSE(model): - expr = ( - model.experiment_outputs[model.y] - - model.response_function[model.experiment_outputs[model.hour]] - ) ** 2 - return expr - # Create an experiment list from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import RooneyBieglerExperiment + + class NewRooneyBieglerExperiment(RooneyBieglerExperiment): + def label_model(self): + m = self.model + + # create the experiment outputs + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update( + [(m.response_function[self.data['hour']], self.data['y'])] + ) + + # create the unknown parameters + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update((k, pyo.value(k)) for k in [m.asymptote, m.rate_constant]) + + # create the measurement error + m.measurement_error = pyo.Suffix(direction = pyo.Suffix.LOCAL) + m.measurement_error.update([(m.response_function[self.data['hour']], None)]) + exp_list = [] for i in range(data.shape[0]): - exp_list.append(RooneyBieglerExperiment(data.loc[i, :])) + exp_list.append(NewRooneyBieglerExperiment(data.loc[i, :])) .. doctest:: :skipif: not __import__('pyomo.contrib.parmest.parmest').contrib.parmest.parmest.parmest_available >>> import pyomo.contrib.parmest.parmest as parmest - >>> pest = parmest.Estimator(exp_list, obj_function=SSE) + >>> pest = parmest.Estimator(exp_list, obj_function="SSE") Optionally, solver options can be supplied, e.g., @@ -91,14 +104,14 @@ Optionally, solver options can be supplied, e.g., :skipif: not __import__('pyomo.contrib.parmest.parmest').contrib.parmest.parmest.parmest_available >>> solver_options = {"max_iter": 6000} - >>> pest = parmest.Estimator(exp_list, obj_function=SSE, solver_options=solver_options) + >>> pest = parmest.Estimator(exp_list, obj_function="SSE", solver_options=solver_options) List of experiment objects -------------------------- The first argument is a list of experiment objects which is used to -create one labeled model for each expeirment. +create one labeled model for each experiment. The template :class:`~pyomo.contrib.parmest.experiment.Experiment` can be used to generate a list of experiment objects. @@ -137,17 +150,20 @@ expressions that are used to build an objective for the two-stage stochastic programming problem. If the Pyomo model is not written as a two-stage stochastic programming problem in -this format, and/or if the user wants to use an objective that is -different than the original model, a custom objective function can be -defined for parameter estimation. The objective function has a single argument, -which is the model from a single experiment. +this format, the user can select the "SSE" or "SSE_weighted" built-in objective +functions. If the user wants to use an objective that is different from the built-in +options, a custom objective function can be defined for parameter estimation. However, +covariance matrix estimation will not support this custom objective function. The objective +function (built-in or custom) has a single argument, which is the model from a single +experiment. The objective function returns a Pyomo expression which is used to define "SecondStageCost". The objective function can be used to customize data points and weights that are used in parameter estimation. -Parmest includes one built in objective function to compute the sum of squared errors ("SSE") between the -``m.experiment_outputs`` model values and data values. +Parmest includes two built-in objective functions ("SSE" and "SSE_weighted") to compute +the sum of squared errors between the ``m.experiment_outputs`` model values and +data values. Suggested initialization procedure for parameter estimation problems -------------------------------------------------------------------- @@ -162,4 +178,11 @@ estimation solve from the square problem solution, set optional argument ``solve argument ``(initialize_parmest_model=True)``. Different initial guess values for the fitted parameters can be provided using optional argument `theta_values` (**Pandas Dataframe**) -3. Solve parameter estimation problem by calling :class:`~pyomo.contrib.parmest.parmest.Estimator.theta_est` +3. Solve parameter estimation problem by calling +:class:`~pyomo.contrib.parmest.parmest.Estimator.theta_est`, e.g., + +.. doctest:: + :skipif: not __import__('pyomo.contrib.parmest.parmest').contrib.parmest.parmest.parmest_available + + >>> pest = parmest.Estimator(exp_list, obj_function="SSE") + >>> obj_val, theta_val = pest.theta_est() diff --git a/pyomo/contrib/parmest/parmest.py b/pyomo/contrib/parmest/parmest.py index de80c44baac..90edbe1f59c 100644 --- a/pyomo/contrib/parmest/parmest.py +++ b/pyomo/contrib/parmest/parmest.py @@ -37,9 +37,11 @@ import pyomo.contrib.parmest.utils.create_ef as local_ef import pyomo.contrib.parmest.utils.scenario_tree as scenario_tree +from enum import Enum import re import importlib as im import logging +import warnings import types import json from collections.abc import Callable @@ -61,6 +63,8 @@ from pyomo.opt import SolverFactory from pyomo.environ import Block, ComponentUID +from pyomo.contrib.sensitivity_toolbox.sens import get_dsdp + import pyomo.contrib.parmest.utils as utils import pyomo.contrib.parmest.graphics as graphics from pyomo.dae import ContinuousSet @@ -229,12 +233,531 @@ def _experiment_instance_creation_callback( def SSE(model): """ - Sum of squared error between `experiment_output` model and data values + Returns an expression that is used to compute the sum of squared errors + ('SSE') objective, assuming Gaussian i.i.d. errors + + Argument: + model: annotated Pyomo model """ - expr = sum((y - y_hat) ** 2 for y, y_hat in model.experiment_outputs.items()) + # check if the model has all the required suffixes + _check_model_labels_helper(model, logging_level=logging.ERROR) + + # SSE between the prediction and observation of the measured variables + expr = sum((y - y_hat) ** 2 for y_hat, y in model.experiment_outputs.items()) return expr +def SSE_weighted(model): + """ + Returns an expression that is used to compute the 'SSE_weighted' objective, + assuming Gaussian i.i.d. errors, with measurement error standard deviation + defined in the annotated Pyomo model + + Argument: + model: annotated Pyomo model + """ + # check if the model has all the required suffixes + _check_model_labels_helper(model, logging_level=logging.ERROR) + + # Check that measurement errors exist + if not hasattr(model, "measurement_error"): + raise AttributeError( + 'Experiment model does not have suffix "measurement_error". ' + '"measurement_error" is a required suffix for the "SSE_weighted" ' + 'objective.' + ) + + # check if all the values of the measurement error standard deviation + # have been supplied + all_known_errors = all( + model.measurement_error[y_hat] is not None for y_hat in model.experiment_outputs + ) + + if all_known_errors: + # calculate the weighted SSE between the prediction + # and observation of the measured variables + try: + expr = (1 / 2) * sum( + ((y - y_hat) / model.measurement_error[y_hat]) ** 2 + for y_hat, y in model.experiment_outputs.items() + ) + return expr + except ZeroDivisionError: + raise ValueError( + 'Division by zero encountered in the "SSE_weighted" objective. ' + 'One or more values of the measurement error are zero.' + ) + else: + raise ValueError( + 'One or more values are missing from "measurement_error". All values of ' + 'the measurement errors are required for the "SSE_weighted" objective.' + ) + + +def _check_model_labels_helper(model, logging_level): + """ + Checks if the annotated Pyomo model contains the necessary suffixes + + Argument: + model: annotated Pyomo model for suffix checking + logging_level: logging level specified by the user, e.g., logging.INFO + """ + required_attrs = ("experiment_outputs", "unknown_parameters") + + # check if any of the required attributes are missing + missing_attr = [attr for attr in required_attrs if not hasattr(model, attr)] + if missing_attr: + missing_str = ", ".join(f'"{attr}"' for attr in missing_attr) + raise AttributeError( + f"Experiment model is missing required attribute(s): {missing_str}" + ) + + # set the logging + logger.setLevel(level=logging_level) + if logging_level == logging.INFO: + logger.info("Model has expected labels.") + + +def _get_labeled_model_helper(experiment): + """ + Checks if the Experiment class object has a "get_labeled_model" function + + Argument: + experiment: Estimator class object that contains the Pyomo model + for a particular experimental condition + + Returns: + Annotated Pyomo model + """ + get_model = getattr(experiment, "get_labeled_model", None) + if not callable(get_model): + raise AttributeError( + 'The experiment object must have a "get_labeled_model" ' 'function.' + ) + + try: + return get_model().clone() + except Exception as exc: + raise RuntimeError(f"Failed to clone labeled model: {exc}") + + +class CovarianceMethodLib(Enum): + finite_difference = "finite_difference" + automatic_differentiation_kaug = "automatic_differentiation_kaug" + reduced_hessian = "reduced_hessian" + + +class ObjectiveLib(Enum): + SSE = "SSE" + SSE_weighted = "SSE_weighted" + + +class UnsupportedArgsLib(Enum): + calc_cov = "calc_cov" + cov_n = "cov_n" + + +# Compute the Jacobian matrix of measured variables with respect to the parameters +def _compute_jacobian(experiment, theta_vals, step, solver, tee, logging_level): + """ + Computes the Jacobian matrix of the measured variables with respect to the + parameters using the central finite difference scheme + + Arguments: + experiment: Estimator class object that contains the Pyomo model + for a particular experimental condition + theta_vals: dictionary containing the estimates of the unknown parameters + step: float used for relative perturbation of the parameters, + e.g., step=0.02 is a 2% perturbation + solver: string ``solver`` object specified by the user, e.g., 'ipopt' + tee: boolean solver option to be passed for verbose output + logging_level: logging level specified by the user, e.g., logging.INFO + + Returns: + J: Jacobian matrix + """ + # grab the model + model = _get_labeled_model_helper(experiment) + + # check if the model has all the required suffixes + _check_model_labels_helper(model, logging_level) + + # fix the value of the unknown parameters to the estimated values + params = [k for k, v in model.unknown_parameters.items()] + for param in params: + param.fix(theta_vals[param.name]) + + # re-solve the model with the estimated parameters + try: + solver = pyo.SolverFactory(solver) + solver.solve(model, tee=tee) + except Exception as e: + raise RuntimeError( + f"Model from experiment did not solve appropriately. Make sure the " + f"model is well-posed. The original error was {e}." + ) + + # get the measured variables + y_hat_list = [y_hat for y_hat, y in model.experiment_outputs.items()] + + # get the estimated parameter values + param_values = [p.value for p in params] + + # get the number of parameters and measured variables + n_params = len(param_values) + n_outputs = len(y_hat_list) + + # compute the sensitivity of measured variables to the parameters (Jacobian) + J = np.zeros((n_outputs, n_params)) + + for i, param in enumerate(params): + # store original value of the parameter + orig_value = param_values[i] + + # calculate the relative perturbation + relative_perturbation = step * orig_value + + # Forward perturbation + param.fix(orig_value + relative_perturbation) + + # solve the model + try: + solver.solve(model, tee=tee) + except Exception as e: + raise RuntimeError( + f"Model from experiment did not solve appropriately. Make sure the " + f"model is well-posed. The original error was {e}." + ) + + # forward perturbation measured variables + y_hat_plus = [pyo.value(y_hat) for y_hat, y in model.experiment_outputs.items()] + + # Backward perturbation + param.fix(orig_value - relative_perturbation) + + # re-solve the model + try: + solver.solve(model, tee=tee) + except Exception as e: + raise RuntimeError( + f"Model from experiment did not solve appropriately. Make sure the " + f"model is well-posed. The original error was {e}." + ) + + # backward perturbation measured variables + y_hat_minus = [ + pyo.value(y_hat) for y_hat, y in model.experiment_outputs.items() + ] + + # Restore the original parameter value + param.fix(orig_value) + + # Central difference approximation for the Jacobian + J[:, i] = [ + (y_hat_plus[w] - y_hat_minus[w]) / (2 * relative_perturbation) + for w in range(len(y_hat_plus)) + ] + + return J + + +# Compute the covariance matrix of the estimated parameters +def compute_covariance_matrix( + experiment_list, + method, + theta_vals, + step, + solver, + tee, + logging_level, + estimated_var=None, +): + """ + Computes the covariance matrix of the estimated parameters using + 'finite_difference' and 'automatic_differentiation_kaug' methods + + Arguments: + experiment_list: list of Estimator class objects containing the Pyomo model + for different experimental conditions + method: string ``method`` object specified by the user, + e.g., 'finite_difference' + theta_vals: dictionary containing the estimates of the unknown parameters + step: float used for relative perturbation of the parameters, + e.g., step=0.02 is a 2% perturbation + solver: string ``solver`` object specified by the user, e.g., 'ipopt' + tee: boolean solver option to be passed for verbose output + logging_level: logging level specified by the user, e.g., logging.INFO + estimated_var: value of the estimated variance of the measurement error + in cases where the user does not supply the + measurement error standard deviation + + Returns: + cov: covariance matrix of the estimated parameters + """ + if method == CovarianceMethodLib.finite_difference.value: + # store the FIM of all experiments + FIM_all_exp = [] + for ( + experiment + ) in experiment_list: # loop through the experiments and compute the FIM + FIM_all_exp.append( + _finite_difference_FIM( + experiment, + theta_vals=theta_vals, + step=step, + solver=solver, + tee=tee, + logging_level=logging_level, + estimated_var=estimated_var, + ) + ) + + FIM = np.sum(FIM_all_exp, axis=0) + + # covariance matrix + try: + cov = np.linalg.inv(FIM) + except np.linalg.LinAlgError: + cov = np.linalg.pinv(FIM) + if logging_level == logging.INFO: + logger.info("The FIM is singular. Using pseudo-inverse instead.") + else: + print("The FIM is singular. Using pseudo-inverse instead.") + + cov = pd.DataFrame(cov, index=theta_vals.keys(), columns=theta_vals.keys()) + elif method == CovarianceMethodLib.automatic_differentiation_kaug.value: + # store the FIM of all experiments + FIM_all_exp = [] + for ( + experiment + ) in experiment_list: # loop through the experiments and compute the FIM + FIM_all_exp.append( + _kaug_FIM( + experiment, + theta_vals=theta_vals, + solver=solver, + tee=tee, + estimated_var=estimated_var, + ) + ) + + FIM = np.sum(FIM_all_exp, axis=0) + + # covariance matrix + try: + cov = np.linalg.inv(FIM) + except np.linalg.LinAlgError: + cov = np.linalg.pinv(FIM) + if logging_level == logging.INFO: + logger.info("The FIM is singular. Using pseudo-inverse instead.") + else: + print("The FIM is singular. Using pseudo-inverse instead.") + + cov = pd.DataFrame(cov, index=theta_vals.keys(), columns=theta_vals.keys()) + + return cov + + +# compute the Fisher information matrix of the estimated parameters using +# 'finite_difference' +def _finite_difference_FIM( + experiment, theta_vals, step, solver, tee, logging_level, estimated_var=None +): + """ + Computes the Fisher information matrix from 'finite_difference' Jacobian matrix + and measurement errors standard deviation defined in the annotated Pyomo model + + Arguments: + experiment: Estimator class object that contains the Pyomo model + for a particular experimental condition + theta_vals: dictionary containing the estimates of the unknown parameters + step: float used for relative perturbation of the parameters, + e.g., step=0.02 is a 2% perturbation + solver: string ``solver`` object specified by the user, e.g., 'ipopt' + tee: boolean solver option to be passed for verbose output + logging_level: logging level specified by the user, e.g., logging.INFO + estimated_var: value of the estimated variance of the measurement error in + cases where the user does not supply the + measurement error standard deviation + + Returns: + FIM: Fisher information matrix about the parameters + """ + # compute the Jacobian matrix using finite difference + J = _compute_jacobian(experiment, theta_vals, step, solver, tee, logging_level) + + # computing the condition number of the Jacobian matrix + cond_number_jac = np.linalg.cond(J) + if logging_level == logging.INFO: + logger.info(f"The condition number of the Jacobian matrix is {cond_number_jac}") + + # grab the model + model = _get_labeled_model_helper(experiment) + + # extract the measured variables and measurement errors + y_hat_list = [y_hat for y_hat, y in model.experiment_outputs.items()] + + # check if the model has a 'measurement_error' attribute and + # the measurement error standard deviation has been supplied + all_known_errors = all( + model.measurement_error[y_hat] is not None for y_hat in model.experiment_outputs + ) + + if hasattr(model, "measurement_error") and all_known_errors: + error_list = [ + model.measurement_error[y_hat] for y_hat in model.experiment_outputs + ] + + # compute the matrix of the inverse of the measurement error variance + # the following assumes independent and identically distributed + # measurement errors + try: + W = np.diag([1 / (err**2) for err in error_list]) + except ZeroDivisionError: + raise ValueError( + 'Division by zero encountered in computing the covariance matrix. ' + 'One or more values of the measurement error are zero.' + ) + + # check if the error list is consistent + if len(error_list) == 0 or len(y_hat_list) == 0: + raise ValueError( + "Experiment outputs and measurement errors cannot be empty." + ) + + # check if the dimension of error_list is the same with that of y_hat_list + if len(error_list) != len(y_hat_list): + raise ValueError( + "Experiment outputs and measurement errors are not the same length." + ) + + # calculate the FIM using the formula in our future paper + # Lilonfe et al. (2025) + FIM = J.T @ W @ J + else: + FIM = (1 / estimated_var) * (J.T @ J) + + return FIM + + +# compute the Fisher information matrix of the estimated parameters using +# 'automatic_differentiation_kaug' +def _kaug_FIM(experiment, theta_vals, solver, tee, estimated_var=None): + """ + Computes the FIM using 'automatic_differentiation_kaug', a sensitivity-based + approach that uses the annotated Pyomo model optimality condition and + user-defined measurement errors standard deviation + + Disclaimer - code adopted from the kaug function implemented in Pyomo.DoE + + Arguments: + experiment: Estimator class object that contains the Pyomo model + for a particular experimental condition + theta_vals: dictionary containing the estimates of the unknown parameters + solver: string ``solver`` object specified by the user, e.g., 'ipopt' + tee: boolean solver option to be passed for verbose output + estimated_var: value of the estimated variance of the measurement error in + cases where the user does not supply the + measurement error standard deviation + + Returns: + FIM: Fisher information matrix about the parameters + """ + # grab the model + model = _get_labeled_model_helper(experiment) + + # fix the parameter values to the estimated values + params = [k for k, v in model.unknown_parameters.items()] + for param in params: + param.fix(theta_vals[param.name]) + + # re-solve the model with the estimated parameters + try: + solver = pyo.SolverFactory(solver) + solver.solve(model, tee=tee) + except Exception as e: + raise RuntimeError( + f"Model from experiment did not solve appropriately. Make sure the " + f"model is well-posed. The original error was {e}." + ) + + # add zero (dummy/placeholder) objective function + if not hasattr(model, "objective"): + model.objective = pyo.Objective(expr=0, sense=pyo.minimize) + + solver.solve(model, tee=tee) + + # Probe the solved model for dsdp results (sensitivities s.t. parameters) + params_dict = {k.name: v for k, v in model.unknown_parameters.items()} + params_names = list(params_dict.keys()) + + dsdp_re, col = get_dsdp(model, params_names, params_dict, tee=tee) + + # analyze result + dsdp_array = dsdp_re.toarray().T + + # store dsdp returned + dsdp_extract = [] + + # get right lines from results + measurement_index = [] + + # loop over measurement variables and their time points + for k, v in model.experiment_outputs.items(): + name = k.name + try: + kaug_no = col.index(name) + measurement_index.append(kaug_no) + # get right line of dsdp + dsdp_extract.append(dsdp_array[kaug_no]) + except: + # k_aug does not provide value for fixed variables + logging.getLogger(__name__).debug("The variable is fixed: %s", name) + # produce the sensitivity for fixed variables + zero_sens = np.zeros(len(params_names)) + # for fixed variables, the sensitivity are a zero vector + dsdp_extract.append(zero_sens) + + # Extract and calculate sensitivity if scaled by constants or parameters. + jac = [[] for _ in params_names] + + for d in range(len(dsdp_extract)): + for k, v in model.unknown_parameters.items(): + p = params_names.index(k.name) # Index of parameter in np array + sensi = dsdp_extract[d][p] + jac[p].append(sensi) + + # record kaug jacobian + kaug_jac = np.array(jac).T + + # compute FIM + # compute the matrix of the inverse of the measurement error variance + # the following assumes independent and identically distributed + # measurement errors + W = np.zeros((len(model.measurement_error), len(model.measurement_error))) + all_known_errors = all( + model.measurement_error[y_hat] is not None for y_hat in model.experiment_outputs + ) + + count = 0 + for k, v in model.measurement_error.items(): + if all_known_errors: + try: + W[count, count] = 1 / (v**2) + except ZeroDivisionError: + raise ValueError( + 'Division by zero encountered in computing the covariance matrix. ' + 'One or more values of the measurement error are zero.' + ) + else: + W[count, count] = 1 / estimated_var + count += 1 + + FIM = kaug_jac.T @ W @ kaug_jac + + return FIM + + class Estimator(object): """ Parameter estimation class @@ -245,14 +768,16 @@ class Estimator(object): A list of experiment objects which creates one labeled model for each experiment obj_function: string or function (optional) - Built in objective (currently only "SSE") or custom function used to - formulate parameter estimation objective. + Built-in objective ("SSE" or "SSE_weighted") or custom function + used to formulate parameter estimation objective. If no function is specified, the model is used "as is" and should be defined with a "FirstStageCost" and "SecondStageCost" expression that are used to build an objective. Default is None. tee: bool, optional If True, print the solver output to the screen. Default is False. + logging_level: logging level, optional + e.g., logging.INFO. Default is logging.ERROR. diagnostic_mode: bool, optional If True, print diagnostics from the solver. Default is False. solver_options: dict, optional @@ -271,6 +796,7 @@ def __init__( experiment_list, obj_function=None, tee=False, + logging_level=logging.ERROR, diagnostic_mode=False, solver_options=None, ): @@ -279,24 +805,36 @@ def __init__( assert isinstance(experiment_list, list) self.exp_list = experiment_list - # check that an experiment has experiment_outputs and unknown_parameters - model = self.exp_list[0].get_labeled_model() - try: - outputs = [k.name for k, v in model.experiment_outputs.items()] - except: - raise RuntimeError( - 'Experiment list model does not have suffix ' + '"experiment_outputs".' - ) - try: - params = [k.name for k, v in model.unknown_parameters.items()] - except: - raise RuntimeError( - 'Experiment list model does not have suffix ' + '"unknown_parameters".' - ) + # check if the experiment has a ``get_labeled_model`` function + model = _get_labeled_model_helper(self.exp_list[0]) + + # check if the model has all the required suffixes + _check_model_labels_helper(model, logging_level) # populate keyword argument options - self.obj_function = obj_function + if isinstance(obj_function, str): + try: + self.obj_function = ObjectiveLib(obj_function) + except ValueError: + raise ValueError( + f"Invalid objective function: '{obj_function}'. " + f"Choose from: {[e.value for e in ObjectiveLib]}." + ) + else: + if obj_function is None: + self.obj_function = obj_function + else: + deprecation_warning( + "You're using a deprecated input to the `obj_function` argument by " + "passing a custom function. This usage will be removed in a " + "future release. Please update to the new parmest interface using " + "the built-in 'SSE' and 'SSE_weighted' objectives.", + version="6.9.3.dev0", + ) + self.obj_function = obj_function + self.tee = tee + self.logging_level = logging_level self.diagnostic_mode = diagnostic_mode self.solver_options = solver_options @@ -307,7 +845,7 @@ def __init__( # We could collect the union (or intersect?) of thetas when the models are built theta_names = [] for experiment in self.exp_list: - model = experiment.get_labeled_model() + model = _get_labeled_model_helper(experiment) theta_names.extend([k.name for k, v in model.unknown_parameters.items()]) # Utilize list(dict.fromkeys(theta_names)) to preserve parameter # order compared with list(set(theta_names)), which had @@ -400,7 +938,7 @@ def _create_parmest_model(self, experiment_number): Modify the Pyomo model for parameter estimation """ - model = self.exp_list[experiment_number].get_labeled_model() + model = _get_labeled_model_helper(self.exp_list[experiment_number]) if len(model.unknown_parameters) == 0: model.parmest_dummy_var = pyo.Var(initialize=1.0) @@ -425,8 +963,11 @@ def _create_parmest_model(self, experiment_number): # TODO, this needs to be turned into an enum class of options that still support # custom functions - if self.obj_function == 'SSE': - second_stage_rule = SSE + if isinstance(self.obj_function, Enum): + if self.obj_function == ObjectiveLib.SSE: + second_stage_rule = SSE + else: + second_stage_rule = SSE_weighted else: # A custom function uses model.experiment_outputs as data second_stage_rule = self.obj_function @@ -457,8 +998,7 @@ def _Q_opt( solver="ef_ipopt", return_values=[], bootlist=None, - calc_cov=False, - cov_n=None, + **kwargs, ): """ Set up all thetas as first stage Vars, return resulting theta @@ -507,7 +1047,7 @@ def _Q_opt( # Solve the extensive form with ipopt if solver == "ef_ipopt": - if not calc_cov: + if not kwargs: # Do not calculate the reduced hessian solver = SolverFactory('ipopt') @@ -517,27 +1057,57 @@ def _Q_opt( solve_result = solver.solve(self.ef_instance, tee=self.tee) - # The import error will be raised when we attempt to use - # inv_reduced_hessian_barrier below. - # - # elif not asl_available: - # raise ImportError("parmest requires ASL to calculate the " - # "covariance matrix with solver 'ipopt'") - else: - # parmest makes the fitted parameters stage 1 variables - ind_vars = [] - for ndname, Var, solval in ef_nonants(ef): - ind_vars.append(Var) - # calculate the reduced hessian - (solve_result, inv_red_hes) = ( - inverse_reduced_hessian.inv_reduced_hessian_barrier( - self.ef_instance, - independent_variables=ind_vars, - solver_options=self.solver_options, - tee=self.tee, - ) + # The import error will be raised when we attempt to use + # inv_reduced_hessian_barrier below. + # + # elif not asl_available: + # raise ImportError("parmest requires ASL to calculate the " + # "covariance matrix with solver 'ipopt'") + elif kwargs and all(arg.value in kwargs for arg in UnsupportedArgsLib): + deprecation_warning( + "You're using a deprecated call to the `theta_est()` function " + "with the `calc_cov` and `cov_n` arguments. This usage will be " + "removed in a future release. Please update to the new parmest " + "interface using `cov_est()` function for covariance calculation.", + version="6.9.3.dev0", ) + calc_cov = kwargs[UnsupportedArgsLib.calc_cov.value] + cov_n = kwargs[UnsupportedArgsLib.cov_n.value] + if not isinstance(calc_cov, bool): + raise TypeError("Expected a boolean for 'calc_cov' argument.") + + if not calc_cov: + # Do not calculate the reduced hessian + + solver = SolverFactory('ipopt') + if self.solver_options is not None: + for key in self.solver_options: + solver.options[key] = self.solver_options[key] + + solve_result = solver.solve(self.ef_instance, tee=self.tee) + + # The import error will be raised when we attempt to use + # inv_reduced_hessian_barrier below. + # + # elif not asl_available: + # raise ImportError("parmest requires ASL to calculate the " + # "covariance matrix with solver 'ipopt'") + else: + # parmest makes the fitted parameters stage 1 variables + ind_vars = [] + for nd_name, Var, sol_val in ef_nonants(ef): + ind_vars.append(Var) + # calculate the reduced hessian + (solve_result, inv_red_hes) = ( + inverse_reduced_hessian.inv_reduced_hessian_barrier( + self.ef_instance, + independent_variables=ind_vars, + solver_options=self.solver_options, + tee=self.tee, + ) + ) + if self.diagnostic_mode: print( ' Solver termination condition = ', @@ -545,44 +1115,60 @@ def _Q_opt( ) # assume all first stage are thetas... - thetavals = {} - for ndname, Var, solval in ef_nonants(ef): + theta_vals = {} + for nd_name, Var, sol_val in ef_nonants(ef): # process the name # the scenarios are blocks, so strip the scenario name - vname = Var.name[Var.name.find(".") + 1 :] - thetavals[vname] = solval + var_name = Var.name[Var.name.find(".") + 1 :] + theta_vals[var_name] = sol_val - objval = pyo.value(ef.EF_Obj) + obj_val = pyo.value(ef.EF_Obj) - if calc_cov: - # Calculate the covariance matrix + # add the estimated theta to the class + self.estimated_theta = theta_vals - # Number of data points considered - n = cov_n + if kwargs and all(arg.value in kwargs for arg in UnsupportedArgsLib): + if calc_cov: + if not isinstance(cov_n, int): + raise TypeError("Expected an integer for 'cov_n' argument.") + num_unknowns = max( + [ + len(experiment.get_labeled_model().unknown_parameters) + for experiment in self.exp_list + ] + ) + assert cov_n > num_unknowns, ( + "The number of datapoints must be greater than the " + "number of parameters to estimate" + ) - # Extract number of fitted parameters - l = len(thetavals) + # Number of data points considered + n = cov_n - # Assumption: Objective value is sum of squared errors - sse = objval + # Extract number of fitted parameters + l = len(theta_vals) - '''Calculate covariance assuming experimental observation errors are - independent and follow a Gaussian - distribution with constant variance. + # Assumption: Objective value is sum of squared errors + sse = obj_val - The formula used in parmest was verified against equations (7-5-15) and - (7-5-16) in "Nonlinear Parameter Estimation", Y. Bard, 1974. + '''Calculate covariance assuming experimental observation errors + are independent and follow a Gaussian distribution + with constant variance. - This formula is also applicable if the objective is scaled by a constant; - the constant cancels out. (was scaled by 1/n because it computes an - expected value.) - ''' - cov = 2 * sse / (n - l) * inv_red_hes - cov = pd.DataFrame( - cov, index=thetavals.keys(), columns=thetavals.keys() - ) + The formula used in parmest was verified against equations + (7-5-15) and (7-5-16) in "Nonlinear Parameter Estimation", + Y. Bard, 1974. - thetavals = pd.Series(thetavals) + This formula is also applicable if the objective is scaled by a + constant; the constant cancels out. + (was scaled by 1/n because it computes an expected value.) + ''' + cov = 2 * sse / (n - l) * inv_red_hes + cov = pd.DataFrame( + cov, index=theta_vals.keys(), columns=theta_vals.keys() + ) + + theta_vals = pd.Series(theta_vals) if len(return_values) > 0: var_values = [] @@ -612,18 +1198,233 @@ def _Q_opt( if len(vals) > 0: var_values.append(vals) var_values = pd.DataFrame(var_values) + + if not kwargs: + return obj_val, theta_vals, var_values + elif kwargs and all(arg.value in kwargs for arg in UnsupportedArgsLib): + if calc_cov: + return obj_val, theta_vals, var_values, cov + else: + return obj_val, theta_vals, var_values + + if not kwargs: + return obj_val, theta_vals + elif kwargs and all(arg.value in kwargs for arg in UnsupportedArgsLib): if calc_cov: - return objval, thetavals, var_values, cov + return obj_val, theta_vals, cov else: - return objval, thetavals, var_values + return obj_val, theta_vals - if calc_cov: - return objval, thetavals, cov + else: + raise RuntimeError("Unknown solver in Q_Opt=" + solver) + + def _cov_at_theta(self, method, solver, cov_n, step): + """ + Covariance matrix calculation using all scenarios in the data + + Argument: + method: string ``method`` object specified by the user, + e.g., 'finite_difference' + solver: string ``solver`` object specified by the user, e.g., 'ipopt' + cov_n: integer, number of datapoints specified by the user which is used + in the objective function + step: float used for relative perturbation of the parameters, + e.g., step=0.02 is a 2% perturbation + + Returns: + cov: pd.DataFrame, covariance matrix of the estimated parameters + """ + if method == CovarianceMethodLib.reduced_hessian.value: + # compute the inverse reduced hessian to be used + # in the "reduced_hessian" method + # parmest makes the fitted parameters stage 1 variables + ind_vars = [] + for nd_name, Var, sol_val in ef_nonants(self.ef_instance): + ind_vars.append(Var) + # calculate the reduced hessian + (solve_result, inv_red_hes) = ( + inverse_reduced_hessian.inv_reduced_hessian_barrier( + self.ef_instance, + independent_variables=ind_vars, + solver_options=self.solver_options, + tee=self.tee, + ) + ) + + self.inv_red_hes = inv_red_hes + + # Number of data points considered + n = cov_n + + # Extract the number of fitted parameters + l = len(self.estimated_theta) + + # calculate the sum of squared errors at the estimated parameter values + sse_vals = [] + for experiment in self.exp_list: + model = _get_labeled_model_helper(experiment) + + # fix the value of the unknown parameters to the estimated values + params = [k for k, v in model.unknown_parameters.items()] + for param in params: + param.fix(self.estimated_theta[param.name]) + + # re-solve the model with the estimated parameters + try: + pyo.SolverFactory(solver).solve(model, tee=self.tee) + except Exception as e: + raise RuntimeError( + f"Model from experiment did not solve appropriately. Make sure the " + f"model is well-posed. The original error was {e}." + ) + + # choose and evaluate the sum of squared errors expression + if self.obj_function == ObjectiveLib.SSE: + sse_expr = SSE(model) + elif self.obj_function == ObjectiveLib.SSE_weighted: + sse_expr = SSE_weighted(model) else: - return objval, thetavals + raise ValueError( + f"Invalid objective function for covariance calculation: " + f"{self.obj_function}. Choose from: " + f"{[e.value for e in ObjectiveLib]} in the Estimator object." + ) + + # evaluate the numerical SSE and store it + sse_val = pyo.value(sse_expr) + sse_vals.append(sse_val) + + sse = sum(sse_vals) # total SSE + + """Calculate covariance assuming experimental observation errors are + independent and follow a Gaussian distribution with constant variance. + + The formula used in parmest was verified against equations (7-5-15) and + (7-5-16) in "Nonlinear Parameter Estimation", Y. Bard, 1974. + + This formula is also applicable if the objective is scaled by a constant; + the constant cancels out. (was scaled by 1/n because it computes an + expected value.) + """ + # check if the user-supplied covariance method is supported + try: + cov_method = CovarianceMethodLib(method) + except ValueError: + raise ValueError( + f"Invalid method: '{method}'. Choose " + f"from: {[e.value for e in CovarianceMethodLib]}." + ) + + # check if the user specified 'SSE' or 'SSE_weighted' as the objective function + if self.obj_function == ObjectiveLib.SSE: + # check if the user defined the 'measurement_error' attribute + if hasattr(model, "measurement_error"): + # get the measurement errors + meas_error = [ + model.measurement_error[y_hat] + for y_hat, y in model.experiment_outputs.items() + ] + + # check if the user supplied the values of the measurement errors + if all(item is None for item in meas_error): + measurement_var = sse / ( + n - l + ) # estimate of the measurement variance + if cov_method == CovarianceMethodLib.reduced_hessian: + cov = ( + 2 * measurement_var * self.inv_red_hes + ) # covariance matrix + cov = pd.DataFrame( + cov, + index=self.estimated_theta.keys(), + columns=self.estimated_theta.keys(), + ) + else: + cov = compute_covariance_matrix( + self.exp_list, + method, + theta_vals=self.estimated_theta, + solver=solver, + step=step, + tee=self.tee, + logging_level=self.logging_level, + estimated_var=measurement_var, + ) + elif all(item is not None for item in meas_error): + if cov_method == CovarianceMethodLib.reduced_hessian: + cov = 2 * (meas_error[0] ** 2) * self.inv_red_hes + cov = pd.DataFrame( + cov, + index=self.estimated_theta.keys(), + columns=self.estimated_theta.keys(), + ) + else: + cov = compute_covariance_matrix( + self.exp_list, + method, + theta_vals=self.estimated_theta, + solver=solver, + step=step, + tee=self.tee, + logging_level=self.logging_level, + ) + else: + raise ValueError( + "One or more values of the measurement errors have " + "not been supplied." + ) + else: + raise AttributeError( + 'Experiment model does not have suffix "measurement_error".' + ) + elif self.obj_function == ObjectiveLib.SSE_weighted: + # check if the user defined the 'measurement_error' attribute + if hasattr(model, "measurement_error"): + meas_error = [ + model.measurement_error[y_hat] + for y_hat, y in model.experiment_outputs.items() + ] + # check if the user supplied the values for the measurement errors + if all(item is not None for item in meas_error): + if ( + cov_method == CovarianceMethodLib.finite_difference + or cov_method + == CovarianceMethodLib.automatic_differentiation_kaug + ): + cov = compute_covariance_matrix( + self.exp_list, + method, + theta_vals=self.estimated_theta, + step=step, + solver=solver, + tee=self.tee, + logging_level=self.logging_level, + ) + else: + cov = self.inv_red_hes + cov = pd.DataFrame( + cov, + index=self.estimated_theta.keys(), + columns=self.estimated_theta.keys(), + ) + else: + raise ValueError( + 'One or more values of the measurement errors have not been ' + 'supplied. All values of the measurement errors are required ' + 'for the "SSE_weighted" objective.' + ) + else: + raise AttributeError( + 'Experiment model does not have suffix "measurement_error".' + ) else: - raise RuntimeError("Unknown solver in Q_Opt=" + solver) + raise NotImplementedError( + 'Covariance calculation is only supported for "SSE" and ' + '"SSE_weighted" objectives.' + ) + + return cov def _Q_at_theta(self, thetavals, initialize_parmest_model=False): """ @@ -854,9 +1655,7 @@ def _get_sample_list(self, samplesize, num_samples, replacement=True): return samplelist - def theta_est( - self, solver="ef_ipopt", return_values=[], calc_cov=False, cov_n=None - ): + def theta_est(self, solver="ef_ipopt", return_values=[], **kwargs): """ Parameter estimation using all scenarios in the data @@ -866,59 +1665,90 @@ def theta_est( Currently only "ef_ipopt" is supported. Default is "ef_ipopt". return_values: list, optional List of Variable names, used to return values from the model for data reconciliation - calc_cov: boolean, optional - If True, calculate and return the covariance matrix (only for "ef_ipopt" solver). - Default is False. - cov_n: int, optional - If calc_cov=True, then the user needs to supply the number of datapoints - that are used in the objective function. Returns ------- objectiveval: float The objective function value - thetavals: pd.Series + theta_vals: pd.Series Estimated values for theta variable values: pd.DataFrame Variable values for each variable name in return_values (only for solver='ef_ipopt') - cov: pd.DataFrame - Covariance matrix of the fitted parameters (only for solver='ef_ipopt') """ # check if we are using deprecated parmest if self.pest_deprecated is not None: - return self.pest_deprecated.theta_est( - solver=solver, - return_values=return_values, - calc_cov=calc_cov, - cov_n=cov_n, - ) + if not kwargs: + return self.pest_deprecated.theta_est( + solver=solver, return_values=return_values + ) + elif kwargs and all(arg.value in kwargs for arg in UnsupportedArgsLib): + calc_cov = kwargs[UnsupportedArgsLib.calc_cov.value] + cov_n = kwargs[UnsupportedArgsLib.cov_n.value] + return self.pest_deprecated.theta_est( + solver=solver, + return_values=return_values, + calc_cov=calc_cov, + cov_n=cov_n, + ) assert isinstance(solver, str) assert isinstance(return_values, list) - assert isinstance(calc_cov, bool) - if calc_cov: - num_unknowns = max( - [ - len(experiment.get_labeled_model().unknown_parameters) - for experiment in self.exp_list - ] - ) - assert isinstance(cov_n, int), ( - "The number of datapoints that are used in the objective function is " - "required to calculate the covariance matrix" - ) - assert ( - cov_n > num_unknowns - ), "The number of datapoints must be greater than the number of parameters to estimate" return self._Q_opt( - solver=solver, - return_values=return_values, - bootlist=None, - calc_cov=calc_cov, - cov_n=cov_n, + solver=solver, return_values=return_values, bootlist=None, **kwargs + ) + + def cov_est( + self, method="finite_difference", solver="ipopt", cov_n=None, step=1e-3 + ): + """ + Covariance matrix calculation using all scenarios in the data + + Parameters + ---------- + method: string, optional + Options - 'finite_difference', 'reduced_hessian', + and 'automatic_differentiation_kaug' + solver: string, optional + E.g., 'ipopt' + cov_n: integer, required + Number of datapoints used in the covariance calculation + step: float, optional + The value used for relative perturbation of the parameters, e.g., + step=0.02 is a 2% perturbation + + Returns + ------- + cov: pd.DataFrame, covariance matrix of the estimated parameters + """ + # check if the solver input is a string + if not isinstance(solver, str): + raise TypeError("Expected a string for the solver, e.g., 'ipopt'") + + # check if the method input is a string + if not isinstance(method, str): + raise TypeError( + "Expected a string for the method, e.g., 'finite_difference'" + ) + + # check if the user-supplied number of datapoints is an integer + if not isinstance(cov_n, int): + raise TypeError("Expected an integer for the 'cov_n' argument.") + + # number of unknown parameters + num_unknowns = max( + [ + len(experiment.get_labeled_model().unknown_parameters) + for experiment in self.exp_list + ] ) + assert cov_n > num_unknowns, ( + "The number of datapoints must be greater than the " + "number of parameters to estimate." + ) + + return self._cov_at_theta(method=method, solver=solver, cov_n=cov_n, step=step) def theta_est_bootstrap( self, diff --git a/pyomo/contrib/parmest/tests/test_parmest.py b/pyomo/contrib/parmest/tests/test_parmest.py index 9a381f97fea..f786f14bb37 100644 --- a/pyomo/contrib/parmest/tests/test_parmest.py +++ b/pyomo/contrib/parmest/tests/test_parmest.py @@ -9,12 +9,13 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -import platform import sys import os import subprocess from itertools import product +import pytest +from parameterized import parameterized, parameterized_class import pyomo.common.unittest as unittest import pyomo.contrib.parmest.parmest as parmest import pyomo.contrib.parmest.graphics as graphics @@ -26,10 +27,8 @@ from pyomo.common.fileutils import this_file_dir from pyomo.contrib.parmest.experiment import Experiment from pyomo.contrib.pynumero.asl import AmplInterface -from pyomo.opt import SolverFactory -is_osx = platform.mac_ver()[0] != "" -ipopt_available = SolverFactory("ipopt").available() +ipopt_available = pyo.SolverFactory("ipopt").available() pynumero_ASL_available = AmplInterface.available() testdir = this_file_dir() @@ -37,6 +36,449 @@ _RANDOM_SEED_FOR_TESTING = 524 +# Test class for the built-in "SSE" and "SSE_weighted" objective functions +# validated the results using the Rooney-Biegler paper example +# Rooney-Biegler paper example is the case when the measurement error is None +# we considered another case when the user supplies the value of the measurement error +@unittest.skipIf( + not parmest.parmest_available, + "Cannot test parmest: required dependencies are missing", +) +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + +# we use parameterized_class to test the two objective functions +# over the two cases of the measurement error. Included a third objective function +# to test the error message when an incorrect objective function is supplied +@parameterized_class( + ("measurement_std", "objective_function"), + [ + (None, "SSE"), + (None, "SSE_weighted"), + (None, "incorrect_obj"), + (0.1, "SSE"), + (0.1, "SSE_weighted"), + (0.1, "incorrect_obj"), + ], +) +class TestRooneyBieglerWSSE(unittest.TestCase): + + def setUp(self): + self.data = pd.DataFrame( + data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0], [5, 15.6], [7, 19.8]], + columns=["hour", "y"], + ) + + # create the Rooney-Biegler model + def rooney_biegler_model(): + """ + Formulates the Pyomo model of the Rooney-Biegler example + + Returns: + m: Pyomo model + """ + m = pyo.ConcreteModel() + + m.asymptote = pyo.Var(within=pyo.NonNegativeReals, initialize=10) + m.rate_constant = pyo.Var(within=pyo.NonNegativeReals, initialize=0.2) + + m.hour = pyo.Var(within=pyo.PositiveReals, initialize=0.1) + m.y = pyo.Var(within=pyo.NonNegativeReals) + + @m.Constraint() + def response_rule(m): + return m.y == m.asymptote * (1 - pyo.exp(-m.rate_constant * m.hour)) + + return m + + # create the Experiment class + class RooneyBieglerExperimentWSSE(Experiment): + def __init__(self, hour, y, measurement_error_std): + self.y = y + self.hour = hour + self.model = None + self.measurement_error_std = measurement_error_std + + def get_labeled_model(self): + self.create_model() + self.finalize_model() + self.label_model() + + return self.model + + def create_model(self): + m = self.model = rooney_biegler_model() + + return m + + def finalize_model(self): + m = self.model + + # fix the input variable + m.hour.fix(self.hour) + + return m + + def label_model(self): + m = self.model + + # add experiment outputs + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, self.y)]) + + # add unknown parameters + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update( + (k, pyo.value(k)) for k in [m.asymptote, m.rate_constant] + ) + + # add measurement error + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, self.measurement_error_std)]) + + return m + + # extract the input and output variables + hour_data = self.data["hour"] + y_data = self.data["y"] + + # create the experiments list + exp_list = [] + for i in range(self.data.shape[0]): + exp_list.append( + RooneyBieglerExperimentWSSE( + hour_data[i], y_data[i], self.measurement_std + ) + ) + + self.exp_list = exp_list + + if self.objective_function == "incorrect_obj": + with pytest.raises( + ValueError, + match=r"Invalid objective function: 'incorrect_obj'\. " + r"Choose from: \['SSE', 'SSE_weighted'\]\.", + ): + self.pest = parmest.Estimator( + self.exp_list, obj_function=self.objective_function, tee=True + ) + else: + self.pest = parmest.Estimator( + self.exp_list, obj_function=self.objective_function, tee=True + ) + + def check_rooney_biegler_parameters( + self, obj_val, theta_vals, obj_function, measurement_error + ): + """ + Checks if the objective value and parameter estimates are equal to the + expected values and agree with the results of the Rooney-Biegler paper + + Argument: + obj_val: float or integer value of the objective function + theta_vals: dictionary of the estimated parameters + obj_function: string objective function supplied by the user, + e.g., 'SSE' + measurement_error: float or integer value of the measurement error + standard deviation + """ + if obj_function == "SSE": + self.assertAlmostEqual(obj_val, 4.33171, places=2) + elif obj_function == "SSE_weighted" and measurement_error is not None: + self.assertAlmostEqual(obj_val, 216.58556, places=2) + + self.assertAlmostEqual( + theta_vals["asymptote"], 19.1426, places=2 + ) # 19.1426 from the paper + self.assertAlmostEqual( + theta_vals["rate_constant"], 0.5311, places=2 + ) # 0.5311 from the paper + + def check_rooney_biegler_covariance( + self, cov, cov_method, obj_function, measurement_error + ): + """ + Checks if the covariance matrix elements are equal to the expected + values and agree with the results of the Rooney-Biegler paper + + Argument: + cov: pd.DataFrame, covariance matrix of the estimated parameters + cov_method: string ``method`` object specified by the user + Options - 'finite_difference', 'reduced_hessian', + and 'automatic_differentiation_kaug' + obj_function: string objective function supplied by the user, + e.g., 'SSE' + measurement_error: float or integer value of the measurement error + standard deviation + """ + + # get indices in covariance matrix + cov_cols = cov.columns.to_list() + asymptote_index = [idx for idx, s in enumerate(cov_cols) if "asymptote" in s][0] + rate_constant_index = [ + idx for idx, s in enumerate(cov_cols) if "rate_constant" in s + ][0] + + if measurement_error is None: + if obj_function == "SSE": + if ( + cov_method == "finite_difference" + or cov_method == "automatic_differentiation_kaug" + ): + self.assertAlmostEqual( + cov.iloc[asymptote_index, asymptote_index], 6.229612, places=2 + ) # 6.22864 from paper + self.assertAlmostEqual( + cov.iloc[asymptote_index, rate_constant_index], + -0.432265, + places=2, + ) # -0.4322 from paper + self.assertAlmostEqual( + cov.iloc[rate_constant_index, asymptote_index], + -0.432265, + places=2, + ) # -0.4322 from paper + self.assertAlmostEqual( + cov.iloc[rate_constant_index, rate_constant_index], + 0.041242, + places=2, + ) # 0.04124 from paper + else: + self.assertAlmostEqual( + cov.iloc[asymptote_index, asymptote_index], 36.935351, places=2 + ) # 6.22864 from paper + self.assertAlmostEqual( + cov.iloc[asymptote_index, rate_constant_index], + -2.551392, + places=2, + ) # -0.4322 from paper + self.assertAlmostEqual( + cov.iloc[rate_constant_index, asymptote_index], + -2.551392, + places=2, + ) # -0.4322 from paper + self.assertAlmostEqual( + cov.iloc[rate_constant_index, rate_constant_index], + 0.243428, + places=2, + ) # 0.04124 from paper + else: + if obj_function == "SSE" or obj_function == "SSE_weighted": + if ( + cov_method == "finite_difference" + or cov_method == "automatic_differentiation_kaug" + ): + self.assertAlmostEqual( + cov.iloc[asymptote_index, asymptote_index], 0.009588, places=4 + ) + self.assertAlmostEqual( + cov.iloc[asymptote_index, rate_constant_index], + -0.000665, + places=4, + ) + self.assertAlmostEqual( + cov.iloc[rate_constant_index, asymptote_index], + -0.000665, + places=4, + ) + self.assertAlmostEqual( + cov.iloc[rate_constant_index, rate_constant_index], + 0.000063, + places=4, + ) + else: + self.assertAlmostEqual( + cov.iloc[asymptote_index, asymptote_index], 0.056845, places=4 + ) + self.assertAlmostEqual( + cov.iloc[asymptote_index, rate_constant_index], + -0.003927, + places=4, + ) + self.assertAlmostEqual( + cov.iloc[rate_constant_index, asymptote_index], + -0.003927, + places=4, + ) + self.assertAlmostEqual( + cov.iloc[rate_constant_index, rate_constant_index], + 0.000375, + places=4, + ) + + # test the covariance calculation of the three supported methods + # added a 'unsupported_method' to test the error message when the method supplied + # is not supported + @parameterized.expand( + [ + ("finite_difference"), + ("automatic_differentiation_kaug"), + ("reduced_hessian"), + ("unsupported_method"), + ] + ) + def test_parmest_covariance(self, cov_method): + """ + Estimates the parameters and covariance matrix and compares them + with the results of the Rooney-Biegler paper + + Argument: + cov_method: string ``method`` specified by the user + Options - 'finite_difference', 'reduced_hessian', + and 'automatic_differentiation_kaug' + """ + if self.measurement_std is None: + if self.objective_function == "SSE": + + # estimate the parameters + obj_val, theta_vals = self.pest.theta_est() + + # check the parameter estimation result + self.check_rooney_biegler_parameters( + obj_val, + theta_vals, + obj_function=self.objective_function, + measurement_error=self.measurement_std, + ) + + # calculate the covariance matrix + if cov_method in ( + "finite_difference", + "automatic_differentiation_kaug", + "reduced_hessian", + ): + cov = self.pest.cov_est(cov_n=6, method=cov_method) + + # check the covariance calculation results + self.check_rooney_biegler_covariance( + cov, + cov_method, + obj_function=self.objective_function, + measurement_error=self.measurement_std, + ) + else: + with pytest.raises( + ValueError, + match=r"Invalid method: 'unsupported_method'\. Choose from: " + r"\['finite_difference', " + r"'automatic_differentiation_kaug', " + r"'reduced_hessian'\]\.", + ): + cov = self.pest.cov_est(cov_n=6, method=cov_method) + elif self.objective_function == "SSE_weighted": + with pytest.raises( + ValueError, + match='One or more values are missing from ' + '"measurement_error". All values of the measurement errors are ' + 'required for the "SSE_weighted" objective.', + ): + # we expect this error when estimating the parameters + obj_val, theta_vals = self.pest.theta_est() + else: + if ( + self.objective_function == "SSE" + or self.objective_function == "SSE_weighted" + ): + # estimate the parameters + obj_val, theta_vals = self.pest.theta_est() + + # check the parameter estimation results + self.check_rooney_biegler_parameters( + obj_val, + theta_vals, + obj_function=self.objective_function, + measurement_error=self.measurement_std, + ) + + # calculate the covariance matrix + if cov_method in ( + "finite_difference", + "automatic_differentiation_kaug", + "reduced_hessian", + ): + cov = self.pest.cov_est(cov_n=6, method=cov_method) + + # check the covariance calculation results + self.check_rooney_biegler_covariance( + cov, + cov_method, + obj_function=self.objective_function, + measurement_error=self.measurement_std, + ) + else: + with pytest.raises( + ValueError, + match=r"Invalid method: 'unsupported_method'\. Choose from: " + r"\['finite_difference', " + r"'automatic_differentiation_kaug', " + r"'reduced_hessian'\]\.", + ): + cov = self.pest.cov_est(cov_n=6, method=cov_method) + + def test_cov_scipy_least_squares_comparison(self): + """ + Estimates the unknown parameters and covariance matrix from the measurement + error standard deviation using Scipy least_squares function. + """ + if self.measurement_std is None or self.objective_function == "incorrect_obj": + self.skipTest( + "This test only applies to 'SSE' and 'SSE_weighted' " + "objectives with user-supplied measurement error" + ) + + def model(theta, t): + """ + Model to be fitted y = model(theta, t) + Arguments: + theta: vector of fitted parameters + t: independent variable [hours] + + Returns: + y: model predictions [need to check paper for units] + """ + asymptote = theta[0] + rate_constant = theta[1] + + return asymptote * (1 - np.exp(-rate_constant * t)) + + def residual(theta, t, y): + """ + Calculate residuals + Arguments: + theta: vector of fitted parameters + t: independent variable [hours] + y: dependent variable [?] + """ + return y - model(theta, t) + + # define data + t = self.data["hour"].to_numpy() + y = self.data["y"].to_numpy() + + # define initial guess + theta_guess = np.array([15, 0.5]) + + ## solve with optimize.least_squares + sol = scipy.optimize.least_squares( + residual, theta_guess, method="trf", args=(t, y), verbose=2 + ) + theta_hat = sol.x + + self.assertAlmostEqual( + theta_hat[0], 19.1426, places=2 + ) # 19.1426 from the paper + self.assertAlmostEqual(theta_hat[1], 0.5311, places=2) # 0.5311 from the paper + + # calculate the variance of the measurement error + sigre = self.measurement_std**2 + + cov = sigre * np.linalg.inv(np.matmul(sol.jac.T, sol.jac)) + + self.assertAlmostEqual(cov[0, 0], 0.009588, places=4) + self.assertAlmostEqual(cov[0, 1], -0.000665, places=4) + self.assertAlmostEqual(cov[1, 0], -0.000665, places=4) + self.assertAlmostEqual(cov[1, 1], 0.000063, places=4) + + @unittest.skipIf( not parmest.parmest_available, "Cannot test parmest: required dependencies are missing", @@ -108,7 +550,7 @@ def label_model(self): # check the exception raised by parmest due to not defining # the "experiment_outputs" - with self.assertRaises(RuntimeError) as context: + with self.assertRaises(AttributeError) as context: parmest.Estimator(exp_list, obj_function="SSE", tee=True) self.assertIn("experiment_outputs", str(context.exception)) @@ -969,12 +1411,12 @@ def test_parmest_exception(self): Test the exception raised by parmest when the "unknown_parameters" attribute is not defined in the model """ - with self.assertRaises(RuntimeError) as context: + with self.assertRaises(AttributeError) as context: parmest.Estimator(self.exp_list_df_no_params, obj_function="SSE") self.assertIn("unknown_parameters", str(context.exception)) - with self.assertRaises(RuntimeError) as context: + with self.assertRaises(AttributeError) as context: parmest.Estimator(self.exp_list_dict_no_params, obj_function="SSE") self.assertIn("unknown_parameters", str(context.exception))