diff --git a/_delphi_utils_python/delphi_utils/weekday.py b/_delphi_utils_python/delphi_utils/weekday.py index ba5d75815..c30923812 100644 --- a/_delphi_utils_python/delphi_utils/weekday.py +++ b/_delphi_utils_python/delphi_utils/weekday.py @@ -19,9 +19,10 @@ def get_params(data, denominator_col, numerator_cols, date_col, scales, logger): series column in the data. """ tmp = data.reset_index() - denoms = tmp.groupby(date_col).sum()[denominator_col] - nums = tmp.groupby(date_col).sum()[numerator_cols] - + denoms = tmp.groupby(date_col).sum(numeric_only = True)[denominator_col] + nums = tmp.groupby(date_col).sum(numeric_only = True)[numerator_cols] + if nums.shape[0] < 7: + logger.warning("Trying to handle weekday effects with fewer than 7 days worth of data. This will probably not work.") # Construct design matrix to have weekday indicator columns and then day # indicators. X = np.zeros((nums.shape[0], 6 + nums.shape[0])) @@ -40,7 +41,15 @@ def get_params(data, denominator_col, numerator_cols, date_col, scales, logger): logger.error("Unable to calculate weekday correction") else: params[i,:] = result - + # the values multiplied by in calc_adjustment + # Sunday, the last day here, is a sum of the other days + effective_multipliers = np.exp(np.concatenate([-params[:,:6], params[:,:6].sum(axis=0, keepdims = True)])) + + if effective_multipliers.max() == np.inf or effective_multipliers.min() == -np.inf: + logger.warning("largest weekday correction is infinite. Defaulting to no correction") + params = np.zeros((nums.shape[1], X.shape[1])) + elif effective_multipliers.max() > 1000: + logger.warning("largest weekday correction is over a factor of 1000. This is probably an error, but may be associated with a zero value.") return params @staticmethod @@ -93,11 +102,12 @@ def _fit(X, scales, npnums, npdenoms): for scale in scales: try: prob = cp.Problem(cp.Minimize((-ll + lmbda * penalty) / scale)) - _ = prob.solve() + _ = prob.solve(solver = cp.ECOS) return b.value except SolverError: # If the magnitude of the objective function is too large, an error is # thrown; Rescale the objective function by going through loop + print(f"Solver didn't work at {scale}x") continue return None diff --git a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py index 002d4a7c9..ee11f80a4 100644 --- a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py +++ b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py @@ -61,12 +61,16 @@ def download(ftp_credentials, out_path, logger): client = paramiko.SSHClient() client.set_missing_host_key_policy(AllowAnythingPolicy()) - client.connect(ftp_credentials["host"], - username=ftp_credentials["user"], - password=ftp_credentials["pass"], - port=ftp_credentials["port"]) + client.connect( + ftp_credentials["host"], + username=ftp_credentials["user"], + password=ftp_credentials["pass"], + port=ftp_credentials["port"], + allow_agent=False, + look_for_keys=False, + ) sftp = client.open_sftp() - sftp.chdir('/optum/receiving') + sftp.chdir("/optum/receiving") # go through files in recieving dir files_to_download = []