Skip to content

Anomaly tools changes #133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
kiihne-noaa wants to merge 20 commits into
base: main
Choose a base branch
from
Draft

Anomaly tools changes #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
f052068
creating save_refmodel
kiihne-noaa Feb 18, 2026
eb5cba7
Update epmt_query.py
kiihne-noaa Feb 25, 2026
eab48d5
Update epmt_query.py
kiihne-noaa Feb 25, 2026
ef473e9
Update epmt_query.py
kiihne-noaa Feb 26, 2026
ff70178
Update epmt_query.py
kiihne-noaa Feb 26, 2026
d4c5eaf
Update epmt_query.py
kiihne-noaa Feb 26, 2026
39de787
Update epmt_query.py
kiihne-noaa Mar 3, 2026
3135e6d
Create test_save_refmodel.py
kiihne-noaa Mar 6, 2026
b9fb9d1
fixed orm import
kiihne-noaa Mar 6, 2026
aca0802
Update test_save_refmodel.py
kiihne-noaa Mar 6, 2026
2855601
Update epmt_query.py
kiihne-noaa Mar 10, 2026
d211bf0
Update epmt_query.py
kiihne-noaa Mar 10, 2026
74d6a20
changing tag variable to account for new settup
kiihne-noaa Mar 11, 2026
6e8276f
Update epmt_query.py
kiihne-noaa Mar 11, 2026
011ea21
adding fmt to save_refmodel default
kiihne-noaa Mar 12, 2026
7695e43
Update epmt_query.py
kiihne-noaa Mar 12, 2026
9578eb4
tag to {} if =None
kiihne-noaa Mar 12, 2026
e968ace
Update src/epmt/epmt_query.py
kiihne-noaa Mar 12, 2026
1281d9f
Update src/epmt/epmt_query.py
kiihne-noaa Mar 12, 2026
b2b38dc
Merge branch 'main' into anomaly_tools_changes
ilaflott Mar 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions src/epmt/epmt_query.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1107,7 +1107,6 @@ def get_refmodels(name=None, tag={}, fltr=None, limit=0, order=None, before=None
# we assume the user wants the output in the form of a list of dicts
return out_list


# This function computes a dict such as:
# for univariate classifiers:
#
Expand Down Expand Up @@ -1158,7 +1157,6 @@ def _refmodel_scores(col, methods, features):
return ret
#


@db_session
def create_refmodel(jobs, name=None, tag={}, op_tags=[],
methods=[],
Expand Down Expand Up @@ -1274,6 +1272,8 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[],

if isinstance(tag, str):
tag = tag_from_string(tag)
elif tag is None:
tag = {}

methods = methods or uvod_classifiers()

Expand Down Expand Up @@ -1361,6 +1361,8 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[],
info_dict['pca'] = {'inp_features': orig_features, 'out_features': pca_features}

# now save the ref model
save_refmodel(ReferenceModel, jobs=jobs, computed=computed, info_dict = info_dict, enabled=enabled, name=name, tags=tag, op_tags=op_tags, fmt=fmt)
'''
r = orm_create(ReferenceModel, jobs=jobs, name=name, tags=tag, op_tags=op_tags,
computed=computed, info_dict = info_dict, enabled=enabled)
orm_commit()
Expand All @@ -1370,9 +1372,18 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[],
return r.id
r_dict = orm_to_dict(r, with_collections=True)
return pd.Series(r_dict) if fmt == 'pandas' else r_dict

'''
# returns the number of models deleted.

#jobs, computed, info_dict, and enabled can all be retrieved from the ReferenceModel object
def save_refmodel(ReferenceModel, jobs, computed, info_dict, enabled, name=None, tags={}, op_tags=[], fmt='dict'):
r = orm_create(ReferenceModel, jobs=jobs, computed=computed, info_dict = info_dict, enabled=enabled, name=name, tags=tags, op_tags=op_tags)
orm_commit()
if fmt == 'orm':
return r
elif fmt == 'terse':
return r.id
r_dict = orm_to_dict(r, with_collections=True)
return pd.Series(r_dict) if fmt == 'pandas' else r_dict

@db_session
def delete_refmodels(*ref_ids):
Expand Down
23 changes: 23 additions & 0 deletions test_save_refmodel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import numpy as np
import epmt
from epmt import epmt_query
# epmt_outliers contains the EPMT Outlier Detection API
from epmt import epmt_outliers as eod
# epmt_stat contains statistical functions
from epmt import epmt_stat as es
from epmt import epmt_query as eq
from epmt.orm import *
import pandas
from pandas import DataFrame

feature_list = [ 'duration', 'rchar', 'syscr', 'syscw', 'wchar', 'cstime', 'cutime', 'majflt', 'cpu_time', 'minflt', 'rssmax', 'cmajflt','cminflt', 'inblock', 'outblock', 'usertime', 'num_procs', 'starttime', 'vol_ctxsw', 'read_bytes', 'systemtime', 'time_oncpu', 'timeslices', 'invol_ctxsw', 'write_bytes', 'time_waiting', 'cancelled_write_bytes']
random_jobs = eq.get_jobs(limit = 100, fmt = 'dict', trigger_post_process=False)

try:
r = eq.get_refmodels("bronx_test_model")
eq.delete_refmodels(r[0]['id'])
finally:
print("ready for new bronx model")
r = eq.create_refmodel(random_jobs, methods=es.mvod_classifiers(), features = feature_list, name = "bronx_test_model")

eq.save_refmodel(ReferenceModel, r['jobs'], r['computed'], r['info_dict'], r['enabled'], name='test_name', tag={}, op_tags=[])
Comment on lines +12 to +23
Copy link

Copilot AI Mar 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file is named like a unit test but behaves like a destructive integration script: it performs DB reads/writes, deletes existing models, and has no assertions. This will be flaky and potentially harmful in CI/dev environments.\n\nSuggested direction (pick one):\n- Convert it to a real automated test (e.g., pytest) with assertions and isolated DB/fixtures (and avoid deleting arbitrary existing models), or\n- Move it to a scripts/ or examples/ location and keep it out of the test discovery path.\n\nAlso, the call uses tag={} but save_refmodel() expects tags=... (per its signature).

Suggested change
feature_list = [ 'duration', 'rchar', 'syscr', 'syscw', 'wchar', 'cstime', 'cutime', 'majflt', 'cpu_time', 'minflt', 'rssmax', 'cmajflt','cminflt', 'inblock', 'outblock', 'usertime', 'num_procs', 'starttime', 'vol_ctxsw', 'read_bytes', 'systemtime', 'time_oncpu', 'timeslices', 'invol_ctxsw', 'write_bytes', 'time_waiting', 'cancelled_write_bytes']
random_jobs = eq.get_jobs(limit = 100, fmt = 'dict', trigger_post_process=False)
try:
r = eq.get_refmodels("bronx_test_model")
eq.delete_refmodels(r[0]['id'])
finally:
print("ready for new bronx model")
r = eq.create_refmodel(random_jobs, methods=es.mvod_classifiers(), features = feature_list, name = "bronx_test_model")
eq.save_refmodel(ReferenceModel, r['jobs'], r['computed'], r['info_dict'], r['enabled'], name='test_name', tag={}, op_tags=[])
import uuid
feature_list = [ 'duration', 'rchar', 'syscr', 'syscw', 'wchar', 'cstime', 'cutime', 'majflt', 'cpu_time', 'minflt', 'rssmax', 'cmajflt','cminflt', 'inblock', 'outblock', 'usertime', 'num_procs', 'starttime', 'vol_ctxsw', 'read_bytes', 'systemtime', 'time_oncpu', 'timeslices', 'invol_ctxsw', 'write_bytes', 'time_waiting', 'cancelled_write_bytes']
def test_save_refmodel_persists_refmodel():
"""
Integration-style test that creates a reference model, saves it,
verifies it can be retrieved, and then cleans it up.
"""
model_name = "bronx_test_model_" + uuid.uuid4().hex
random_jobs = eq.get_jobs(limit=100, fmt='dict', trigger_post_process=False)
r = eq.create_refmodel(
random_jobs,
methods=es.mvod_classifiers(),
features=feature_list,
name=model_name,
)
try:
eq.save_refmodel(
ReferenceModel,
r['jobs'],
r['computed'],
r['info_dict'],
r['enabled'],
name=model_name,
tags={},
op_tags=[],
)
saved_models = eq.get_refmodels(model_name)
assert saved_models, "Expected at least one saved reference model"
finally:
# Clean up only the models created for this test
try:
models_to_delete = eq.get_refmodels(model_name)
except Exception:
models_to_delete = []
for m in models_to_delete:
if 'id' in m:
eq.delete_refmodels(m['id'])

Copilot uses AI. Check for mistakes.
Loading