From a1cf3bdcea4de9d6d43182a58b8a4d7c4d5442d1 Mon Sep 17 00:00:00 2001 From: Patrick Vu Date: Fri, 29 Aug 2025 19:14:44 +0700 Subject: [PATCH 1/2] Ability to store metadata with an identifier #1051 --- src/keri/app/connecting.py | 107 ++++++++++----- src/keri/db/basing.py | 25 +++- tests/app/test_connecting.py | 246 +++++++++++++++++++++++++++++++++++ 3 files changed, 345 insertions(+), 33 deletions(-) diff --git a/src/keri/app/connecting.py b/src/keri/app/connecting.py index e6a2f545c..b66478fd2 100644 --- a/src/keri/app/connecting.py +++ b/src/keri/app/connecting.py @@ -11,16 +11,24 @@ from keri import kering -class Organizer: - """ Organizes contacts relating contact information to AIDs """ +class BaseOrganizer: + """ Base class for organizing contact or identifier information """ - def __init__(self, hby): - """ Create contact Organizer + def __init__(self, hby, cigsdb, datadb, fielddb, imgsdb): + """ Create base Organizer Parameters: - hby (Habery): database environment for contact information + hby (Habery): database environment + cigsdb: database for storing signatures + datadb: database for storing main data + fielddb: database for storing individual fields + imgsdb: database for storing images """ self.hby = hby + self.cigsdb = cigsdb + self.datadb = datadb + self.fielddb = fielddb + self.imgsdb = imgsdb def update(self, pre, data): """ Add or update contact information in data for the identifier prefix @@ -39,11 +47,11 @@ def update(self, pre, data): raw = json.dumps(existing).encode("utf-8") cigar = self.hby.signator.sign(ser=raw) - self.hby.db.ccigs.pin(keys=(pre,), val=cigar) - self.hby.db.cons.pin(keys=(pre,), val=raw) + self.cigsdb.pin(keys=(pre,), val=cigar) + self.datadb.pin(keys=(pre,), val=raw) for field, val in data.items(): - self.hby.db.cfld.pin(keys=(pre, field), val=val) + self.fielddb.pin(keys=(pre, field), val=val) def replace(self, pre, data): """ Replace all contact information for identifier prefix with data @@ -68,7 +76,7 @@ def set(self, pre, field, val): data = self.get(pre) or dict() data[field] = val self.replace(pre, data) - self.hby.db.cfld.pin(keys=(pre, field), val=val) + self.fielddb.pin(keys=(pre, field), val=val) def unset(self, pre, field): """ Remove field from contact information for identifier prefix @@ -81,7 +89,7 @@ def unset(self, pre, field): data = self.get(pre) del data[field] self.replace(pre, data) - self.hby.db.cfld.rem(keys=(pre, field)) + self.fielddb.rem(keys=(pre, field)) def rem(self, pre): """ Remove all contact information for identifier prefix @@ -92,9 +100,9 @@ def rem(self, pre): Returns: """ - self.hby.db.ccigs.rem(keys=(pre,)) - self.hby.db.cons.rem(keys=(pre,)) - return self.hby.db.cfld.trim(keys=(pre,)) + self.cigsdb.rem(keys=(pre,)) + self.datadb.rem(keys=(pre,)) + return self.fielddb.trim(keys=(pre,)) def get(self, pre, field=None): """ Retrieve all contact information for identifier prefix @@ -107,10 +115,10 @@ def get(self, pre, field=None): dict: Contact data """ - raw = self.hby.db.cons.get(keys=(pre,)) + raw = self.datadb.get(keys=(pre,)) if raw is None: return None - cigar = self.hby.db.ccigs.get(keys=(pre,)) + cigar = self.cigsdb.get(keys=(pre,)) if not self.hby.signator.verify(ser=raw.encode("utf-8"), cigar=cigar): raise kering.ValidationError(f"failed signature on {pre} contact data") @@ -135,7 +143,7 @@ def list(self): key = "" data = None contacts = [] - for (pre, field), val in self.hby.db.cfld.getItemIter(): + for (pre, field), val in self.fielddb.getItemIter(): if pre != key: if data is not None: contacts.append(data) @@ -162,7 +170,7 @@ def find(self, field, val): """ pres = [] prog = re.compile(f".*{val}.*", re.I) - for (pre, f), v in self.hby.db.cfld.getItemIter(): + for (pre, f), v in self.fielddb.getItemIter(): if f == field and prog.match(v): pres.append(pre) @@ -182,10 +190,9 @@ def values(self, field, val=None): prog = re.compile(f".*{val}.*", re.I) if val is not None else None vals = oset() - for (pre, f), v in self.hby.db.cfld.getItemIter(): - if f == field: - if prog is None or prog.match(v): - vals.add(v) + for (pre, f), v in self.fielddb.getItemIter(): + if f == field and (prog is None or prog.match(v)): + vals.add(v) return list(vals) @@ -201,10 +208,10 @@ def setImg(self, pre, typ, stream): stream (file): file-like stream of image data """ - self.hby.db.delTopVal(db=self.hby.db.imgs, top=pre.encode("utf-8")) + self.hby.db.delTopVal(db=self.imgsdb, top=pre.encode("utf-8")) key = f"{pre}.content-type".encode("utf-8") - self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=typ.encode("utf-8")) + self.hby.db.setVal(db=self.imgsdb, key=key, val=typ.encode("utf-8")) idx = 0 size = 0 @@ -213,12 +220,12 @@ def setImg(self, pre, typ, stream): if not chunk: break key = f"{pre}.{idx}".encode("utf-8") - self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=chunk) + self.hby.db.setVal(db=self.imgsdb, key=key, val=chunk) idx += 1 size += len(chunk) key = f"{pre}.content-length".encode("utf-8") - self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=size.to_bytes(4, "big")) + self.hby.db.setVal(db=self.imgsdb, key=key, val=size.to_bytes(4, "big")) def getImgData(self, pre): """ Get image metadata for identifier image if one exists @@ -231,19 +238,19 @@ def getImgData(self, pre): """ key = f"{pre}.content-length".encode("utf-8") - size = self.hby.db.getVal(db=self.hby.db.imgs, key=key) + size = self.hby.db.getVal(db=self.imgsdb, key=key) if size is None: return None key = f"{pre}.content-type".encode("utf-8") - typ = self.hby.db.getVal(db=self.hby.db.imgs, key=key) + typ = self.hby.db.getVal(db=self.imgsdb, key=key) if typ is None: return None - return dict( - type=bytes(typ).decode("utf-8"), - length=int.from_bytes(size, "big") - ) + return { + "type": bytes(typ).decode("utf-8"), + "length": int.from_bytes(size, "big") + } def getImg(self, pre): """ Generator that yields image data in 4k chunks for identifier @@ -255,8 +262,44 @@ def getImg(self, pre): idx = 0 while True: key = f"{pre}.{idx}".encode("utf-8") - chunk = self.hby.db.getVal(db=self.hby.db.imgs, key=key) + chunk = self.hby.db.getVal(db=self.imgsdb, key=key) if not chunk: break yield bytes(chunk) idx += 1 + + +class Organizer(BaseOrganizer): + """ Organizes contacts relating contact information to AIDs """ + + def __init__(self, hby): + """ Create contact Organizer + + Parameters: + hby (Habery): database environment for contact information + """ + super().__init__( + hby=hby, + cigsdb=hby.db.ccigs, + datadb=hby.db.cons, + fielddb=hby.db.cfld, + imgsdb=hby.db.imgs + ) + + +class IdentifierOrganizer(BaseOrganizer): + """ Organizes identifier information for local identifiers """ + + def __init__(self, hby): + """ Create identifier Organizer + + Parameters: + hby (Habery): database environment for identifier information + """ + super().__init__( + hby=hby, + cigsdb=hby.db.icigs, + datadb=hby.db.icons, + fielddb=hby.db.ifld, + imgsdb=hby.db.iimgs + ) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0188fd417..058a3f87d 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1281,6 +1281,24 @@ def reopen(self, **kwa): # TODO: clean self.imgs = self.env.open_db(key=b'imgs.') + # Field values for identifier information for local identifiers. Keyed by prefix/field + # TODO: clean + self.ifld = subing.Suber(db=self, + subkey="ifld.") + + # Signed identifier data, keys by prefix + # TODO: clean + self.icons = subing.Suber(db=self, + subkey="icons.") + + # Transferable signatures on identifier data + # TODO: clean + self.icigs = subing.CesrSuber(db=self, subkey='icigs.', klas=coring.Cigar) + + # Chunked image data for identifier information for local identifiers + # TODO: clean + self.iimgs = self.env.open_db(key=b'iimgs.') + # Delegation escrow dbs # # delegated partial witness escrow self.dpwe = subing.SerderSuber(db=self, subkey='dpwe.') @@ -1499,7 +1517,8 @@ def clean(self): # reprocess them. We need a more secure method in the future unsecured = ["hbys", "schema", "states", "rpys", "eans", "tops", "cgms", "exns", "erpy", "kdts", "ksns", "knas", "oobis", "roobi", "woobi", "moobi", "mfa", "rmfa", - "cfld", "cons", "ccigs", "cdel", "migs"] + "cfld", "cons", "ccigs", "cdel", "migs", + "ifld", "icons", "icigs"] for name in unsecured: srcdb = getattr(self, name) @@ -1521,6 +1540,10 @@ def clean(self): for (key, val) in self.getTopItemIter(self.imgs): copy.imgs.setVal(key=key, val=val) + # Insecure raw iimgs database copy. + for (key, val) in self.getTopItemIter(self.iimgs): + copy.iimgs.setVal(key=key, val=val) + # clone .habs habitat name prefix Komer subdb # copy.habs = koming.Komer(db=copy, schema=HabitatRecord, subkey='habs.') # copy for keys, val in self.habs.getItemIter(): diff --git a/tests/app/test_connecting.py b/tests/app/test_connecting.py index 448cbc37a..c7ee8c88d 100644 --- a/tests/app/test_connecting.py +++ b/tests/app/test_connecting.py @@ -185,3 +185,249 @@ def test_organizer_imgs(): img.extend(chunk) assert len(img) == 0 + + +def test_base_organizer(): + """Test BaseOrganizer with custom database configuration""" + joe = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + bob = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + + joed = {"first": "Joe", "last": "Jury", "address": "9934 Glen Creek St.", "city": "Lawrence", "state": "MA", "zip": "01841", + "company": "HCF", "alias": "joe"} + bobd = {"first": "Bob", "last": "Burns", "address": "37 East Shadow Brook St.", "city": "Sebastian", "state": "FL", + "zip": "32958", "company": "HCF", "alias": "bob"} + + with habbing.openHby(name="test", temp=True) as hby: + # Test BaseOrganizer with contact databases (same as Organizer) + base_org = connecting.BaseOrganizer( + hby=hby, + cigsdb=hby.db.ccigs, + datadb=hby.db.cons, + fielddb=hby.db.cfld, + imgsdb=hby.db.imgs + ) + + # Test basic CRUD operations + base_org.replace(pre=joe, data=joed) + retrieved = base_org.get(pre=joe) + assert retrieved["first"] == "Joe" + assert retrieved["last"] == "Jury" + assert retrieved["id"] == joe + + # Test update + base_org.update(pre=bob, data=bobd) + retrieved = base_org.get(pre=bob) + assert retrieved["first"] == "Bob" + + # Test list + contacts = base_org.list() + assert len(contacts) == 2 + + # Test find + results = base_org.find(field="company", val="HCF") + assert len(results) == 2 + + # Test values + companies = base_org.values(field="company") + assert "HCF" in companies + + # Test set + base_org.set(pre=joe, field="phone", val="555-1234") + retrieved = base_org.get(pre=joe, field="phone") + assert retrieved == "555-1234" + + # Test unset + base_org.unset(pre=joe, field="phone") + retrieved = base_org.get(pre=joe, field="phone") + assert retrieved is None + + # Test rem + base_org.rem(pre=joe) + retrieved = base_org.get(pre=joe) + assert retrieved is None + + +def test_identifier_organizer(): + """Test IdentifierOrganizer with identifier databases""" + aid1 = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + aid2 = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + + # Sample identifier metadata + id1_data = {"name": "Primary ID", "description": "Main identifier", "role": "controller", + "created": "2025-08-29T00:00:00Z", "status": "active"} + id2_data = {"name": "Secondary ID", "description": "Backup identifier", "role": "witness", + "created": "2025-08-29T01:00:00Z", "status": "active"} + + with habbing.openHby(name="test", temp=True) as hby: + # Test IdentifierOrganizer + id_org = connecting.IdentifierOrganizer(hby=hby) + + # Test basic CRUD operations + id_org.replace(pre=aid1, data=id1_data) + retrieved = id_org.get(pre=aid1) + assert retrieved["name"] == "Primary ID" + assert retrieved["role"] == "controller" + assert retrieved["id"] == aid1 + + # Test update + id_org.update(pre=aid2, data=id2_data) + retrieved = id_org.get(pre=aid2) + assert retrieved["name"] == "Secondary ID" + + # Test list + identifiers = id_org.list() + assert len(identifiers) == 2 + + # Test find by role + controllers = id_org.find(field="role", val="controller") + assert len(controllers) == 1 + assert controllers[0]["name"] == "Primary ID" + + witnesses = id_org.find(field="role", val="witness") + assert len(witnesses) == 1 + assert witnesses[0]["name"] == "Secondary ID" + + # Test values + roles = id_org.values(field="role") + assert "controller" in roles + assert "witness" in roles + + statuses = id_org.values(field="status") + assert "active" in statuses + + # Test set field + id_org.set(pre=aid1, field="version", val="1.0") + retrieved = id_org.get(pre=aid1, field="version") + assert retrieved == "1.0" + + # Test unset field + id_org.unset(pre=aid1, field="version") + retrieved = id_org.get(pre=aid1, field="version") + assert retrieved is None + + # Test rem + id_org.rem(pre=aid1) + retrieved = id_org.get(pre=aid1) + assert retrieved is None + + # Verify only one identifier remains + identifiers = id_org.list() + assert len(identifiers) == 1 + assert identifiers[0]["name"] == "Secondary ID" + + +def test_organizer_vs_identifier_organizer_separation(): + """Test that Organizer and IdentifierOrganizer store data separately""" + contact_id = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + identifier_id = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + + contact_data = {"first": "John", "last": "Doe", "company": "ACME Corp"} + identifier_data = {"name": "Test ID", "role": "controller", "status": "active"} + + with habbing.openHby(name="test", temp=True) as hby: + # Create both organizers + contact_org = connecting.Organizer(hby=hby) + id_org = connecting.IdentifierOrganizer(hby=hby) + + # Add data to both + contact_org.replace(pre=contact_id, data=contact_data) + id_org.replace(pre=identifier_id, data=identifier_data) + + # Verify contact organizer only has contact data + contacts = contact_org.list() + assert len(contacts) == 1 + assert contacts[0]["first"] == "John" + assert contacts[0]["company"] == "ACME Corp" + + # Verify identifier organizer only has identifier data + identifiers = id_org.list() + assert len(identifiers) == 1 + assert identifiers[0]["name"] == "Test ID" + assert identifiers[0]["role"] == "controller" + + # Verify cross-contamination doesn't occur + assert contact_org.get(pre=identifier_id) is None + assert id_org.get(pre=contact_id) is None + + # Test with same ID in both systems (should be separate) + same_id = "EFC7f_MEPE5dboc_E4yG15fnpMD34YaU3ue6vnDLodJU" + contact_org.replace(pre=same_id, data={"name": "Contact Person"}) + id_org.replace(pre=same_id, data={"name": "Identifier Name"}) + + contact_data = contact_org.get(pre=same_id) + id_data = id_org.get(pre=same_id) + + assert contact_data["name"] == "Contact Person" + assert id_data["name"] == "Identifier Name" + # They should be completely separate + assert contact_data != id_data + + +def test_identifier_organizer_imgs(): + """Test IdentifierOrganizer image functionality""" + with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + id_org = connecting.IdentifierOrganizer(hby=hby) + pre = "EFC7f_MEPE5dboc_E4yG15fnpMD34YaU3ue6vnDLodJU" + + # Create test image data + data = bytearray(os.urandom(50000)) + assert len(data) == 50000 + stream = io.BytesIO(data) + + # Test setImg + id_org.setImg(pre, "image/jpeg", stream) + + # Test getImg + img = bytearray() + for chunk in id_org.getImg(pre): + img.extend(chunk) + + assert img == data + + # Test getImgData + md = id_org.getImgData(pre=pre) + assert md["type"] == "image/jpeg" + assert md["length"] == len(data) + + # Test non-existent image + non_existent = "Eo60ITGA69z4jNBU4RsvbgsjfAHFcTM2HVEXea1SvnXk" + md = id_org.getImgData(pre=non_existent) + assert md is None + + img = bytearray() + for chunk in id_org.getImg(non_existent): + img.extend(chunk) + + assert len(img) == 0 + + +def test_base_organizer_inheritance(): + """Test that Organizer and IdentifierOrganizer properly inherit from BaseOrganizer""" + with habbing.openHby(name="test", temp=True) as hby: + contact_org = connecting.Organizer(hby=hby) + id_org = connecting.IdentifierOrganizer(hby=hby) + + # Test inheritance + assert isinstance(contact_org, connecting.BaseOrganizer) + assert isinstance(id_org, connecting.BaseOrganizer) + + # Test that they have all the expected methods + expected_methods = ['update', 'replace', 'set', 'unset', 'rem', 'get', 'list', 'find', 'values', + 'setImg', 'getImgData', 'getImg'] + + for method in expected_methods: + assert hasattr(contact_org, method) + assert hasattr(id_org, method) + assert callable(getattr(contact_org, method)) + assert callable(getattr(id_org, method)) + + # Test that database attributes are set correctly + assert contact_org.cigsdb == hby.db.ccigs + assert contact_org.datadb == hby.db.cons + assert contact_org.fielddb == hby.db.cfld + assert contact_org.imgsdb == hby.db.imgs + + assert id_org.cigsdb == hby.db.icigs + assert id_org.datadb == hby.db.icons + assert id_org.fielddb == hby.db.ifld + assert id_org.imgsdb == hby.db.iimgs From a9e80cd9d21b715f9855b2db32ebb8728eb86cec Mon Sep 17 00:00:00 2001 From: Patrick Vu Date: Wed, 3 Sep 2025 09:45:52 +0700 Subject: [PATCH 2/2] rename signed identifier data to icons + return dict instead of dictionary literal --- src/keri/app/connecting.py | 10 +++++----- src/keri/db/basing.py | 6 +++--- tests/app/test_connecting.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/keri/app/connecting.py b/src/keri/app/connecting.py index b66478fd2..9b4f3d385 100644 --- a/src/keri/app/connecting.py +++ b/src/keri/app/connecting.py @@ -247,10 +247,10 @@ def getImgData(self, pre): if typ is None: return None - return { - "type": bytes(typ).decode("utf-8"), - "length": int.from_bytes(size, "big") - } + return dict( + type=bytes(typ).decode("utf-8"), + length=int.from_bytes(size, "big") + ) def getImg(self, pre): """ Generator that yields image data in 4k chunks for identifier @@ -299,7 +299,7 @@ def __init__(self, hby): super().__init__( hby=hby, cigsdb=hby.db.icigs, - datadb=hby.db.icons, + datadb=hby.db.sids, fielddb=hby.db.ifld, imgsdb=hby.db.iimgs ) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 058a3f87d..241e76ffa 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1288,8 +1288,8 @@ def reopen(self, **kwa): # Signed identifier data, keys by prefix # TODO: clean - self.icons = subing.Suber(db=self, - subkey="icons.") + self.sids = subing.Suber(db=self, + subkey="sids.") # Transferable signatures on identifier data # TODO: clean @@ -1518,7 +1518,7 @@ def clean(self): unsecured = ["hbys", "schema", "states", "rpys", "eans", "tops", "cgms", "exns", "erpy", "kdts", "ksns", "knas", "oobis", "roobi", "woobi", "moobi", "mfa", "rmfa", "cfld", "cons", "ccigs", "cdel", "migs", - "ifld", "icons", "icigs"] + "ifld", "sids", "icigs"] for name in unsecured: srcdb = getattr(self, name) diff --git a/tests/app/test_connecting.py b/tests/app/test_connecting.py index c7ee8c88d..e5f1a62dc 100644 --- a/tests/app/test_connecting.py +++ b/tests/app/test_connecting.py @@ -428,6 +428,6 @@ def test_base_organizer_inheritance(): assert contact_org.imgsdb == hby.db.imgs assert id_org.cigsdb == hby.db.icigs - assert id_org.datadb == hby.db.icons + assert id_org.datadb == hby.db.sids assert id_org.fielddb == hby.db.ifld assert id_org.imgsdb == hby.db.iimgs