escrow metrics

src/keri/app/indirecting.py

@@ -20,6 +20,7 @@
 
 import keri.app.oobiing
 from . import directing, storing, httping, forwarding, agenting, oobiing
+from ..metric import EscrowEnd
 from .habbing import GroupHab
 from .. import help, kering
 from ..core import (eventing, parsing, routing, coring, serdering,
@@ -92,6 +93,8 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo
     app.add_route("/receipts", receiptEnd)
     queryEnd = QueryEnd(hab=hab)
     app.add_route("/query", queryEnd)
+    metricsEnd = EscrowEnd(hby=hby, reger=reger)
+    app.add_route("/metrics", metricsEnd)
 
     server = createHttpServer(host, httpPort, app, keypath, certpath, cafilepath)
     if not server.reopen():

src/keri/metric/__init__.py

@@ -0,0 +1,7 @@
+# -*- encoding: utf-8 -*-
+"""
+KERI
+keri.metric Package
+"""
+
+from .metricing import EscrowEnd

src/keri/metric/metricing.py

@@ -0,0 +1,95 @@
+# -*- encoding: utf-8 -*-
+"""
+KERI
+keri.metric.metricing module
+
+Prometheus metrics endpoints for KERI escrow monitoring
+"""
+import falcon
+
+
+class EscrowEnd:
+    """Prometheus metrics endpoint for escrow counts.
+
+    Exposes escrow counts in Prometheus text format for monitoring/alerting.
+    """
+
+    def __init__(self, hby, reger):
+        """Initialize EscrowEnd.
+
+        Parameters:
+            hby (Habery): Habery instance for accessing KEL escrows
+            reger (Reger): Registry for accessing TEL escrows
+        """
+        self.hby = hby
+        self.reger = reger
+
+    def on_get(self, req, rep):
+        """GET /metrics - Returns Prometheus format metrics.
+
+        Parameters:
+            req (Request): Falcon HTTP request object
+            rep (Response): Falcon HTTP response object
+        """
+        lines = []
+
+        # Header
+        lines.append("# HELP keri_escrow_count Number of items in each escrow type")
+        lines.append("# TYPE keri_escrow_count gauge")
+
+        # KEL / Baser escrows
+        escrow_counts = [
+            ("unverified_receipts", sum(1 for _ in self.hby.db.getUreItemIter())),
+            ("verified_receipts", sum(1 for _ in self.hby.db.getVreItemIter())),
+            ("out_of_order_events", sum(1 for _ in self.hby.db.getOoeItemIter())),
+            ("partially_witnessed_events", sum(1 for _ in self.hby.db.getPweItemIter())),
+            ("partially_signed_events", sum(1 for _ in self.hby.db.getPseItemIter())),
+            ("likely_duplicitous_events", sum(1 for _ in self.hby.db.getLdeItemIter())),
+            ("unverified_event_indexed_couples", sum(1 for _ in self.hby.db.uwes.getItemIter())),
+            ("query_not_found", sum(1 for _ in self.hby.db.qnfs.getItemIter())),
+            ("partially_delegated_events", sum(1 for _ in self.hby.db.pdes.getItemIter())),
+            ("reply", sum(1 for _ in self.hby.db.rpes.getItemIter())),
+            ("failed_oobi", sum(1 for _ in self.hby.db.eoobi.getItemIter())),
+            ("group_partial_witness", sum(1 for _ in self.hby.db.gpwe.getItemIter())),
+            ("group_delegate", sum(1 for _ in self.hby.db.gdee.getItemIter())),
+            ("delegated_partial_witness", sum(1 for _ in self.hby.db.dpwe.getItemIter())),
+            ("group_partial_signed", sum(1 for _ in self.hby.db.gpse.getItemIter())),
+            ("exchange_partial_signed", sum(1 for _ in self.hby.db.epse.getItemIter())),
+            ("delegated_unanchored", sum(1 for _ in self.hby.db.dune.getItemIter())),
+        ]
+
+        for name, count in escrow_counts:
+            lines.append(f'keri_escrow_count{{type="{name}",layer="kel"}} {count}')
+
+        # TEL / Reger escrows
+        tel_escrow_counts = [
+            ("out_of_order", sum(1 for _ in self.reger.getOotItemIter())),
+            ("partially_witnessed", sum(1 for _ in self.reger.getTweItemIter())),
+            ("anchorless", sum(1 for _ in self.reger.getTaeItemIter())),
+            ("missing_registry", sum(1 for _ in self.reger.mre.getItemIter())),
+            ("broken_chain", sum(1 for _ in self.reger.mce.getItemIter())),
+            ("missing_schema", sum(1 for _ in self.reger.mse.getItemIter())),
+            ("missing_signature", sum(1 for _ in self.reger.cmse.getItemIter())),
+            ("partial_witness", sum(1 for _ in self.reger.tpwe.getItemIter())),
+            ("multisig", sum(1 for _ in self.reger.tmse.getItemIter())),
+            ("event_dissemination", sum(1 for _ in self.reger.tede.getItemIter())),
+        ]
+
+        for name, count in tel_escrow_counts:
+            lines.append(f'keri_escrow_count{{type="{name}",layer="tel"}} {count}')
+
+        # Registry transaction escrows
+        registry_escrow_counts = [
+            ("registry_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", "")))),
+            ("registry_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", "")))),
+            ("credential_missing_registry", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", "")))),
+            ("credential_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", "")))),
+            ("credential_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", "")))),
+        ]
+
+        for name, count in registry_escrow_counts:
+            lines.append(f'keri_escrow_count{{type="{name}",layer="registry"}} {count}')
+
+        rep.content_type = "text/plain; version=0.0.4; charset=utf-8"
+        rep.status = falcon.HTTP_200
+        rep.text = "\n".join(lines) + "\n"

src/keri/vdr/viring.py

@@ -798,6 +798,13 @@ def delTwe(self, key):
         """
         return self.delVal(self.twes, key)
 
+    def getTweItemIter(self):
+        """
+        Return iterator of all items in .twes
+
+        """
+        return self.getTopItemIter(self.twes)
+
     def putTae(self, key, val):
         """
         Use snKey()

tests/app/test_indirecting.py

@@ -19,6 +19,8 @@
 from keri import kering
 from keri import core
 from keri.app import indirecting, storing, habbing, agenting
+from keri.metric import EscrowEnd
+from keri.vdr import viring
 
 
 def test_mailbox_iter():
@@ -294,9 +296,49 @@ def test_createHttpServer(monkeypatch):
     assert isinstance(server.servant, MockServerTls)
 
 
+def test_metrics_end():
+    """Test MetricsEnd returns Prometheus format metrics"""
+    with habbing.openHby(name="test", salt=core.Salter(raw=b'0123456789abcdef').qb64, temp=True) as hby:
+        reger = viring.Reger(name=hby.name, db=hby.db, temp=True)
+
+        app = falcon.App()
+        metricsEnd = EscrowEnd(hby=hby, reger=reger)
+        app.add_route("/metrics", metricsEnd)
+
+        client = testing.TestClient(app)
+
+        # Test GET /metrics
+        res = client.simulate_get("/metrics")
+        assert res.status_code == 200
+        assert "text/plain" in res.headers['Content-Type']
+
+        # Verify Prometheus format
+        body = res.text
+        assert "# HELP keri_escrow_count" in body
+        assert "# TYPE keri_escrow_count gauge" in body
+
+        # Verify KEL escrow metrics present
+        assert 'keri_escrow_count{type="out_of_order_events",layer="kel"}' in body
+        assert 'keri_escrow_count{type="partially_witnessed_events",layer="kel"}' in body
+        assert 'keri_escrow_count{type="unverified_receipts",layer="kel"}' in body
+
+        # Verify TEL escrow metrics present
+        assert 'keri_escrow_count{type="out_of_order",layer="tel"}' in body
+        assert 'keri_escrow_count{type="missing_registry",layer="tel"}' in body
+
+        # Verify registry escrow metrics present
+        assert 'keri_escrow_count{type="registry_missing_anchor",layer="registry"}' in body
+
+        # All counts should be 0 for empty db
+        lines = [l for l in body.split('\n') if l and not l.startswith('#')]
+        for line in lines:
+            assert line.endswith(' 0'), f"Expected count 0, got: {line}"
+
+        reger.close()
 
 
 if __name__ == "__main__":
     test_mailbox_iter()
     test_qrymailbox_iter()
     test_wit_query_ends()
+    test_metrics_end()