diff --git a/Makefile b/Makefile index 13c3a83b0..2528106e0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.6 +VERSION=1.2.7 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index c027f6c98..04e100e0e 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.6', # also change in src/keri/__init__.py + version='1.2.7', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index cb99ceb42..9826257d2 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.6' # also change in setup.py +__version__ = '1.2.7' # also change in setup.py diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 0f19fc775..578912ddc 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -23,6 +23,7 @@ """ import math +import os from collections import namedtuple, deque from dataclasses import dataclass, asdict, field @@ -33,8 +34,11 @@ from .. import core from ..core import coring from ..db import dbing, subing, koming +from .. import help from ..help import helping +logger = help.ogler.getLogger() + Algoage = namedtuple("Algoage", 'randy salty group extern') Algos = Algoage(randy='randy', salty='salty', group="group", extern="extern") # randy is rerandomize, salty is use salt @@ -251,6 +255,15 @@ def __init__(self, headDirPath=None, perm=None, reopen=False, **kwa): if perm is None: perm = self.Perm # defaults to restricted permissions for non temp + mapSize = os.getenv(dbing.KERIKeeperMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIKeeperMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Keeper, self).__init__(headDirPath=headDirPath, perm=perm, reopen=reopen, **kwa) diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index 09ac24db7..4fcaecd79 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -3,15 +3,18 @@ keri.app.notifying module """ +import os from collections.abc import Iterable from typing import Union, Type -from keri import kering +from keri import kering, help from keri.help import helping from keri.app import signaling from keri.core import coring from keri.db import dbing, subing +logger = help.ogler.getLogger() + def notice(attrs, dt=None, read=False): """ @@ -221,6 +224,15 @@ def __init__(self, name="not", headDirPath=None, reopen=True, **kwa): self.nidx = None self.ncigs = None + mapSize = os.getenv(dbing.KERINoterMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERINoterMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Noter, self).__init__(name=name, headDirPath=headDirPath, reopen=reopen, **kwa) def reopen(self, **kwa): diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index e7124f044..ebfe0ece9 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -3,6 +3,7 @@ keri.app.storing module """ +import os from hio.base import doing from hio.help import decking @@ -49,6 +50,15 @@ def __init__(self, name="mbx", headDirPath=None, reopen=True, **kwa): self.tpcs = None self.msgs = None + mapSize = os.getenv(dbing.KERIMailboxerMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIMailboxerMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Mailboxer, self).__init__(name=name, headDirPath=headDirPath, reopen=reopen, **kwa) def reopen(self, **kwa): diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 59f32bcae..b34c0f4a0 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -595,9 +595,6 @@ def reopenDB(db, clear=False, **kwa): db.close(clear=clear) -KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" - - class Baser(dbing.LMDBer): """ Baser sets up named sub databases with Keri Event Logs within main database @@ -967,11 +964,13 @@ def __init__(self, headDirPath=None, reopen=False, **kwa): self._kevers = dbdict() self._kevers.db = self # assign db for read through cache of kevers - if (mapSize := os.getenv(KERIBaserMapSizeKey)) is not None: + mapSize = os.getenv(dbing.KERIBaserMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: try: self.MapSize = int(mapSize) except ValueError: - logger.error("KERI_BASER_MAP_SIZE must be an integer value >1!") + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIBaserMapSizeKey} or {dbing.KERILMDBMapSizeKey}") raise super(Baser, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 667edb2b2..55ff69a12 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -57,14 +57,24 @@ from hio.base import filing import keri +from .. import help from ..kering import MaxON # maximum ordinal number for seqence or first seen from ..help import helping +logger = help.ogler.getLogger() + ProemSize = 32 # does not include trailing separator MaxProem = int("f"*(ProemSize), 16) SuffixSize = 32 # does not include trailing separator MaxSuffix = int("f"*(SuffixSize), 16) +KERILMDBMapSizeKey = "KERI_LMDB_MAP_SIZE" # fallback +KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" +KERIRegerMapSizeKey = "KERI_REGER_MAP_SIZE" +KERIKeeperMapSizeKey = "KERI_KEEPER_MAP_SIZE" +KERIMailboxerMapSizeKey = "KERI_MAILBOXER_MAP_SIZE" +KERINoterMapSizeKey = "KERI_NOTER_MAP_SIZE" + def onKey(top, on, *, sep=b'.'): """ Returns: @@ -382,6 +392,7 @@ def __init__(self, readonly=False, **kwa): self.env = None self._version = None self.readonly = True if readonly else False + super(LMDBer, self).__init__(**kwa) def reopen(self, readonly=False, **kwa): diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index e0d0c45ae..886af906a 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -7,6 +7,7 @@ Provides public simple Verifiable Credential Issuance/Revocation Registry A special purpose Verifiable Data Registry (VDR) """ +import os from dataclasses import dataclass, field, asdict @@ -276,6 +277,15 @@ def __init__(self, headDirPath=None, reopen=True, **kwa): else: self._tevers = dict() + mapSize = os.getenv(dbing.KERIRegerMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIRegerMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Reger, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) diff --git a/tests/app/test_keeper_mapsize.py b/tests/app/test_keeper_mapsize.py new file mode 100644 index 000000000..318c4d8ec --- /dev/null +++ b/tests/app/test_keeper_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_keeper_mapsize module + +""" +import os +import pytest + + +def test_keeper_specific_env_var(): + from keri.app import keeping + + os.environ['KERI_KEEPER_MAP_SIZE'] = '150000000' + + try: + keeper = keeping.Keeper(name='test_keeper', temp=True) + assert keeper.MapSize == 150000000 + keeper.close() + finally: + os.environ.pop('KERI_KEEPER_MAP_SIZE', None) + + +def test_keeper_general_env_var_fallback(): + from keri.app import keeping + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + keeper = keeping.Keeper(name='test_keeper_general', temp=True) + assert keeper.MapSize == 250000000 + keeper.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_keeper_specific_takes_precedence(): + from keri.app import keeping + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_KEEPER_MAP_SIZE'] = '200000000' + + try: + keeper = keeping.Keeper(name='test_keeper_precedence', temp=True) + assert keeper.MapSize == 200000000 + keeper.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_KEEPER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_keeper_specific_env_var() + test_keeper_general_env_var_fallback() + test_keeper_specific_takes_precedence() diff --git a/tests/app/test_mailboxer_mapsize.py b/tests/app/test_mailboxer_mapsize.py new file mode 100644 index 000000000..65d3115be --- /dev/null +++ b/tests/app/test_mailboxer_mapsize.py @@ -0,0 +1,55 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_mailboxer_mapsize module + +""" +import os +import pytest + + +def test_mailboxer_specific_env_var(): + from keri.app import storing + + os.environ['KERI_MAILBOXER_MAP_SIZE'] = '150000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer', temp=True) + assert mbx.MapSize == 150000000 + mbx.close() + finally: + os.environ.pop('KERI_MAILBOXER_MAP_SIZE', None) + + +def test_mailboxer_general_env_var_fallback(): + from keri.app import storing + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer_general', temp=True) + assert mbx.MapSize == 250000000 + mbx.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_mailboxer_specific_takes_precedence(): + """Test that KERI_MAILBOXER_MAP_SIZE takes precedence over KERI_LMDB_MAP_SIZE""" + from keri.app import storing + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_MAILBOXER_MAP_SIZE'] = '200000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer_precedence', temp=True) + assert mbx.MapSize == 200000000 + mbx.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_MAILBOXER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_mailboxer_specific_env_var() + test_mailboxer_general_env_var_fallback() + test_mailboxer_specific_takes_precedence() diff --git a/tests/app/test_noter_mapsize.py b/tests/app/test_noter_mapsize.py new file mode 100644 index 000000000..4748092a4 --- /dev/null +++ b/tests/app/test_noter_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_noter_mapsize module + +""" +import os +import pytest + + +def test_noter_specific_env_var(): + from keri.app import notifying + + os.environ['KERI_NOTER_MAP_SIZE'] = '150000000' + + try: + noter = notifying.Noter(name='test_noter', temp=True) + assert noter.MapSize == 150000000 + noter.close() + finally: + os.environ.pop('KERI_NOTER_MAP_SIZE', None) + + +def test_noter_general_env_var_fallback(): + from keri.app import notifying + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + noter = notifying.Noter(name='test_noter_general', temp=True) + assert noter.MapSize == 250000000 + noter.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_noter_specific_takes_precedence(): + from keri.app import notifying + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_NOTER_MAP_SIZE'] = '200000000' + + try: + noter = notifying.Noter(name='test_noter_precedence', temp=True) + assert noter.MapSize == 200000000 + noter.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_NOTER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_noter_specific_env_var() + test_noter_general_env_var_fallback() + test_noter_specific_takes_precedence() diff --git a/tests/db/test_baser_mapsize.py b/tests/db/test_baser_mapsize.py new file mode 100644 index 000000000..3c23d6bb8 --- /dev/null +++ b/tests/db/test_baser_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.db.test_baser_mapsize module + +""" +import os +import pytest + + +def test_baser_specific_env_var(): + from keri.db import basing + + os.environ['KERI_BASER_MAP_SIZE'] = '150000000' + + try: + baser = basing.Baser(name='test_baser', temp=True) + assert baser.MapSize == 150000000 + baser.close() + finally: + os.environ.pop('KERI_BASER_MAP_SIZE', None) + + +def test_baser_general_env_var_fallback(): + from keri.db import basing + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + baser = basing.Baser(name='test_baser_general', temp=True) + assert baser.MapSize == 250000000 + baser.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_baser_specific_takes_precedence(): + from keri.db import basing + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_BASER_MAP_SIZE'] = '200000000' + + try: + baser = basing.Baser(name='test_baser_precedence', temp=True) + assert baser.MapSize == 200000000 + baser.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_BASER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_baser_specific_env_var() + test_baser_general_env_var_fallback() + test_baser_specific_takes_precedence() diff --git a/tests/vdr/test_reger_mapsize.py b/tests/vdr/test_reger_mapsize.py new file mode 100644 index 000000000..df9c4bb47 --- /dev/null +++ b/tests/vdr/test_reger_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.vdr.test_reger_mapsize module + +""" +import os +import pytest + + +def test_reger_specific_env_var(): + from keri.vdr import viring + + os.environ['KERI_REGER_MAP_SIZE'] = '150000000' + + try: + reger = viring.Reger(name='test_reger', temp=True) + assert reger.MapSize == 150000000 + reger.close() + finally: + os.environ.pop('KERI_REGER_MAP_SIZE', None) + + +def test_reger_general_env_var_fallback(): + from keri.vdr import viring + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + reger = viring.Reger(name='test_reger_general', temp=True) + assert reger.MapSize == 250000000 + reger.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_reger_specific_takes_precedence(): + from keri.vdr import viring + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_REGER_MAP_SIZE'] = '200000000' + + try: + reger = viring.Reger(name='test_reger_precedence', temp=True) + assert reger.MapSize == 200000000 + reger.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_REGER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_reger_specific_env_var() + test_reger_general_env_var_fallback() + test_reger_specific_takes_precedence()