Merge pull request #21 from xoeye/feature/stack-context-and-oncall-de…

…faults

CHANGES.md

@@ -1,3 +1,9 @@
+## 1.14.0
+
+`StackContext` and `OnCallDefault` utilities for providing new ways of
+injecting keyword arguments across a call stack without polluting all
+your function signatures in between.
+
 ### 1.13.1
 
 Fix regression in `backoff` and associated implementation.

tests/xoto3/dynamodb/get_test.py

@@ -5,6 +5,7 @@
 from xoto3.dynamodb.exceptions import ItemNotFoundException, get_item_exception_type
 from xoto3.dynamodb.get import (
     GetItem,
+    GetItem_kwargs,
     retry_notfound_consistent_read,
     strongly_consistent_get_item,
     strongly_consistent_get_item_if_exists,
@@ -86,3 +87,13 @@ def _fake_get(**kw):
         test_get(ConsistentRead=True)
 
     assert calls == 1
+
+
+def test_consistent_read_via_kwargs(integration_test_id_table, integration_test_id_table_put):
+    item_key = dict(id="item-will-not-immediately-exist")
+    item = dict(item_key, val="felicity")
+
+    integration_test_id_table_put(item)
+
+    with GetItem_kwargs.set_default(dict(ConsistentRead=True)):
+        assert item == GetItem(integration_test_id_table, item_key)

tests/xoto3/utils/contextual_default.py

@@ -0,0 +1,19 @@
+from xoto3.utils.contextual_default import ContextualDefault
+
+IntDefault = ContextualDefault("i", 1)
+
+
+def test_that_the_name_is_used_and_everything_works():
+    @IntDefault.apply
+    def f(a: str, i: int = 2):
+        return i
+
+    assert f("a") == 1
+    with IntDefault.set_default(4):
+        assert f("b") == 4
+        with IntDefault.set_default(7):
+            assert f("c") == 7
+            assert f("c", 8) == 8
+        assert f("d") == 4
+    assert f("e") == 1
+    assert f("f", i=3) == 3

tests/xoto3/utils/oncall_default_test.py

@@ -0,0 +1,90 @@
+# pylint: disable=unused-argument,unused-variable
+from datetime import datetime
+
+import pytest
+
+from xoto3.utils.oncall_default import NotSafeToDefaultError, OnCallDefault
+
+utcnow = OnCallDefault(datetime.utcnow)
+
+
+def test_oncall_default_works_with_pos_or_kw():
+    @utcnow.apply_to("when")
+    def final(a: str, when: datetime = utcnow(), f: float = 1.2):
+        return when
+
+    assert final("a") <= utcnow()
+
+    val = datetime(1888, 8, 8, 8, 8, 8)
+    assert val == final("a", when=val)
+    assert val == final("c", f=4.2, when=val)
+
+
+def test_oncall_default_works_with_kw_only():
+    @utcnow.apply_to("when")
+    def f(a: str, *, when: datetime = utcnow()):
+        return when
+
+    val = datetime(1900, 1, 1, 11, 11, 11)
+    assert val == f("3", when=val)
+
+
+def test_deco_works_with_var_kwargs():
+    @utcnow.apply_to("when")
+    def f(**kwargs):
+        return kwargs["when"]
+
+    assert datetime.utcnow() <= f()
+    assert f() <= datetime.utcnow()
+
+    direct = datetime(2012, 12, 12, 12, 12, 12)
+    assert direct == f(when=direct)
+
+
+def test_disallow_positional_without_default():
+    """A positional-possible argument without a default could have a
+    positional argument provided after it and then we'd be unable to tell
+    for sure whether it had been provided intentionally.
+    """
+
+    with pytest.raises(NotSafeToDefaultError):
+
+        @utcnow.apply_to("when")
+        def nope(when: datetime, a: int):
+            pass
+
+
+def test_disallow_not_found_without_var_kwargs():
+
+    with pytest.raises(NotSafeToDefaultError):
+
+        @utcnow.apply_to("notthere")
+        def steve(a: str, *args, b=1, c=2):
+            pass
+
+
+def test_disallow_var_args_name_matches():
+    with pytest.raises(NotSafeToDefaultError):
+        # *args itself has the default value 'new empty tuple', and if
+        # you want to provide a positional default you should give it
+        # a real name.
+        @utcnow.apply_to("args")
+        def felicity(a: str, *args):
+            pass
+
+
+GeorgeKwargs = OnCallDefault(lambda: dict(b=2, c=3))
+
+
+def test_allow_var_kwargs_merge():
+    # kwargs itself is a dict,
+    # and we will perform top-level merging
+    # for you if that's what you want
+
+    @GeorgeKwargs.apply_to("kwargs")
+    def george(a: str, **kwargs):
+        return kwargs
+
+    assert george("1") == dict(b=2, c=3)
+    assert george("2", b=3) == dict(b=3, c=3)
+    assert george("3", c=5, d=78) == dict(b=2, c=5, d=78)

tests/xoto3/utils/stack_context_test.py

@@ -0,0 +1,69 @@
+from contextvars import ContextVar
+from datetime import datetime
+
+from xoto3.utils.oncall_default import OnCallDefault
+from xoto3.utils.stack_context import StackContext, stack_context, unwrap
+
+NowContext = ContextVar("UtcNow", default=datetime.utcnow)
+
+
+def test_stack_context():
+    def final():
+        return NowContext.get()()
+
+    def intermediate():
+        return final()
+
+    outer_when = datetime(2018, 9, 9, 9, 9, 9)
+
+    def outer():
+        with stack_context(NowContext, lambda: outer_when):
+            return intermediate()
+
+    way_outer_when = datetime(2019, 12, 12, 8, 0, 0)
+    with stack_context(NowContext, lambda: way_outer_when):
+        assert way_outer_when == intermediate()
+        assert outer_when == outer()
+
+    assert NowContext.get() != outer_when
+    assert NowContext.get() != way_outer_when
+
+
+def test_composes_with_oncall_default():
+
+    when = OnCallDefault(unwrap(NowContext.get))
+
+    @when.apply_to("now")
+    def f(now: datetime = when()):
+        assert isinstance(now, datetime)
+        return now
+
+    val = datetime(1922, 8, 3, 1, 2, 1)
+    assert f(now=val) == val
+
+    with stack_context(NowContext, lambda: val):
+        assert val == f()
+        new_val = datetime(888, 8, 8, 8, 8, 8)
+        with stack_context(NowContext, lambda: new_val):
+            assert new_val == f()
+        assert val == f()
+        assert new_val == f(new_val)
+
+    assert f(val) == val
+    assert f(val) <= datetime.utcnow()
+
+
+ConsistentReadContext = StackContext("ConsistentRead", False)
+
+
+def test_StackContext_interface():
+    def f():
+        return ConsistentReadContext()
+
+    def g():
+        return f()
+
+    assert g() is False
+    with ConsistentReadContext.set(True):
+        assert g() is True
+    assert g() is False

xoto3/__about__.py

@@ -1,4 +1,4 @@
 """xoto3"""
-__version__ = "1.13.1"
+__version__ = "1.14.0"
 __author__ = "Peter Gaultney"
 __author_email__ = "[email protected]"

xoto3/dynamodb/batch_get.py

@@ -9,6 +9,7 @@
 from xoto3.backoff import backoff
 from xoto3.dynamodb.types import TableResource
 from xoto3.lazy_session import tll_from_session
+from xoto3.utils.contextual_default import ContextualDefault
 from xoto3.utils.iter import grouper_it, peek
 from xoto3.utils.lazy import Lazy
 
@@ -30,8 +31,14 @@ class KeyItemPair(ty.NamedTuple):
     item: Item  # if empty, the key was not found
 
 
+BatchGetItem_kwargs: ContextualDefault[dict] = ContextualDefault(
+    "batch_get_item_kwargs", dict(), "xoto3-"
+)
+
+
+@BatchGetItem_kwargs.apply
 def BatchGetItem(
-    table: TableResource, keys: ty.Iterable[ItemKey], **kwargs
+    table: TableResource, keys: ty.Iterable[ItemKey], **batch_get_item_kwargs
 ) -> Iterable[KeyItemPair]:
     """Abstracts threading, pagination, and limited deduplication for BatchGetItem.
 
@@ -63,22 +70,26 @@ def key_tuple_item_pair_to_key_item_pair(ktip: KeyTupleItemPair) -> KeyItemPair:
     return (
         key_tuple_item_pair_to_key_item_pair(ktip)
         for ktip in BatchGetItemTupleKeys(
-            table.name, (key_translator(key) for key in keys), canonical_key_attrs_order, **kwargs
+            table.name,
+            (key_translator(key) for key in keys),
+            canonical_key_attrs_order,
+            **batch_get_item_kwargs,
         )
     )
 
 
 KeyTupleItemPair = Tuple[KeyTuple, Item]
 
 
+@BatchGetItem_kwargs.apply
 def BatchGetItemTupleKeys(
     table_name: str,
     key_value_tuples: Iterable[Tuple[KeyAttributeType, ...]],
     key_attr_names: ty.Sequence[str] = ("id",),
     *,
     dynamodb_resource=None,
     thread_pool=None,
-    **kwargs,
+    **batch_get_item_kwargs,
 ) -> Iterable[KeyTupleItemPair]:
     """Gets multiple items from the same table in as few round trips as possible.
 
@@ -145,7 +156,7 @@ def partial_get_single_batch(key_values_batch: Set[Tuple[KeyAttributeType, ...]]
                 key_values_batch,
                 key_attr_names,
                 dynamodb_resource=_DYNAMODB_RESOURCE(),
-                **kwargs,
+                **batch_get_item_kwargs,
             )
 
         # threaded implementation
@@ -160,7 +171,11 @@ def partial_get_single_batch(key_values_batch: Set[Tuple[KeyAttributeType, ...]]
         # single-threaded serial batches
         for key_values_batch_set in batches_of_100_iter:
             results = _get_single_batch(
-                table_name, key_values_batch_set, key_attr_names, dynamodb_resource=ddbr, **kwargs
+                table_name,
+                key_values_batch_set,
+                key_attr_names,
+                dynamodb_resource=ddbr,
+                **batch_get_item_kwargs,
             )
             for key_value_tuple, item in results:
                 total_count += 1
@@ -180,7 +195,7 @@ def _get_single_batch(
     key_attr_names: ty.Sequence[str] = ("id",),
     *,
     dynamodb_resource=None,
-    **kwargs,
+    **batch_get_item_kwargs,
 ) -> List[KeyTupleItemPair]:
     """Does a BatchGetItem of a single batch of 100.
 
@@ -200,7 +215,7 @@ def _get_single_batch(
 
     table_request = {
         "Keys": [_kv_tuple_to_key(kt, key_attr_names) for kt in key_values_batch],
-        **kwargs,
+        **batch_get_item_kwargs,
     }
     output: List[KeyTupleItemPair] = list()
     while table_request and table_request.get("Keys", []):

xoto3/dynamodb/get.py

@@ -2,37 +2,50 @@
 from logging import getLogger
 from typing import Callable, TypeVar, cast
 
+from xoto3.utils.contextual_default import ContextualDefault
+
 from .constants import DEFAULT_ITEM_NAME
 from .exceptions import ItemNotFoundException, raise_if_empty_getitem_response
 from .types import Item, ItemKey, TableResource
 
 logger = getLogger(__name__)
 
 
-def GetItem(Table: TableResource, Key: ItemKey, nicename=DEFAULT_ITEM_NAME, **kwargs) -> Item:
-    """Use this if possible instead of get_item directly
+GetItem_kwargs: ContextualDefault[dict] = ContextualDefault("get_item_kwargs", dict(), "xoto3-")
 
-    because the default behavior of the boto3 get_item is bad (doesn't
-    fail if no item was found).
 
+@GetItem_kwargs.apply
+def GetItem(
+    Table: TableResource, Key: ItemKey, nicename=DEFAULT_ITEM_NAME, **get_item_kwargs,
+) -> Item:
+    """Use this instead of get_item to raise
+    {nicename/Item}NotFoundException when an item is not found.
+
+    ```
+    with GetItem_kwargs.set_default(dict(ConsistentRead=True)):
+        function_that_calls_GetItem(...)
+    ```
+
+    to set the default for ConsistentRead differently in different
+    contexts without drilling parameters all the way down here. Note
+    that an explicitly-provided parameter will always override the
+    default.
     """
     nicename = nicename or DEFAULT_ITEM_NAME  # don't allow empty string
     logger.debug(f"Get{nicename} {Key} from Table {Table.name}")
-    response = Table.get_item(Key={**Key}, **kwargs)
+    response = Table.get_item(Key={**Key}, **get_item_kwargs)
     raise_if_empty_getitem_response(response, nicename=nicename, key=Key, table_name=Table.name)
     return response["Item"]
 
 
 def strongly_consistent_get_item(
     table: TableResource, key: ItemKey, *, nicename: str = DEFAULT_ITEM_NAME,
 ) -> Item:
-    """This is the default getter for a reason.
-
-    GetItem raises if the item does not exist, preventing you from updating
-    something that does not exist.
+    """Shares ItemNotFoundException-raising behavior with GetItem.
 
-    Strongly consistent reads are important when performing updates - if you
-    read a stale copy you will be guaranteed to fail your update.
+    Strongly consistent reads are important when performing
+    transactional updates - if you read a stale copy you will be
+    likely to fail a transaction retry.
     """
     return GetItem(table, key, ConsistentRead=True, nicename=nicename)