better reaction handling

Author: Tsche

runtime/src/api/client.ts

@@ -6,6 +6,8 @@ interface CounterState {
   id: string | null;
   up: number;
   down: number;
+  upvotes: number;
+  reactions: Record<string, number>;
   age: number;
   stale: boolean;
 }
@@ -363,6 +365,8 @@ function emptyState(): ReactionState {
     id: null,
     up: 0,
     down: 0,
+    upvotes: 0,
+    reactions: {},
     age: 0,
     stale: false,
     viewer: "none",
@@ -384,6 +388,8 @@ function isCounterState(value: unknown): value is CounterState {
     (state.id === null || typeof state.id === "string") &&
     Number.isSafeInteger(state.up) && (state.up ?? -1) >= 0 &&
     Number.isSafeInteger(state.down) && (state.down ?? -1) >= 0 &&
+    Number.isSafeInteger(state.upvotes) && (state.upvotes ?? -1) >= 0 &&
+    !!state.reactions && typeof state.reactions === "object" &&
     Number.isSafeInteger(state.age) && (state.age ?? -1) >= 0 &&
     typeof state.stale === "boolean"
   );

src/feedback/api/routes.py

@@ -48,17 +48,31 @@ async def reactions(request: Request) -> Response:
         await container.refresh_stale(site, cached)
     cached = container.databases[site.id].reactions(keys)
     now = int(container.clock())
-    items: dict[str, dict[str, str | int | bool | None]] = {}
+    items: dict[str, dict[str, object]] = {}
     for key in keys:
         item = cached.get(key)
         if item is None:
-            items[key] = {"id": None, "up": 0, "down": 0, "age": 0, "stale": False}
+            items[key] = {
+                "id": None, "up": 0, "down": 0, "upvotes": 0,
+                "reactions": {name: 0 for name in site.reaction_counters},
+                "age": 0, "stale": False,
+            }
             continue
         age = max(0, now - item.fetched_at)
+        reactions = {
+            name: item.reactions.get(name, 0)
+            for name in site.reaction_counters
+        }
+        thumbsup = item.up
+        up = item.up if site.upvote_source == "thumbsup" else item.upvotes
+        if site.upvote_source == "both":
+            up = item.up + item.upvotes
         items[key] = {
             "id": item.node_id,
-            "up": item.up,
-            "down": item.down,
+            "up": up,
+            "down": item.down if site.downvotes else 0,
+            "upvotes": item.upvotes,
+            "reactions": {**reactions, "THUMBS_UP": thumbsup},
             "age": age,
             "stale": age >= site.cache_fresh_seconds,
         }

src/feedback/config.py

@@ -16,6 +16,8 @@ class ConfigError(ValueError):
 
 _SITE_ID = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\Z")
 _MAPPINGS = frozenset({"key", "title", "url", "pathname", "custom", "number"})
+_UPVOTE_SOURCES = frozenset({"thumbsup", "native", "both"})
+_REACTIONS = frozenset({"LAUGH", "HOORAY", "CONFUSED", "HEART", "ROCKET", "EYES"})
 _SERVICE_KEYS = frozenset(
     {
         "public_origin",
@@ -41,6 +43,9 @@ class ConfigError(ValueError):
         "refresh_cooldown_seconds",
         "refresh_sweep_seconds",
         "max_batch_size",
+        "downvotes",
+        "upvote_source",
+        "reaction_counters",
     }
 )
 
@@ -71,6 +76,9 @@ class SiteConfig:
     refresh_cooldown_seconds: int = 5
     refresh_sweep_seconds: int = 86_400
     max_batch_size: int = 100
+    downvotes: bool = True
+    upvote_source: str = "thumbsup"
+    reaction_counters: tuple[str, ...] = tuple(sorted(_REACTIONS))
 
 
 @dataclass(frozen=True, slots=True)
@@ -138,6 +146,18 @@ def load_config(path: Path | str, *, data_directory: Path | None = None) -> Conf
         repository = _string(value, "repository")
         if repository.count("/") != 1 or any(not part for part in repository.split("/")):
             raise ConfigError(f"sites.{site_id}.repository must be owner/name")
+        downvotes = _bool(value, "downvotes", True)
+        upvote_source = _string(value, "upvote_source") if "upvote_source" in value else "thumbsup"
+        if upvote_source not in _UPVOTE_SOURCES:
+            raise ConfigError(f"sites.{site_id}.upvote_source is unsupported")
+        reaction_counters = (
+            tuple(item.upper() for item in _string_list(value, "reaction_counters"))
+            if "reaction_counters" in value else tuple(sorted(_REACTIONS))
+        )
+        if not reaction_counters or any(item not in _REACTIONS for item in reaction_counters):
+            raise ConfigError(f"sites.{site_id}.reaction_counters contains an unsupported reaction")
+        if len(set(reaction_counters)) != len(reaction_counters):
+            raise ConfigError(f"sites.{site_id}.reaction_counters contains duplicates")
 
         sites[site_id] = SiteConfig(
             id=site_id,
@@ -154,6 +174,9 @@ def load_config(path: Path | str, *, data_directory: Path | None = None) -> Conf
                 value, "refresh_sweep_seconds", 3600, 604_800, 86_400
             ),
             max_batch_size=_bounded_int(value, "max_batch_size", 1, 100, 100),
+            downvotes=downvotes,
+            upvote_source=upvote_source,
+            reaction_counters=reaction_counters,
         )
 
     return Config(service, MappingProxyType(sites))
@@ -186,6 +209,13 @@ def _string_list(value: Mapping[str, Any], key: str) -> list[str]:
     return result
 
 
+def _bool(value: Mapping[str, Any], key: str, default: bool) -> bool:
+    result = value.get(key, default)
+    if not isinstance(result, bool):
+        raise ConfigError(f"{key} must be a boolean")
+    return result
+
+
 def _bounded_int(
     value: Mapping[str, Any],
     key: str,

src/feedback/database/migrations/0002_reaction_intents.sql

@@ -0,0 +1,7 @@
+BEGIN IMMEDIATE;
+
+ALTER TABLE discussions ADD COLUMN upvotes INTEGER NOT NULL DEFAULT 0 CHECK (upvotes >= 0);
+
+PRAGMA user_version = 2;
+
+COMMIT;

src/feedback/database/migrations/__init__.py

@@ -1,6 +1,6 @@
 from importlib.resources import files
 
-NAMES = ("0001_initial.sql",)
+NAMES = ("0001_initial.sql", "0002_reaction_intents.sql")
 
 
 def scripts() -> tuple[str, ...]:

src/feedback/database/queries/reactions.sql

@@ -1,3 +1,7 @@
-SELECT resource_id, github_node_id, up_count, down_count, fetched_at
-FROM discussions
-WHERE resource_id IN (SELECT value FROM json_each(?))
+SELECT d.resource_id, d.github_node_id, d.up_count, d.down_count,
+	d.upvotes, d.fetched_at,
+	   COALESCE(json_group_object(rc.reaction, rc.count) FILTER (WHERE rc.reaction IS NOT NULL), '{}')
+FROM discussions d
+LEFT JOIN reaction_counts rc ON rc.resource_id = d.resource_id
+WHERE d.resource_id IN (SELECT value FROM json_each(?))
+GROUP BY d.resource_id

src/feedback/database/queries/tracked_reactions.sql

@@ -1,5 +1,9 @@
-SELECT resource_id, github_node_id, up_count, down_count, fetched_at
-FROM discussions
-WHERE resource_id > ? AND fetched_at <= ?
-ORDER BY resource_id
+SELECT d.resource_id, d.github_node_id, d.up_count, d.down_count,
+	d.upvotes, d.fetched_at,
+	   COALESCE(json_group_object(rc.reaction, rc.count) FILTER (WHERE rc.reaction IS NOT NULL), '{}')
+FROM discussions d
+LEFT JOIN reaction_counts rc ON rc.resource_id = d.resource_id
+WHERE d.resource_id > ? AND d.fetched_at <= ?
+GROUP BY d.resource_id
+ORDER BY d.resource_id
 LIMIT ?

src/feedback/database/queries/update_reactions.sql

@@ -2,6 +2,7 @@ UPDATE discussions
 SET
     up_count = ?,
     down_count = ?,
+    upvotes = ?,
     locked = ?,
     github_updated_at = ?,
     fetched_at = ?

src/feedback/database/sqlite.py

@@ -28,7 +28,9 @@ class ReactionCounts:
     node_id: str
     up: int
     down: int
+    upvotes: int
     fetched_at: int
+    reactions: dict[str, int]
 
 
 @dataclass(frozen=True, slots=True)
@@ -71,7 +73,10 @@ def reactions(self, resource_ids: Iterable[str]) -> dict[str, ReactionCounts]:
         with self.connect() as connection:
             rows = connection.execute(REACTIONS, (json.dumps(keys),)).fetchall()
         return {
-            row[0]: ReactionCounts(node_id=row[1], up=row[2], down=row[3], fetched_at=row[4])
+            row[0]: ReactionCounts(
+                node_id=row[1], up=row[2], down=row[3], upvotes=row[4], fetched_at=row[5],
+                reactions=json.loads(row[6]),
+            )
             for row in rows
         }
 
@@ -86,7 +91,10 @@ def tracked_reactions(
         with self.connect() as connection:
             rows = connection.execute(TRACKED_REACTIONS, (after, fetched_before, limit)).fetchall()
         return [
-            (row[0], ReactionCounts(node_id=row[1], up=row[2], down=row[3], fetched_at=row[4]))
+            (row[0], ReactionCounts(
+                node_id=row[1], up=row[2], down=row[3], upvotes=row[4], fetched_at=row[5],
+                reactions=json.loads(row[6]),
+            ))
             for row in rows
         ]
 
@@ -101,6 +109,8 @@ def put_discussion(
         url: str,
         up: int = 0,
         down: int = 0,
+        upvotes: int = 0,
+        reactions: dict[str, int] | None = None,
         fetched_at: int | None = None,
     ) -> None:
         fetched = int(time.time()) if fetched_at is None else fetched_at
@@ -119,22 +129,42 @@ def put_discussion(
                     fetched,
                 ),
             )
+            connection.execute(
+                "UPDATE discussions SET upvotes = ? WHERE resource_id = ?",
+                (upvotes, resource_id),
+            )
+            if reactions:
+                connection.executemany(
+                    "INSERT OR REPLACE INTO reaction_counts (resource_id, reaction, count) VALUES (?, ?, ?)",
+                    [(resource_id, name, count) for name, count in reactions.items()],
+                )
 
     def update_reactions(
         self,
         *,
         node_id: str,
         up: int,
         down: int,
+        upvotes: int,
+        reactions: dict[str, int],
         locked: bool,
         github_updated_at: int | None,
         fetched_at: int,
     ) -> bool:
         with self.connect() as connection:
             cursor = connection.execute(
                 UPDATE_REACTIONS,
-                (up, down, locked, github_updated_at, fetched_at, node_id),
+                (up, down, upvotes, locked, github_updated_at, fetched_at, node_id),
             )
+            resource = connection.execute(
+                "SELECT resource_id FROM discussions WHERE github_node_id = ?", (node_id,)
+            ).fetchone()
+            if resource is not None:
+                connection.execute("DELETE FROM reaction_counts WHERE resource_id = ?", (resource[0],))
+                connection.executemany(
+                    "INSERT INTO reaction_counts (resource_id, reaction, count) VALUES (?, ?, ?)",
+                    [(resource[0], name, count) for name, count in reactions.items()],
+                )
         return cursor.rowcount == 1
 
     def adjust_reactions(

src/feedback/service/reaction_cache.py

@@ -52,19 +52,21 @@ async def refresh(
                 parsed = _parse_node(node)
                 if parsed is None:
                     continue
-                node_id, up, down, locked, updated_at = parsed
+                node_id, up, down, upvotes, reactions, locked, updated_at = parsed
                 refreshed += database.update_reactions(
                     node_id=node_id,
                     up=up,
                     down=down,
+                    upvotes=upvotes,
+                    reactions=reactions,
                     locked=locked,
                     github_updated_at=updated_at,
                     fetched_at=fetched_at,
                 )
         return refreshed
 
 
-def _parse_node(value: object) -> tuple[str, int, int, bool, int | None] | None:
+def _parse_node(value: object) -> tuple[str, int, int, int, dict[str, int], bool, int | None] | None:
     if value is None:
         return None
     if not isinstance(value, dict):
@@ -94,4 +96,11 @@ def _parse_node(value: object) -> tuple[str, int, int, bool, int | None] | None:
             updated_at = int(datetime.fromisoformat(updated.replace("Z", "+00:00")).timestamp())
         except ValueError as exc:
             raise GitHubError("github_malformed_response") from exc
-    return node_id, counts.get("THUMBS_UP", 0), counts.get("THUMBS_DOWN", 0), locked, updated_at
+    upvotes = value.get("upvoteCount", 0)
+    if isinstance(upvotes, bool) or not isinstance(upvotes, int) or upvotes < 0:
+        raise GitHubError("github_malformed_response")
+    return (
+        node_id, counts.get("THUMBS_UP", 0), counts.get("THUMBS_DOWN", 0), upvotes,
+        {name: count for name, count in counts.items() if name != "THUMBS_UP" and name != "THUMBS_DOWN"},
+        locked, updated_at,
+    )