From 54c6d0b685c198d055eef3db93ccd81b6058c3f6 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 15:58:55 -0700 Subject: [PATCH 01/24] Add schema changes for status and game. --- statbot/game.py | 22 ++++++++++++++++++++++ statbot/sql.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 statbot/game.py diff --git a/statbot/game.py b/statbot/game.py new file mode 100644 index 0000000..34489ed --- /dev/null +++ b/statbot/game.py @@ -0,0 +1,22 @@ +# +# game.py +# +# statbot - Store Discord records for later analysis +# Copyright (c) 2017 Ammon Smith +# +# statbot is available free of charge under the terms of the MIT +# License. You are free to redistribute and/or modify it under those +# terms. It is distributed in the hopes that it will be useful, but +# WITHOUT ANY WARRANTY. See the LICENSE file for more details. +# + +from enum import Enum + +__all__ = [ + 'GameType', +] + +class GameType(Enum): + NOTHING = -1 + PLAYING = 0 + STREAMING = 1 diff --git a/statbot/sql.py b/statbot/sql.py index 3d05223..5005322 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -25,6 +25,7 @@ from .audit_log import AuditLogData from .cache import LruCache from .emoji import EmojiData +from .game import GameType from .mention import MentionType from .util import null_logger @@ -173,6 +174,34 @@ def reaction_values(reaction, user, current): 'guild_id': reaction.message.guild.id, } +def game_values(member): + values = { + 'timestamp': datetime.now(), + 'user_id': member.id, + } + + if member.game is None: + values.update( + type=GameType.NOTHING, + name=None, + url=None, + ) + else: + values.update( + type=GameType(member.game.type), + name=member.game.name, + url=member.game.url, + ) + + return values + +def status_values(member): + return { + 'timestamp': datetime.now(), + 'user_id': member.id, + 'status': member.status, + } + class _Transaction: __slots__ = ( 'conn', @@ -225,6 +254,8 @@ class DiscordSqlHandler: 'tb_messages', 'tb_reactions', 'tb_typing', + 'tb_playing', + 'tb_status', 'tb_pins', 'tb_mentions', 'tb_guilds', @@ -290,6 +321,18 @@ def __init__(self, addr, cache_size, logger=null_logger): Column('guild_id', BigInteger, ForeignKey('guilds.guild_id')), UniqueConstraint('timestamp', 'user_id', 'channel_id', 'guild_id', name='uq_typing')) + self.tb_playing = Table('playing', meta, + Column('timestamp', DateTime), + Column('user_id', BigInteger, ForeignKey('users.user_id')), + Column('type', Enum(GameType)), + Column('name', String, nullable=True), + Column('url', String, nullable=True), + UniqueConstraint('timestamp', 'user_id', name='uq_playing')) + self.tb_status = Table('status', meta, + Column('timestamp', DateTime), + Column('user_id', BigInteger, ForeignKey('users.user_id')), + Column('status', Enum(discord.Status)), + UniqueConstraint('timestamp', 'user_id', name='uq_status')) self.tb_pins = Table('pins', meta, Column('pin_id', BigInteger, primary_key=True), Column('message_id', BigInteger, ForeignKey('messages.message_id'), From dd590cd4c9ab6e5a94fe2817a749635146b237b1 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:13:54 -0700 Subject: [PATCH 02/24] Hook in events for status and game changes. --- statbot/client.py | 8 +++++++- statbot/sql.py | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/statbot/client.py b/statbot/client.py index 34042a7..2a8fcda 100644 --- a/statbot/client.py +++ b/statbot/client.py @@ -29,7 +29,7 @@ def member_needs_update(before, after): change we will ignore. ''' - for attr in ('name', 'discriminator', 'nick', 'avatar', 'roles'): + for attr in ('name', 'discriminator', 'nick', 'avatar', 'roles', 'game', 'status'): if getattr(before, attr) != getattr(after, attr): return True return False @@ -398,6 +398,12 @@ async def on_member_update(self, before, after): self.sql.update_user(trans, after) self.sql.update_member(trans, after) + if before.game != after.game: + self.sql.playing(trans, after) + + if before.status != after.status: + self.sql.status_change(trans, after) + async def on_guild_role_create(self, role): self._log_ignored(f"Role {role.id} was created in guild {role.guild.id}") if not await self._accept_guild(role.guild): diff --git a/statbot/sql.py b/statbot/sql.py index 5005322..e8f7913 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -174,9 +174,9 @@ def reaction_values(reaction, user, current): 'guild_id': reaction.message.guild.id, } -def game_values(member): +def game_values(member, when): values = { - 'timestamp': datetime.now(), + 'timestamp': when, 'user_id': member.id, } @@ -273,6 +273,8 @@ class DiscordSqlHandler: 'message_cache', 'typing_cache', + 'playing_cache', + 'status_cache', 'guild_cache', 'channel_cache', 'voice_channel_cache', @@ -458,6 +460,8 @@ def __init__(self, addr, cache_size, logger=null_logger): # Caches self.message_cache = LruCache(cache_size['event-size']) self.typing_cache = LruCache(cache_size['event-size']) + self.playing_cache = LruCache(cache_size['event-size']) + self.status_cache = LruCache(cache_size['event-size']) self.guild_cache = LruCache(cache_size['lookup-size']) self.channel_cache = LruCache(cache_size['lookup-size']) self.voice_channel_cache = LruCache(cache_size['lookup-size']) @@ -608,7 +612,7 @@ def insert_mentions(self, trans, message): def typing(self, trans, channel, user, when): key = (when, user.id, channel.id) if self.typing_cache.get(key, False): - self.logger.debug(f"Typing lookup is up-to-date") + self.logger.debug("Typing lookup is up-to-date") return self.logger.info(f"Inserting typing event for user {user.id}") @@ -623,6 +627,43 @@ def typing(self, trans, channel, user, when): trans.execute(ins) self.typing_cache[key] = True + # Playing + def playing(self, trans, member): + timestamp = datetime.now() + key = (timestamp, member.id) + values = game_values(member, timestamp) + + if self.playing_cache.get(key, None): + self.logger.debug("Playing lookup is up-to-date") + return + + self.logger.info(f"Inserting playing event for user {member.id}") + ins = self.tb_playing \ + .insert() \ + .values(values) + trans.execute(ins) + self.playing_cache[key] = values + + # Status + def status_change(self, trans, member): + timestamp = datetime.now() + key = (timestamp, member.id) + + if self.status_cache.get(key, None): + self.logger.debug("Status change lookup is up-to-date") + return + + self.logger.info(f"Inserting status change event for user {member.id}") + ins = self.tb_status \ + .insert() \ + .values({ + 'timestamp': timestamp, + 'user_id': member.id, + 'status': member.status, + }) + trans.execute(ins) + self.status_cache[key] = member.status + # Reactions def add_reaction(self, trans, reaction, user): self.logger.info(f"Inserting live reaction for user {user.id} on message {reaction.message.id}") From 300f6c0b936a47a74f88b0b1bb4ec573aed0289a Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:40:10 -0700 Subject: [PATCH 03/24] Fix SQL name conflict. --- statbot/sql.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/statbot/sql.py b/statbot/sql.py index e8f7913..bbe4d59 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -182,26 +182,19 @@ def game_values(member, when): if member.game is None: values.update( - type=GameType.NOTHING, + game_type=GameType.NOTHING, name=None, url=None, ) else: values.update( - type=GameType(member.game.type), + game_type=GameType(member.game.type), name=member.game.name, url=member.game.url, ) return values -def status_values(member): - return { - 'timestamp': datetime.now(), - 'user_id': member.id, - 'status': member.status, - } - class _Transaction: __slots__ = ( 'conn', @@ -255,7 +248,7 @@ class DiscordSqlHandler: 'tb_reactions', 'tb_typing', 'tb_playing', - 'tb_status', + 'tb_status_changes', 'tb_pins', 'tb_mentions', 'tb_guilds', @@ -326,15 +319,15 @@ def __init__(self, addr, cache_size, logger=null_logger): self.tb_playing = Table('playing', meta, Column('timestamp', DateTime), Column('user_id', BigInteger, ForeignKey('users.user_id')), - Column('type', Enum(GameType)), + Column('game_type', Enum(GameType)), Column('name', String, nullable=True), Column('url', String, nullable=True), UniqueConstraint('timestamp', 'user_id', name='uq_playing')) - self.tb_status = Table('status', meta, + self.tb_status_changes = Table('status_changes', meta, Column('timestamp', DateTime), Column('user_id', BigInteger, ForeignKey('users.user_id')), - Column('status', Enum(discord.Status)), - UniqueConstraint('timestamp', 'user_id', name='uq_status')) + Column('user_status', Enum(discord.Status)), + UniqueConstraint('timestamp', 'user_id', name='uq_status_changes')) self.tb_pins = Table('pins', meta, Column('pin_id', BigInteger, primary_key=True), Column('message_id', BigInteger, ForeignKey('messages.message_id'), @@ -654,12 +647,12 @@ def status_change(self, trans, member): return self.logger.info(f"Inserting status change event for user {member.id}") - ins = self.tb_status \ + ins = self.tb_status_changes \ .insert() \ .values({ 'timestamp': timestamp, 'user_id': member.id, - 'status': member.status, + 'user_status': member.status, }) trans.execute(ins) self.status_cache[key] = member.status From e72d79389dd73d63b72ed8d0f6b380341f95b93d Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:40:29 -0700 Subject: [PATCH 04/24] Add game and status it _init_sql() --- statbot/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/statbot/client.py b/statbot/client.py index 2a8fcda..ea8d30b 100644 --- a/statbot/client.py +++ b/statbot/client.py @@ -155,6 +155,8 @@ def _init_sql(self, trans): self.logger.info(f"Processing {len(guild.members)} members...") for member in guild.members: self.sql.upsert_member(trans, member) + self.sql.playing(trans, member) + self.sql.status_change(trans, member) # In case people left while the bot was down self.sql.remove_old_members(trans, guild) From b3ee9d2eda8022c4a209f7e9df4986eda483214c Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:45:36 -0700 Subject: [PATCH 05/24] Fix bug in remove_all_members(). --- statbot/sql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/statbot/sql.py b/statbot/sql.py index bbe4d59..26ce965 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -1045,13 +1045,13 @@ def _insert_role_membership(self, trans, member): .values(values) trans.execute(ins) - def remove_member(self, trans, member): - self.logger.debug(f"Removing member {member.id} from guild {member.guild.id}") + def remove_member(self, trans, user_id, guild_id): + self.logger.debug(f"Removing member {user_id} from guild {guild_id}") upd = self.tb_guild_membership \ .update() \ .where(and_( - self.tb_guild_membership.c.user_id == member.id, - self.tb_guild_membership.c.guild_id == member.guild.id, + self.tb_guild_membership.c.user_id == user_id, + self.tb_guild_membership.c.guild_id == guild_id, )) \ .values(is_member=False) trans.execute(upd) @@ -1079,7 +1079,7 @@ def remove_old_members(self, trans, guild): user_id = row[0] member = guild.get_member(user_id) if member is None: - self.remove_member(trans, FakeMember(id=user_id, guild=guild)) + self.remove_member(trans, user_id, guild.id) def upsert_member(self, trans, member): self.logger.debug(f"Upserting member data for {member.id}") From 660a9da8a3c37404d27a9935e99c0e5c58204c4a Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:50:06 -0700 Subject: [PATCH 06/24] Fix constraint in special insertion. --- statbot/sql.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/statbot/sql.py b/statbot/sql.py index 26ce965..b65481f 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -690,9 +690,7 @@ def insert_reaction(self, trans, reaction, users): self.logger.debug(f"Inserting single reaction {data} from {user.id}") ins = p_insert(self.tb_reactions) \ .values(values) \ - .on_conflict_do_nothing(index_elements=[ - 'message_id', 'emoji_id', 'emoji_unicode', 'user_id', 'created_at', - ]) + .on_conflict_do_nothing(constraint='uq_reactions') trans.execute(ins) def clear_reactions(self, trans, message): From e433b2468b9cb040a30a992d422d82a32b4428df Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 15:58:55 -0700 Subject: [PATCH 07/24] Add schema changes for status and game. --- statbot/sql.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/statbot/sql.py b/statbot/sql.py index b65481f..331c606 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -195,6 +195,13 @@ def game_values(member, when): return values +def status_values(member): + return { + 'timestamp': datetime.now(), + 'user_id': member.id, + 'status': member.status, + } + class _Transaction: __slots__ = ( 'conn', From bdfcb11a7784c70383ea3f5d8f0b4f94a9760938 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:40:10 -0700 Subject: [PATCH 08/24] Fix SQL name conflict. --- statbot/sql.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/statbot/sql.py b/statbot/sql.py index 331c606..b65481f 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -195,13 +195,6 @@ def game_values(member, when): return values -def status_values(member): - return { - 'timestamp': datetime.now(), - 'user_id': member.id, - 'status': member.status, - } - class _Transaction: __slots__ = ( 'conn', From 02382b6df94b0faed78381f64768793f9acfbca3 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 15:58:55 -0700 Subject: [PATCH 09/24] Add schema changes for status and game. --- statbot/sql.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/statbot/sql.py b/statbot/sql.py index b65481f..331c606 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -195,6 +195,13 @@ def game_values(member, when): return values +def status_values(member): + return { + 'timestamp': datetime.now(), + 'user_id': member.id, + 'status': member.status, + } + class _Transaction: __slots__ = ( 'conn', From dd2b464bfd4267d7f57fce2e365d2f79b65d1ac2 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:40:10 -0700 Subject: [PATCH 10/24] Fix SQL name conflict. --- statbot/sql.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/statbot/sql.py b/statbot/sql.py index 331c606..b65481f 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -195,13 +195,6 @@ def game_values(member, when): return values -def status_values(member): - return { - 'timestamp': datetime.now(), - 'user_id': member.id, - 'status': member.status, - } - class _Transaction: __slots__ = ( 'conn', From 85b108f424449e9361a461f6d53d7cb776b6fd68 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:45:36 -0700 Subject: [PATCH 11/24] Fix bug in remove_all_members(). --- statbot/sql.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/statbot/sql.py b/statbot/sql.py index b65481f..67f8c1f 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -10,7 +10,6 @@ # WITHOUT ANY WARRANTY. See the LICENSE file for more details. # -from collections import namedtuple from datetime import datetime import functools @@ -30,7 +29,6 @@ from .util import null_logger Column = functools.partial(Column, nullable=False) -FakeMember = namedtuple('FakeMember', ('guild', 'id')) MAX_ID = 2 ** 63 - 1 From 06013697cd2753cd08b9174ad6e1421872219725 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 15:58:55 -0700 Subject: [PATCH 12/24] Add schema changes for status and game. --- statbot/client.py | 7 ++-- statbot/game.py | 22 ---------- statbot/sql.py | 101 +++++++++++++++++++++++++--------------------- 3 files changed, 58 insertions(+), 72 deletions(-) delete mode 100644 statbot/game.py diff --git a/statbot/client.py b/statbot/client.py index ea8d30b..bcb65c9 100644 --- a/statbot/client.py +++ b/statbot/client.py @@ -157,6 +157,7 @@ def _init_sql(self, trans): self.sql.upsert_member(trans, member) self.sql.playing(trans, member) self.sql.status_change(trans, member) + self.sql.activity_change(trans, member) # In case people left while the bot was down self.sql.remove_old_members(trans, guild) @@ -400,12 +401,12 @@ async def on_member_update(self, before, after): self.sql.update_user(trans, after) self.sql.update_member(trans, after) - if before.game != after.game: - self.sql.playing(trans, after) - if before.status != after.status: self.sql.status_change(trans, after) + if before.activity != after.activity: + self.sql.activity_change(trans, after) + async def on_guild_role_create(self, role): self._log_ignored(f"Role {role.id} was created in guild {role.guild.id}") if not await self._accept_guild(role.guild): diff --git a/statbot/game.py b/statbot/game.py deleted file mode 100644 index 34489ed..0000000 --- a/statbot/game.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# game.py -# -# statbot - Store Discord records for later analysis -# Copyright (c) 2017 Ammon Smith -# -# statbot is available free of charge under the terms of the MIT -# License. You are free to redistribute and/or modify it under those -# terms. It is distributed in the hopes that it will be useful, but -# WITHOUT ANY WARRANTY. See the LICENSE file for more details. -# - -from enum import Enum - -__all__ = [ - 'GameType', -] - -class GameType(Enum): - NOTHING = -1 - PLAYING = 0 - STREAMING = 1 diff --git a/statbot/sql.py b/statbot/sql.py index 67f8c1f..8464ca5 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -10,6 +10,7 @@ # WITHOUT ANY WARRANTY. See the LICENSE file for more details. # +from collections import defaultdict from datetime import datetime import functools @@ -24,7 +25,6 @@ from .audit_log import AuditLogData from .cache import LruCache from .emoji import EmojiData -from .game import GameType from .mention import MentionType from .util import null_logger @@ -172,25 +172,26 @@ def reaction_values(reaction, user, current): 'guild_id': reaction.message.guild.id, } -def game_values(member, when): - values = { - 'timestamp': when, - 'user_id': member.id, - } +def activity_values(member, when): + values = defaultdict(lambda: None, + timestamp=when, + user_id=member.id, + other={}, + ) - if member.game is None: - values.update( - game_type=GameType.NOTHING, - name=None, - url=None, - ) - else: + if member.activity is not None: values.update( - game_type=GameType(member.game.type), - name=member.game.name, - url=member.game.url, + type=member.activity.type, + start_time=member.activity.start, + end_time=member.activity.end, ) + for attr in ('url', 'state', 'details', 'twitch_name'): + values[attr] = getattr(member.activity, attr, None) + + for attr in ('timestamps', 'assets', 'party'): + values['other'][attr] = getattr(member.activity, 'assets', None) + return values class _Transaction: @@ -245,8 +246,8 @@ class DiscordSqlHandler: 'tb_messages', 'tb_reactions', 'tb_typing', - 'tb_playing', - 'tb_status_changes', + 'tb_status', + 'tb_activities', 'tb_pins', 'tb_mentions', 'tb_guilds', @@ -264,8 +265,8 @@ class DiscordSqlHandler: 'message_cache', 'typing_cache', - 'playing_cache', 'status_cache', + 'activity_cache', 'guild_cache', 'channel_cache', 'voice_channel_cache', @@ -314,18 +315,24 @@ def __init__(self, addr, cache_size, logger=null_logger): Column('guild_id', BigInteger, ForeignKey('guilds.guild_id')), UniqueConstraint('timestamp', 'user_id', 'channel_id', 'guild_id', name='uq_typing')) - self.tb_playing = Table('playing', meta, + self.tb_status = Table('status', meta, Column('timestamp', DateTime), Column('user_id', BigInteger, ForeignKey('users.user_id')), - Column('game_type', Enum(GameType)), - Column('name', String, nullable=True), - Column('url', String, nullable=True), - UniqueConstraint('timestamp', 'user_id', name='uq_playing')) - self.tb_status_changes = Table('status_changes', meta, + Column('status', Enum(discord.Status)), + UniqueConstraint('timestamp', 'user_id', name='uq_status')) + self.tb_activities = Table('activities', meta, Column('timestamp', DateTime), Column('user_id', BigInteger, ForeignKey('users.user_id')), - Column('user_status', Enum(discord.Status)), - UniqueConstraint('timestamp', 'user_id', name='uq_status_changes')) + Column('type', Enum(discord.ActivityType), nullable=True), + Column('name', String, nullable=True), + Column('start_time', DateTime, nullable=True), + Column('end_time', DateTime, nullable=True), + Column('url', String, nullable=True), + Column('state', String, nullable=True), + Column('details', String, nullable=True), + Column('twitch_name', String, nullable=True), + Column('other', JSON), + UniqueConstraint('timestamp', 'user_id', name='uq_activities')) self.tb_pins = Table('pins', meta, Column('pin_id', BigInteger, primary_key=True), Column('message_id', BigInteger, ForeignKey('messages.message_id'), @@ -451,8 +458,8 @@ def __init__(self, addr, cache_size, logger=null_logger): # Caches self.message_cache = LruCache(cache_size['event-size']) self.typing_cache = LruCache(cache_size['event-size']) - self.playing_cache = LruCache(cache_size['event-size']) self.status_cache = LruCache(cache_size['event-size']) + self.activity_cache = LruCache(cache_size['event-size']) self.guild_cache = LruCache(cache_size['lookup-size']) self.channel_cache = LruCache(cache_size['lookup-size']) self.voice_channel_cache = LruCache(cache_size['lookup-size']) @@ -618,23 +625,6 @@ def typing(self, trans, channel, user, when): trans.execute(ins) self.typing_cache[key] = True - # Playing - def playing(self, trans, member): - timestamp = datetime.now() - key = (timestamp, member.id) - values = game_values(member, timestamp) - - if self.playing_cache.get(key, None): - self.logger.debug("Playing lookup is up-to-date") - return - - self.logger.info(f"Inserting playing event for user {member.id}") - ins = self.tb_playing \ - .insert() \ - .values(values) - trans.execute(ins) - self.playing_cache[key] = values - # Status def status_change(self, trans, member): timestamp = datetime.now() @@ -645,16 +635,33 @@ def status_change(self, trans, member): return self.logger.info(f"Inserting status change event for user {member.id}") - ins = self.tb_status_changes \ + ins = self.tb_status \ .insert() \ .values({ 'timestamp': timestamp, 'user_id': member.id, - 'user_status': member.status, + 'status': member.status, }) trans.execute(ins) self.status_cache[key] = member.status + # Activity + def activity_change(self, trans, member): + timestamp = datetime.now() + key = (timestamp, member.id) + + if self.activity_cache.get(key, None): + self.logger.debug("Activity change lookup is up-to-date") + return + + self.logger.info(f"Inserting activity change event for user {member.id}") + values = activity_values(member) + ins = self.tb_activities \ + .insert() \ + .values(values) + trans.execute(ins) + self.activity_cache[key] = values + # Reactions def add_reaction(self, trans, reaction, user): self.logger.info(f"Inserting live reaction for user {user.id} on message {reaction.message.id}") From 508ed5ee1537b8f3d9c943bc09f19a32f44b5860 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:13:54 -0700 Subject: [PATCH 13/24] Hook in events for status and game changes. --- statbot/sql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/statbot/sql.py b/statbot/sql.py index 8464ca5..ed23cf0 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -662,6 +662,8 @@ def activity_change(self, trans, member): trans.execute(ins) self.activity_cache[key] = values +======= +>>>>>>> Hook in events for status and game changes. # Reactions def add_reaction(self, trans, reaction, user): self.logger.info(f"Inserting live reaction for user {user.id} on message {reaction.message.id}") From d8475cfd9d005f7b883303733931b56c874b2690 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:40:10 -0700 Subject: [PATCH 14/24] Fix SQL name conflict. --- statbot/sql.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/statbot/sql.py b/statbot/sql.py index ed23cf0..68737cb 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -640,7 +640,7 @@ def status_change(self, trans, member): .values({ 'timestamp': timestamp, 'user_id': member.id, - 'status': member.status, + 'user_status': member.status, }) trans.execute(ins) self.status_cache[key] = member.status @@ -662,8 +662,6 @@ def activity_change(self, trans, member): trans.execute(ins) self.activity_cache[key] = values -======= ->>>>>>> Hook in events for status and game changes. # Reactions def add_reaction(self, trans, reaction, user): self.logger.info(f"Inserting live reaction for user {user.id} on message {reaction.message.id}") From 4b6cad718a16bb59c671654c60a52ad8bd783132 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 18 Oct 2017 16:40:29 -0700 Subject: [PATCH 15/24] Add game and status it _init_sql() --- statbot/sql.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/statbot/sql.py b/statbot/sql.py index 68737cb..cdf319c 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -1078,8 +1078,7 @@ def remove_old_members(self, trans, guild): )) result = trans.execute(sel) - for row in result.fetchall(): - user_id = row[0] + for user_id, _, _, _, _ in result.fetchall(): member = guild.get_member(user_id) if member is None: self.remove_member(trans, user_id, guild.id) From 51f01228219e4f2192b4ec96f37d486c63aaa98b Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Tue, 3 Apr 2018 16:44:05 -0700 Subject: [PATCH 16/24] Rename 'status' column to avoid conflict. --- statbot/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statbot/sql.py b/statbot/sql.py index cdf319c..842b49a 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -318,7 +318,7 @@ def __init__(self, addr, cache_size, logger=null_logger): self.tb_status = Table('status', meta, Column('timestamp', DateTime), Column('user_id', BigInteger, ForeignKey('users.user_id')), - Column('status', Enum(discord.Status)), + Column('user_status', Enum(discord.Status)), UniqueConstraint('timestamp', 'user_id', name='uq_status')) self.tb_activities = Table('activities', meta, Column('timestamp', DateTime), From 5d3bd67d8d53f7c2b838cd9d8ee72d54b7114fbc Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Tue, 3 Apr 2018 16:51:04 -0700 Subject: [PATCH 17/24] Add UserStatus enum. --- statbot/sql.py | 11 ++++++----- statbot/status.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 statbot/status.py diff --git a/statbot/sql.py b/statbot/sql.py index 842b49a..804763e 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -26,6 +26,7 @@ from .cache import LruCache from .emoji import EmojiData from .mention import MentionType +from .status import UserStatus from .util import null_logger Column = functools.partial(Column, nullable=False) @@ -246,7 +247,7 @@ class DiscordSqlHandler: 'tb_messages', 'tb_reactions', 'tb_typing', - 'tb_status', + 'tb_statuses', 'tb_activities', 'tb_pins', 'tb_mentions', @@ -315,10 +316,10 @@ def __init__(self, addr, cache_size, logger=null_logger): Column('guild_id', BigInteger, ForeignKey('guilds.guild_id')), UniqueConstraint('timestamp', 'user_id', 'channel_id', 'guild_id', name='uq_typing')) - self.tb_status = Table('status', meta, + self.tb_statuses = Table('statuses', meta, Column('timestamp', DateTime), Column('user_id', BigInteger, ForeignKey('users.user_id')), - Column('user_status', Enum(discord.Status)), + Column('user_status', Enum(UserStatus)), UniqueConstraint('timestamp', 'user_id', name='uq_status')) self.tb_activities = Table('activities', meta, Column('timestamp', DateTime), @@ -635,12 +636,12 @@ def status_change(self, trans, member): return self.logger.info(f"Inserting status change event for user {member.id}") - ins = self.tb_status \ + ins = self.tb_statuses \ .insert() \ .values({ 'timestamp': timestamp, 'user_id': member.id, - 'user_status': member.status, + 'user_status': UserStatus.convert(member.status), }) trans.execute(ins) self.status_cache[key] = member.status diff --git a/statbot/status.py b/statbot/status.py new file mode 100644 index 0000000..3b9d3a7 --- /dev/null +++ b/statbot/status.py @@ -0,0 +1,40 @@ +# +# status.py +# +# statbot - Store Discord records for later analysis +# Copyright (c) 2017-2018 Ammon Smith +# +# statbot is available free of charge under the terms of the MIT +# License. You are free to redistribute and/or modify it under those +# terms. It is distributed in the hopes that it will be useful, but +# WITHOUT ANY WARRANTY. See the LICENSE file for more details. +# + +from enum import Enum, unique + +import discord + +__all__ = [ + 'UserStatus', +] + +# Type "discord.Status" type conflicts with some Postgres thing, +# so we duplicate it here under a different name. + +@unique +class UserStatus(Enum): + ONLINE = 'online' + OFFLINE = 'offline' + IDLE = 'idle' + DO_NOT_DISTURB = 'dnd' + + @staticmethod + def convert(status): + return { + discord.Status.online: UserStatus.ONLINE, + discord.Status.offline: UserStatus.OFFLINE, + discord.Status.idle: UserStatus.IDLE, + discord.Status.dnd: UserStatus.DO_NOT_DISTURB, + discord.Status.do_not_disturb: UserStatus.DO_NOT_DISTURB, + discord.Status.invisible: UserStatus.OFFLINE, + }[status] From eb8212e100f5e8a6571d2e981ab51d953bb2cded Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Tue, 3 Apr 2018 17:10:38 -0700 Subject: [PATCH 18/24] Running fixes. --- statbot/client.py | 11 +++++------ statbot/sql.py | 2 +- statbot/status.py | 8 ++++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/statbot/client.py b/statbot/client.py index bcb65c9..4cfced9 100644 --- a/statbot/client.py +++ b/statbot/client.py @@ -29,7 +29,7 @@ def member_needs_update(before, after): change we will ignore. ''' - for attr in ('name', 'discriminator', 'nick', 'avatar', 'roles', 'game', 'status'): + for attr in ('name', 'discriminator', 'nick', 'avatar', 'roles', 'activity', 'status'): if getattr(before, attr) != getattr(after, attr): return True return False @@ -155,7 +155,6 @@ def _init_sql(self, trans): self.logger.info(f"Processing {len(guild.members)} members...") for member in guild.members: self.sql.upsert_member(trans, member) - self.sql.playing(trans, member) self.sql.status_change(trans, member) self.sql.activity_change(trans, member) @@ -191,12 +190,12 @@ async def on_ready(self): self.logger.info("Recording activity in the following guilds:") for id in self.config['guild-ids']: guild = self.get_guild(id) - if guild is not None: - self.logger.info(f"* {guild.name} ({id})") - else: - self.logger.error(f"Unable to find guild ID {id}") + if guild is None: + self.logger.error(f"No guild with id {id}!") exit(1) + self.logger.info(f"* {guild.name} ({id})") + if not self.sql_init: self.logger.info("Initializing SQL lookup tables...") with self.sql.transaction() as trans: diff --git a/statbot/sql.py b/statbot/sql.py index 804763e..2b7374f 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -656,7 +656,7 @@ def activity_change(self, trans, member): return self.logger.info(f"Inserting activity change event for user {member.id}") - values = activity_values(member) + values = activity_values(member, timestamp) ins = self.tb_activities \ .insert() \ .values(values) diff --git a/statbot/status.py b/statbot/status.py index 3b9d3a7..ba93719 100644 --- a/statbot/status.py +++ b/statbot/status.py @@ -23,10 +23,10 @@ @unique class UserStatus(Enum): - ONLINE = 'online' - OFFLINE = 'offline' - IDLE = 'idle' - DO_NOT_DISTURB = 'dnd' + ONLINE = 'ONLINE' + OFFLINE = 'OFFLINE' + IDLE = 'IDLE' + DO_NOT_DISTURB = 'DO_NOT_DISTURB' @staticmethod def convert(status): From 925754f9f19ca93ab4340b0ca4ef1d2f24b688e6 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Tue, 3 Apr 2018 17:10:47 -0700 Subject: [PATCH 19/24] Only add fields to JSON if they exist. --- statbot/sql.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/statbot/sql.py b/statbot/sql.py index 2b7374f..8e9d776 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -191,7 +191,10 @@ def activity_values(member, when): values[attr] = getattr(member.activity, attr, None) for attr in ('timestamps', 'assets', 'party'): - values['other'][attr] = getattr(member.activity, 'assets', None) + try: + values['other'][attr] = getattr(member.activity, attr) + except KeyError: + pass return values From 3b182ee1117960643d7f12359e7fe5297587342e Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Tue, 3 Apr 2018 17:11:00 -0700 Subject: [PATCH 20/24] Other update improvements. --- statbot/sql.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/statbot/sql.py b/statbot/sql.py index 8e9d776..4774877 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -221,6 +221,7 @@ def __exit__(self, type, value, traceback): if (type, value, traceback) == (None, None, None): self.logger.debug("Committing transaction...") self.trans.commit() + self.logger.debug("Committed") else: self.logger.error("Exception occurred in 'with' scope!", exc_info=1) self.logger.debug("Rolling back transaction...") @@ -516,6 +517,9 @@ def add_message(self, trans, message): self.upsert_user(trans, message.author) self.insert_mentions(trans, message) + if isinstance(message.author, discord.Member): + self.upsert_member(trans, message.author) + def edit_message(self, trans, before, after): self.logger.info(f"Updating message {after.id}") upd = self.tb_messages \ @@ -1031,6 +1035,9 @@ def update_member(self, trans, member): .values(nick=member.nick) trans.execute(upd) + self.status_change(trans, member) + self.activity_change(trans, member) + self._delete_role_membership(trans, member) self._insert_role_membership(trans, member) From 89efb3589aebc8373aee9ac9186e3e7019d24ec5 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Fri, 13 Apr 2018 18:32:53 -0700 Subject: [PATCH 21/24] Use constant dictionary for UserStatus conversion. --- statbot/status.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/statbot/status.py b/statbot/status.py index ba93719..f7b9d57 100644 --- a/statbot/status.py +++ b/statbot/status.py @@ -21,6 +21,15 @@ # Type "discord.Status" type conflicts with some Postgres thing, # so we duplicate it here under a different name. +USER_STATUS_CONVERSION = { + discord.Status.online: UserStatus.ONLINE, + discord.Status.offline: UserStatus.OFFLINE, + discord.Status.idle: UserStatus.IDLE, + discord.Status.dnd: UserStatus.DO_NOT_DISTURB, + discord.Status.do_not_disturb: UserStatus.DO_NOT_DISTURB, + discord.Status.invisible: UserStatus.OFFLINE, +} + @unique class UserStatus(Enum): ONLINE = 'ONLINE' @@ -30,11 +39,4 @@ class UserStatus(Enum): @staticmethod def convert(status): - return { - discord.Status.online: UserStatus.ONLINE, - discord.Status.offline: UserStatus.OFFLINE, - discord.Status.idle: UserStatus.IDLE, - discord.Status.dnd: UserStatus.DO_NOT_DISTURB, - discord.Status.do_not_disturb: UserStatus.DO_NOT_DISTURB, - discord.Status.invisible: UserStatus.OFFLINE, - }[status] + return USER_STATUS_CONVERSION[status] From 6e29251cfa523381b8a3bdfeddf1cc6b3cfc1410 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Fri, 13 Apr 2018 19:33:32 -0700 Subject: [PATCH 22/24] Fix pylint issue. --- statbot/status.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/statbot/status.py b/statbot/status.py index f7b9d57..45b5b92 100644 --- a/statbot/status.py +++ b/statbot/status.py @@ -21,15 +21,6 @@ # Type "discord.Status" type conflicts with some Postgres thing, # so we duplicate it here under a different name. -USER_STATUS_CONVERSION = { - discord.Status.online: UserStatus.ONLINE, - discord.Status.offline: UserStatus.OFFLINE, - discord.Status.idle: UserStatus.IDLE, - discord.Status.dnd: UserStatus.DO_NOT_DISTURB, - discord.Status.do_not_disturb: UserStatus.DO_NOT_DISTURB, - discord.Status.invisible: UserStatus.OFFLINE, -} - @unique class UserStatus(Enum): ONLINE = 'ONLINE' @@ -40,3 +31,12 @@ class UserStatus(Enum): @staticmethod def convert(status): return USER_STATUS_CONVERSION[status] + +USER_STATUS_CONVERSION = { + discord.Status.online: UserStatus.ONLINE, + discord.Status.offline: UserStatus.OFFLINE, + discord.Status.idle: UserStatus.IDLE, + discord.Status.dnd: UserStatus.DO_NOT_DISTURB, + discord.Status.do_not_disturb: UserStatus.DO_NOT_DISTURB, + discord.Status.invisible: UserStatus.OFFLINE, +} From a5c697899e0458855984332a2ebb3d248c754b85 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Fri, 13 Apr 2018 22:17:06 -0700 Subject: [PATCH 23/24] Add tracking for voice event changes. --- statbot/client.py | 14 ++++++++++++++ statbot/sql.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/statbot/client.py b/statbot/client.py index 4cfced9..5e763e8 100644 --- a/statbot/client.py +++ b/statbot/client.py @@ -406,6 +406,14 @@ async def on_member_update(self, before, after): if before.activity != after.activity: self.sql.activity_change(trans, after) + async def on_voice_state_update(self, member, before, after): + self._log_ignored(f"Member {member.id} updated their voice state in guild {member.guild.id}") + if not await self._accept_guild(member.guild): + return + + with self.sql.transaction() as trans: + self.sql.voice_state_change(trans, member, after) + async def on_guild_role_create(self, role): self._log_ignored(f"Role {role.id} was created in guild {role.guild.id}") if not await self._accept_guild(role.guild): @@ -449,3 +457,9 @@ async def on_guild_emojis_update(self, guild, before, after): self.sql.add_emoji(trans, emoji) for emoji in before - after: self.sql.remove_emoji(trans, emoji) + + async def on_guild_available(self, guild): + self.logger.info(f"Guild {guild.id} '{guild.name}' is now available.") + + async def on_guild_unavailable(self, guild): + self.logger.info(f"Guild {guild.id} '{guild.name}' is unavailable.") diff --git a/statbot/sql.py b/statbot/sql.py index 4774877..0f7aa10 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -193,11 +193,24 @@ def activity_values(member, when): for attr in ('timestamps', 'assets', 'party'): try: values['other'][attr] = getattr(member.activity, attr) - except KeyError: + except AttributeError: pass return values +def voice_event_values(member, when, voice_state): + return { + 'timestamp': when, + 'user_id': member.id, + 'guild_id': member.guild.id, + 'self_deaf': voice_state.self_deaf, + 'self_mute': voice_state.self_mute, + 'guild_deaf': voice_state.deaf, + 'guild_mute': voice_state.mute, + 'afk': voice_state.afk, + 'voice_channel_id': getattr(voice_state.channel, 'id', None), + } + class _Transaction: __slots__ = ( 'conn', @@ -253,6 +266,7 @@ class DiscordSqlHandler: 'tb_typing', 'tb_statuses', 'tb_activities', + 'tb_voice_events', 'tb_pins', 'tb_mentions', 'tb_guilds', @@ -272,6 +286,7 @@ class DiscordSqlHandler: 'typing_cache', 'status_cache', 'activity_cache', + 'voice_event_cache', 'guild_cache', 'channel_cache', 'voice_channel_cache', @@ -338,6 +353,18 @@ def __init__(self, addr, cache_size, logger=null_logger): Column('twitch_name', String, nullable=True), Column('other', JSON), UniqueConstraint('timestamp', 'user_id', name='uq_activities')) + self.tb_voice_events = Table('voice_events', meta, + Column('timestamp', DateTime, primary_key=True), + Column('user_id', BigInteger, ForeignKey('users.user_id'), primary_key=True), + Column('guild_id', BigInteger, ForeignKey('guilds.guild_id'), primary_key=True), + Column('self_deaf', Boolean), + Column('self_mute', Boolean), + Column('guild_deaf', Boolean), + Column('guild_mute', Boolean), + Column('afk', Boolean), + Column('voice_channel_id', BigInteger, + ForeignKey('voice_channels.voice_channel_id'), nullable=True), + UniqueConstraint('timestamp', 'user_id', 'guild_id', name='uq_voice_events')) self.tb_pins = Table('pins', meta, Column('pin_id', BigInteger, primary_key=True), Column('message_id', BigInteger, ForeignKey('messages.message_id'), @@ -465,6 +492,7 @@ def __init__(self, addr, cache_size, logger=null_logger): self.typing_cache = LruCache(cache_size['event-size']) self.status_cache = LruCache(cache_size['event-size']) self.activity_cache = LruCache(cache_size['event-size']) + self.voice_event_cache = LruCache(cache_size['event-size']) self.guild_cache = LruCache(cache_size['lookup-size']) self.channel_cache = LruCache(cache_size['lookup-size']) self.voice_channel_cache = LruCache(cache_size['lookup-size']) @@ -670,6 +698,23 @@ def activity_change(self, trans, member): trans.execute(ins) self.activity_cache[key] = values + # Voice state + def voice_state_change(self, trans, member, voice_state): + timestamp = datetime.now() + key = (timestamp, member.id, member.guild.id) + + if self.voice_event_cache.get(key, None): + self.logger.debug("Voice state change lookup is up-to-date") + return + + self.logger.info(f"Inserting voice state change event for user {member.id}") + values = voice_event_values(member, timestamp, voice_state) + ins = self.tb_voice_events \ + .insert() \ + .values(values) + trans.execute(ins) + self.voice_event_cache[key] = values + # Reactions def add_reaction(self, trans, reaction, user): self.logger.info(f"Inserting live reaction for user {user.id} on message {reaction.message.id}") From 2e4b2e34f6f0c3d8f457ca54e7f7e7f7e19c7490 Mon Sep 17 00:00:00 2001 From: Ammon Smith Date: Wed, 16 May 2018 21:11:35 -0700 Subject: [PATCH 24/24] Use with_only_columns() to restrict SELECT. --- statbot/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/statbot/sql.py b/statbot/sql.py index 0f7aa10..2631edd 100644 --- a/statbot/sql.py +++ b/statbot/sql.py @@ -1132,9 +1132,10 @@ def remove_old_members(self, trans, guild): self.tb_guild_membership.c.guild_id == guild.id, self.tb_guild_membership.c.is_member == True, )) + sel = sel.with_only_columns([self.tb_guild_membership.c.user_id]) result = trans.execute(sel) - for user_id, _, _, _, _ in result.fetchall(): + for user_id, in result.fetchall(): member = guild.get_member(user_id) if member is None: self.remove_member(trans, user_id, guild.id)