Follow hashtags

README.md

@@ -68,6 +68,14 @@ If someone sends you a direct message in a private 1:1 conversation, it will be
 
 and the chat with "[email protected]" will pop up.
 
+You can follow hashtags and group of hashtags. To do that:
+
+1. Create a group with the bot
+2. Set the name of the group with the hashtags you want to follow separated by a space, for example "#deltachat #chatmail"
+3. Send a message to send the group creation to the bot.
+
+The bot will set the group avatar to the same as the "Home" and "Notifications" avatar. Every message that matches any of the hashtags will be sent in the group. To follow other hashtags you can modify the name or create other groups. To stop any following in a chat, just leave the chat.
+
 To logout from your account:
 
 ```

chatmail2mastodon/hooks.py

@@ -4,6 +4,7 @@
 from argparse import Namespace
 from pathlib import Path
 from threading import Thread
+import re
 
 import mastodon
 from deltachat2 import (
@@ -22,7 +23,7 @@
 
 from .cli import cli
 from .migrations import run_migrations
-from .orm import Account, DmChat, OAuth, initdb, session_scope
+from .orm import Account, DmChat, OAuth, Hashtags, initdb, session_scope
 from .util import (
     TOOT_SEP,
     Visibility,
@@ -82,9 +83,44 @@ def log_event(bot: Bot, accid: int, event: CoreEvent) -> None:
                 chatid = bot.rpc.create_chat_by_contact_id(accid, event.contact_id)
                 send_help(bot, accid, chatid)
 
+@cli.on(events.RawEvent(types=EventType.CHAT_MODIFIED))
+def on_added(bot: Bot, accid: int, event: CoreEvent) -> None:
+    """Process member-added messages"""
+    chatid = event.chat_id
+
+    with session_scope() as session:
+        dmchat = session.query(DmChat).filter_by(chat_id=chatid).first()
+        if dmchat:
+            return
+        home = session.query(Account).filter_by(home=chatid).first()
+        if home:
+            return
+        notif = session.query(Account).filter_by(notifications=chatid).first()
+        if notif:
+            return
+
+        contact_ids = [c for c in bot.rpc.get_chat_contacts(accid, chatid) if c != SpecialContactId.SELF]
+        if len(contact_ids) != 1:
+            return
+
+        info = bot.rpc.get_basic_chat_info(accid, chatid)
+        tags = re.split(r'[ ,]+', info.name)
+        if len(info.name.strip()) == 0 or False in [tag.startswith('#') for tag in tags]:
+            return
+
+        hashtags = session.query(Hashtags).filter_by(chat_id=chatid).first()
+        if not hashtags:
+            contact_id = contact_ids[0]
+            session.add(Hashtags(chat_id=chatid, contactid=contact_id))
+
+            try:
+                bot.rpc.set_chat_profile_image(accid, chatid, MASTODON_LOGO)
+            except Exception as err:
+                bot.logger.exception(err)
+
 
 @cli.on(events.NewMessage(is_info=True))
-def on_info_msg(bot: Bot, accid: int, event: NewMsgEvent) -> None:
+def on_removed(bot: Bot, accid: int, event: NewMsgEvent) -> None:
     """Process member-removed messages"""
     msg = event.msg
     chatid = msg.chat_id
@@ -120,6 +156,11 @@ def on_info_msg(bot: Bot, accid: int, event: NewMsgEvent) -> None:
             if dmchat:
                 chats.append(chatid)
                 session.delete(dmchat)
+            else:
+                hashtags = session.query(Hashtags).filter_by(chat_id=chatid).first()
+                if hashtags:
+                    chats.append(chatid)
+                    session.delete(hashtags)
 
     for chatid in chats:
         try:
@@ -746,6 +787,8 @@ def send_help(bot: Bot, accid: int, chatid: int) -> None:
 
     When a Mastodon user writes a private/direct message to you, a chat will be created for your private conversation with that user.
 
+    To follow hashtags, create a group with you and the bot, set the name to the hashtags you want to follow separated by a space (for example "#deltachat #chatmail", you can change anytime). You can create as many groups as you want.
+
 **Available commands**
 
 /login - Login on Mastodon. Example:

chatmail2mastodon/migrations.py

@@ -149,3 +149,16 @@ def migrate3(bot: Bot, database: sqlite3.Connection) -> None:
                 (conid, row["url"], row["user"], row["client_id"], row["client_secret"]),
             )
         database.execute("DROP TABLE old_oauth")
+
+
+def migrate4(bot: Bot, database: sqlite3.Connection) -> None:
+    database.execute(
+        """"
+        CREATE TABLE hashtags (
+                    chat_id INTEGER PRIMARY KEY,
+                    contactid INTEGER NOT NULL,
+                    last VARCHAR(1000),
+                    Foreign KEY(contactid) REFERENCES account (id)
+            )
+        """
+    )

chatmail2mastodon/orm.py

@@ -24,14 +24,19 @@ class Account(Base):
     muted_notif = Column(Boolean)
 
     dm_chats = relationship("DmChat", backref="account", cascade="all, delete, delete-orphan")
-
+    hashtags = relationship("Hashtags", backref="account", cascade="all, delete, delete-orphan")
 
 class DmChat(Base):
     __tablename__ = "dmchat"
     chat_id = Column(Integer, primary_key=True)
     contactid = Column(Integer, ForeignKey("account.id"), nullable=False)
     contact = Column(String(1000), nullable=False)
 
+class Hashtags(Base):
+    __tablename__ = "hashtags"
+    chat_id = Column(Integer, primary_key=True)
+    contactid = Column(Integer, ForeignKey("account.id"), nullable=False)
+    last = Column(String(1000))
 
 class OAuth(Base):
     __tablename__ = "oauth"

chatmail2mastodon/util.py

@@ -9,6 +9,8 @@
 from enum import Enum
 from tempfile import NamedTemporaryFile
 from typing import Any, Dict, Generator, Iterable, List, Optional
+import json
+import random
 
 import requests
 from bs4 import BeautifulSoup
@@ -20,10 +22,11 @@
     MastodonNetworkError,
     MastodonServerError,
     MastodonUnauthorizedError,
+    MastodonRatelimitError
 )
 from pydub import AudioSegment
 
-from .orm import Account, Client, DmChat, session_scope
+from .orm import Account, Client, Hashtags, DmChat, session_scope
 
 SPAM = [
     "/fediversechick/",
@@ -98,8 +101,12 @@ def toot2reply(toot: AttribAccessDict) -> MsgData:
         reply.override_sender_name = _get_name(toot.account)
 
     if toot.media_attachments:
-        reply.file = toot.media_attachments.pop(0).url
-        text += "\n".join(media.url for media in toot.media_attachments) + "\n\n"
+        first = toot.media_attachments.pop(0)
+        reply.file = first.preview_url or first.preview_remote_url or first.url
+        if reply.file != first.url:
+            text += first.url + "\n"
+        text += f"[{first.description or 'no alt'}]\n\n"
+        text += "\n\n".join(f"{media.url}\n[{media.description or 'no alt'}]" for media in toot.media_attachments) + "\n\n"
 
     soup = BeautifulSoup(toot.content, "html.parser")
     if toot.mentions:
@@ -308,7 +315,10 @@ def _check_mastodon(bot: Bot, args: Namespace) -> None:
                     if muted_home:
                         bot.logger.debug(f"contactid={conid}: Ignoring Home timeline (muted)")
                     else:
-                        _check_home(bot, accid, masto, conid, home_chat, last_home)
+                        _check_home(bot, accid, masto, conid, home_chat, last_home, muted_home)
+
+                    _check_hashtags(bot, accid, masto, conid)
+
                     bot.logger.debug(f"contactid={conid}: Done checking account")
                 except MastodonUnauthorizedError as ex:
                     bot.logger.exception(ex)
@@ -329,7 +339,7 @@ def _check_mastodon(bot: Bot, args: Namespace) -> None:
                     chatid = bot.rpc.create_chat_by_contact_id(accid, conid)
                     text = f"❌ ERROR Your account was logged out: {ex}"
                     bot.rpc.send_msg(accid, chatid, MsgData(text=text))
-                except (MastodonNetworkError, MastodonServerError) as ex:
+                except (MastodonNetworkError, MastodonServerError, MastodonRatelimitError) as ex:
                     bot.logger.exception(ex)
                 except Exception as ex:  # noqa
                     bot.logger.exception(ex)
@@ -570,7 +580,7 @@ def _check_notifications(
 
 
 def _check_home(
-    bot: Bot, accid: int, masto: Mastodon, conid: int, home_chat: int, last_id: str
+    bot: Bot, accid: int, masto: Mastodon, conid: int, home_chat: int, last_id: str, muted_home: bool
 ) -> None:
     me = masto.me()
     bot.logger.debug(f"contactid={conid}: Getting Home timeline (last_id={last_id})")
@@ -580,7 +590,43 @@ def _check_home(
             acc = session.query(Account).filter_by(id=conid).first()
             acc.last_home = last_id = toots[0].id
         toots = [toot for toot in toots if me.id not in [acc.id for acc in toot.mentions]]
+
     bot.logger.debug(f"contactid={conid}: Home: {len(toots)} new entries (last_id={last_id})")
     if toots:
+	    for reply in toots2replies(bot, reversed(toots)):
+	        bot.rpc.send_msg(accid, home_chat, reply)
+
+def _check_hashtags(
+    bot: Bot, accid: int, masto: Mastodon, conid: int
+) -> None:
+    chats = []
+    with session_scope() as session:
+        hashtags_chats = session.query(Hashtags).filter_by(contactid=conid)
+        for chat in hashtags_chats:
+            chats.append((chat.last, chat.chat_id))
+
+    # Randomize chats to have a chance to check them all in case of throttling
+    for (last, chat_id) in random.sample(chats, k=len(chats)):
+        toots = []
+        info = bot.rpc.get_basic_chat_info(accid, chat_id)
+        tags = [tag for tag in re.split(r'\W+', info.name) if tag != '']
+        bot.logger.debug(f"contactid={conid}: Getting {len(tags)} hashtag timelines in {len(chats)} chats")
+
+        lasts = json.loads(last if last else "{}")
+        newlasts = {}
+
+        for tag in tags:
+            t = masto.timeline_hashtag(tag, min_id=lasts.get(tag), limit=100)
+            toots.extend([tt for tt in t if tt.id not in [to.id for to in toots]]) # Remove duplicates
+            newlasts[tag] = t[0].id if t else lasts.get(tag)
+
+        # re-sort
+        toots.sort(key=lambda s: s.edited_at if s.edited_at else s.created_at)
+
+        bot.logger.debug(f"{len(toots)} toots matching {info.name}")
         for reply in toots2replies(bot, reversed(toots)):
-            bot.rpc.send_msg(accid, home_chat, reply)
+            bot.rpc.send_msg(accid, chat_id, reply)
+
+        with session_scope() as session:
+            chat = session.query(Hashtags).filter_by(chat_id=chat_id).first()
+            chat.last = json.dumps(newlasts)