From 1f3a702ae341937a2972cb7d3620e14cbeb3c3b3 Mon Sep 17 00:00:00 2001 From: rako Date: Sun, 7 Dec 2025 00:20:47 +0200 Subject: [PATCH 1/9] Add hashtags manage --- README.md | 8 ++++++ chatmail2mastodon/hooks.py | 42 ++++++++++++++++++++++++++++-- chatmail2mastodon/migrations.py | 13 ++++++++++ chatmail2mastodon/orm.py | 7 ++++- chatmail2mastodon/util.py | 46 +++++++++++++++++++++++++++++---- 5 files changed, 108 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 35988fb..4d87cc4 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,14 @@ If someone sends you a direct message in a private 1:1 conversation, it will be and the chat with "friend@example.com" will pop up. +You can follow hashtags and group of hashtags. To do that: + +1. Create a group with the bot +2. Set the name of the group with the hashtags you want to follow separated by a space, for example "#deltachat #chatmail" +3. Send a message to send the group creation to the bot. + +The bot will set the group avatar to the same as the "Home" and "Notifications" avatar. Every message that matches any of the hashtags will be sent in the group. To follow other hashtags you can modify the name or create other groups. To stop any following in a chat, just leave the chat. + To logout from your account: ``` diff --git a/chatmail2mastodon/hooks.py b/chatmail2mastodon/hooks.py index 10412ad..6ec7a59 100644 --- a/chatmail2mastodon/hooks.py +++ b/chatmail2mastodon/hooks.py @@ -22,7 +22,7 @@ from .cli import cli from .migrations import run_migrations -from .orm import Account, DmChat, OAuth, initdb, session_scope +from .orm import Account, DmChat, OAuth, Hashtags, initdb, session_scope from .util import ( TOOT_SEP, Visibility, @@ -82,9 +82,40 @@ def log_event(bot: Bot, accid: int, event: CoreEvent) -> None: chatid = bot.rpc.create_chat_by_contact_id(accid, event.contact_id) send_help(bot, accid, chatid) +@cli.on(events.RawEvent(types=EventType.CHAT_MODIFIED)) +def on_added(bot: Bot, accid: int, event: CoreEvent) -> None: + """Process member-added messages""" + chatid = event.chat_id + + with session_scope() as session: + dmchat = session.query(DmChat).filter_by(chat_id=chatid).first() + if dmchat: + return + home = session.query(Account).filter_by(home=chatid).first() + if home: + return + notif = session.query(Account).filter_by(notifications=chatid).first() + if notif: + return + + contact_ids = [c for c in bot.rpc.get_chat_contacts(accid, chatid) if c != SpecialContactId.SELF] + if len(contact_ids) != 1: + return + + hashtags = session.query(Hashtags).filter_by(chat_id=chatid).first() + if not hashtags: + + contact_id = contact_ids[0] + session.add(Hashtags(chat_id=chatid, contactid=contact_id)) + + try: + bot.rpc.set_chat_profile_image(accid, chatid, MASTODON_LOGO) + except Exception as err: + bot.logger.exception(err) + @cli.on(events.NewMessage(is_info=True)) -def on_info_msg(bot: Bot, accid: int, event: NewMsgEvent) -> None: +def on_removed(bot: Bot, accid: int, event: NewMsgEvent) -> None: """Process member-removed messages""" msg = event.msg chatid = msg.chat_id @@ -120,6 +151,11 @@ def on_info_msg(bot: Bot, accid: int, event: NewMsgEvent) -> None: if dmchat: chats.append(chatid) session.delete(dmchat) + else: + hashtags = session.query(Hashtags).filter_by(chat_id=chatid).first() + if hashtags: + chats.append(chatid) + session.delete(hashtags) for chatid in chats: try: @@ -746,6 +782,8 @@ def send_help(bot: Bot, accid: int, chatid: int) -> None: When a Mastodon user writes a private/direct message to you, a chat will be created for your private conversation with that user. + To follow hashtags, create a group with you and the bot, set the name to the hashtags you want to follow separated by a space (for example "#deltachat #chatmail", you can change anytime). You can create as many groups as you want. + **Available commands** /login - Login on Mastodon. Example: diff --git a/chatmail2mastodon/migrations.py b/chatmail2mastodon/migrations.py index 63946fd..174635d 100644 --- a/chatmail2mastodon/migrations.py +++ b/chatmail2mastodon/migrations.py @@ -149,3 +149,16 @@ def migrate3(bot: Bot, database: sqlite3.Connection) -> None: (conid, row["url"], row["user"], row["client_id"], row["client_secret"]), ) database.execute("DROP TABLE old_oauth") + + +def migrate4(bot: Bot, database: sqlite3.Connection) -> None: + database.execute( + """" + CREATE TABLE hashtags ( + chat_id INTEGER PRIMARY KEY, + contactid INTEGER NOT NULL, + last VARCHAR(1000), + Foreign KEY(contactid) REFERENCES account (id) + ) + """ + ) \ No newline at end of file diff --git a/chatmail2mastodon/orm.py b/chatmail2mastodon/orm.py index 4be5a3d..758c5e8 100644 --- a/chatmail2mastodon/orm.py +++ b/chatmail2mastodon/orm.py @@ -24,7 +24,7 @@ class Account(Base): muted_notif = Column(Boolean) dm_chats = relationship("DmChat", backref="account", cascade="all, delete, delete-orphan") - + hashtags = relationship("Hashtags", backref="account", cascade="all, delete, delete-orphan") class DmChat(Base): __tablename__ = "dmchat" @@ -32,6 +32,11 @@ class DmChat(Base): contactid = Column(Integer, ForeignKey("account.id"), nullable=False) contact = Column(String(1000), nullable=False) +class Hashtags(Base): + __tablename__ = "hashtags" + chat_id = Column(Integer, primary_key=True) + contactid = Column(Integer, ForeignKey("account.id"), nullable=False) + last = Column(String(1000)) class OAuth(Base): __tablename__ = "oauth" diff --git a/chatmail2mastodon/util.py b/chatmail2mastodon/util.py index 62e9745..a462e95 100644 --- a/chatmail2mastodon/util.py +++ b/chatmail2mastodon/util.py @@ -9,6 +9,7 @@ from enum import Enum from tempfile import NamedTemporaryFile from typing import Any, Dict, Generator, Iterable, List, Optional +import json import requests from bs4 import BeautifulSoup @@ -23,7 +24,7 @@ ) from pydub import AudioSegment -from .orm import Account, Client, DmChat, session_scope +from .orm import Account, Client, Hashtags, DmChat, session_scope SPAM = [ "/fediversechick/", @@ -308,7 +309,10 @@ def _check_mastodon(bot: Bot, args: Namespace) -> None: if muted_home: bot.logger.debug(f"contactid={conid}: Ignoring Home timeline (muted)") else: - _check_home(bot, accid, masto, conid, home_chat, last_home) + _check_home(bot, accid, masto, conid, home_chat, last_home, muted_home) + + _check_hashtags(bot, accid, masto, conid) + bot.logger.debug(f"contactid={conid}: Done checking account") except MastodonUnauthorizedError as ex: bot.logger.exception(ex) @@ -392,7 +396,7 @@ def get_mastodon(api_url: str, token: Optional[str] = None, **kwargs) -> Mastodo return Mastodon( access_token=token, api_base_url=api_url, - ratelimit_method="throw", + ratelimit_method="wait", session=web, **kwargs, ) @@ -570,7 +574,7 @@ def _check_notifications( def _check_home( - bot: Bot, accid: int, masto: Mastodon, conid: int, home_chat: int, last_id: str + bot: Bot, accid: int, masto: Mastodon, conid: int, home_chat: int, last_id: str, muted_home: bool ) -> None: me = masto.me() bot.logger.debug(f"contactid={conid}: Getting Home timeline (last_id={last_id})") @@ -580,7 +584,39 @@ def _check_home( acc = session.query(Account).filter_by(id=conid).first() acc.last_home = last_id = toots[0].id toots = [toot for toot in toots if me.id not in [acc.id for acc in toot.mentions]] + bot.logger.debug(f"contactid={conid}: Home: {len(toots)} new entries (last_id={last_id})") if toots: + for reply in toots2replies(bot, reversed(toots)): + bot.rpc.send_msg(accid, home_chat, reply) + +def _check_hashtags( + bot: Bot, accid: int, masto: Mastodon, conid: int +) -> None: + chats = [] + with session_scope() as session: + hashtags_chats = session.query(Hashtags).filter_by(contactid=conid) + for chat in hashtags_chats: + chats.append((chat.last, chat.chat_id)) + + for (last, chat_id) in chats: + toots = [] + info = bot.rpc.get_basic_chat_info(accid, chat_id) + tags = [tag for tag in re.split(r'\W+', info.name) if tag != ''] + bot.logger.debug(f"contactid={conid}: Getting {len(tags)} hashtag timelines in {len(chats)} chats") + + lasts = json.loads(last if last else "{}") + newlasts = {} + + for tag in tags: + t = masto.timeline_hashtag(tag, min_id=lasts.get(tag), limit=100) + toots.extend(t) + newlasts[tag] = t[0].id if t else lasts.get(tag) + + bot.logger.debug(f"{len(toots)} toots matching {info.name}") for reply in toots2replies(bot, reversed(toots)): - bot.rpc.send_msg(accid, home_chat, reply) + bot.rpc.send_msg(accid, chat_id, reply) + + with session_scope() as session: + chat = session.query(Hashtags).filter_by(chat_id=chat_id).first() + chat.last = json.dumps(newlasts) \ No newline at end of file From 5b78b62fafd6c7d047097e9fce2dfa81170edebb Mon Sep 17 00:00:00 2001 From: rako Date: Sun, 7 Dec 2025 19:19:39 +0200 Subject: [PATCH 2/9] Actually do throw and catch rate limit error --- chatmail2mastodon/util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chatmail2mastodon/util.py b/chatmail2mastodon/util.py index a462e95..1d76fd7 100644 --- a/chatmail2mastodon/util.py +++ b/chatmail2mastodon/util.py @@ -21,6 +21,7 @@ MastodonNetworkError, MastodonServerError, MastodonUnauthorizedError, + MastodonRatelimitError ) from pydub import AudioSegment @@ -333,7 +334,7 @@ def _check_mastodon(bot: Bot, args: Namespace) -> None: chatid = bot.rpc.create_chat_by_contact_id(accid, conid) text = f"❌ ERROR Your account was logged out: {ex}" bot.rpc.send_msg(accid, chatid, MsgData(text=text)) - except (MastodonNetworkError, MastodonServerError) as ex: + except (MastodonNetworkError, MastodonServerError, MastodonRatelimitError) as ex: bot.logger.exception(ex) except Exception as ex: # noqa bot.logger.exception(ex) @@ -396,7 +397,7 @@ def get_mastodon(api_url: str, token: Optional[str] = None, **kwargs) -> Mastodo return Mastodon( access_token=token, api_base_url=api_url, - ratelimit_method="wait", + ratelimit_method="throw", session=web, **kwargs, ) From 8966150e195a2859a8e9af97394c2373f3a03588 Mon Sep 17 00:00:00 2001 From: rako Date: Sun, 7 Dec 2025 19:31:26 +0200 Subject: [PATCH 3/9] Remove duplicates from multiple hashtags and re-sort them --- chatmail2mastodon/util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/chatmail2mastodon/util.py b/chatmail2mastodon/util.py index 1d76fd7..6f9ff4a 100644 --- a/chatmail2mastodon/util.py +++ b/chatmail2mastodon/util.py @@ -611,9 +611,12 @@ def _check_hashtags( for tag in tags: t = masto.timeline_hashtag(tag, min_id=lasts.get(tag), limit=100) - toots.extend(t) + toots.extend([tt for tt in t if tt.id not in [to.id for to in toots]]) # Remove duplicates newlasts[tag] = t[0].id if t else lasts.get(tag) + # re-sort + toots.sort(key=lambda s: s.edited_at if s.edited_at else s.created_at) + bot.logger.debug(f"{len(toots)} toots matching {info.name}") for reply in toots2replies(bot, reversed(toots)): bot.rpc.send_msg(accid, chat_id, reply) From 22f802c45e49087e207ffd486758bca99ee0a7bb Mon Sep 17 00:00:00 2001 From: rako Date: Mon, 8 Dec 2025 13:37:24 +0200 Subject: [PATCH 4/9] Randomize list of chats When there are too many hashtags timelines to check we get throttled and the process stops for the user. If the order is always the same some hashtags are never being checked. By randomizing we give all of them a chance --- chatmail2mastodon/util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chatmail2mastodon/util.py b/chatmail2mastodon/util.py index 6f9ff4a..c77fbee 100644 --- a/chatmail2mastodon/util.py +++ b/chatmail2mastodon/util.py @@ -10,6 +10,7 @@ from tempfile import NamedTemporaryFile from typing import Any, Dict, Generator, Iterable, List, Optional import json +import random import requests from bs4 import BeautifulSoup @@ -600,7 +601,8 @@ def _check_hashtags( for chat in hashtags_chats: chats.append((chat.last, chat.chat_id)) - for (last, chat_id) in chats: + # Randomize chats to have a chance to check them all in case of throttling + for (last, chat_id) in random.sample(chats, k=len(chats)): toots = [] info = bot.rpc.get_basic_chat_info(accid, chat_id) tags = [tag for tag in re.split(r'\W+', info.name) if tag != ''] From abeb5d74931de9f060f72a210dab12e6f2af15f3 Mon Sep 17 00:00:00 2001 From: rako Date: Tue, 16 Dec 2025 20:07:27 +0300 Subject: [PATCH 5/9] Don't consider a chat a list of hashtags if not all words starts with a # --- chatmail2mastodon/hooks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chatmail2mastodon/hooks.py b/chatmail2mastodon/hooks.py index 6ec7a59..bbf3ed6 100644 --- a/chatmail2mastodon/hooks.py +++ b/chatmail2mastodon/hooks.py @@ -102,6 +102,10 @@ def on_added(bot: Bot, accid: int, event: CoreEvent) -> None: if len(contact_ids) != 1: return + info = bot.rpc.get_basic_chat_info(accid, chatid) + tags = re.split(r'[ ,]+', info.name) + if False in [tag.startswith('#') for tag in tags]: + return hashtags = session.query(Hashtags).filter_by(chat_id=chatid).first() if not hashtags: From c47033f28f1873b18d780d610289ab76712a1504 Mon Sep 17 00:00:00 2001 From: rako Date: Thu, 18 Dec 2025 22:26:36 +0300 Subject: [PATCH 6/9] Fix import and skip newly created chats with no name yet --- chatmail2mastodon/hooks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chatmail2mastodon/hooks.py b/chatmail2mastodon/hooks.py index bbf3ed6..e503542 100644 --- a/chatmail2mastodon/hooks.py +++ b/chatmail2mastodon/hooks.py @@ -4,6 +4,7 @@ from argparse import Namespace from pathlib import Path from threading import Thread +import re import mastodon from deltachat2 import ( @@ -104,11 +105,11 @@ def on_added(bot: Bot, accid: int, event: CoreEvent) -> None: info = bot.rpc.get_basic_chat_info(accid, chatid) tags = re.split(r'[ ,]+', info.name) - if False in [tag.startswith('#') for tag in tags]: + if len(info.name.strip()) == 0 or False in [tag.startswith('#') for tag in tags]: return + hashtags = session.query(Hashtags).filter_by(chat_id=chatid).first() if not hashtags: - contact_id = contact_ids[0] session.add(Hashtags(chat_id=chatid, contactid=contact_id)) From cc3d00057cb5142f3bc6b2afac1eb147071e2c78 Mon Sep 17 00:00:00 2001 From: rako Date: Wed, 24 Dec 2025 22:33:45 +0300 Subject: [PATCH 7/9] Add description of image to text --- chatmail2mastodon/util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chatmail2mastodon/util.py b/chatmail2mastodon/util.py index c77fbee..754db2e 100644 --- a/chatmail2mastodon/util.py +++ b/chatmail2mastodon/util.py @@ -101,8 +101,10 @@ def toot2reply(toot: AttribAccessDict) -> MsgData: reply.override_sender_name = _get_name(toot.account) if toot.media_attachments: - reply.file = toot.media_attachments.pop(0).url - text += "\n".join(media.url for media in toot.media_attachments) + "\n\n" + first = toot.media_attachments.pop(0) + reply.file = first.url + text += f"[{first.description}]\n\n" + text += "\n\n".join(f"{media.url}\n[{media.description}]" for media in toot.media_attachments) + "\n\n" soup = BeautifulSoup(toot.content, "html.parser") if toot.mentions: From 8a68df430b4becbf01e8e03dff62c75f6a860c48 Mon Sep 17 00:00:00 2001 From: rako Date: Fri, 26 Dec 2025 23:12:50 +0300 Subject: [PATCH 8/9] Add default "no alt" if there is none --- chatmail2mastodon/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chatmail2mastodon/util.py b/chatmail2mastodon/util.py index 754db2e..115951b 100644 --- a/chatmail2mastodon/util.py +++ b/chatmail2mastodon/util.py @@ -103,8 +103,8 @@ def toot2reply(toot: AttribAccessDict) -> MsgData: if toot.media_attachments: first = toot.media_attachments.pop(0) reply.file = first.url - text += f"[{first.description}]\n\n" - text += "\n\n".join(f"{media.url}\n[{media.description}]" for media in toot.media_attachments) + "\n\n" + text += f"[{first.description or 'no alt'}]\n\n" + text += "\n\n".join(f"{media.url}\n[{media.description or 'no alt'}]" for media in toot.media_attachments) + "\n\n" soup = BeautifulSoup(toot.content, "html.parser") if toot.mentions: @@ -627,4 +627,4 @@ def _check_hashtags( with session_scope() as session: chat = session.query(Hashtags).filter_by(chat_id=chat_id).first() - chat.last = json.dumps(newlasts) \ No newline at end of file + chat.last = json.dumps(newlasts) From d689162c62fa04e091dd527c9561834feda85b5d Mon Sep 17 00:00:00 2001 From: rako Date: Sat, 27 Dec 2025 00:30:08 +0300 Subject: [PATCH 9/9] Use preview url before raw url --- chatmail2mastodon/util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chatmail2mastodon/util.py b/chatmail2mastodon/util.py index 115951b..bbe2cd0 100644 --- a/chatmail2mastodon/util.py +++ b/chatmail2mastodon/util.py @@ -102,7 +102,9 @@ def toot2reply(toot: AttribAccessDict) -> MsgData: if toot.media_attachments: first = toot.media_attachments.pop(0) - reply.file = first.url + reply.file = first.preview_url or first.preview_remote_url or first.url + if reply.file != first.url: + text += first.url + "\n" text += f"[{first.description or 'no alt'}]\n\n" text += "\n\n".join(f"{media.url}\n[{media.description or 'no alt'}]" for media in toot.media_attachments) + "\n\n"