From 13b3d245d49a579f53cf29d9b6f14f22c446a518 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 22 May 2023 13:26:56 +1200 Subject: [PATCH] feat: Summarizer (#15) --- package.json | 6 +- pnpm-lock.yaml | 40 ++++--- src/AutoThreads.ts | 2 +- src/MemberCache.ts | 53 ++++++++++ src/Mentions.ts | 5 +- src/NoEmbed.ts | 5 +- src/Summarizer.ts | 252 ++++++++++++++++++++++++++++++++++++++++++++ src/_common.ts | 6 ++ src/main.ts | 9 +- src/utils/Errors.ts | 4 +- 10 files changed, 355 insertions(+), 27 deletions(-) create mode 100644 src/MemberCache.ts create mode 100644 src/Summarizer.ts diff --git a/package.json b/package.json index f49ca38..41dd0ef 100644 --- a/package.json +++ b/package.json @@ -16,15 +16,17 @@ "author": "", "license": "ISC", "devDependencies": { - "@types/node": "^20.1.7", + "@types/node": "^20.2.3", "prettier": "^2.8.8", "tsc-watch": "^6.0.4", "typescript": "^5.0.4" }, "dependencies": { + "@effect-http/client": "^0.26.1", "@effect/data": "^0.12.2", "@effect/io": "^0.25.12", - "dfx": "^0.45.7", + "@effect/stream": "^0.21.1", + "dfx": "^0.45.8", "dotenv": "^16.0.3", "openai": "^3.2.1" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d94731..73b02e1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,15 +1,21 @@ lockfileVersion: '6.0' dependencies: + '@effect-http/client': + specifier: ^0.26.1 + version: 0.26.1 '@effect/data': specifier: ^0.12.2 version: 0.12.2 '@effect/io': specifier: ^0.25.12 version: 0.25.12 + '@effect/stream': + specifier: ^0.21.1 + version: 0.21.1 dfx: - specifier: ^0.45.7 - version: 0.45.7 + specifier: ^0.45.8 + version: 0.45.8 dotenv: specifier: ^16.0.3 version: 16.0.3 @@ -19,8 +25,8 @@ dependencies: devDependencies: '@types/node': - specifier: ^20.1.7 - version: 20.1.7 + specifier: ^20.2.3 + version: 20.2.3 prettier: specifier: ^2.8.8 version: 2.8.8 @@ -33,12 +39,12 @@ devDependencies: packages: - /@effect-http/client@0.25.2: - resolution: {integrity: sha512-gKZ4A2i6n+PK9DZ/jt8fJ8n80eOmYXcYI2Vp8ztj1PQjwXcylvzmoLfeQD15JmX7vwy+X0fccgrMunbT2Q1W3Q==} + /@effect-http/client@0.26.1: + resolution: {integrity: sha512-QVT5PWM4t0PmTyHMvlK12mzULQ4NEIhE8Kw1/Zdcm/7AoBSUzJzEj2Tbdg1EL3OJlDnNjZS6zahW9jcckkXvJQ==} dependencies: '@effect/data': 0.12.2 '@effect/io': 0.25.12 - '@effect/schema': 0.19.0 + '@effect/schema': 0.19.3 '@effect/stream': 0.21.1 dev: false @@ -52,12 +58,12 @@ packages: '@effect/data': 0.12.2 dev: false - /@effect/schema@0.19.0: - resolution: {integrity: sha512-BCmDLvFJol1uAyCJbCi+Ss0HFf+k/qb4tyvc8Iu4O/7ChNnBiACDWukU3f+7lHZPIh729arzdBoBSQqyKX64iw==} + /@effect/schema@0.19.3: + resolution: {integrity: sha512-jR7o19HBX7PBYRLV1G6ZlM90giRe+lZkA6BqVNamkhNfVsHU4MnGr2lqM2H4fy4MgR744GGXuxCDjVdydkMvAw==} dependencies: '@effect/data': 0.12.2 '@effect/io': 0.25.12 - fast-check: 3.8.1 + fast-check: 3.9.0 dev: false /@effect/stream@0.21.1: @@ -67,8 +73,8 @@ packages: '@effect/io': 0.25.12 dev: false - /@types/node@20.1.7: - resolution: {integrity: sha512-WCuw/o4GSwDGMoonES8rcvwsig77dGCMbZDrZr2x4ZZiNW4P/gcoZXe/0twgtobcTkmg9TuKflxYL/DuwDyJzg==} + /@types/node@20.2.3: + resolution: {integrity: sha512-pg9d0yC4rVNWQzX8U7xb4olIOFuuVL9za3bzMT2pu2SU0SNEi66i2qrvhE2qt0HvkhuCaWJu7pLNOt/Pj8BIrw==} dev: true /asynckit@0.4.0: @@ -113,10 +119,10 @@ packages: engines: {node: '>=0.4.0'} dev: false - /dfx@0.45.7: - resolution: {integrity: sha512-lICO2m8Pq1Wbvip9bR3vVyTcpjYQWqoIzNj+KgFvAkmqsyHeQo5Btk/JRZXdX08ulpn3mtQkyjWxHGEgyRxFvg==} + /dfx@0.45.8: + resolution: {integrity: sha512-DzUTrnyhYzVkNdFKnx2OswaQME9vBBQyNomnDSu7dXyRKeKmraJ/0O7BKeYiQ77qNH5Ap9O7JqLCZ2dYp9C/9g==} dependencies: - '@effect-http/client': 0.25.2 + '@effect-http/client': 0.26.1 '@effect/data': 0.12.2 '@effect/io': 0.25.12 '@effect/stream': 0.21.1 @@ -149,8 +155,8 @@ packages: through: 2.3.8 dev: true - /fast-check@3.8.1: - resolution: {integrity: sha512-WRll9CUIz6jWKgByFlHT2M/1BY3F7lewKl5BBIz5VHAy7B8y5iklK9rVm922kx+0x1hJqdkffuTs008xfIgytQ==} + /fast-check@3.9.0: + resolution: {integrity: sha512-5vBtAK9lt7eQ/g18iBHETnbRK0jskxISmdlkp0Lz9JghEg+TqDL1q26xiK3jQzW5D27Q0V88NeZwkoIi52Qqcw==} engines: {node: '>=8.0.0'} dependencies: pure-rand: 6.0.2 diff --git a/src/AutoThreads.ts b/src/AutoThreads.ts index 419be86..4698b46 100644 --- a/src/AutoThreads.ts +++ b/src/AutoThreads.ts @@ -122,7 +122,7 @@ const make = ({ topicKeyword }: AutoThreadsOptions) => ), Effect.catchTags({ NotValidMessageError: () => Effect.unit(), - DiscordRESTError: logRESTError, + DiscordRESTError: logRESTError(log), }), Effect.catchAllCause(Effect.logErrorCause), ), diff --git a/src/MemberCache.ts b/src/MemberCache.ts new file mode 100644 index 0000000..6bf15c4 --- /dev/null +++ b/src/MemberCache.ts @@ -0,0 +1,53 @@ +import { + Cache, + Data, + Duration, + Effect, + Layer, + Request, + RequestResolver, + Tag, + pipe, +} from "bot/_common" +import { Discord, DiscordREST } from "dfx" +import { DiscordRESTError, ResponseDecodeError } from "dfx/DiscordREST" + +export class GetMember extends Data.TaggedClass("GetMember")<{ + readonly guildId: Discord.Snowflake + readonly userId: Discord.Snowflake +}> {} + +const make = Effect.gen(function* (_) { + const rest = yield* _(DiscordREST) + + interface GetMember + extends Request.Request< + DiscordRESTError | ResponseDecodeError, + Discord.GuildMember + > { + readonly _tag: "GetMember" + readonly guildId: Discord.Snowflake + readonly userId: Discord.Snowflake + } + const GetMember = Request.tagged("GetMember") + + const resolver = RequestResolver.fromFunctionEffect( + ({ guildId, userId }: GetMember) => + Effect.flatMap(rest.getGuildMember(guildId, userId), _ => _.json), + ) + + const cache = yield* _(Request.makeCache(1000, Duration.days(1))) + + return { + get: (guildId: Discord.Snowflake, userId: Discord.Snowflake) => + pipe( + Effect.request(GetMember({ guildId, userId }), resolver), + Effect.withRequestCache(cache), + Effect.withRequestCaching("on"), + ), + } as const +}) + +export interface MemberCache extends Effect.Effect.Success {} +export const MemberCache = Tag() +export const MemberCacheLive = Layer.effect(MemberCache, make) diff --git a/src/Mentions.ts b/src/Mentions.ts index 4250ba6..127e738 100644 --- a/src/Mentions.ts +++ b/src/Mentions.ts @@ -3,7 +3,7 @@ import { OpenAI, OpenAIMessage } from "bot/OpenAI" import { Data, Effect, Layer, pipe } from "bot/_common" import { logRESTError } from "bot/utils/Errors" import * as Str from "bot/utils/String" -import { Discord, DiscordREST } from "dfx" +import { Discord, DiscordREST, Log } from "dfx" import { DiscordGateway } from "dfx/DiscordGateway" class NonEligibleMessage extends Data.TaggedClass("NonEligibleMessage")<{ @@ -15,6 +15,7 @@ const make = Effect.gen(function* (_) { const gateway = yield* _(DiscordGateway) const channels = yield* _(ChannelsCache) const openai = yield* _(OpenAI) + const log = yield* _(Log.Log) const botUser = yield* _( rest.getCurrentUser(), @@ -96,7 +97,7 @@ ${msg.content}`, Effect.catchTags({ NonEligibleMessage: _ => Effect.unit(), NoSuchElementException: _ => Effect.unit(), - DiscordRESTError: logRESTError, + DiscordRESTError: logRESTError(log), }), Effect.catchAllCause(Effect.logErrorCause), ), diff --git a/src/NoEmbed.ts b/src/NoEmbed.ts index 67e283e..329cf0a 100644 --- a/src/NoEmbed.ts +++ b/src/NoEmbed.ts @@ -1,7 +1,7 @@ import { ChannelsCache, ChannelsCacheLive } from "bot/ChannelsCache" import { Config, Data, Effect, Layer, pipe } from "bot/_common" import { logRESTError } from "bot/utils/Errors" -import { Discord, DiscordREST } from "dfx" +import { Discord, DiscordREST, Log } from "dfx" import { DiscordGateway } from "dfx/gateway" class NotValidMessageError extends Data.TaggedClass("NotValidMessageError")<{ @@ -17,6 +17,7 @@ const make = ({ topicKeyword }: NoEmbedOptions) => const gateway = yield* _(DiscordGateway) const rest = yield* _(DiscordREST) const channels = yield* _(ChannelsCache) + const log = yield* _(Log.Log) const getChannel = (guildId: string, id: string) => Effect.flatMap(channels.get(guildId, id), _ => @@ -57,7 +58,7 @@ const make = ({ topicKeyword }: NoEmbedOptions) => ), Effect.catchTags({ NotValidMessageError: () => Effect.unit(), - DiscordRESTError: logRESTError, + DiscordRESTError: logRESTError(log), }), Effect.catchAllCause(Effect.logErrorCause), ) diff --git a/src/Summarizer.ts b/src/Summarizer.ts new file mode 100644 index 0000000..6c3eaf7 --- /dev/null +++ b/src/Summarizer.ts @@ -0,0 +1,252 @@ +import { ChannelsCache, ChannelsCacheLive } from "bot/ChannelsCache" +import { MemberCache, MemberCacheLive } from "bot/MemberCache" +import { + Chunk, + Data, + Effect, + Http, + Layer, + Option, + Stream, + pipe, +} from "bot/_common" +import { Discord, DiscordREST, Ix, Log } from "dfx" +import { InteractionsRegistry, InteractionsRegistryLive } from "dfx/gateway" + +export class NotInThreadError extends Data.TaggedClass( + "NotInThreadError", +)<{}> {} + +export class PermissionsError extends Data.TaggedClass("PermissionsError")<{ + readonly action: string + readonly subject: string +}> {} + +const make = Effect.gen(function* (_) { + const rest = yield* _(DiscordREST) + const channels = yield* _(ChannelsCache) + const registry = yield* _(InteractionsRegistry) + const members = yield* _(MemberCache) + const application = yield* _( + Effect.flatMap(rest.getCurrentBotApplicationInformation(), _ => _.json), + ) + const scope = yield* _(Effect.scope()) + + const getAllMessages = (channelId: string) => + pipe( + Stream.paginateChunkEffect(Option.none(), before => + pipe( + rest.getChannelMessages(channelId, { + limit: 100, + before: Option.getOrUndefined(before), + }), + Effect.flatMap(_ => _.json), + Effect.map(messages => + messages.length < 100 + ? ([ + Chunk.unsafeFromArray(messages), + Option.none>(), + ] as const) + : ([ + Chunk.unsafeFromArray(messages), + Option.some(Option.some(messages[messages.length - 1].id)), + ] as const), + ), + ), + ), + + // only include normal messages + Stream.flatMap(_ => { + if (_.type === Discord.MessageType.THREAD_STARTER_MESSAGE) { + return Stream.succeed(_) + } else if ( + _.content !== "" && + (_.type === Discord.MessageType.REPLY || + _.type === Discord.MessageType.DEFAULT) + ) { + return Stream.succeed(_) + } + + return Stream.empty + }), + + Stream.mapEffectPar(Number.MAX_SAFE_INTEGER, message => { + if (message.type !== Discord.MessageType.THREAD_STARTER_MESSAGE) { + return Effect.succeed(message) + } + + return Effect.flatMap( + rest.getChannelMessage( + message.message_reference!.channel_id!, + message.message_reference!.message_id!, + ), + _ => _.json, + ) + }), + ) + + const summarize = ( + channel: Discord.Channel, + thread: Discord.Channel, + messages: Chunk.Chunk, + ) => + Effect.gen(function* (_) { + const messageContent = yield* _( + Effect.forEachParWithIndex(messages, (message, index) => { + const reply = pipe( + Option.fromNullable(message.message_reference), + Option.flatMap(ref => + Chunk.findFirstIndex(messages, _ => _.id === ref.message_id), + ), + Option.map( + index => [Chunk.unsafeGet(messages, index), index + 1] as const, + ), + ) + return summarizeMessage(thread, index + 1, message, reply) + }), + ) + + return `# ${thread.name} + +Thread started in: #${channel.name}
+Thread started at: ${new Date( + thread.thread_metadata!.create_timestamp!, + ).toUTCString()} + +${messageContent.join("\n\n")}` + }) + + const summarizeMessage = ( + thread: Discord.Channel, + index: number, + message: Discord.Message, + replyTo: Option.Option, + ) => + Effect.gen(function* (_) { + const user = message.author + const member = yield* _(members.get(thread.guild_id!, message.author.id)) + const username = member.nick ?? user.username + const content = `${index}: **${username}**, ${new Date( + message.timestamp, + ).toUTCString()}${Option.match( + replyTo, + () => "", + ([, index]) => ` (in reply to \\#${index})`, + )}
+${message.content}` + + const mentions = yield* _( + Effect.forEachPar(content.matchAll(/<@(\d+)>/g), ([, userId]) => + Effect.option( + members.get(thread.guild_id!, userId as Discord.Snowflake), + ), + ), + ) + + return mentions.reduce( + (content, member) => + Option.match( + member, + () => content, + member => + content.replace( + new RegExp(`<@${member.user!.id}>`, "g"), + `**@${member.nick ?? member.user!.username}**`, + ), + ), + content, + ) + }) + + const followUpResponse = ( + context: Discord.Interaction, + channel: Discord.Channel, + ) => + pipe( + Effect.all({ + parentChannel: channels.get(channel.guild_id!, channel.parent_id!), + }), + Effect.bind("messages", () => + Effect.map( + Stream.runCollect(getAllMessages(channel.id)), + Chunk.reverse, + ), + ), + Effect.bind("summary", ({ parentChannel, messages }) => + summarize(parentChannel, channel, messages), + ), + Effect.tap(({ summary }) => { + const formData = new FormData() + + formData.append( + "file", + new Blob([summary], { type: "text/plain" }), + `${channel.name} Summary.md`, + ) + formData.append( + "payload_json", + JSON.stringify({ + content: "Here is your summary!", + flags: Discord.MessageFlag.EPHEMERAL, + }), + ) + + return rest.editOriginalInteractionResponse( + application.id, + context.token, + { body: Http.body.formData(formData) }, + ) + }), + ) + + const command = Ix.global( + { + name: "summarize", + description: "Create a summary of the current thread", + }, + pipe( + Effect.all({ context: Ix.Interaction }), + Effect.bind("channel", ({ context }) => + channels.get(context.guild_id!, context.channel_id!), + ), + Effect.filterOrFail( + ({ channel }) => channel.type === Discord.ChannelType.PUBLIC_THREAD, + () => new NotInThreadError(), + ), + Effect.tap(({ context, channel }) => + Effect.forkIn(followUpResponse(context, channel), scope), + ), + Effect.as( + Ix.response({ + type: Discord.InteractionCallbackType.CHANNEL_MESSAGE_WITH_SOURCE, + data: { + content: "Creating summary...", + flags: Discord.MessageFlag.EPHEMERAL, + }, + }), + ), + ), + ) + + const ix = Ix.builder + .add(command) + .catchTagRespond("NotInThreadError", () => + Effect.succeed( + Ix.response({ + type: Discord.InteractionCallbackType.CHANNEL_MESSAGE_WITH_SOURCE, + data: { + content: "This command can only be used in a thread", + flags: Discord.MessageFlag.EPHEMERAL, + }, + }), + ), + ) + .catchAllCause(Effect.logErrorCause) + + yield* _(registry.register(ix)) +}) + +export const SummarizerLive = Layer.provide( + Layer.mergeAll(ChannelsCacheLive, InteractionsRegistryLive, MemberCacheLive), + Layer.scopedDiscard(make), +) diff --git a/src/_common.ts b/src/_common.ts index 5ef394f..cc74d92 100644 --- a/src/_common.ts +++ b/src/_common.ts @@ -1,3 +1,5 @@ +export * as Http from "@effect-http/client" +export * as Chunk from "@effect/data/Chunk" export { Tag } from "@effect/data/Context" export * as Data from "@effect/data/Data" export { millis, seconds } from "@effect/data/Duration" @@ -5,10 +7,14 @@ export * as Duration from "@effect/data/Duration" export { flow, identity, pipe } from "@effect/data/Function" export * as HashMap from "@effect/data/HashMap" export * as Option from "@effect/data/Option" +export * as Cache from "@effect/io/Cache" export * as Cause from "@effect/io/Cause" export * as Config from "@effect/io/Config" export * as ConfigSecret from "@effect/io/Config/Secret" export * as Effect from "@effect/io/Effect" export * as Layer from "@effect/io/Layer" +export * as Request from "@effect/io/Request" +export * as RequestResolver from "@effect/io/RequestResolver" export * as Schedule from "@effect/io/Schedule" +export * as Stream from "@effect/stream/Stream" export { Discord } from "dfx" diff --git a/src/main.ts b/src/main.ts index 823c6cb..e10825a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,6 +7,7 @@ import { Intents } from "dfx" import { makeLive } from "dfx/gateway" import * as Dotenv from "dotenv" import { MentionsLive } from "./Mentions.js" +import { SummarizerLive } from "./Summarizer.js" Dotenv.config() @@ -42,7 +43,13 @@ const NoEmbedLive = NoEmbed.makeLayer({ const MainLive = pipe( Layer.mergeAll(DiscordLive, OpenAILive), Layer.provide( - Layer.mergeAll(AutoThreadsLive, NoEmbedLive, MentionsLive, BotLive), + Layer.mergeAll( + AutoThreadsLive, + NoEmbedLive, + MentionsLive, + SummarizerLive, + BotLive, + ), ), ) diff --git a/src/utils/Errors.ts b/src/utils/Errors.ts index adea8ee..8ad6f64 100644 --- a/src/utils/Errors.ts +++ b/src/utils/Errors.ts @@ -2,9 +2,9 @@ import { Effect } from "bot/_common" import { Log } from "dfx" import { DiscordRESTError } from "dfx/DiscordREST" -export const logRESTError = (_: DiscordRESTError) => +export const logRESTError = (log: Log.Log) => (_: DiscordRESTError) => "response" in _.error ? Effect.flatMap(_.error.response.json, _ => Effect.logInfo(JSON.stringify(_, null, 2)), ) - : Effect.tap(Log.Log, log => log.info(_.error)) + : log.info(_.error)