chore: add author and source_id to reference tweet

Author: rebelchris

__tests__/workers/postUpdated.ts

@@ -659,15 +659,15 @@ it('should create referenced social post for quote and set sharedPostId', async
   expect(post.type).toEqual(PostType.SocialTwitter);
   expect(post.subType).toEqual('quote');
   expect(post.sharedPostId).toBeTruthy();
-  expect(post.showOnFeed).toEqual(false);
-  expect(post.private).toEqual(true);
 
   const referencedPost = await con.getRepository(Post).findOneByOrFail({
     id: post.sharedPostId,
   });
   expect(referencedPost.type).toEqual(PostType.SocialTwitter);
   expect(referencedPost.subType).toEqual('tweet');
   expect(referencedPost.title).toEqual('Referenced quoted tweet');
+  expect(referencedPost.showOnFeed).toEqual(false);
+  expect(referencedPost.private).toEqual(true);
 });
 
 it('should reuse existing referenced social post for quote when status id already exists', async () => {
@@ -755,15 +755,15 @@ it('should create referenced social post for repost and set sharedPostId', async
   expect(post.type).toEqual(PostType.SocialTwitter);
   expect(post.subType).toEqual('repost');
   expect(post.sharedPostId).toBeTruthy();
-  expect(post.showOnFeed).toEqual(false);
-  expect(post.private).toEqual(true);
 
   const referencedPost = await con.getRepository(Post).findOneByOrFail({
     id: post.sharedPostId,
   });
   expect(referencedPost.type).toEqual(PostType.SocialTwitter);
   expect(referencedPost.subType).toEqual('tweet');
   expect(referencedPost.title).toEqual('Referenced repost tweet');
+  expect(referencedPost.showOnFeed).toEqual(false);
+  expect(referencedPost.private).toEqual(true);
 });
 
 it('should map social twitter tweet subtype as plain tweet', async () => {

src/common/schema/socialTwitter.ts

@@ -44,6 +44,8 @@ export const twitterSocialReferenceSchema = z
     content_html: z.string().nullish(),
     sub_type: z.string().nullish(),
     media: z.array(twitterSocialMediaSchema).nullish(),
+    source_id: z.string().nullish(),
+    author_username: z.string().nullish(),
   })
   .passthrough();
 

src/common/twitterSocial.ts

@@ -1,7 +1,7 @@
 import { ValidationError } from 'apollo-server-errors';
 import type { EntityManager } from 'typeorm';
 import { generateShortId } from '../ids';
-import { UNKNOWN_SOURCE } from '../entity/Source';
+import { Source, SourceType, UNKNOWN_SOURCE } from '../entity/Source';
 import { Post, PostOrigin, PostType } from '../entity/posts/Post';
 import { SocialTwitterPost } from '../entity/posts/SocialTwitterPost';
 import { markdown } from './markdown';
@@ -30,6 +30,7 @@ export interface TwitterReferencePost {
   contentHtml?: string | null;
   image?: string | null;
   videoId?: string | null;
+  authorUsername?: string | null;
 }
 
 export interface TwitterMappingResult {
@@ -41,9 +42,7 @@ export interface TwitterMappingResult {
 export interface TwitterReferenceUpsertParams {
   entityManager: EntityManager;
   reference: TwitterReferencePost;
-  sourceId?: string | null;
   language?: string | null;
-  isPrivate?: boolean;
 }
 
 const getStringOrUndefined = (value?: string | null): string | undefined => {
@@ -270,6 +269,7 @@ const extractTwitterReference = (
     contentHtml: getStringOrUndefined(reference.content_html),
     image: pickPrimaryImage(reference.media || []),
     videoId: pickPrimaryVideoId(reference.media || []),
+    authorUsername: getStringOrUndefined(reference.author_username),
   };
 };
 
@@ -374,12 +374,38 @@ export const mapTwitterSocialPayload = ({
   };
 };
 
+const resolveSourceByTwitterUsername = async ({
+  entityManager,
+  authorUsername,
+}: {
+  entityManager: EntityManager;
+  authorUsername?: string | null;
+}): Promise<{ id: string; isPrivate: boolean } | undefined> => {
+  if (!authorUsername) {
+    return undefined;
+  }
+
+  const matchedSource = await entityManager
+    .getRepository(Source)
+    .createQueryBuilder('source')
+    .select(['source.id', 'source.private'])
+    .where('source.type = :type', { type: SourceType.Machine })
+    .andWhere('LOWER(source.twitter) = :twitter', {
+      twitter: authorUsername.toLowerCase(),
+    })
+    .getOne();
+
+  if (!matchedSource) {
+    return undefined;
+  }
+
+  return { id: matchedSource.id, isPrivate: matchedSource.private };
+};
+
 export const upsertTwitterReferencedPost = async ({
   entityManager,
   reference,
-  sourceId,
   language,
-  isPrivate = false,
 }: TwitterReferenceUpsertParams): Promise<string | undefined> => {
   const referenceUrl = getStringOrUndefined(reference.url);
   if (!referenceUrl) {
@@ -425,7 +451,12 @@ export const upsertTwitterReferencedPost = async ({
     getStringOrUndefined(reference.contentHtml) ||
     (content ? markdown.render(content) : undefined);
   const visible = !!(title || content);
-  const referenceSourceId = sourceId || UNKNOWN_SOURCE;
+  const resolvedSource = await resolveSourceByTwitterUsername({
+    entityManager,
+    authorUsername: reference.authorUsername,
+  });
+  const referenceSourceId = resolvedSource?.id || UNKNOWN_SOURCE;
+  const isPrivate = resolvedSource?.isPrivate ?? true;
 
   const repository = entityManager.getRepository(SocialTwitterPost);
 

src/workers/postUpdated.ts

@@ -831,20 +831,8 @@ const worker: Worker = {
           fixedData.sharedPostId = await upsertTwitterReferencedPost({
             entityManager,
             reference: twitterReference,
-            sourceId: fixedData.sourceId,
             language: fixedData.language,
-            isPrivate: fixedData.private,
           });
-
-          if (fixedData.sharedPostId) {
-            fixedData.showOnFeed = false;
-            fixedData.private = true;
-            fixedData.flags = {
-              ...fixedData.flags,
-              showOnFeed: false,
-              private: true,
-            };
-          }
         }
 
         // See if post id is not available