Skip to content

Commit 5c46bbe

Browse files
codethewebcodetheweb
committed
Update previous comics
1 parent 43c35ad commit 5c46bbe

File tree

3 files changed

+75
-37
lines changed

3 files changed

+75
-37
lines changed

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
"eslint": "^7.22.0",
3636
"husky": "^4.2.5",
3737
"lint-staged": "^10.5.3",
38-
"prisma": "2.23.0",
38+
"prisma": "2.25.0",
3939
"rimraf": "^3.0.2",
4040
"typescript": "^4.2.3",
4141
"xo": "^0.38.2"
@@ -71,7 +71,7 @@
7171
"@nestjs/platform-fastify": "^7.6.15",
7272
"@nestjs/serve-static": "^2.1.4",
7373
"@nestjs/swagger": "^4.8.0",
74-
"@prisma/client": "2.23.0",
74+
"@prisma/client": "2.25.0",
7575
"apn": "^2.2.0",
7676
"bullmq": "^1.15.1",
7777
"class-transformer": "^0.4.0",

‎src/comic/processors/scrape-comics.ts‎

Lines changed: 55 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import got, {HTTPError} from 'got';
33
import probe, {ProbeResult} from 'probe-image-size';
44
import {Logger} from '@nestjs/common';
55
import * as exifr from 'exifr';
6-
import {ComicImg, PrismaClient} from '@prisma/client';
6+
import {ComicImg, Prisma, PrismaClient} from '@prisma/client';
77

88
const PARALLEL_SCRAPES = 5;
99

@@ -44,7 +44,8 @@ const processJob = async (_: Job) => {
4444
let fetchingId = 1;
4545

4646
if (latestComic) {
47-
fetchingId = latestComic.id + 1;
47+
// Check the latest two for updates
48+
fetchingId = latestComic.id - 2;
4849
}
4950

5051
// Require 2 not found errors in a row to terminate
@@ -116,22 +117,59 @@ const processJob = async (_: Job) => {
116117
}
117118
}
118119

119-
await prisma.comic.create({
120-
data: {
121-
id: comic.num,
122-
publishedAt: new Date(`${comic.month}-${comic.day}-${comic.year}`),
123-
news: comic.news,
124-
safeTitle: comic.safe_title,
125-
title: comic.title,
126-
transcript: comic.transcript,
127-
alt: comic.alt,
128-
sourceUrl: `https://xkcd.com/${comic.num}`,
129-
explainUrl: `https://www.explainxkcd.com/wiki/index.php/${comic.num}`,
130-
imgs: {
131-
create: imgs
132-
}
133-
}
120+
const currentComic = await prisma.comic.findUnique({
121+
where: {id: comic.num},
122+
select: {imgs: true}
134123
});
124+
125+
const data: Prisma.ComicCreateInput = {
126+
id: comic.num,
127+
publishedAt: new Date(`${comic.month}-${comic.day}-${comic.year}`),
128+
news: comic.news,
129+
safeTitle: comic.safe_title,
130+
title: comic.title,
131+
transcript: comic.transcript,
132+
alt: comic.alt,
133+
sourceUrl: `https://xkcd.com/${comic.num}`,
134+
explainUrl: `https://www.explainxkcd.com/wiki/index.php/${comic.num}`
135+
};
136+
137+
if (currentComic) {
138+
await Promise.all<any>([
139+
prisma.comic.update({
140+
where: {id: data.id},
141+
data
142+
}),
143+
...imgs.map(async img => {
144+
const existingImage = await prisma.comicImg.findFirst({
145+
where: {
146+
comic_id: data.id,
147+
size: img.size
148+
}
149+
});
150+
151+
if (existingImage) {
152+
await prisma.comicImg.update({
153+
where: {id: existingImage.id},
154+
data: img
155+
});
156+
} else {
157+
await prisma.comicImg.create({
158+
data: {
159+
...img,
160+
comic_id: data.id
161+
}
162+
});
163+
}
164+
})
165+
]);
166+
} else {
167+
data.imgs = {
168+
create: imgs
169+
};
170+
171+
await prisma.comic.create({data});
172+
}
135173
} catch (error: unknown) {
136174
if (error instanceof HTTPError) {
137175
if (error.response.statusCode === 404) {

‎yarn.lock‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -479,22 +479,22 @@
479479
consola "^2.15.0"
480480
node-fetch "^2.6.1"
481481

482-
"@prisma/client@2.23.0":
483-
version "2.23.0"
484-
resolved "https://registry.yarnpkg.com/@prisma/client/-/client-2.23.0.tgz#4bf16ab19b140873ba79bd159da86842b1746e0a"
485-
integrity sha512-xsHdo3+wIH0hJVGfKHYTEKtifStjKH0b5t8t7hV32Fypq6+3uxhAi3F25yxuI4XSHXg21nb7Ha82lNwU/0TERA==
482+
"@prisma/client@2.25.0":
483+
version "2.25.0"
484+
resolved "https://registry.yarnpkg.com/@prisma/client/-/client-2.25.0.tgz#a81cdf93ce93128eb35298cf8935480f3da6cca3"
485+
integrity sha512-JDrAJ+oemiYAwgpYNJvCVT59S9bMbqkx78q2OT54xmmBoyYWWnn6t6oS6q8gKMiKHS6rzm/jdh3sy+2E0R+NAQ==
486486
dependencies:
487-
"@prisma/engines-version" "2.23.0-36.adf5e8cba3daf12d456d911d72b6e9418681b28b"
487+
"@prisma/engines-version" "2.25.0-36.c838e79f39885bc8e1611849b1eb28b5bb5bc922"
488488

489-
"@prisma/engines-version@2.23.0-36.adf5e8cba3daf12d456d911d72b6e9418681b28b":
490-
version "2.23.0-36.adf5e8cba3daf12d456d911d72b6e9418681b28b"
491-
resolved "https://registry.yarnpkg.com/@prisma/engines-version/-/engines-version-2.23.0-36.adf5e8cba3daf12d456d911d72b6e9418681b28b.tgz#c813279bbea48dedad039b0bc3b044117d2dbaa1"
492-
integrity sha512-VNgnOe+oPQKmy3HOtWi/Q1fvcKZUQkf1OfTD1pzrLBx9tJPejyxt1Mq54L+OOAuYvfrua6bmfojFVLh7uXuWVw==
489+
"@prisma/engines-version@2.25.0-36.c838e79f39885bc8e1611849b1eb28b5bb5bc922":
490+
version "2.25.0-36.c838e79f39885bc8e1611849b1eb28b5bb5bc922"
491+
resolved "https://registry.yarnpkg.com/@prisma/engines-version/-/engines-version-2.25.0-36.c838e79f39885bc8e1611849b1eb28b5bb5bc922.tgz#b353576a97d0c1952fd4f4201189e845aaafbea8"
492+
integrity sha512-uZaonv3ZzLYAi99AooOe2BOBmb3k+ibVsJyZ5J3F6U1uFHTtTI9AVzC51mE09iNcgq3ZBt2CZNi5CDQZedMWyA==
493493

494-
"@prisma/engines@2.23.0-36.adf5e8cba3daf12d456d911d72b6e9418681b28b":
495-
version "2.23.0-36.adf5e8cba3daf12d456d911d72b6e9418681b28b"
496-
resolved "https://registry.yarnpkg.com/@prisma/engines/-/engines-2.23.0-36.adf5e8cba3daf12d456d911d72b6e9418681b28b.tgz#440abe0ebef44b6e1bdaf2b4d14fcde9fe74f18c"
497-
integrity sha512-Tgk3kggO5B9IT6mimJAw6HSxbFoDAuDKL3sHHSS41EnQm76j/nf4uhGZFPzOQwZWOLeT5ZLO2khr4/FCA9Nkhw==
494+
"@prisma/engines@2.25.0-36.c838e79f39885bc8e1611849b1eb28b5bb5bc922":
495+
version "2.25.0-36.c838e79f39885bc8e1611849b1eb28b5bb5bc922"
496+
resolved "https://registry.yarnpkg.com/@prisma/engines/-/engines-2.25.0-36.c838e79f39885bc8e1611849b1eb28b5bb5bc922.tgz#68d7850d311df6d017e1b878adb19ec21483bcf0"
497+
integrity sha512-vjLCk8AFRZu3D8h/SMcWDzTo0xkMuUDyXQzXekn8gzAGjb47B6LQXGR6rDoZ3/uPM13JNTLPvF62mtVaY6fVeQ==
498498

499499
"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2":
500500
version "1.1.2"
@@ -5801,12 +5801,12 @@ prettier@^2.2.1:
58015801
resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5"
58025802
integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==
58035803

5804-
prisma@2.23.0:
5805-
version "2.23.0"
5806-
resolved "https://registry.yarnpkg.com/prisma/-/prisma-2.23.0.tgz#6464cca0e085ed23b1815013a67c868eff07a7d2"
5807-
integrity sha512-3c/lmDy8nsPcEsfCufvCTJUEuwmAcTPbeGg9fL1qjlvS314duLUA/k2nm3n1rq4ImKqzeC5uaKfvI2IoAfwrJA==
5804+
prisma@2.25.0:
5805+
version "2.25.0"
5806+
resolved "https://registry.yarnpkg.com/prisma/-/prisma-2.25.0.tgz#1ebfef3e945a22c673b3e3c5100f098da475700d"
5807+
integrity sha512-AdAlP+PShvugljIx62Omu+eLKu6Cozz06dehmClIHSb0/yFiVnyBtrRVV4LZus+QX6Ayg7CTDvtzroACAWl+Zw==
58085808
dependencies:
5809-
"@prisma/engines" "2.23.0-36.adf5e8cba3daf12d456d911d72b6e9418681b28b"
5809+
"@prisma/engines" "2.25.0-36.c838e79f39885bc8e1611849b1eb28b5bb5bc922"
58105810

58115811
probe-image-size@^5.0.0:
58125812
version "5.0.0"

0 commit comments

Comments
 (0)