Skip to content

Commit 853b93e

Browse files
waybackarchiverwaybackarchiver
committed
Add support Ghostarchive
1 parent 30dc536 commit 853b93e

File tree

7 files changed

+94
-38
lines changed

7 files changed

+94
-38
lines changed

‎cmd/wayback/main.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ var (
2626
is bool
2727
ip bool
2828
ph bool
29+
ga bool
2930

3031
daemon []string
3132

@@ -73,6 +74,7 @@ func init() {
7374
rootCmd.Flags().BoolVarP(&is, "is", "", true, "Wayback webpages to Archive Today")
7475
rootCmd.Flags().BoolVarP(&ip, "ip", "", true, "Wayback webpages to IPFS")
7576
rootCmd.Flags().BoolVarP(&ph, "ph", "", true, "Wayback webpages to Telegraph")
77+
rootCmd.Flags().BoolVarP(&ga, "ga", "", true, "Wayback webpages to Ghostarchive")
7678
rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc")
7779
rootCmd.Flags().StringVarP(&host, "ipfs-host", "", "127.0.0.1", "IPFS daemon host, do not require, unless enable ipfs")
7880
rootCmd.Flags().UintVarP(&port, "ipfs-port", "p", 5001, "IPFS daemon port")
@@ -128,6 +130,9 @@ func setToEnv(cmd *cobra.Command) {
128130
if flags.Changed("ph") {
129131
os.Setenv("WAYBACK_ENABLE_PH", fmt.Sprint(ph))
130132
}
133+
if flags.Changed("ga") {
134+
os.Setenv("WAYBACK_ENABLE_GA", fmt.Sprint(ga))
135+
}
131136
if flags.Changed("token") {
132137
os.Setenv("WAYBACK_TELEGRAM_TOKEN", token)
133138
}
@@ -153,11 +158,12 @@ func setToEnv(cmd *cobra.Command) {
153158

154159
// nolint:gocyclo
155160
func run(cmd *cobra.Command, args []string) {
156-
if !ia && !is && !ip && !ph {
161+
if !ia && !is && !ip && !ph && !ga {
157162
ia, is, ip = true, true, true
158163
os.Setenv("WAYBACK_ENABLE_IA", "true")
159164
os.Setenv("WAYBACK_ENABLE_IS", "true")
160165
os.Setenv("WAYBACK_ENABLE_IP", "true")
166+
os.Setenv("WAYBACK_ENABLE_GA", "true")
161167
}
162168

163169
setToEnv(cmd)

‎config/config.go

+3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const (
1313
SLOT_IS = "is" // archive.today
1414
SLOT_IP = "ip" // IPFS
1515
SLOT_PH = "ph" // Telegraph
16+
SLOT_GA = "ga" // Ghostarchive
1617
SLOT_TT = "tt" // Time Travel
1718
SLOT_GC = "gc" // Google Cache
1819

@@ -68,6 +69,7 @@ func SlotName(s string) string {
6869
SLOT_IS: "archive.today",
6970
SLOT_IP: "IPFS",
7071
SLOT_PH: "Telegraph",
72+
SLOT_GA: "Ghostarchive",
7173
SLOT_TT: "Time Travel",
7274
SLOT_GC: "Google Cache",
7375
}
@@ -86,6 +88,7 @@ func SlotExtra(s string) string {
8688
SLOT_IS: "https://archive.today/",
8789
SLOT_IP: "https://ipfs.github.io/public-gateway-checker/",
8890
SLOT_PH: "https://telegra.ph/",
91+
SLOT_GA: "https://ghostarchive.org/",
8992
SLOT_TT: "http://timetravel.mementoweb.org/",
9093
SLOT_GC: "https://webcache.googleusercontent.com/",
9194
}

‎config/config_test.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,7 @@ func TestEnableSlots(t *testing.T) {
397397
os.Setenv("WAYBACK_ENABLE_IS", "true")
398398
os.Setenv("WAYBACK_ENABLE_IP", "true")
399399
os.Setenv("WAYBACK_ENABLE_PH", "true")
400+
os.Setenv("WAYBACK_ENABLE_GA", "true")
400401

401402
parser := NewParser()
402403
opts, err := parser.ParseEnvironmentVariables()
@@ -409,10 +410,11 @@ func TestEnableSlots(t *testing.T) {
409410
SLOT_IS: true,
410411
SLOT_IP: true,
411412
SLOT_PH: true,
413+
SLOT_GA: true,
412414
}
413415
got := opts.Slots()
414416

415-
if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] || !got[SLOT_PH] {
417+
if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] || !got[SLOT_PH] || !got[SLOT_GA] {
416418
t.Fatalf(`Unexpected default slots, got %v instead of %v`, got, expected)
417419
}
418420
}
@@ -431,6 +433,7 @@ func TestDefaultSlots(t *testing.T) {
431433
SLOT_IS: true,
432434
SLOT_IP: true,
433435
SLOT_PH: true,
436+
SLOT_GA: true,
434437
}
435438
got := opts.Slots()
436439

‎config/options.go

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ const (
3434
defEnabledIS = true
3535
defEnabledIP = true
3636
defEnabledPH = true
37+
defEnabledGA = true
3738

3839
defTelegramToken = ""
3940
defTelegramChannel = ""
@@ -282,6 +283,7 @@ func NewOptions() *Options {
282283
SLOT_IS: defEnabledIS,
283284
SLOT_IP: defEnabledIP,
284285
SLOT_PH: defEnabledPH,
286+
SLOT_GA: defEnabledGA,
285287
},
286288
telegram: &telegram{
287289
token: defTelegramToken,

‎go.mod

+12-8
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ module github.com/wabarc/wayback
22

33
// +heroku goVersion go1.20
44

5-
go 1.20
5+
go 1.21
6+
7+
toolchain go1.22.0
68

79
require (
8-
github.com/PuerkitoBio/goquery v1.8.1
10+
github.com/PuerkitoBio/goquery v1.9.0
911
github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030
1012
github.com/cretz/bine v0.2.0
1113
github.com/davecgh/go-spew v1.1.1
@@ -35,13 +37,14 @@ require (
3537
github.com/spf13/cobra v1.6.1
3638
github.com/wabarc/archive.is v1.4.0
3739
github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896
40+
github.com/wabarc/ghostarchive v0.1.1
3841
github.com/wabarc/go-anonfile v0.1.0
3942
github.com/wabarc/go-catbox v0.1.0
4043
github.com/wabarc/helper v0.0.0-20230418130954-be7440352bcb
4144
github.com/wabarc/imgbb v1.0.0
4245
github.com/wabarc/ipfs-pinner v1.1.1-0.20230502052510-dc378f9e202b
4346
github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792
44-
github.com/wabarc/playback v0.0.0-20230331122619-84484ab4d599
47+
github.com/wabarc/playback v0.0.0-20240229124108-a6ddc935de3c
4548
github.com/wabarc/proxier v0.0.0-20230610135141-b55fe1536465
4649
github.com/wabarc/rivet v0.1.4-0.20230505152228-2c5c81b4bd10
4750
github.com/wabarc/screenshot v1.6.1-0.20240214000834-3820163034f4
@@ -62,7 +65,7 @@ require (
6265
github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 // indirect
6366
github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20230201052002-6c5833b989be // indirect
6467
github.com/VividCortex/ewma v1.2.0 // indirect
65-
github.com/andybalholm/brotli v1.0.5 // indirect
68+
github.com/andybalholm/brotli v1.1.0 // indirect
6669
github.com/andybalholm/cascadia v1.3.2 // indirect
6770
github.com/benbjohnson/clock v1.3.5 // indirect
6871
github.com/beorn7/perks v1.0.1 // indirect
@@ -77,6 +80,7 @@ require (
7780
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 // indirect
7881
github.com/chromedp/chromedp v0.9.5 // indirect
7982
github.com/chromedp/sysutil v1.0.0 // indirect
83+
github.com/cloudflare/circl v1.3.7 // indirect
8084
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
8185
github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
8286
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
@@ -85,7 +89,6 @@ require (
8589
github.com/dop251/goja v0.0.0-20221115122301-6c0d9883792e // indirect
8690
github.com/fatih/color v1.16.0 // indirect
8791
github.com/fortytw2/leaktest v1.3.0 // indirect
88-
github.com/gaukas/godicttls v0.0.3 // indirect
8992
github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect
9093
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
9194
github.com/gobwas/httphead v0.1.0 // indirect
@@ -107,7 +110,7 @@ require (
107110
github.com/kallydev/telegraph-go v1.0.1-0.20230318133700-df034d9eed50 // indirect
108111
github.com/kennygrant/sanitize v1.2.4 // indirect
109112
github.com/kkdai/youtube/v2 v2.7.18 // indirect
110-
github.com/klauspost/compress v1.17.6 // indirect
113+
github.com/klauspost/compress v1.17.7 // indirect
111114
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
112115
github.com/kr/pretty v0.3.1 // indirect
113116
github.com/kr/text v0.2.0 // indirect
@@ -134,7 +137,8 @@ require (
134137
github.com/oliamb/cutter v0.2.2 // indirect
135138
github.com/prometheus/client_model v0.5.0 // indirect
136139
github.com/prometheus/procfs v0.12.0 // indirect
137-
github.com/refraction-networking/utls v1.3.2 // indirect
140+
github.com/quic-go/quic-go v0.41.0 // indirect
141+
github.com/refraction-networking/utls v1.6.3 // indirect
138142
github.com/rivo/uniseg v0.4.3 // indirect
139143
github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f // indirect
140144
github.com/rogpeppe/go-internal v1.10.0 // indirect
@@ -151,7 +155,7 @@ require (
151155
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
152156
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
153157
github.com/ybbus/httpretry v1.0.2 // indirect
154-
golang.org/x/crypto v0.19.0 // indirect
158+
golang.org/x/crypto v0.20.0 // indirect
155159
golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect
156160
golang.org/x/mod v0.15.0 // indirect
157161
golang.org/x/sys v0.17.0 // indirect

0 commit comments

Comments
 (0)