From 41bf02c92587f4c0e7db989206db90f70a75a0e8 Mon Sep 17 00:00:00 2001 From: Fady Anwar Date: Sun, 9 Jan 2022 00:01:59 +0000 Subject: [PATCH 1/5] adding a simple command line to upload files --- w3scli/main.go | 149 +++++++++++++++++++++++++++++++++++++++++++++++ w3scli/w3scli.go | 149 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 w3scli/main.go create mode 100644 w3scli/w3scli.go diff --git a/w3scli/main.go b/w3scli/main.go new file mode 100644 index 0000000..00ba7a9 --- /dev/null +++ b/w3scli/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + + "github.com/ipfs/go-cid" + "github.com/web3-storage/go-w3s-client" + w3fs "github.com/web3-storage/go-w3s-client/fs" +) + +// Usage: +// TOKEN="API_TOKEN" go run ./main.go +func main() { + c, err := w3s.NewClient( + w3s.WithEndpoint(os.Getenv("ENDPOINT")), + w3s.WithToken(os.Getenv("TOKEN")), + ) + if err != nil { + panic(err) + } + + // cid := putSingleFile(c) + // getStatusForCid(c, cid) + // getStatusForKnownCid(c) + getFiles(c) + // listUploads(c) +} + +func putSingleFile(c w3s.Client) cid.Cid { + file, err := os.Open("images/donotresist.jpg") + if err != nil { + panic(err) + } + return putFile(c, file) +} + +func putMultipleFiles(c w3s.Client) cid.Cid { + f0, err := os.Open("images/donotresist.jpg") + if err != nil { + panic(err) + } + f1, err := os.Open("images/pinpie.jpg") + if err != nil { + panic(err) + } + dir := w3fs.NewDir("comic", []fs.File{f0, f1}) + return putFile(c, dir) +} + +func putMultipleFilesAndDirectories(c w3s.Client) cid.Cid { + f0, err := os.Open("images/donotresist.jpg") + if err != nil { + panic(err) + } + f1, err := os.Open("images/pinpie.jpg") + if err != nil { + panic(err) + } + d0 := w3fs.NewDir("one", []fs.File{f0}) + d1 := w3fs.NewDir("two", []fs.File{f1}) + rootdir := w3fs.NewDir("comic", []fs.File{d0, d1}) + return putFile(c, rootdir) +} + +func putDirectory(c w3s.Client) cid.Cid { + dir, err := os.Open("images") + if err != nil { + panic(err) + } + return putFile(c, dir) +} + +func putFile(c w3s.Client, f fs.File, opts ...w3s.PutOption) cid.Cid { + cid, err := c.Put(context.Background(), f, opts...) + if err != nil { + panic(err) + } + fmt.Printf("https://%v.ipfs.dweb.link\n", cid) + return cid +} + +func getStatusForCid(c w3s.Client, cid cid.Cid) { + s, err := c.Status(context.Background(), cid) + if err != nil { + panic(err) + } + fmt.Printf("Status: %+v", s) +} + +func getStatusForKnownCid(c w3s.Client) { + cid, _ := cid.Parse("bafybeiauyddeo2axgargy56kwxirquxaxso3nobtjtjvoqu552oqciudrm") + getStatusForCid(c, cid) +} + +func getFiles(c w3s.Client) { + cid, _ := cid.Parse("bafybeide43vps6vt2oo7nbqfwn5zz6l2alyi64mym3sb7reqhmypjnmej4") + + res, err := c.Get(context.Background(), cid) + if err != nil { + panic(err) + } + + f, fsys, err := res.Files() + if err != nil { + panic(err) + } + + info, err := f.Stat() + if err != nil { + panic(err) + } + + if info.IsDir() { + err = fs.WalkDir(fsys, "/", func(path string, d fs.DirEntry, err error) error { + info, _ := d.Info() + fmt.Printf("%s (%d bytes)\n", path, info.Size()) + return err + }) + if err != nil { + panic(err) + } + } else { + fmt.Printf("%s (%d bytes)\n", cid.String(), info.Size()) + } +} + +func listUploads(c w3s.Client) { + uploads, err := c.List(context.Background()) + if err != nil { + panic(err) + } + + for { + u, err := uploads.Next() + if err != nil { + // finished successfully + if err == io.EOF { + break + } + panic(err) + } + + fmt.Printf("%s %s Size: %d Deals: %d Pins: %d\n", u.Created.Format("2006-01-02 15:04:05"), u.Cid, u.DagSize, len(u.Deals), len(u.Pins)) + } +} diff --git a/w3scli/w3scli.go b/w3scli/w3scli.go new file mode 100644 index 0000000..00ba7a9 --- /dev/null +++ b/w3scli/w3scli.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + + "github.com/ipfs/go-cid" + "github.com/web3-storage/go-w3s-client" + w3fs "github.com/web3-storage/go-w3s-client/fs" +) + +// Usage: +// TOKEN="API_TOKEN" go run ./main.go +func main() { + c, err := w3s.NewClient( + w3s.WithEndpoint(os.Getenv("ENDPOINT")), + w3s.WithToken(os.Getenv("TOKEN")), + ) + if err != nil { + panic(err) + } + + // cid := putSingleFile(c) + // getStatusForCid(c, cid) + // getStatusForKnownCid(c) + getFiles(c) + // listUploads(c) +} + +func putSingleFile(c w3s.Client) cid.Cid { + file, err := os.Open("images/donotresist.jpg") + if err != nil { + panic(err) + } + return putFile(c, file) +} + +func putMultipleFiles(c w3s.Client) cid.Cid { + f0, err := os.Open("images/donotresist.jpg") + if err != nil { + panic(err) + } + f1, err := os.Open("images/pinpie.jpg") + if err != nil { + panic(err) + } + dir := w3fs.NewDir("comic", []fs.File{f0, f1}) + return putFile(c, dir) +} + +func putMultipleFilesAndDirectories(c w3s.Client) cid.Cid { + f0, err := os.Open("images/donotresist.jpg") + if err != nil { + panic(err) + } + f1, err := os.Open("images/pinpie.jpg") + if err != nil { + panic(err) + } + d0 := w3fs.NewDir("one", []fs.File{f0}) + d1 := w3fs.NewDir("two", []fs.File{f1}) + rootdir := w3fs.NewDir("comic", []fs.File{d0, d1}) + return putFile(c, rootdir) +} + +func putDirectory(c w3s.Client) cid.Cid { + dir, err := os.Open("images") + if err != nil { + panic(err) + } + return putFile(c, dir) +} + +func putFile(c w3s.Client, f fs.File, opts ...w3s.PutOption) cid.Cid { + cid, err := c.Put(context.Background(), f, opts...) + if err != nil { + panic(err) + } + fmt.Printf("https://%v.ipfs.dweb.link\n", cid) + return cid +} + +func getStatusForCid(c w3s.Client, cid cid.Cid) { + s, err := c.Status(context.Background(), cid) + if err != nil { + panic(err) + } + fmt.Printf("Status: %+v", s) +} + +func getStatusForKnownCid(c w3s.Client) { + cid, _ := cid.Parse("bafybeiauyddeo2axgargy56kwxirquxaxso3nobtjtjvoqu552oqciudrm") + getStatusForCid(c, cid) +} + +func getFiles(c w3s.Client) { + cid, _ := cid.Parse("bafybeide43vps6vt2oo7nbqfwn5zz6l2alyi64mym3sb7reqhmypjnmej4") + + res, err := c.Get(context.Background(), cid) + if err != nil { + panic(err) + } + + f, fsys, err := res.Files() + if err != nil { + panic(err) + } + + info, err := f.Stat() + if err != nil { + panic(err) + } + + if info.IsDir() { + err = fs.WalkDir(fsys, "/", func(path string, d fs.DirEntry, err error) error { + info, _ := d.Info() + fmt.Printf("%s (%d bytes)\n", path, info.Size()) + return err + }) + if err != nil { + panic(err) + } + } else { + fmt.Printf("%s (%d bytes)\n", cid.String(), info.Size()) + } +} + +func listUploads(c w3s.Client) { + uploads, err := c.List(context.Background()) + if err != nil { + panic(err) + } + + for { + u, err := uploads.Next() + if err != nil { + // finished successfully + if err == io.EOF { + break + } + panic(err) + } + + fmt.Printf("%s %s Size: %d Deals: %d Pins: %d\n", u.Created.Format("2006-01-02 15:04:05"), u.Cid, u.DagSize, len(u.Deals), len(u.Pins)) + } +} From 50ac4eede8a0300008579434aef6b3858c273413 Mon Sep 17 00:00:00 2001 From: Fady Anwar Date: Sun, 9 Jan 2022 00:06:59 +0000 Subject: [PATCH 2/5] adding w3scli.go file --- w3scli/main.go | 149 ----------------------------------------------- w3scli/w3scli.go | 121 +++----------------------------------- 2 files changed, 8 insertions(+), 262 deletions(-) delete mode 100644 w3scli/main.go diff --git a/w3scli/main.go b/w3scli/main.go deleted file mode 100644 index 00ba7a9..0000000 --- a/w3scli/main.go +++ /dev/null @@ -1,149 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io" - "io/fs" - "os" - - "github.com/ipfs/go-cid" - "github.com/web3-storage/go-w3s-client" - w3fs "github.com/web3-storage/go-w3s-client/fs" -) - -// Usage: -// TOKEN="API_TOKEN" go run ./main.go -func main() { - c, err := w3s.NewClient( - w3s.WithEndpoint(os.Getenv("ENDPOINT")), - w3s.WithToken(os.Getenv("TOKEN")), - ) - if err != nil { - panic(err) - } - - // cid := putSingleFile(c) - // getStatusForCid(c, cid) - // getStatusForKnownCid(c) - getFiles(c) - // listUploads(c) -} - -func putSingleFile(c w3s.Client) cid.Cid { - file, err := os.Open("images/donotresist.jpg") - if err != nil { - panic(err) - } - return putFile(c, file) -} - -func putMultipleFiles(c w3s.Client) cid.Cid { - f0, err := os.Open("images/donotresist.jpg") - if err != nil { - panic(err) - } - f1, err := os.Open("images/pinpie.jpg") - if err != nil { - panic(err) - } - dir := w3fs.NewDir("comic", []fs.File{f0, f1}) - return putFile(c, dir) -} - -func putMultipleFilesAndDirectories(c w3s.Client) cid.Cid { - f0, err := os.Open("images/donotresist.jpg") - if err != nil { - panic(err) - } - f1, err := os.Open("images/pinpie.jpg") - if err != nil { - panic(err) - } - d0 := w3fs.NewDir("one", []fs.File{f0}) - d1 := w3fs.NewDir("two", []fs.File{f1}) - rootdir := w3fs.NewDir("comic", []fs.File{d0, d1}) - return putFile(c, rootdir) -} - -func putDirectory(c w3s.Client) cid.Cid { - dir, err := os.Open("images") - if err != nil { - panic(err) - } - return putFile(c, dir) -} - -func putFile(c w3s.Client, f fs.File, opts ...w3s.PutOption) cid.Cid { - cid, err := c.Put(context.Background(), f, opts...) - if err != nil { - panic(err) - } - fmt.Printf("https://%v.ipfs.dweb.link\n", cid) - return cid -} - -func getStatusForCid(c w3s.Client, cid cid.Cid) { - s, err := c.Status(context.Background(), cid) - if err != nil { - panic(err) - } - fmt.Printf("Status: %+v", s) -} - -func getStatusForKnownCid(c w3s.Client) { - cid, _ := cid.Parse("bafybeiauyddeo2axgargy56kwxirquxaxso3nobtjtjvoqu552oqciudrm") - getStatusForCid(c, cid) -} - -func getFiles(c w3s.Client) { - cid, _ := cid.Parse("bafybeide43vps6vt2oo7nbqfwn5zz6l2alyi64mym3sb7reqhmypjnmej4") - - res, err := c.Get(context.Background(), cid) - if err != nil { - panic(err) - } - - f, fsys, err := res.Files() - if err != nil { - panic(err) - } - - info, err := f.Stat() - if err != nil { - panic(err) - } - - if info.IsDir() { - err = fs.WalkDir(fsys, "/", func(path string, d fs.DirEntry, err error) error { - info, _ := d.Info() - fmt.Printf("%s (%d bytes)\n", path, info.Size()) - return err - }) - if err != nil { - panic(err) - } - } else { - fmt.Printf("%s (%d bytes)\n", cid.String(), info.Size()) - } -} - -func listUploads(c w3s.Client) { - uploads, err := c.List(context.Background()) - if err != nil { - panic(err) - } - - for { - u, err := uploads.Next() - if err != nil { - // finished successfully - if err == io.EOF { - break - } - panic(err) - } - - fmt.Printf("%s %s Size: %d Deals: %d Pins: %d\n", u.Created.Format("2006-01-02 15:04:05"), u.Cid, u.DagSize, len(u.Deals), len(u.Pins)) - } -} diff --git a/w3scli/w3scli.go b/w3scli/w3scli.go index 00ba7a9..863f18f 100644 --- a/w3scli/w3scli.go +++ b/w3scli/w3scli.go @@ -1,20 +1,20 @@ -package main +package w3scli import ( "context" "fmt" - "io" "io/fs" "os" - "github.com/ipfs/go-cid" "github.com/web3-storage/go-w3s-client" - w3fs "github.com/web3-storage/go-w3s-client/fs" + ) // Usage: // TOKEN="API_TOKEN" go run ./main.go func main() { + + c, err := w3s.NewClient( w3s.WithEndpoint(os.Getenv("ENDPOINT")), w3s.WithToken(os.Getenv("TOKEN")), @@ -23,56 +23,16 @@ func main() { panic(err) } - // cid := putSingleFile(c) - // getStatusForCid(c, cid) - // getStatusForKnownCid(c) - getFiles(c) - // listUploads(c) -} - -func putSingleFile(c w3s.Client) cid.Cid { - file, err := os.Open("images/donotresist.jpg") - if err != nil { - panic(err) - } - return putFile(c, file) -} - -func putMultipleFiles(c w3s.Client) cid.Cid { - f0, err := os.Open("images/donotresist.jpg") - if err != nil { - panic(err) - } - f1, err := os.Open("images/pinpie.jpg") + arg := os.Args[1] + file, err := os.Open(arg) if err != nil { panic(err) } - dir := w3fs.NewDir("comic", []fs.File{f0, f1}) - return putFile(c, dir) -} + + putFile(c, file) -func putMultipleFilesAndDirectories(c w3s.Client) cid.Cid { - f0, err := os.Open("images/donotresist.jpg") - if err != nil { - panic(err) - } - f1, err := os.Open("images/pinpie.jpg") - if err != nil { - panic(err) - } - d0 := w3fs.NewDir("one", []fs.File{f0}) - d1 := w3fs.NewDir("two", []fs.File{f1}) - rootdir := w3fs.NewDir("comic", []fs.File{d0, d1}) - return putFile(c, rootdir) } -func putDirectory(c w3s.Client) cid.Cid { - dir, err := os.Open("images") - if err != nil { - panic(err) - } - return putFile(c, dir) -} func putFile(c w3s.Client, f fs.File, opts ...w3s.PutOption) cid.Cid { cid, err := c.Put(context.Background(), f, opts...) @@ -82,68 +42,3 @@ func putFile(c w3s.Client, f fs.File, opts ...w3s.PutOption) cid.Cid { fmt.Printf("https://%v.ipfs.dweb.link\n", cid) return cid } - -func getStatusForCid(c w3s.Client, cid cid.Cid) { - s, err := c.Status(context.Background(), cid) - if err != nil { - panic(err) - } - fmt.Printf("Status: %+v", s) -} - -func getStatusForKnownCid(c w3s.Client) { - cid, _ := cid.Parse("bafybeiauyddeo2axgargy56kwxirquxaxso3nobtjtjvoqu552oqciudrm") - getStatusForCid(c, cid) -} - -func getFiles(c w3s.Client) { - cid, _ := cid.Parse("bafybeide43vps6vt2oo7nbqfwn5zz6l2alyi64mym3sb7reqhmypjnmej4") - - res, err := c.Get(context.Background(), cid) - if err != nil { - panic(err) - } - - f, fsys, err := res.Files() - if err != nil { - panic(err) - } - - info, err := f.Stat() - if err != nil { - panic(err) - } - - if info.IsDir() { - err = fs.WalkDir(fsys, "/", func(path string, d fs.DirEntry, err error) error { - info, _ := d.Info() - fmt.Printf("%s (%d bytes)\n", path, info.Size()) - return err - }) - if err != nil { - panic(err) - } - } else { - fmt.Printf("%s (%d bytes)\n", cid.String(), info.Size()) - } -} - -func listUploads(c w3s.Client) { - uploads, err := c.List(context.Background()) - if err != nil { - panic(err) - } - - for { - u, err := uploads.Next() - if err != nil { - // finished successfully - if err == io.EOF { - break - } - panic(err) - } - - fmt.Printf("%s %s Size: %d Deals: %d Pins: %d\n", u.Created.Format("2006-01-02 15:04:05"), u.Cid, u.DagSize, len(u.Deals), len(u.Pins)) - } -} From e38bb085ded275c3db52968521088e4814c25cba Mon Sep 17 00:00:00 2001 From: Fady Anwar Date: Sun, 9 Jan 2022 00:17:25 +0000 Subject: [PATCH 3/5] Updating comment --- w3scli/w3scli.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/w3scli/w3scli.go b/w3scli/w3scli.go index 863f18f..46ae13e 100644 --- a/w3scli/w3scli.go +++ b/w3scli/w3scli.go @@ -11,7 +11,7 @@ import ( ) // Usage: -// TOKEN="API_TOKEN" go run ./main.go +// TOKEN="API_TOKEN" go run ./w3scli.go func main() { From f5f0bacedf523a186882abd4ae3964b1fd45d38b Mon Sep 17 00:00:00 2001 From: Fady Anwar Date: Thu, 13 Jan 2022 19:29:01 +0000 Subject: [PATCH 4/5] Adding feature to retry on failure --- put.go | 9 +++++++-- w3scli/w3scli.go | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/put.go b/put.go index d495797..4f0a6bb 100644 --- a/put.go +++ b/put.go @@ -94,6 +94,7 @@ func (c *client) PutCar(ctx context.Context, car io.Reader) (cid.Cid, error) { var root cid.Cid for { r, err := spltr.Next() + if err != nil { if err == io.EOF { break @@ -112,8 +113,8 @@ func (c *client) PutCar(ctx context.Context, car io.Reader) (cid.Cid, error) { return root, nil } -// TODO: retry func (c *client) sendCar(ctx context.Context, r io.Reader) (cid.Cid, error) { + req, err := http.NewRequestWithContext(ctx, "POST", c.cfg.endpoint+"/car", r) if err != nil { return cid.Undef, err @@ -126,7 +127,11 @@ func (c *client) sendCar(ctx context.Context, r io.Reader) (cid.Cid, error) { return cid.Undef, err } if res.StatusCode != 200 { - return cid.Undef, fmt.Errorf("unexpected response status: %d", res.StatusCode) + + //retry on failure + fmt.Printf("unexpected response status: %d, retrying..\n", res.StatusCode) + c.sendCar(ctx, r) + } d := json.NewDecoder(res.Body) var out struct { diff --git a/w3scli/w3scli.go b/w3scli/w3scli.go index 46ae13e..ca5c0e5 100644 --- a/w3scli/w3scli.go +++ b/w3scli/w3scli.go @@ -1,4 +1,4 @@ -package w3scli +package main import ( "context" From b204f77c5c64053f57c84ceee8b8158ab0c39931 Mon Sep 17 00:00:00 2001 From: Fady Anwar Date: Wed, 6 Jul 2022 19:09:24 +0100 Subject: [PATCH 5/5] adding missing imports on readme sample code --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 332a19a..2c2e793 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ package main import ( "io/fs" "os" + "fmt" + "context" "github.com/web3-storage/go-w3s-client" )