From 7962e804e8f854bc719814db958bafc97b1422ac Mon Sep 17 00:00:00 2001 From: slhmy Date: Sun, 16 Feb 2025 10:35:43 +0800 Subject: [PATCH] Add storage_module --- modules/storage/minio.go | 228 +++++++++++++++++++++++++++++++++++++ modules/storage/storage.go | 73 ++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 modules/storage/minio.go create mode 100644 modules/storage/storage.go diff --git a/modules/storage/minio.go b/modules/storage/minio.go new file mode 100644 index 0000000..590289b --- /dev/null +++ b/modules/storage/minio.go @@ -0,0 +1,228 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "context" + "io" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + core_module "github.com/oj-lab/platform/modules/core" +) + +var ( + _ ObjectStorage = &MinioStorage{} + + quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") +) + +type minioObject struct { + *minio.Object +} + +func (m *minioObject) Stat() (os.FileInfo, error) { + oi, err := m.Object.Stat() + if err != nil { + return nil, convertMinioErr(err) + } + + return &minioFileInfo{oi}, nil +} + +// MinioStorage returns a minio bucket storage +type MinioStorage struct { + ctx context.Context + client *minio.Client + bucket string + basePath string +} + +func convertMinioErr(err error) error { + if err == nil { + return nil + } + errResp, ok := err.(minio.ErrorResponse) + if !ok { + return err + } + + // Convert two responses to standard analogues + switch errResp.Code { + case "NoSuchKey": + return os.ErrNotExist + case "AccessDenied": + return os.ErrPermission + } + + return err +} + +// NewMinioStorage returns a minio storage +func NewMinioStorage(ctx context.Context) (ObjectStorage, error) { + endpoint := core_module.Config.GetString(objectStorageEndpointConfigKey) + accessKeyID := core_module.Config.GetString(objectStorageAccessKeyConfigKey) + secretAccessKey := core_module.Config.GetString(objectStorageSecretKeyConfigKey) + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + }) + if err != nil { + return nil, convertMinioErr(err) + } + + bucket := core_module.Config.GetString(objectStorageBucketConfigKey) + basePath := core_module.Config.GetString(objectStorageBasePathConfigKey) + + // Check to see if we already own this bucket + exists, err := client.BucketExists(ctx, bucket) + if err != nil { + return nil, convertMinioErr(err) + } + if !exists { + if err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { + return nil, convertMinioErr(err) + } + } + + return &MinioStorage{ + ctx: ctx, + client: client, + bucket: bucket, + basePath: basePath, + }, nil +} + +func (m *MinioStorage) buildMinioPath(p string) string { + p = strings.TrimPrefix(path.Join(m.basePath, p), "/") // object store doesn't use slash for root path + if p == "." { + p = "" // object store doesn't use dot as relative path + } + return p +} + +func (m *MinioStorage) buildMinioDirPrefix(p string) string { + // ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo" + p = m.buildMinioPath(p) + "/" + if p == "/" { + p = "" // object store doesn't use slash for root path + } + return p +} + +// Open opens a file +func (m *MinioStorage) Open(path string) (Object, error) { + opts := minio.GetObjectOptions{} + object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts) + if err != nil { + return nil, convertMinioErr(err) + } + return &minioObject{object}, nil +} + +// Save saves a file to minio +func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) { + uploadInfo, err := m.client.PutObject( + m.ctx, + m.bucket, + m.buildMinioPath(path), + r, + size, + minio.PutObjectOptions{ + ContentType: "application/octet-stream", + }, + ) + if err != nil { + return 0, convertMinioErr(err) + } + return uploadInfo.Size, nil +} + +type minioFileInfo struct { + minio.ObjectInfo +} + +func (m minioFileInfo) Name() string { + return path.Base(m.ObjectInfo.Key) +} + +func (m minioFileInfo) Size() int64 { + return m.ObjectInfo.Size +} + +func (m minioFileInfo) ModTime() time.Time { + return m.LastModified +} + +func (m minioFileInfo) IsDir() bool { + return strings.HasSuffix(m.ObjectInfo.Key, "/") +} + +func (m minioFileInfo) Mode() os.FileMode { + return os.ModePerm +} + +func (m minioFileInfo) Sys() any { + return nil +} + +// Stat returns the stat information of the object +func (m *MinioStorage) Stat(path string) (os.FileInfo, error) { + info, err := m.client.StatObject( + m.ctx, + m.bucket, + m.buildMinioPath(path), + minio.StatObjectOptions{}, + ) + if err != nil { + return nil, convertMinioErr(err) + } + return &minioFileInfo{info}, nil +} + +// Delete delete a file +func (m *MinioStorage) Delete(path string) error { + err := m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{}) + + return convertMinioErr(err) +} + +// URL gets the redirect URL to a file. The presigned link is valid for 5 minutes. +func (m *MinioStorage) URL(path, name string, serveDirectReqParams url.Values) (*url.URL, error) { + // copy serveDirectReqParams + reqParams, err := url.ParseQuery(serveDirectReqParams.Encode()) + if err != nil { + return nil, err + } + // TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we? + reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"") + u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams) + return u, convertMinioErr(err) +} + +// IterateObjects iterates across the objects in the miniostorage +func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error { + opts := minio.GetObjectOptions{} + for mObjInfo := range m.client.ListObjects(m.ctx, m.bucket, minio.ListObjectsOptions{ + Prefix: m.buildMinioDirPrefix(dirName), + Recursive: true, + }) { + object, err := m.client.GetObject(m.ctx, m.bucket, mObjInfo.Key, opts) + if err != nil { + return convertMinioErr(err) + } + if err := func(object *minio.Object, fn func(path string, obj Object) error) error { + defer object.Close() + return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object}) + }(object, fn); err != nil { + return convertMinioErr(err) + } + } + return nil +} diff --git a/modules/storage/storage.go b/modules/storage/storage.go new file mode 100644 index 0000000..0cff234 --- /dev/null +++ b/modules/storage/storage.go @@ -0,0 +1,73 @@ +package storage + +import ( + "io" + "net/url" + "os" +) + +const ( + objectStorageEndpointConfigKey = "object_storage.endpoint" + objectStorageAccessKeyConfigKey = "object_storage.access_key" + objectStorageSecretKeyConfigKey = "object_storage.secret_key" + objectStorageBucketConfigKey = "object_storage.bucket" + objectStorageBasePathConfigKey = "object_storage.base_path" +) + +// Object represents the object on the storage +type Object interface { + io.ReadCloser + io.Seeker + Stat() (os.FileInfo, error) +} + +// ObjectStorage represents an object storage to handle a bucket and files +type ObjectStorage interface { + Open(path string) (Object, error) + // Save store a object, if size is unknown set -1 + Save(path string, r io.Reader, size int64) (int64, error) + Stat(path string) (os.FileInfo, error) + Delete(path string) error + URL(path, name string, reqParams url.Values) (*url.URL, error) + IterateObjects(path string, iterator func(path string, obj Object) error) error +} + +// Copy copies a file from source ObjectStorage to dest ObjectStorage +func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, srcPath string) (int64, error) { + f, err := srcStorage.Open(srcPath) + if err != nil { + return 0, err + } + defer f.Close() + + size := int64(-1) + fsinfo, err := f.Stat() + if err == nil { + size = fsinfo.Size() + } + + return dstStorage.Save(dstPath, f, size) +} + +// Clean delete all the objects in this storage +func Clean(storage ObjectStorage) error { + return storage.IterateObjects("", func(path string, obj Object) error { + _ = obj.Close() + return storage.Delete(path) + }) +} + +// SaveFrom saves data to the ObjectStorage with path p from the callback +func SaveFrom(objStorage ObjectStorage, path string, callback func(w io.Writer) error) error { + pr, pw := io.Pipe() + defer pr.Close() + go func() { + defer pw.Close() + if err := callback(pw); err != nil { + _ = pw.CloseWithError(err) + } + }() + + _, err := objStorage.Save(path, pr, -1) + return err +}