Skip to content

fix: fput specified domain name is invalid #423

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# 2.16.1
## 修复
1. 修复 fput 指定域名不生效问题

# 2.16.0
## 新增
1. create-share / share-ls / share-cp 命令支持创建对文件夹的共享,列举和下载
Expand Down
9 changes: 5 additions & 4 deletions cmd_test/bucket_mk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
package cmd

import (
"github.com/qiniu/qshell/v2/cmd_test/test"
"strings"
"testing"

"github.com/qiniu/qshell/v2/cmd_test/test"
)

func TestMkBucket(t *testing.T) {
Expand All @@ -15,7 +16,7 @@ func TestMkBucket(t *testing.T) {
t.Fatal("should return bucket exists")
}

if !strings.Contains(errs, "error:the bucket already exists") {
if !strings.Contains(errs, "the bucket already exists") {
t.Fatal("expected error:bucket exists, but:" + errs)
}

Expand All @@ -29,8 +30,8 @@ func TestMkBucketNotExistRegion(t *testing.T) {
t.Fatal("should return bucket exists")
}

if !strings.Contains(errs, "error:invalid region parameter") {
t.Fatal("expected error:error:invalid region parameter, but:" + errs)
if !strings.Contains(errs, "invalid region parameter") {
t.Fatal("expected error:invalid region parameter, but:" + errs)
}

return
Expand Down
5 changes: 0 additions & 5 deletions cmd_test/upload_form_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,6 @@ func TestFormUploadWithUploadHost(t *testing.T) {
if !strings.Contains(result, "MimeType: image/jpg") {
t.Fatal(result)
}

path, err = test.CreateTempFile(1*1024 + 1)
if err != nil {
t.Fatal("create form upload file error:", err)
}
}

func TestFormUploadWithWrongUploadHost(t *testing.T) {
Expand Down
10 changes: 3 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ require (
github.com/astaxie/beego v1.12.3
github.com/aws/aws-sdk-go v1.37.31
github.com/mitchellh/go-homedir v1.1.0
github.com/qiniu/go-sdk/v7 v7.24.0
github.com/qiniu/go-sdk/v7 v7.25.4
github.com/schollz/progressbar/v3 v3.8.6
github.com/spf13/cast v1.3.1
github.com/spf13/cobra v1.1.3
Expand All @@ -20,19 +20,14 @@ require (
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect
github.com/elastic/go-sysinfo v1.15.0 // indirect
github.com/elastic/go-windows v1.0.0 // indirect
github.com/gammazero/toposort v0.1.1 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/google/uuid v1.1.1 // indirect
github.com/huandu/xstrings v1.3.1 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/reflectwalk v1.0.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
modernc.org/fileutil v1.3.0 // indirect
)

Expand Down Expand Up @@ -68,5 +63,6 @@ require (
gopkg.in/yaml.v3 v3.0.0 // indirect
)

go 1.21
go 1.23.0

toolchain go1.24.1
19 changes: 2 additions & 17 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY=
github.com/elastic/go-sysinfo v1.15.0 h1:54pRFlAYUlVNQ2HbXzLVZlV+fxS7Eax49stzg95M4Xw=
github.com/elastic/go-sysinfo v1.15.0/go.mod h1:jPSuTgXG+dhhh0GKIyI2Cso+w5lPJ5PvVqKlL8LV/Hk=
github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY=
github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
Expand Down Expand Up @@ -131,8 +126,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
Expand Down Expand Up @@ -186,7 +179,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
Expand Down Expand Up @@ -263,7 +255,6 @@ github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHu
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand All @@ -281,16 +272,13 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk=
github.com/qiniu/go-sdk/v7 v7.24.0 h1:1Vq8Xb1GrwqpeLp1HAnKv6xi0d9GyyiAapJb5/X2na8=
github.com/qiniu/go-sdk/v7 v7.24.0/go.mod h1:uZE85Pi0ftIHT/UNLShosdzwsovqpdas0LwAGO7cPao=
github.com/qiniu/go-sdk/v7 v7.25.4 h1:ulCKlTEyrZzmNytXweOrnva49+Q4+ASjYBCSXhkRWTo=
github.com/qiniu/go-sdk/v7 v7.25.4/go.mod h1:dmKtJ2ahhPWFVi9o1D5GemmWoh/ctuB9peqTowyTO8o=
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
Expand Down Expand Up @@ -440,7 +428,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -553,8 +540,6 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8=
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
Expand Down
10 changes: 10 additions & 0 deletions iqshell/common/flow/flow.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,26 +104,32 @@ func (f *Flow) Start() {
workList := make([]*WorkInfo, 0, f.doWorkInfoListCount)
for {
hasMore, workInfo, err := f.WorkProvider.Provide()
log.DebugF("work producer get work, hasMore:%v, workInfo: %+v, err: %+v", hasMore, workInfo, err)
if err != nil {
if err.Code == data.ErrorCodeParamMissing ||
err.Code == data.ErrorCodeLineHeader {
log.DebugF("work producer get work, skip:%s because:%s", workInfo, err)
f.notifyWorkSkip(workInfo, nil, err)
} else {
log.DebugF("work producer get work fail, error:%s info:%s", err, workInfo)
f.notifyWorkFail(workInfo, err)
}
continue
}

if workInfo == nil || workInfo.Work == nil {
if !hasMore {
log.Info("work producer get work completed")
break
} else {
log.Info("work producer get work fail: work in empty")
continue
}
}

// 检测 work 是否需要过
if skip, cause := f.shouldWorkSkip(workInfo); skip {
log.DebugF("work producer get work, skip:%s cause:%s", workInfo.Data, cause)
f.notifyWorkSkip(workInfo, nil, cause)
continue
}
Expand Down Expand Up @@ -192,11 +198,15 @@ func (f *Flow) Start() {

workCount := len(workList)

log.InfoF("work consumer get works, count:%d", workCount)

_ = f.limitAcquire(workCount)
// workRecordList 有数据则长度和 workList 长度相同
workRecordList, workErr := worker.DoWork(workList)
f.limitRelease(workCount)

log.InfoF("work consumer handle works, count:%d error:%+v", workCount, workErr)

if len(workRecordList) == 0 && workErr != nil {
log.ErrorF("Do Worker Error:%+v", workErr)
for _, workInfo := range workList {
Expand Down
21 changes: 13 additions & 8 deletions iqshell/storage/bucket/operations/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,15 +178,20 @@ func parseDate(dateString string) (time.Time, *data.CodeError) {
return time.Time{}, data.NewEmptyError().AppendDescF("date format must be year-month-day-hour-minute-second")
}

var dateItems [6]int
for ind, field := range fields {
field, err := strconv.Atoi(field)
if err != nil {
return time.Time{}, data.NewEmptyError().AppendDescF("date format must be year-month-day-hour-minute-second, each field must be integer")
}
dateItems[ind] = field
layouts := []string{
"2006",
"2006-01",
"2006-01-02",
"2006-01-02-15",
"2006-01-02-15-04",
"2006-01-02-15-04-05",
}
layout := layouts[len(fields)-1]
date, err := time.ParseInLocation(layout, dateString, time.Local)
if err != nil {
return time.Time{}, data.NewEmptyError().AppendDescF("date format invalid:%s parse date error:%s", dateString, err)
}
return time.Date(dateItems[0], time.Month(dateItems[1]), dateItems[2], dateItems[3], dateItems[4], dateItems[5], 0, time.Local), nil
return date, nil
}

func (info *ListInfo) getStartDate() (time.Time, *data.CodeError) {
Expand Down
15 changes: 12 additions & 3 deletions iqshell/storage/object/download/operations/work_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ func (w *workProvider) getWorkInfoFromFile() {
var keys []string
for {
if len(keys) == 300 {
w.getWorkInfoOfKeys(keys)
if !w.getWorkInfoOfKeys(keys) {
break
}
keys = nil
}

Expand Down Expand Up @@ -140,9 +142,9 @@ func (w *workProvider) getWorkInfoFromFile() {
}()
}

func (w *workProvider) getWorkInfoOfKeys(keys []string) {
func (w *workProvider) getWorkInfoOfKeys(keys []string) bool {
if len(keys) == 0 {
return
return true
}

operations := make([]batch.Operation, 0, len(keys))
Expand All @@ -155,6 +157,11 @@ func (w *workProvider) getWorkInfoOfKeys(keys []string) {
}
}

if len(operations) == 0 {
log.Error("get file info error: no valid key found")
return false
}

results, err := batch.Some(operations)
if len(results) == len(operations) {
for i, result := range results {
Expand Down Expand Up @@ -203,6 +210,8 @@ func (w *workProvider) getWorkInfoOfKeys(keys []string) {
}
time.Sleep(10 * time.Second)
}

return true
}

func (w *workProvider) getWorkInfoFromBucket() {
Expand Down