Skip to content

APP-14607: Update CLI to handle binary data file too large error #5635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gloriacai01 merged 3 commits into from
Jan 6, 2026
Merged

APP-14607: Update CLI to handle binary data file too large error #5635

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3e88106
add error check
gloriacai01 Jan 2, 2026
0eecfe3
Update cli/data.go
gloriacai01 Jan 5, 2026
a97fe5c
add INCLUDE_BINARY_TOO_LARGE check
gloriacai01 Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions cli/data.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,14 @@ func getMatchingBinaryIDs(ctx context.Context, client datapb.DataServiceClient,
}
}

// Check for the errors returned from server that we send if the requested file is too large.
// Resource exhausted is returned when the message we're receiving exceeds the GRPC maximum limit
// Unavailable (such as error 'upstream connect error or disconnect/reset before headers. reset reason: connection termination')
// can also be returned when the file is too large.
func isLargeFileError(err error) bool {
return strings.Contains(err.Error(), "querying binary data with include_binary=true but one or more objects exceed the maximum size of 100 MiB") || status.Code(err) == codes.ResourceExhausted || status.Code(err) == codes.Unavailable
}

func (c *viamClient) downloadBinary(dst string, timeout uint, ids ...string) error {
args, err := getGlobalArgs(c.c)
if err != nil {
Expand All @@ -523,17 +531,15 @@ func (c *viamClient) downloadBinary(dst string, timeout uint, ids ...string) err
IncludeBinary: !largeFile,
})
// If any file is too large, we break and try a different pathway for downloading
if err == nil || status.Code(err) == codes.ResourceExhausted || status.Code(err) == codes.Unavailable {
if err == nil || isLargeFileError(err) {
debugf(c.c.App.Writer, args.Debug, "Small file download for files %v: attempt %d/%d succeeded", ids, count+1, maxRetryCount)
break
}
debugf(c.c.App.Writer, args.Debug, "Small file download for files %v: attempt %d/%d failed", ids, count+1, maxRetryCount)
}

// For large files, we get the metadata but not the binary itself
// Resource exhausted is returned when the message we're receiving exceeds the GRPC maximum limit
// Unavailable (such as error 'upstream connect error or disconnect/reset before headers. reset reason: connection termination')
// can also be returned when the file is too large.
if err != nil && (status.Code(err) == codes.ResourceExhausted || status.Code(err) == codes.Unavailable) {
if err != nil && isLargeFileError(err) {
largeFile = true
for count := 0; count < maxRetryCount; count++ {
resp, err = c.dataClient.BinaryDataByIDs(c.c.Context, &datapb.BinaryDataByIDsRequest{
Expand Down
Loading