[no-release-notes] archive clone/fetch

go/libraries/doltcore/remotesrv/grpc.go

@@ -295,7 +295,12 @@ func (rs *RemoteChunkStore) StreamDownloadLocations(stream remotesapi.ChunkStore
 			var ranges []*remotesapi.RangeChunk
 			for h, r := range hashToRange {
 				hCpy := h
-				ranges = append(ranges, &remotesapi.RangeChunk{Hash: hCpy[:], Offset: r.Offset, Length: r.Length})
+				ranges = append(ranges, &remotesapi.RangeChunk{
+					Hash:             hCpy[:],
+					Offset:           r.Offset,
+					Length:           r.Length,
+					DictionaryOffset: r.DictOffset,
+					DictionaryLength: r.DictLength})
 			}
 
 			url := rs.getDownloadUrl(md, prefix+"/"+loc)
@@ -606,7 +611,7 @@ func getTableFileInfo(
 	}
 	appendixTableFileInfo := make([]*remotesapi.TableFileInfo, 0)
 	for _, t := range tableList {
-		url := rs.getDownloadUrl(md, prefix+"/"+t.LocationPrefix()+t.FileID())
+		url := rs.getDownloadUrl(md, prefix+"/"+t.LocationPrefix()+t.FileID()+t.LocationSuffix())
 		url, err = rs.sealer.Seal(url)
 		if err != nil {
 			return nil, status.Error(codes.Internal, "failed to get seal download url for "+t.FileID())

go/libraries/doltcore/remotesrv/http.go

@@ -35,6 +35,7 @@ import (
 
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 	"github.com/dolthub/dolt/go/store/hash"
+	"github.com/dolthub/dolt/go/store/nbs"
 	"github.com/dolthub/dolt/go/store/types"
 )
 
@@ -94,12 +95,18 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
-		_, ok := hash.MaybeParse(path[i+1:])
+
+		fileName := path[i+1:]
+		if strings.HasSuffix(fileName, nbs.ArchiveFileSuffix) {
+			fileName = fileName[:len(fileName)-len(nbs.ArchiveFileSuffix)]
+		}
+		_, ok := hash.MaybeParse(fileName)
 		if !ok {
-			logger.WithField("last_path_component", path[i+1:]).Warn("bad request with unparseable last path component")
+			logger.WithField("last_path_component", fileName).Warn("bad request with unparseable last path component")
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
+
 		abs, err := fh.fs.Abs(path)
 		if err != nil {
 			logger.WithError(err).Error("could not get absolute path")

go/libraries/doltcore/remotesrv/server.go

@@ -17,7 +17,6 @@ package remotesrv
 import (
 	"context"
 	"crypto/tls"
-	"errors"
 	"net"
 	"net/http"
 	"strings"
@@ -29,7 +28,6 @@ import (
 	"google.golang.org/grpc"
 
 	remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1"
-	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 )
 
@@ -80,14 +78,6 @@ func NewServer(args ServerArgs) (*Server, error) {
 		args.Logger = logrus.NewEntry(logrus.StandardLogger())
 	}
 
-	storageMetadata, err := env.GetMultiEnvStorageMetadata(args.FS)
-	if err != nil {
-		return nil, err
-	}
-	if storageMetadata.ArchiveFilesPresent() {
-		return nil, errors.New("archive files present. Please run `dolt archive --revert` before running the server.")
-	}
-
 	s := new(Server)
 	s.stopChan = make(chan struct{})
 

go/libraries/doltcore/remotestorage/chunk_cache.go

@@ -21,21 +21,20 @@ import (
 
 // ChunkCache is an interface used for caching chunks
 type ChunkCache interface {
-	// Put puts a slice of chunks into the cache.
-	Put(c []nbs.CompressedChunk) bool
+	// Put puts a slice of chunks into the cache. Error returned if the cache capacity has been exceeded.
+	Put(c []nbs.ToChunker) error
 
 	// Get gets a map of hash to chunk for a set of hashes.  In the event that a chunk is not in the cache, chunks.Empty.
 	// is put in it's place
-	Get(h hash.HashSet) map[hash.Hash]nbs.CompressedChunk
+	Get(h hash.HashSet) map[hash.Hash]nbs.ToChunker
 
 	// Has takes a set of hashes and returns the set of hashes that the cache currently does not have in it.
 	Has(h hash.HashSet) (absent hash.HashSet)
 
-	// PutChunk puts a single chunk in the cache.  true returns in the event that the chunk was cached successfully
-	// and false is returned if that chunk is already is the cache.
-	PutChunk(chunk nbs.CompressedChunk) bool
+	// PutChunk puts a single chunk in the cache. Returns an error if the cache capacity has been exceeded.
+	PutChunk(chunk nbs.ToChunker) error
 
 	// GetAndClearChunksToFlush gets a map of hash to chunk which includes all the chunks that were put in the cache
 	// between the last time GetAndClearChunksToFlush was called and now.
-	GetAndClearChunksToFlush() map[hash.Hash]nbs.CompressedChunk
+	GetAndClearChunksToFlush() map[hash.Hash]nbs.ToChunker
 }