day5 multi node

Author: geektutu

gee-cache/day2-single-node/geecache/geecache.go

@@ -68,12 +68,18 @@ func cloneBytes(b []byte) []byte {
 }
 
 func (g *Group) load(key string) (value ByteView, err error) {
+	return g.getLocally(key)
+}
+
+func (g *Group) getLocally(key string) (ByteView, error) {
 	bytes, err := g.getter.Get(key)
-	if err == nil {
-		value = ByteView{cloneBytes(bytes)}
-		g.populateCache(key, value)
+	if err != nil {
+		return ByteView{}, err
+
 	}
-	return
+	value := ByteView{b: cloneBytes(bytes)}
+	g.populateCache(key, value)
+	return value, nil
 }
 
 func (g *Group) populateCache(key string, value ByteView) {

gee-cache/day3-http-server/geecache/geecache.go

@@ -68,12 +68,18 @@ func cloneBytes(b []byte) []byte {
 }
 
 func (g *Group) load(key string) (value ByteView, err error) {
+	return g.getLocally(key)
+}
+
+func (g *Group) getLocally(key string) (ByteView, error) {
 	bytes, err := g.getter.Get(key)
-	if err == nil {
-		value = ByteView{cloneBytes(bytes)}
-		g.populateCache(key, value)
+	if err != nil {
+		return ByteView{}, err
+
 	}
-	return
+	value := ByteView{b: cloneBytes(bytes)}
+	g.populateCache(key, value)
+	return value, nil
 }
 
 func (g *Group) populateCache(key string, value ByteView) {

gee-cache/day4-consistent-hash/geecache/geecache.go

@@ -68,12 +68,18 @@ func cloneBytes(b []byte) []byte {
 }
 
 func (g *Group) load(key string) (value ByteView, err error) {
+	return g.getLocally(key)
+}
+
+func (g *Group) getLocally(key string) (ByteView, error) {
 	bytes, err := g.getter.Get(key)
-	if err == nil {
-		value = ByteView{cloneBytes(bytes)}
-		g.populateCache(key, value)
+	if err != nil {
+		return ByteView{}, err
+
 	}
-	return
+	value := ByteView{b: cloneBytes(bytes)}
+	g.populateCache(key, value)
+	return value, nil
 }
 
 func (g *Group) populateCache(key string, value ByteView) {

gee-cache/day5-multi-nodes/geecache/byteview.go

@@ -0,0 +1,21 @@
+package geecache
+
+// A ByteView holds an immutable view of bytes.
+type ByteView struct {
+	b []byte
+}
+
+// Len returns the view's length
+func (v ByteView) Len() int {
+	return len(v.b)
+}
+
+// ByteSlice returns a copy of the data as a byte slice.
+func (v ByteView) ByteSlice() []byte {
+	return cloneBytes(v.b)
+}
+
+// String returns the data as a string, making a copy if necessary.
+func (v ByteView) String() string {
+	return string(v.b)
+}

gee-cache/day5-multi-nodes/geecache/cache.go

@@ -0,0 +1,35 @@
+package geecache
+
+import (
+	"geecache/lru"
+	"sync"
+)
+
+type cache struct {
+	mu         sync.RWMutex
+	lru        *lru.Cache
+	cacheBytes int64
+}
+
+func (c *cache) add(key string, value ByteView) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.lru == nil {
+		c.lru = lru.New(c.cacheBytes, nil)
+	}
+	c.lru.Add(key, value)
+}
+
+func (c *cache) get(key string) (value ByteView, ok bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.lru == nil {
+		return
+	}
+
+	if v, ok := c.lru.Get(key); ok {
+		return v.(ByteView), ok
+	}
+
+	return
+}

gee-cache/day5-multi-nodes/geecache/consistenthash/consistenthash.go

@@ -0,0 +1,58 @@
+package consistenthash
+
+import (
+	"hash/crc32"
+	"sort"
+	"strconv"
+)
+
+// Hash maps bytes to uint32
+type Hash func(data []byte) uint32
+
+// Map constains all hashed keys
+type Map struct {
+	hash     Hash
+	replicas int
+	keys     []int // Sorted
+	hashMap  map[int]string
+}
+
+// New creates a Map instance
+func New(replicas int, fn Hash) *Map {
+	m := &Map{
+		replicas: replicas,
+		hash:     fn,
+		hashMap:  make(map[int]string),
+	}
+	if m.hash == nil {
+		m.hash = crc32.ChecksumIEEE
+	}
+	return m
+}
+
+// Add adds some keys to the hash.
+func (m *Map) Add(keys ...string) {
+	for _, key := range keys {
+		for i := 0; i < m.replicas; i++ {
+			hash := int(m.hash([]byte(strconv.Itoa(i) + key)))
+			m.keys = append(m.keys, hash)
+			m.hashMap[hash] = key
+		}
+	}
+	sort.Ints(m.keys)
+}
+
+// Get gets the closest item in the hash to the provided key.
+func (m *Map) Get(key string) string {
+	if len(m.keys) == 0 {
+		return ""
+	}
+
+	hash := int(m.hash([]byte(key)))
+	// Binary search for appropriate replica.
+	idx := sort.Search(len(m.keys), func(i int) bool {
+		return m.keys[i] >= hash
+	})
+
+	return m.hashMap[m.keys[idx%len(m.keys)]]
+}

gee-cache/day5-multi-nodes/geecache/consistenthash/consistenthash_test.go

@@ -0,0 +1,43 @@
+package consistenthash
+
+import (
+	"strconv"
+	"testing"
+)
+
+func TestHashing(t *testing.T) {
+	hash := New(3, func(key []byte) uint32 {
+		i, _ := strconv.Atoi(string(key))
+		return uint32(i)
+	})
+
+	// Given the above hash function, this will give replicas with "hashes":
+	// 2, 4, 6, 12, 14, 16, 22, 24, 26
+	hash.Add("6", "4", "2")
+
+	testCases := map[string]string{
+		"2":  "2",
+		"11": "2",
+		"23": "4",
+		"27": "2",
+	}
+
+	for k, v := range testCases {
+		if hash.Get(k) != v {
+			t.Errorf("Asking for %s, should have yielded %s", k, v)
+		}
+	}
+
+	// Adds 8, 18, 28
+	hash.Add("8")
+
+	// 27 should now map to 8.
+	testCases["27"] = "8"
+
+	for k, v := range testCases {
+		if hash.Get(k) != v {
+			t.Errorf("Asking for %s, should have yielded %s", k, v)
+		}
+	}
+
+}

gee-cache/day5-multi-nodes/geecache/geecache.go

@@ -0,0 +1,109 @@
+package geecache
+
+import (
+	"sync"
+)
+
+// A Group is a cache namespace and associated data loaded spread over
+type Group struct {
+	name      string
+	getter    Getter
+	mainCache cache
+	peers     PeerPicker
+	peersOnce sync.Once
+}
+
+// A Getter loads data for a key.
+type Getter interface {
+	Get(key string) ([]byte, error)
+}
+
+// A GetterFunc implements Getter with a function.
+type GetterFunc func(key string) ([]byte, error)
+
+// Get implements Getter interface function
+func (f GetterFunc) Get(key string) ([]byte, error) {
+	return f(key)
+}
+
+var (
+	mu     sync.RWMutex
+	groups = make(map[string]*Group)
+)
+
+// NewGroup create a new instance of Group
+func NewGroup(name string, cacheBytes int64, getter Getter) *Group {
+	if getter == nil {
+		panic("nil Getter")
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	g := &Group{
+		name:      name,
+		getter:    getter,
+		mainCache: cache{cacheBytes: cacheBytes},
+	}
+	groups[name] = g
+	return g
+}
+
+// GetGroup returns the named group previously created with NewGroup, or
+// nil if there's no such group.
+func GetGroup(name string) *Group {
+	mu.RLock()
+	g := groups[name]
+	mu.RUnlock()
+	return g
+}
+
+// Get value for a key from cache
+func (g *Group) Get(key string) (ByteView, error) {
+	g.peersOnce.Do(func() {
+		g.peers = getPeers()
+	})
+	if v, ok := g.mainCache.get(key); ok {
+		return v, nil
+	}
+
+	return g.load(key)
+}
+
+func cloneBytes(b []byte) []byte {
+	c := make([]byte, len(b))
+	copy(c, b)
+	return c
+}
+
+func (g *Group) load(key string) (value ByteView, err error) {
+	if peer, ok := g.peers.PickPeer(key); ok {
+		value, err = g.getFromPeer(peer, key)
+		if err == nil {
+			return value, nil
+		}
+	}
+
+	return g.getLocally(key)
+}
+
+func (g *Group) populateCache(key string, value ByteView) {
+	g.mainCache.add(key, value)
+}
+
+func (g *Group) getLocally(key string) (ByteView, error) {
+	bytes, err := g.getter.Get(key)
+	if err != nil {
+		return ByteView{}, err
+
+	}
+	value := ByteView{b: cloneBytes(bytes)}
+	g.populateCache(key, value)
+	return value, nil
+}
+
+func (g *Group) getFromPeer(peer PeerGetter, key string) (ByteView, error) {
+	bytes, err := peer.Get(g.name, key)
+	if err != nil {
+		return ByteView{}, err
+	}
+	return ByteView{b: bytes}, nil
+}

gee-cache/day5-multi-nodes/geecache/geecache_test.go

@@ -0,0 +1,48 @@
+package geecache
+
+import (
+	"fmt"
+	"log"
+	"testing"
+)
+
+var db = map[string]string{
+	"Tom":  "630",
+	"Jack": "589",
+	"Sam":  "567",
+}
+
+func TestGet(t *testing.T) {
+	gee := NewGroup("scores", 2<<10, GetterFunc(
+		func(key string) ([]byte, error) {
+			log.Println("[group scores] search key", key)
+			if v, ok := db[key]; ok {
+				return []byte(v), nil
+			}
+			return nil, fmt.Errorf("%s not exist", key)
+		}))
+
+	for k, v := range db {
+		view, err := gee.Get(k)
+		if err != nil || view.String() != v {
+			t.Fatal("failed to get value of Tom")
+		}
+	}
+
+	if view, err := gee.Get("unknown"); err == nil {
+		t.Fatalf("the value of unknow should be empty, but %s got", view)
+	}
+}
+
+func TestGetGroup(t *testing.T) {
+	groupName := "scores"
+	NewGroup(groupName, 2<<10, GetterFunc(
+		func(key string) (bytes []byte, err error) { return }))
+	if group := GetGroup(groupName); group == nil || group.name != groupName {
+		t.Fatalf("group %s not exist", groupName)
+	}
+
+	if group := GetGroup(groupName + "111"); group != nil {
+		t.Fatalf("expect nil, but %s got", group.name)
+	}
+}

gee-cache/day5-multi-nodes/geecache/go.mod

@@ -0,0 +1,3 @@
+module geecache
+
+go 1.13