Merge pull request #2 from ndv6/refactor/renderman

optimize renderman part II

go.mod

@@ -8,8 +8,10 @@ require (
 	github.com/go-rod/rod v0.82.2
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gocolly/colly/v2 v2.1.0
+	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gookit/cache v0.1.2
 	github.com/joho/godotenv v1.3.0
+	github.com/klauspost/compress v1.11.12
 	github.com/labstack/echo/v4 v4.1.17
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.7.0

go.sum

@@ -103,6 +103,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
+github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk=
+github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

renderman/main.go

@@ -9,19 +9,23 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/gookit/cache"
-	"github.com/gookit/cache/redis"
 	"github.com/joho/godotenv"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	"github.com/sirupsen/logrus"
 )
 
-var hc *Remote
-var enableStaticFile = false
-var bot = `googlebot|bingbot|adidxBot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest\/0\.|pinterestbot|slackbot|slack-imgproxy|vkshare|w3c_validator|whatsapp|collector-agent`
-var asset = `\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)`
+var (
+	hc               *Remote
+	enableStaticFile = false
+	bot              = `googlebot|bingbot|adidxBot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest\/0\.|pinterestbot|slackbot|slack-imgproxy|vkshare|w3c_validator|whatsapp|collector-agent`
+	asset            = `\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)`
+	hashName         string
+	compressr        compressor
+)
 
 func main() {
 	runtime.GOMAXPROCS(runtime.NumCPU())
@@ -33,8 +37,10 @@ func main() {
 		RemoteURL:  os.Getenv("CHROMIUM_URL"),
 		BackendURL: os.Getenv("BACKEND_URL"),
 	}
+	compressr = newZstdCompressor()
 
 	enableStaticFile = os.Getenv("ENABLE_STATIC_SERVE") != ""
+	hashName = strings.ReplaceAll(os.Getenv("APP_URL"), ":", "")
 
 	// Middleware
 	e.Use(middleware.Logger())
@@ -45,7 +51,7 @@ func main() {
 		if err != nil {
 			logrus.Error(err)
 		}
-		cache.Register(cache.DvrRedis, redis.Connect(os.Getenv("REDIS_DSN"), "", rdb))
+		cache.Register(cache.DvrRedis, redishashConnect(os.Getenv("REDIS_DSN"), "", rdb))
 		cache.DefaultUse(cache.DvrRedis)
 	} else {
 		cache.Register(cache.DvrFile, cache.NewFileCache("./tmp"))
@@ -59,47 +65,64 @@ func main() {
 
 func httpHandler(ctx echo.Context) error {
 	uri := fmt.Sprintf("%s%s", os.Getenv("BACKEND_URL"), ctx.Request().URL)
-	hash := fmt.Sprintf("%x", md5.Sum([]byte(uri)))
+	if !enableStaticFile {
+		return fetchAndCacheHeadless(ctx, uri)
+	}
 
-	if enableStaticFile {
-		userAgent := strings.ToLower(ctx.Request().Header.Get("user-agent"))
-		matched, _ := regexp.Match(bot, []byte(userAgent))
-		if matched {
-			return fetchAndCacheHeadless(ctx, uri, hash)
-		}
+	userAgent := strings.ToLower(ctx.Request().Header.Get("user-agent"))
+	matched, _ := regexp.Match(bot, []byte(userAgent))
+	if matched {
+		return fetchAndCacheHeadless(ctx, uri)
+	}
 
-		matched, _ = regexp.Match(asset, []byte(uri))
-		if !matched {
-			return fetchAndCacheHeadless(ctx, uri, hash)
-		}
-		hc.Proxy(ctx)
-		return nil
+	matched, _ = regexp.Match(asset, []byte(uri))
+	if !matched {
+		return fetchAndCacheHeadless(ctx, uri)
 	}
+	hc.Proxy(ctx)
+	return nil
+}
 
-	return fetchAndCacheHeadless(ctx, uri, hash)
+func fetchFromRemote(uri string) (c Content, err error) {
+	c, err = hc.FetchHeadless(uri)
+	if err != nil {
+		logrus.Error(err)
+		return
+	}
+	// replace backend_url to app_url
+	c.Content = strings.ReplaceAll(c.Content, os.Getenv("BACKEND_URL"), os.Getenv("APP_URL"))
+	return
 }
 
-func fetchAndCacheHeadless(ctx echo.Context, uri, hash string) error {
-	c := Content{}
-	ctn := cache.Get(hash)
-	if ctn != nil {
-		c.UnMarshal([]byte(fmt.Sprintf("%s", ctn)))
+func storeToCache(key string, c Content, dur time.Duration) {
+	byts := []byte(c.Marshal())
+	byts = compressr.Compress(byts)
+	_ = cache.Set(key, byts, dur)
+}
 
-		// if user agent is not collector-agent then we can return from cache
-		if ctx.Request().Header.Get("user-agent") != "collector-agent" {
-			return ctx.HTML(http.StatusOK, c.Content)
+func fetchAndCacheHeadless(ctx echo.Context, uri string) error {
+	var (
+		hash = fmt.Sprintf("%x", md5.Sum([]byte(uri)))
+		key  = fmt.Sprintf("%s:%s", hashName, hash)
+		ctn  interface{}
+		byts []byte
+		c    Content
+	)
+	if ctx.Request().Header.Get("user-agent") == "collector-agent" || !cache.Has(key) {
+		c, err := fetchFromRemote(uri)
+		if err != nil {
+			return ctx.HTML(http.StatusInternalServerError, "")
 		}
+		storeToCache(key, c, cache.OneDay)
+		return ctx.HTML(http.StatusOK, c.Content)
 	}
-
-	c, err := hc.FetchHeadless(uri)
+	ctn = cache.Get(key)
+	byts = ctn.([]byte)
+	byts, err := compressr.Decompress(byts)
 	if err != nil {
 		logrus.Error(err)
 		return ctx.HTML(http.StatusInternalServerError, "")
 	}
-	// replace backend_url to app_url
-	c.Content = strings.ReplaceAll(c.Content, os.Getenv("BACKEND_URL"), os.Getenv("APP_URL"))
-
-	// cache
-	_ = cache.Set(hash, c.Marshal(), cache.OneDay)
+	c.UnMarshal(byts)
 	return ctx.HTML(http.StatusOK, c.Content)
 }