divided memory

README.md

@@ -29,6 +29,7 @@ Useful :
 
 - cancel at any time, finished compressions will not need to be redone.
 - manage max threads.
+- max memory is divided by number of Guetzli instances (so 1000mb ram with 4 threads will give 250mb ram for each)
 
 ---
 
@@ -43,7 +44,7 @@ Flags:
       --nomemlimit     Do not limit memory usage.
   -f, --force          Force recompression
       --force-quality  Force recompression if quality changed
-  -t, --threads=3      Max concurrent threads. Default limit is 3
+  -t, --threads=3      Max concurrent threads. Default limit is the number of threads for the cpu minus 1
   -v, --version        Guetzli Version
 
 Args:

logger.go

@@ -48,6 +48,12 @@ var colors = []string{
 	"blue",
 	"magenta",
 	"cyan",
+	"red+h",
+	"green+h",
+	"yellow+h",
+	"blue+h",
+	"magenta+h",
+	"cyan+h",
 }
 
 func logForJob(j *job) func(string) string {

main.go

@@ -1,60 +1,48 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/signal"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/mgutz/ansi"
 )
 
 func main() {
 
-	settings := parseArgs()
-	settings = preflight(settings)
+	var (
+		settings   *settings
+		logger     *logger
+		reports    map[string]guetzliReport
+		newReports map[string]guetzliReport
+		jobs       []*job
+	)
 
-	logger := newLogger(settings.verbose)
+	settings = parseArgs()
+	settings = preflight(settings)
+	logger = newLogger(settings.verbose)
+	reports = getReports(settings.log)
+	jobs, newReports = getJobs(settings, reports, logger)
 
-	files, err := ioutil.ReadDir(settings.source)
-	if err != nil {
-		panic(err)
+	if len(jobs) == 0 {
+		return
 	}
 
-	reports := getReports(settings.log)
-
-	version, err := guetzliVersion()
-	if err != nil {
-		panic(err)
-	}
-
-	jobs := []*job{}
-
-FILE_ITERATOR:
-	for index, f := range files {
-		if !isFile(settings.source + f.Name()) {
-			continue FILE_ITERATOR
-		}
-
-		j := &job{
-			fileName: f.Name(),
-			report:   reports[settings.source+f.Name()],
-			settings: settings,
-			quit:     make(chan bool, 1),
-			done:     make(chan bool, 1),
-			logger:   logger,
-		}
+	settings = adjustSettingsBasedOnJobs(settings, len(jobs))
 
-		j.color = ansi.ColorFunc(colors[index%len(colors)])
-
-		if !needsProc(j) {
-			reports[j.report.Path] = j.report
-			j.logger.log(logForJob(j)("- skipped"))
-			continue FILE_ITERATOR
+	if settings.force {
+		saveReports(settings.version, newReports, settings.log)
+	} else if settings.forceQuality {
+		for path, r := range reports {
+			if r.Quality == settings.quality {
+				newReports[path] = r
+			}
 		}
-
-		jobs = append(jobs, j)
+		saveReports(settings.version, newReports, settings.log)
 	}
 
 	jobsQueue := make(chan *job, len(jobs))
@@ -82,7 +70,7 @@ FILE_ITERATOR:
 				select {
 				case success := <-j.done:
 					if success {
-						reports[j.report.Path] = j.report
+						newReports[j.report.Path] = j.report
 					}
 				case <-cancel:
 					close(j.quit)
@@ -104,5 +92,44 @@ FILE_ITERATOR:
 
 	wg.Wait()
 
-	saveReports(version, reports, settings.log)
+	saveReports(settings.version, newReports, settings.log)
+}
+
+func getJobs(settings *settings, reports map[string]guetzliReport, logger *logger) ([]*job, map[string]guetzliReport) {
+
+	newReports := make(map[string]guetzliReport)
+
+	files, err := ioutil.ReadDir(settings.source)
+	if err != nil {
+		panic(err)
+	}
+
+	jobs := []*job{}
+FILE_ITERATOR:
+	for index, f := range files {
+		if !isFile(settings.source + f.Name()) {
+			continue FILE_ITERATOR
+		}
+
+		j := &job{
+			fileName: f.Name(),
+			report:   reports[settings.source+f.Name()],
+			settings: settings,
+			quit:     make(chan bool, 1),
+			done:     make(chan bool, 1),
+			logger:   logger,
+		}
+
+		j.color = ansi.ColorFunc(colors[index%len(colors)])
+
+		if !needsProc(j) {
+			newReports[j.report.Path] = j.report
+			fmt.Printf("%s %s : \n- skipped\n", time.Now().Format("15:04:05"), j.settings.source+j.fileName)
+			continue FILE_ITERATOR
+		}
+
+		jobs = append(jobs, j)
+	}
+
+	return jobs, newReports
 }

settings.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"runtime"
 	"strings"
 
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
@@ -36,7 +37,7 @@ func parseArgs() *settings {
 	forceQuality := kingpin.Flag("force-quality", "Force recompression if quality changed").
 		Bool()
 
-	maxThreads := kingpin.Flag("threads", fmt.Sprintf("Max concurrent threads. Default limit is %d", defaultMaxThreads)).
+	maxThreads := kingpin.Flag("threads", fmt.Sprintf("Max concurrent threads. Default limit is %d", maxParallelism()-1)).
 		Short('t').
 		Default(fmt.Sprint(defaultMaxThreads)).
 		Uint()
@@ -98,18 +99,6 @@ func parseArgs() *settings {
 		maxThreads: int(*maxThreads),
 	}
 
-	if s.verbose {
-		fmt.Printf("Quality     =>  %d\n", s.quality)
-		fmt.Printf("NoMemLimit  =>  %t\n", s.nomemlimit)
-		fmt.Printf("MemLimit    =>  %d\n", s.memlimit)
-		fmt.Printf("Source      =>  %s\n", s.source)
-		fmt.Printf("Output      =>  %s\n", s.output)
-		fmt.Printf("Log         =>  %s\n", s.log)
-		fmt.Printf("Force       =>  %t\n", s.force)
-		fmt.Printf("Force Q     =>  %t\n", s.forceQuality)
-		fmt.Printf("Threads     =>  %d\n", s.maxThreads)
-	}
-
 	return s
 }
 
@@ -130,3 +119,12 @@ type settings struct {
 	version    string
 	maxThreads int
 }
+
+func maxParallelism() int {
+	maxProcs := runtime.GOMAXPROCS(0)
+	numCPU := runtime.NumCPU()
+	if maxProcs < numCPU {
+		return maxProcs
+	}
+	return numCPU
+}

setup.go

@@ -6,7 +6,6 @@ import (
 )
 
 func preflight(s *settings) *settings {
-
 	createIfMissing(s.log)
 	createIfMissing(s.output)
 
@@ -17,8 +16,30 @@ func preflight(s *settings) *settings {
 
 	s.version = version
 
+	return s
+}
+
+func adjustSettingsBasedOnJobs(s *settings, numberOfJobs int) *settings {
+
+	if s.maxThreads > numberOfJobs && numberOfJobs > 0 {
+		s.maxThreads = numberOfJobs
+	}
+
+	originalMemLimit := s.memlimit
+	adjustedMemLimit := s.memlimit / s.maxThreads
+	s.memlimit = s.memlimit / s.maxThreads
+
 	if s.verbose {
-		fmt.Printf("Version  =>  %s\n", s.version)
+		fmt.Printf("Quality     =>  %d\n", s.quality)
+		fmt.Printf("NoMemLimit  =>  %t\n", s.nomemlimit)
+		fmt.Printf("MemLimit    =>  %d / %d => %d\n", originalMemLimit, s.maxThreads, adjustedMemLimit)
+		fmt.Printf("Source      =>  %s\n", s.source)
+		fmt.Printf("Output      =>  %s\n", s.output)
+		fmt.Printf("Log         =>  %s\n", s.log)
+		fmt.Printf("Force       =>  %t\n", s.force)
+		fmt.Printf("Force Q     =>  %t\n", s.forceQuality)
+		fmt.Printf("Threads     =>  %d\n", s.maxThreads)
+		fmt.Printf("Version     =>  %s\n", s.version)
 	}
 
 	return s