diff --git a/newt/cache/cache.go b/newt/cache/cache.go new file mode 100644 index 000000000..b5d7517db --- /dev/null +++ b/newt/cache/cache.go @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cache + +import ( + "encoding/gob" + "fmt" + "mynewt.apache.org/newt/newt/interfaces" + "mynewt.apache.org/newt/newt/repo" + "mynewt.apache.org/newt/util" + "os" + "path/filepath" +) + +type ProjectCache struct { + baseDir string +} + +func InitCache(projDir string) *ProjectCache { + pc := ProjectCache{} + pc.baseDir = filepath.Join(projDir, ".cache") + + if _, err := os.Stat(pc.baseDir); os.IsNotExist(err) { + if err := os.Mkdir(pc.baseDir, 0754); err != nil { + return nil + } + } + + return &pc +} + +func (pc *ProjectCache) getPackagesFile(repo *repo.Repo) string { + return fmt.Sprintf(filepath.Join(pc.baseDir, repo.Name())) +} + +func (pc *ProjectCache) AddPackages(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface) { + cacheName := pc.getPackagesFile(repo) + var dirList []string + + hash, err := repo.CurrentHash() + if err != nil { + return + } + + for _, v := range pkgMap { + dirList = append(dirList, v.BasePath()) + } + + f, err := os.Create(cacheName) + if err != nil { + util.OneTimeWarning("Failed to create cache file for \"%s\"", repo.Name()) + return + } + + defer f.Close() + + enc := gob.NewEncoder(f) + enc.Encode(hash) + enc.Encode(dirList) +} + +func (pc *ProjectCache) GetPackagesDirs(repo *repo.Repo) []string { + cacheName := pc.getPackagesFile(repo) + var dirList []string + + f, err := os.Open(cacheName) + if err != nil { + if !os.IsNotExist(err) { + util.OneTimeWarning("Failed to open cache file for \"%s\"", repo.Name()) + } + return nil + } + + defer f.Close() + + var hash string + + enc := gob.NewDecoder(f) + err = enc.Decode(&hash) + if err != nil { + util.OneTimeWarning("Failed to read cache for \"%s\"", repo.Name()) + return nil + } + + currHash, _ := repo.CurrentHash() + if hash != currHash { + return nil + } + + err = enc.Decode(&dirList) + if err != nil { + util.OneTimeWarning("Failed to read cache for \"%s\"", repo.Name()) + return nil + } + + return dirList +} diff --git a/newt/pkg/localpackage.go b/newt/pkg/localpackage.go index a68cbf2c7..f566b064f 100644 --- a/newt/pkg/localpackage.go +++ b/newt/pkg/localpackage.go @@ -449,8 +449,34 @@ func LocalPackageSpecialName(dirName string) bool { return ok } +func ReadPackage(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface, + pkgPath string) ([]string, error) { + + var warnings []string + + pkg, err := LoadLocalPackage(repo, pkgPath) + if err != nil { + warnings = append(warnings, err.Error()) + return warnings, nil + } + + if oldPkg, ok := pkgMap[pkg.Name()]; ok { + oldlPkg := oldPkg.(*LocalPackage) + warnings = append(warnings, + fmt.Sprintf("Multiple packages with same pkg.name=%s "+ + "in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(), + oldlPkg.BasePath(), pkg.BasePath())) + + return warnings, nil + } + + pkgMap[pkg.Name()] = pkg + + return warnings, nil +} + func ReadLocalPackageRecursive(repo *repo.Repo, - pkgList map[string]interfaces.PackageInterface, basePath string, + pkgMap map[string]interfaces.PackageInterface, basePath string, pkgName string, searchedMap map[string]struct{}) ([]string, error) { var warnings []string @@ -465,7 +491,7 @@ func ReadLocalPackageRecursive(repo *repo.Repo, continue } - subWarnings, err := ReadLocalPackageRecursive(repo, pkgList, + subWarnings, err := ReadLocalPackageRecursive(repo, pkgMap, basePath, filepath.Join(pkgName, name), searchedMap) warnings = append(warnings, subWarnings...) if err != nil { @@ -477,39 +503,21 @@ func ReadLocalPackageRecursive(repo *repo.Repo, return warnings, nil } - pkg, err := LoadLocalPackage(repo, filepath.Join(basePath, pkgName)) - if err != nil { - warnings = append(warnings, err.Error()) - return warnings, nil - } + var subWarnings []string + subWarnings, err = ReadPackage(repo, pkgMap, filepath.Join(basePath, pkgName)) + warnings = append(warnings, subWarnings...) - if oldPkg, ok := pkgList[pkg.Name()]; ok { - oldlPkg := oldPkg.(*LocalPackage) - warnings = append(warnings, - fmt.Sprintf("Multiple packages with same pkg.name=%s "+ - "in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(), - oldlPkg.BasePath(), pkg.BasePath())) - - return warnings, nil - } - - pkgList[pkg.Name()] = pkg - - return warnings, nil + return warnings, err } -func ReadLocalPackages(repo *repo.Repo, basePath string) ( - *map[string]interfaces.PackageInterface, []string, error) { - - pkgMap := &map[string]interfaces.PackageInterface{} - +func ReadLocalPackages(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface, basePath string) ([]string, error) { // Keep track of which directories we have traversed. Prevent infinite // loops caused by symlink cycles by not inspecting the same directory // twice. searchedMap := map[string]struct{}{} - warnings, err := ReadLocalPackageRecursive(repo, *pkgMap, + warnings, err := ReadLocalPackageRecursive(repo, pkgMap, basePath, "", searchedMap) - return pkgMap, warnings, err + return warnings, err } diff --git a/newt/project/project.go b/newt/project/project.go index 879e3922f..93ad16d89 100644 --- a/newt/project/project.go +++ b/newt/project/project.go @@ -21,6 +21,7 @@ package project import ( "fmt" + "mynewt.apache.org/newt/newt/cache" "os" "path" "path/filepath" @@ -57,6 +58,8 @@ type Project struct { // Base path of the project BasePath string + cache *cache.ProjectCache + packages interfaces.PackageList // Contains all the repos that form this project. Each repo is in one of @@ -634,6 +637,13 @@ func (proj *Project) Init(dir string, download bool) error { return err } + if util.EnableProjectCache { + proj.cache = cache.InitCache(proj.BasePath) + if proj.cache == nil { + util.OneTimeWarning("Failed to initialize project cache") + } + } + return nil } @@ -741,12 +751,37 @@ func (proj *Project) loadPackageList() error { // packages / store them in the project package list. repos := proj.Repos() for name, repo := range repos { - list, warnings, err := pkg.ReadLocalPackages(repo, repo.Path()) - if err == nil { - proj.packages[name] = list + pkgMap := &map[string]interfaces.PackageInterface{} + + var dirList []string + + if proj.cache != nil { + dirList = proj.cache.GetPackagesDirs(repo) } - proj.warnings = append(proj.warnings, warnings...) + if proj.cache != nil && dirList != nil { + log.Debug("Using cache for packages in \"%s\"\n", repo.Name()) + for _, pkgPath := range dirList { + warnings, err := pkg.ReadPackage(repo, *pkgMap, pkgPath) + if err == nil { + proj.packages[name] = pkgMap + } + + proj.warnings = append(proj.warnings, warnings...) + } + } else { + log.Debug("Not using cache for packages in \"%s\"\n", repo.Name()) + warnings, err := pkg.ReadLocalPackages(repo, *pkgMap, repo.Path()) + if err == nil { + proj.packages[name] = pkgMap + } + + if proj.cache != nil { + proj.cache.AddPackages(repo, *pkgMap) + } + + proj.warnings = append(proj.warnings, warnings...) + } } return nil diff --git a/newt/settings/settings.go b/newt/settings/settings.go index c70b58bb5..7137ed411 100644 --- a/newt/settings/settings.go +++ b/newt/settings/settings.go @@ -59,6 +59,7 @@ func processNewtrc(yc ycfg.YCfg) { util.SkipNewtCompat, _ = yc.GetValBoolDflt("skip_newt_compat", nil, false) util.SkipSyscfgRepoHash, _ = yc.GetValBoolDflt("skip_syscfg_repo_hash", nil, false) + util.EnableProjectCache, _ = yc.GetValBoolDflt("project_cache", nil, false) } func readNewtrc() ycfg.YCfg { diff --git a/util/util.go b/util/util.go index 41632bccf..b247915c5 100644 --- a/util/util.go +++ b/util/util.go @@ -51,6 +51,7 @@ var ShallowCloneDepth int var logFile *os.File var SkipNewtCompat bool var SkipSyscfgRepoHash bool +var EnableProjectCache bool func ParseEqualsPair(v string) (string, string, error) { s := strings.Split(v, "=")