diff --git a/chat_content.html b/chat_content.html new file mode 100644 index 000000000..71b1cfc85 --- /dev/null +++ b/chat_content.html @@ -0,0 +1,15 @@ +ChatGPT - Shared Content
Extend greenmask to anonymize filestore
Nov 23greenmaskmain
+320-4
+ + + + diff --git a/chat_head.txt b/chat_head.txt new file mode 100644 index 000000000..d33b022dd Binary files /dev/null and b/chat_head.txt differ diff --git a/docs/filestore.md b/docs/filestore.md new file mode 100644 index 000000000..b1b2457fa --- /dev/null +++ b/docs/filestore.md @@ -0,0 +1,130 @@ +# Filestore configuration + +Greenmask can optionally dump and restore a filestore alongside database data. Instead of maintaining a separate script to archive binary files, you can reuse the same database connection and storage configuration (for example, S3 settings) to back up and restore binaries together with the database, without extra scripts or duplicate credential management. You can also limit the filestore to an explicit file list to apply "anonymization by reduction": after restore, a post-restore script can replace selected binary references with a placeholder. + +## `dump.filestore` section + +In the `dump` section, `filestore` controls how a filesystem directory is packaged and uploaded to storage. + +### Parameters + +* `enabled` — enables or disables filestore dumping. +* `root_path` — **required**. Root directory of the filestore on the source filesystem. +* `include_list_file` — path to a file containing relative paths to include. Each line is a path relative to `root_path`. +* `include_list_query` — SQL query (inline) that returns a list of relative paths to include. +* `include_list_query_file` — path to a file with the SQL query to execute. +* `subdir` — storage subdirectory for filestore artifacts. Default: `filestore`. +* `archive_name` — name of the tar archive produced. Default: `filestore.tar.gz`. +* `metadata_name` — name of the metadata JSON file. Default: `filestore.json`. +* `use_pgzip` — optionally overrides the default compression behavior. Inherits dump `--pgzip`, which is `false` by default. +* `fail_on_missing` — if true, missing files cause the dump to fail. Default: `false`. +* `split.max_size_bytes` — enables archive splitting by maximum size. Default: `0` (disabled). +* `split.max_files` — enables archive splitting by maximum number of files. Default: `0` (disabled). + +!!! note + Filestore archives and metadata are uploaded into the configured storage under the `subdir` path. + +!!! warning + Only one include-list source can be configured at a time: + `include_list_file` **or** `include_list_query` **or** `include_list_query_file`. + If no include list is configured, all files under `root_path` are included recursively. + +### Example + +```yaml title="filestore dump config example" +dump: + filestore: + enabled: true + root_path: "/var/lib/odoo/filestore" + subdir: "filestore" + archive_name: "filestore.tar.gz" + metadata_name: "filestore.json" + + # choose exactly one source of paths: + include_list_file: "/etc/greenmask/filestore-files.txt" + # include_list_query: "SELECT DISTINCT store_fname FROM ir_attachment WHERE mimetype != 'application/pdf'" + # include_list_query_file: "/etc/greenmask/filestore_query.sql" + + fail_on_missing: true + use_pgzip: true + + split: + max_size_bytes: 1073741824 # 1 GiB + max_files: 100000 +``` + +## `restore.filestore` section + +In the `restore` section, `filestore` controls how stored filestore archives are fetched and unpacked. + +### Parameters + +* `enabled` — enables or disables filestore restoration. +* `target_path` — **required**. Destination directory on the target filesystem. +* `subdir` — storage subdirectory where filestore artifacts are stored. Default: `filestore`. +* `metadata_name` — metadata file name. Default: `filestore.json`. +* `use_pgzip` — optionally overrides compression behavior from metadata. +* `clean_target` — if true, removes the target directory before extraction. Default: `false`. +* `skip_existing` — if true, existing files are left untouched. Default: `false`. + +!!! note + If `use_pgzip` is set, it overrides the `use_pgzip` value stored in the filestore metadata. + +!!! warning + If `clean_target` is enabled, the entire `target_path` directory will be removed before restore. + +### Example + +```yaml title="filestore restore config example" +restore: + filestore: + enabled: true + target_path: "/var/lib/odoo/filestore" + subdir: "filestore" + metadata_name: "filestore.json" + clean_target: false + skip_existing: true + use_pgzip: true +``` + +## Include list sources + +When dumping a filestore, you can limit which files are packed using one of the include-list mechanisms: + +* **File list** (`include_list_file`) — a text file with one relative path per line. +* **SQL query** (`include_list_query` / `include_list_query_file`) — a query that returns relative paths. + +All paths are resolved relative to `root_path`. + +!!! tip + Use an SQL query when paths are stored in the database and you want the filestore selection to follow the dataset. + +### Why include lists are useful + +Restricting the filestore to an explicit list lets you implement "anonymization by reduction". Instead of copying +all binaries, you can keep only the necessary files and then use a post-restore script to replace references to +missing binaries with a placeholder or a generic asset (for example, `invoice_placeholder.pdf`). This approach +reduces storage, shortens transfer time, and keeps access credentials and storage handling centralized in Greenmask. + +### Odoo example query + +The following Odoo query excludes all PDF and ZIP attachments from the filestore selection: + +```sql +SELECT DISTINCT store_fname +FROM ir_attachment +WHERE store_fname IS NOT NULL + AND (NOT ((COALESCE(mimetype, '') = 'application/pdf') OR (COALESCE(mimetype, '') = 'application/zip'))) +ORDER BY store_fname +``` + +## Archive splitting + +If `split.max_size_bytes` or `split.max_files` is set, the filestore is split into multiple archives. Each archive is +stored separately, and metadata contains the archive list and statistics. + +Splitting is useful when: +* individual archives must stay below storage limits, +* large filestores should be processed in smaller parts. + +Splitting is not tied to `jobs`: filestore dump/restore does not use multi-threaded workers and processes archives sequentially. diff --git a/internal/db/postgres/cmd/dump.go b/internal/db/postgres/cmd/dump.go index 755af501e..49562ebfb 100644 --- a/internal/db/postgres/cmd/dump.go +++ b/internal/db/postgres/cmd/dump.go @@ -39,6 +39,7 @@ import ( "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/custom" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/domains" + "github.com/greenmaskio/greenmask/internal/filestore" "github.com/greenmaskio/greenmask/internal/storages" "github.com/greenmaskio/greenmask/pkg/toolkit" ) @@ -543,9 +544,61 @@ func (d *Dump) Run(ctx context.Context) (err error) { return fmt.Errorf("writeMetaData stage dumping error: %w", err) } + var includeListExecutor filestore.IncludeListQueryExecutor + if d.config.Dump.Filestore != nil { + includeListExecutor = &filestoreQueryExecutor{dump: d} + } + if err := filestore.Dump(ctx, d.config.Dump.Filestore, d.st, d.pgDumpOptions.Pgzip, includeListExecutor); err != nil { + return fmt.Errorf("filestore dumping error: %w", err) + } + return nil } +type filestoreQueryExecutor struct { + dump *Dump +} + +func (e *filestoreQueryExecutor) RunIncludeListQuery(ctx context.Context, query string) ([]string, error) { + conn, tx, err := e.dump.getWorkerTransaction(ctx) + if err != nil { + return nil, err + } + defer func() { + if err := conn.Close(ctx); err != nil { + log.Debug().Err(err).Msg("error closing include list query connection") + } + }() + defer func() { + if err := tx.Rollback(ctx); err != nil { + log.Debug().Err(err).Msg("unable to rollback include list query transaction") + } + }() + + rows, err := tx.Query(ctx, query) + if err != nil { + return nil, fmt.Errorf("run include_list_query: %w", err) + } + defer rows.Close() + + if len(rows.FieldDescriptions()) != 1 { + return nil, fmt.Errorf("include_list_query must return exactly one column, got %d", len(rows.FieldDescriptions())) + } + + var values []string + for rows.Next() { + var rel string + if scanErr := rows.Scan(&rel); scanErr != nil { + return nil, fmt.Errorf("scan include_list_query result: %w", scanErr) + } + values = append(values, rel) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate include_list_query result: %w", err) + } + return values, nil +} + func (d *Dump) MergeTocEntries(schemaEntries []*toc.Entry, dataEntries []*toc.Entry) ( []*toc.Entry, error, ) { diff --git a/internal/db/postgres/cmd/restore.go b/internal/db/postgres/cmd/restore.go index 324073540..d219e7bc5 100644 --- a/internal/db/postgres/cmd/restore.go +++ b/internal/db/postgres/cmd/restore.go @@ -42,6 +42,7 @@ import ( "github.com/greenmaskio/greenmask/internal/db/postgres/toc" "github.com/greenmaskio/greenmask/internal/db/postgres/utils" "github.com/greenmaskio/greenmask/internal/domains" + "github.com/greenmaskio/greenmask/internal/filestore" "github.com/greenmaskio/greenmask/internal/storages" "github.com/greenmaskio/greenmask/pkg/toolkit" ) @@ -142,6 +143,10 @@ func (r *Restore) Run(ctx context.Context) error { return fmt.Errorf("post-data stage restoration error: %w", err) } + if err := filestore.Restore(ctx, r.cfg.Filestore, r.st); err != nil { + return fmt.Errorf("filestore restoration error: %w", err) + } + return nil } diff --git a/internal/domains/config.go b/internal/domains/config.go index 101e1816b..19c521bd2 100644 --- a/internal/domains/config.go +++ b/internal/domains/config.go @@ -97,12 +97,14 @@ type Dump struct { PgDumpOptions pgdump.Options `mapstructure:"pg_dump_options" yaml:"pg_dump_options" json:"pg_dump_options"` Transformation []*Table `mapstructure:"transformation" yaml:"transformation" json:"transformation,omitempty"` VirtualReferences []*VirtualReference `mapstructure:"virtual_references" yaml:"virtual_references" json:"virtual_references,omitempty"` + Filestore *FilestoreDump `mapstructure:"filestore" yaml:"filestore" json:"filestore,omitempty"` } type Restore struct { PgRestoreOptions pgrestore.Options `mapstructure:"pg_restore_options" yaml:"pg_restore_options" json:"pg_restore_options"` Scripts map[string][]pgrestore.Script `mapstructure:"scripts" yaml:"scripts" json:"scripts,omitempty"` ErrorExclusions *DataRestorationErrorExclusions `mapstructure:"insert_error_exclusions" yaml:"insert_error_exclusions" json:"insert_error_exclusions,omitempty"` + Filestore *FilestoreRestore `mapstructure:"filestore" yaml:"filestore" json:"filestore,omitempty"` } type TablesDataRestorationErrorExclusions struct { @@ -122,6 +124,36 @@ type DataRestorationErrorExclusions struct { Global *GlobalDataRestorationErrorExclusions `mapstructure:"global" yaml:"global" json:"global,omitempty"` } +type FilestoreDump struct { + Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled,omitempty"` + RootPath string `mapstructure:"root_path" yaml:"root_path" json:"root_path,omitempty"` + FileList string `mapstructure:"file_list" yaml:"file_list" json:"file_list,omitempty"` + IncludeListFile string `mapstructure:"include_list_file" yaml:"include_list_file" json:"include_list_file,omitempty"` + IncludeListQuery string `mapstructure:"include_list_query" yaml:"include_list_query" json:"include_list_query,omitempty"` + IncludeListQueryFile string `mapstructure:"include_list_query_file" yaml:"include_list_query_file" json:"include_list_query_file,omitempty"` + Subdir string `mapstructure:"subdir" yaml:"subdir" json:"subdir,omitempty"` + ArchiveName string `mapstructure:"archive_name" yaml:"archive_name" json:"archive_name,omitempty"` + MetadataName string `mapstructure:"metadata_name" yaml:"metadata_name" json:"metadata_name,omitempty"` + UsePgzip *bool `mapstructure:"use_pgzip" yaml:"use_pgzip" json:"use_pgzip,omitempty"` + FailOnMissing bool `mapstructure:"fail_on_missing" yaml:"fail_on_missing" json:"fail_on_missing,omitempty"` + Split FilestoreDumpSplit `mapstructure:"split" yaml:"split" json:"split,omitempty"` +} + +type FilestoreDumpSplit struct { + MaxSizeBytes int64 `mapstructure:"max_size_bytes" yaml:"max_size_bytes" json:"max_size_bytes,omitempty"` + MaxFiles int `mapstructure:"max_files" yaml:"max_files" json:"max_files,omitempty"` +} + +type FilestoreRestore struct { + Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled,omitempty"` + TargetPath string `mapstructure:"target_path" yaml:"target_path" json:"target_path,omitempty"` + Subdir string `mapstructure:"subdir" yaml:"subdir" json:"subdir,omitempty"` + MetadataName string `mapstructure:"metadata_name" yaml:"metadata_name" json:"metadata_name,omitempty"` + UsePgzip *bool `mapstructure:"use_pgzip" yaml:"use_pgzip" json:"use_pgzip,omitempty"` + CleanTarget bool `mapstructure:"clean_target" yaml:"clean_target" json:"clean_target,omitempty"` + SkipExisting bool `mapstructure:"skip_existing" yaml:"skip_existing" json:"skip_existing,omitempty"` +} + type TransformerConfig struct { Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"` ApplyForReferences bool `mapstructure:"apply_for_references" yaml:"apply_for_references" json:"apply_for_references,omitempty"` diff --git a/internal/filestore/dump.go b/internal/filestore/dump.go new file mode 100644 index 000000000..de3cae415 --- /dev/null +++ b/internal/filestore/dump.go @@ -0,0 +1,617 @@ +// Copyright 2023 Greenmask +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filestore + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/rs/zerolog/log" + + "github.com/greenmaskio/greenmask/internal/domains" + "github.com/greenmaskio/greenmask/internal/storages" + "github.com/greenmaskio/greenmask/internal/utils/ioutils" +) + +const ( + defaultFilestoreSubdir = "filestore" + defaultArchiveName = "filestore.tar.gz" + defaultMetadataFileName = "filestore.json" + tarGzDoubleExtension = ".tar.gz" + defaultArchiveSplitWidth = 4 +) + +type dumpSettings struct { + cfg *domains.FilestoreDump + usePgzip bool + splitEnabled bool + subdir string + archiveName string + metadataName string + maxSizeBytes int64 + maxFiles int + failOnMissing bool + fileListPath string + includeListQuery string + includeListQueryFile string + includeListSource includeListSource + queryExecutor IncludeListQueryExecutor + rootPath string +} + +type fileEntry struct { + AbsolutePath string + RelativePath string +} + +type archiveState struct { + index int + name string + tw *tar.Writer + gzWriter ioutils.CountWriteCloser + reader ioutils.CountReadCloser + done chan error + files int + originalBytes int64 +} + +// IncludeListQueryExecutor executes the configured include list SQL query and returns the resulting paths. +type IncludeListQueryExecutor interface { + RunIncludeListQuery(ctx context.Context, query string) ([]string, error) +} + +type includeListSource int + +const ( + includeListSourceNone includeListSource = iota + includeListSourceFile + includeListSourceQueryInline + includeListSourceQueryFile +) + +func (s includeListSource) String() string { + switch s { + case includeListSourceFile: + return "file" + case includeListSourceQueryInline: + return "inline_query" + case includeListSourceQueryFile: + return "query_file" + default: + return "" + } +} + +// Dump packs the configured filestore subset (or whole directory) and uploads it to storage. +func Dump( + ctx context.Context, + cfg *domains.FilestoreDump, + st storages.Storager, + defaultPgzip bool, + executor IncludeListQueryExecutor, +) error { + if cfg == nil || !cfg.Enabled { + return nil + } + settings, err := buildDumpSettings(cfg, defaultPgzip) + if err != nil { + return err + } + settings.queryExecutor = executor + + log.Info().Msg("starting filestore dump") + entries, missing, err := collectEntries(ctx, settings) + if err != nil { + return err + } + if len(entries) == 0 { + log.Warn().Msg("filestore dump skipped because no files matched criteria") + return writeMetadata(ctx, st, settings, missing, nil, 0, 0) + } + + manager := newArchiveManager(st, settings) + for _, entry := range entries { + if err := manager.add(ctx, entry); err != nil { + return err + } + } + + totalCompressed, err := manager.close(ctx) + if err != nil { + return err + } + + if err := writeMetadata(ctx, st, settings, missing, manager.archives, manager.totalFiles, totalCompressed); err != nil { + return err + } + log.Info(). + Int("archives", len(manager.archives)). + Int("files", manager.totalFiles). + Int64("original_bytes", manager.totalOriginalBytes). + Int64("compressed_bytes", totalCompressed). + Msg("filestore dump completed") + return nil +} + +func buildDumpSettings(cfg *domains.FilestoreDump, defaultPgzip bool) (*dumpSettings, error) { + if cfg.RootPath == "" { + return nil, errors.New("dump.filestore.root_path cannot be empty") + } + rootPath := filepath.Clean(cfg.RootPath) + info, err := os.Stat(rootPath) + if err != nil { + return nil, fmt.Errorf("stat filestore root: %w", err) + } + if !info.IsDir() { + return nil, fmt.Errorf("filestore root %s is not a directory", rootPath) + } + subdir := cfg.Subdir + if subdir == "" { + subdir = defaultFilestoreSubdir + } + archiveName := cfg.ArchiveName + if archiveName == "" { + archiveName = defaultArchiveName + } + metadataName := cfg.MetadataName + if metadataName == "" { + metadataName = defaultMetadataFileName + } + usePgzip := defaultPgzip + if cfg.UsePgzip != nil { + usePgzip = *cfg.UsePgzip + } + splitEnabled := cfg.Split.MaxFiles > 0 || cfg.Split.MaxSizeBytes > 0 + + fileListPath := strings.TrimSpace(cfg.FileList) + if fileListPath == "" { + fileListPath = strings.TrimSpace(cfg.IncludeListFile) + } + if fileListPath != "" { + fileListPath = filepath.Clean(fileListPath) + } + + includeQuery, querySource, queryFile, err := resolveIncludeListQuery(cfg) + if err != nil { + return nil, err + } + if fileListPath != "" && includeQuery != "" { + return nil, errors.New("only one of file_list/include_list_file or include_list_query/include_list_query_file can be set") + } + listSource := querySource + if fileListPath != "" { + listSource = includeListSourceFile + } + + return &dumpSettings{ + cfg: cfg, + usePgzip: usePgzip, + splitEnabled: splitEnabled, + subdir: subdir, + archiveName: archiveName, + metadataName: metadataName, + maxSizeBytes: cfg.Split.MaxSizeBytes, + maxFiles: cfg.Split.MaxFiles, + failOnMissing: cfg.FailOnMissing, + fileListPath: fileListPath, + includeListQuery: includeQuery, + includeListQueryFile: queryFile, + includeListSource: listSource, + rootPath: rootPath, + }, nil +} + +func resolveIncludeListQuery(cfg *domains.FilestoreDump) (string, includeListSource, string, error) { + inlineQuery := strings.TrimSpace(cfg.IncludeListQuery) + queryFilePath := strings.TrimSpace(cfg.IncludeListQueryFile) + if inlineQuery != "" && queryFilePath != "" { + return "", includeListSourceNone, "", errors.New("only one of include_list_query or include_list_query_file can be set") + } + if queryFilePath != "" { + cleanPath := filepath.Clean(queryFilePath) + bytes, err := os.ReadFile(cleanPath) + if err != nil { + return "", includeListSourceNone, "", fmt.Errorf("read include_list_query_file: %w", err) + } + inlineQuery = string(bytes) + queryFilePath = cleanPath + } + normalized, err := normalizeIncludeListQuery(inlineQuery) + if err != nil { + return "", includeListSourceNone, "", err + } + if normalized == "" { + return "", includeListSourceNone, "", nil + } + source := includeListSourceQueryInline + if queryFilePath != "" { + source = includeListSourceQueryFile + } + return normalized, source, queryFilePath, nil +} + +func normalizeIncludeListQuery(query string) (string, error) { + trimmed := strings.TrimSpace(query) + if trimmed == "" { + return "", nil + } + for strings.HasSuffix(trimmed, ";") { + trimmed = strings.TrimSpace(strings.TrimSuffix(trimmed, ";")) + } + if strings.Contains(trimmed, ";") { + return "", errors.New("include_list_query must contain a single SELECT statement without additional commands") + } + if !strings.HasPrefix(strings.ToUpper(trimmed), "SELECT") { + return "", errors.New("include_list_query must start with SELECT") + } + return trimmed, nil +} + +func collectEntries(ctx context.Context, settings *dumpSettings) ([]fileEntry, []string, error) { + switch { + case settings.includeListQuery != "": + return collectFromQuery(ctx, settings) + case settings.fileListPath != "": + return collectFromList(settings) + default: + return collectWholeDirectory(settings) + } +} + +func collectFromList(settings *dumpSettings) ([]fileEntry, []string, error) { + file, err := os.Open(settings.fileListPath) + if err != nil { + return nil, nil, fmt.Errorf("open file list: %w", err) + } + defer file.Close() + + var entries []fileEntry + var missing []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if err := appendEntryFromValue(&entries, &missing, scanner.Text(), settings); err != nil { + return nil, nil, err + } + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("read file list: %w", err) + } + return entries, missing, nil +} + +func collectFromQuery(ctx context.Context, settings *dumpSettings) ([]fileEntry, []string, error) { + if settings.queryExecutor == nil { + return nil, nil, errors.New("include_list_query requires a query executor but none was provided") + } + values, err := settings.queryExecutor.RunIncludeListQuery(ctx, settings.includeListQuery) + if err != nil { + return nil, nil, fmt.Errorf("execute include_list_query: %w", err) + } + return collectEntriesFromValues(values, settings) +} + +func collectEntriesFromValues(values []string, settings *dumpSettings) ([]fileEntry, []string, error) { + var entries []fileEntry + var missing []string + for _, raw := range values { + if err := appendEntryFromValue(&entries, &missing, raw, settings); err != nil { + return nil, nil, err + } + } + return entries, missing, nil +} + +func appendEntryFromValue(entries *[]fileEntry, missing *[]string, raw string, settings *dumpSettings) error { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + return nil + } + relPath := filepath.Clean(line) + if filepath.IsAbs(relPath) { + return fmt.Errorf("file list entries must be relative: %s", relPath) + } + fullPath := filepath.Join(settings.rootPath, relPath) + if !strings.HasPrefix(fullPath, settings.rootPath) { + return fmt.Errorf("file list entry exits filestore root: %s", relPath) + } + info, err := os.Lstat(fullPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + if settings.failOnMissing { + return fmt.Errorf("listed file %s not found", relPath) + } + *missing = append(*missing, filepath.ToSlash(relPath)) + return nil + } + return fmt.Errorf("stat %s: %w", relPath, err) + } + if info.IsDir() { + log.Warn().Str("path", filepath.ToSlash(relPath)).Msg("skipping directory entry in file list") + return nil + } + *entries = append(*entries, fileEntry{ + AbsolutePath: fullPath, + RelativePath: filepath.ToSlash(relPath), + }) + return nil +} + +func collectWholeDirectory(settings *dumpSettings) ([]fileEntry, []string, error) { + var entries []fileEntry + err := filepath.WalkDir(settings.rootPath, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() && info.Mode()&os.ModeSymlink == 0 { + log.Warn().Str("path", path).Msg("skipping unsupported file type") + return nil + } + rel, err := filepath.Rel(settings.rootPath, path) + if err != nil { + return err + } + entries = append(entries, fileEntry{ + AbsolutePath: path, + RelativePath: filepath.ToSlash(rel), + }) + return nil + }) + if err != nil { + return nil, nil, fmt.Errorf("walk filestore root: %w", err) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].RelativePath < entries[j].RelativePath + }) + return entries, nil, nil +} + +type archiveManager struct { + settings *dumpSettings + st storages.Storager + filestoreStorage storages.Storager + current *archiveState + archives []archiveMeta + totalFiles int + totalOriginalBytes int64 +} + +func newArchiveManager(st storages.Storager, settings *dumpSettings) *archiveManager { + return &archiveManager{ + settings: settings, + st: st, + filestoreStorage: st.SubStorage(settings.subdir, true), + } +} + +func (m *archiveManager) add(ctx context.Context, entry fileEntry) error { + info, err := os.Lstat(entry.AbsolutePath) + if err != nil { + return fmt.Errorf("stat %s: %w", entry.RelativePath, err) + } + if info.Mode()&os.ModeSymlink != 0 { + return m.addSymlink(ctx, entry, info) + } + if !info.Mode().IsRegular() { + log.Warn().Str("path", entry.RelativePath).Msg("skipping unsupported file type") + return nil + } + if err := m.ensureArchive(ctx); err != nil { + return err + } + if err := m.writeFile(entry, info); err != nil { + return err + } + if m.shouldRotate() { + if err := m.rotate(ctx); err != nil { + return err + } + } + return nil +} + +func (m *archiveManager) addSymlink(ctx context.Context, entry fileEntry, info os.FileInfo) error { + if err := m.ensureArchive(ctx); err != nil { + return err + } + linkTarget, err := os.Readlink(entry.AbsolutePath) + if err != nil { + return fmt.Errorf("readlink %s: %w", entry.RelativePath, err) + } + hdr, err := tar.FileInfoHeader(info, linkTarget) + if err != nil { + return fmt.Errorf("tar header for %s: %w", entry.RelativePath, err) + } + hdr.Name = entry.RelativePath + if err := m.current.tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("write header for %s: %w", entry.RelativePath, err) + } + m.recordFile(info.Size()) + return nil +} + +func (m *archiveManager) writeFile(entry fileEntry, info os.FileInfo) error { + file, err := os.Open(entry.AbsolutePath) + if err != nil { + return fmt.Errorf("open %s: %w", entry.RelativePath, err) + } + defer file.Close() + + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return fmt.Errorf("tar header for %s: %w", entry.RelativePath, err) + } + hdr.Name = entry.RelativePath + if err := m.current.tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("write header for %s: %w", entry.RelativePath, err) + } + if _, err := io.Copy(m.current.tw, file); err != nil { + return fmt.Errorf("copy %s: %w", entry.RelativePath, err) + } + m.recordFile(info.Size()) + return nil +} + +func (m *archiveManager) recordFile(size int64) { + m.current.files++ + m.current.originalBytes += size + m.totalFiles++ + m.totalOriginalBytes += size +} + +func (m *archiveManager) close(ctx context.Context) (int64, error) { + if err := m.rotate(ctx); err != nil { + return 0, err + } + var compressedTotal int64 + for _, archive := range m.archives { + compressedTotal += archive.CompressedBytes + } + return compressedTotal, nil +} + +func (m *archiveManager) ensureArchive(ctx context.Context) error { + if m.current != nil { + return nil + } + nextIndex := len(m.archives) + 1 + name := m.buildArchiveName(nextIndex) + w, r := ioutils.NewGzipPipe(m.settings.usePgzip) + state := &archiveState{ + index: nextIndex, + name: name, + tw: tar.NewWriter(w), + gzWriter: w, + reader: r, + done: make(chan error, 1), + } + go func() { + defer func() { + if err := r.Close(); err != nil { + log.Warn().Err(err).Str("archive", name).Msg("error closing filestore reader") + } + }() + err := m.filestoreStorage.PutObject(ctx, name, r) + state.done <- err + }() + m.current = state + return nil +} + +func (m *archiveManager) rotate(ctx context.Context) error { + if m.current == nil { + return nil + } + if err := m.current.tw.Close(); err != nil { + return fmt.Errorf("close tar archive %s: %w", m.current.name, err) + } + if err := m.current.gzWriter.Close(); err != nil { + return fmt.Errorf("close gzip archive %s: %w", m.current.name, err) + } + if err := <-m.current.done; err != nil { + return fmt.Errorf("upload archive %s: %w", m.current.name, err) + } + + m.archives = append(m.archives, archiveMeta{ + Name: m.current.name, + Files: m.current.files, + OriginalBytes: m.current.originalBytes, + CompressedBytes: m.current.reader.GetCount(), + }) + m.current = nil + return nil +} + +func (m *archiveManager) shouldRotate() bool { + if !m.settings.splitEnabled || m.current == nil { + return false + } + if m.settings.maxFiles > 0 && m.current.files >= m.settings.maxFiles { + return true + } + if m.settings.maxSizeBytes > 0 && m.current.originalBytes >= m.settings.maxSizeBytes { + return true + } + return false +} + +func (m *archiveManager) buildArchiveName(idx int) string { + if !m.settings.splitEnabled && idx == 1 { + return m.settings.archiveName + } + name := m.settings.archiveName + ext := filepath.Ext(name) + base := strings.TrimSuffix(name, ext) + suffix := ext + if strings.HasSuffix(strings.ToLower(name), tarGzDoubleExtension) { + base = strings.TrimSuffix(name, tarGzDoubleExtension) + suffix = tarGzDoubleExtension + } + return fmt.Sprintf("%s-%0*d%s", base, defaultArchiveSplitWidth, idx, suffix) +} + +func writeMetadata(ctx context.Context, st storages.Storager, settings *dumpSettings, missing []string, archives []archiveMeta, totalFiles int, totalCompressed int64) error { + meta := metadata{ + GeneratedAt: time.Now().UTC(), + RootPath: settings.rootPath, + FileList: settings.fileListPath, + IncludeListQuery: settings.includeListQuery, + IncludeListQueryFile: settings.includeListQueryFile, + IncludeListSource: settings.includeListSource.String(), + Subdir: settings.subdir, + ArchiveName: settings.archiveName, + UsePgzip: settings.usePgzip, + TotalFiles: totalFiles, + TotalOriginalBytes: 0, + TotalCompressedBytes: totalCompressed, + Archives: archives, + Missing: missing, + } + var totalOriginal int64 + for _, archive := range archives { + totalOriginal += archive.OriginalBytes + } + meta.TotalOriginalBytes = totalOriginal + meta.Split.MaxFiles = settings.maxFiles + meta.Split.MaxSizeBytes = settings.maxSizeBytes + + metaBytes, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return fmt.Errorf("marshal filestore metadata: %w", err) + } + + filestoreStorage := st.SubStorage(settings.subdir, true) + if err := filestoreStorage.PutObject(ctx, settings.metadataName, bytes.NewReader(metaBytes)); err != nil { + return fmt.Errorf("store filestore metadata: %w", err) + } + return nil +} diff --git a/internal/filestore/dump_test.go b/internal/filestore/dump_test.go new file mode 100644 index 000000000..bf90e81dd --- /dev/null +++ b/internal/filestore/dump_test.go @@ -0,0 +1,285 @@ +// Copyright 2023 Greenmask +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filestore + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/greenmaskio/greenmask/internal/domains" + "github.com/greenmaskio/greenmask/internal/storages" + "github.com/greenmaskio/greenmask/internal/storages/directory" +) + +func TestDumpWholeDirectory(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + writeFile(t, filepath.Join(src, "a.txt"), "a") + writeFile(t, filepath.Join(src, "b.txt"), "bb") + require.NoError(t, os.MkdirAll(filepath.Join(src, "nested"), 0o755)) + writeFile(t, filepath.Join(src, "nested", "c.txt"), "ccc") + + storage, dumpStorage := newStorage(t) + + cfg := &domains.FilestoreDump{ + Enabled: true, + RootPath: src, + } + require.NoError(t, Dump(ctx, cfg, dumpStorage, false, nil)) + + meta := readMetadata(t, storage, "filestore.json") + require.Equal(t, 3, meta.TotalFiles) + require.Len(t, meta.Archives, 1) + + files := readArchive(t, storage, meta.Archives[0].Name) + require.Equal(t, map[string]string{ + "a.txt": "a", + "b.txt": "bb", + "nested/c.txt": "ccc", + }, files) +} + +func TestDumpSplitByMaxFiles(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + writeFile(t, filepath.Join(src, "a.txt"), "a") + writeFile(t, filepath.Join(src, "b.txt"), "b") + + storage, dumpStorage := newStorage(t) + + cfg := &domains.FilestoreDump{ + Enabled: true, + RootPath: src, + Split: domains.FilestoreDumpSplit{ + MaxFiles: 1, + }, + } + require.NoError(t, Dump(ctx, cfg, dumpStorage, false, nil)) + + meta := readMetadata(t, storage, "filestore.json") + require.Len(t, meta.Archives, 2) + require.True(t, meta.TotalFiles == 2) +} + +func TestRestoreExtractsArchives(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + writeFile(t, filepath.Join(src, "a.txt"), "payload") + + _, dumpStorage := newStorage(t) + + cfg := &domains.FilestoreDump{ + Enabled: true, + RootPath: src, + } + require.NoError(t, Dump(ctx, cfg, dumpStorage, false, nil)) + + target := t.TempDir() + restoreCfg := &domains.FilestoreRestore{ + Enabled: true, + TargetPath: target, + } + require.NoError(t, Restore(ctx, restoreCfg, dumpStorage)) + + bytes, err := os.ReadFile(filepath.Join(target, "a.txt")) + require.NoError(t, err) + require.Equal(t, "payload", string(bytes)) +} + +func TestDumpIncludeListQuery(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + writeFile(t, filepath.Join(src, "a.txt"), "payload-a") + writeFile(t, filepath.Join(src, "nested", "b.txt"), "payload-b") + + storage, dumpStorage := newStorage(t) + + query := "SELECT path FROM keep_list" + executor := &stubIncludeListQueryExecutor{ + t: t, + expectedQuery: query, + result: []string{"a.txt", "nested/b.txt"}, + } + + cfg := &domains.FilestoreDump{ + Enabled: true, + RootPath: src, + IncludeListQuery: query, + } + require.NoError(t, Dump(ctx, cfg, dumpStorage, false, executor)) + + meta := readMetadata(t, storage, "filestore.json") + require.Equal(t, query, meta.IncludeListQuery) + require.Equal(t, "inline_query", meta.IncludeListSource) + require.Empty(t, meta.IncludeListQueryFile) + + files := readArchive(t, storage, meta.Archives[0].Name) + require.Equal(t, map[string]string{ + "a.txt": "payload-a", + "nested/b.txt": "payload-b", + }, files) +} + +func TestDumpIncludeListQueryFile(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + writeFile(t, filepath.Join(src, "only.txt"), "payload") + writeFile(t, filepath.Join(src, "skip.txt"), "nope") + + storage, dumpStorage := newStorage(t) + + queryFile := filepath.Join(t.TempDir(), "include.sql") + require.NoError(t, os.WriteFile(queryFile, []byte("SELECT path FROM keep_source;\n"), 0o644)) + + executor := &stubIncludeListQueryExecutor{ + t: t, + expectedQuery: "SELECT path FROM keep_source", + result: []string{"only.txt"}, + } + + cfg := &domains.FilestoreDump{ + Enabled: true, + RootPath: src, + IncludeListQueryFile: queryFile, + } + require.NoError(t, Dump(ctx, cfg, dumpStorage, false, executor)) + + meta := readMetadata(t, storage, "filestore.json") + require.Equal(t, "SELECT path FROM keep_source", meta.IncludeListQuery) + require.Equal(t, filepath.Clean(queryFile), meta.IncludeListQueryFile) + require.Equal(t, "query_file", meta.IncludeListSource) + files := readArchive(t, storage, meta.Archives[0].Name) + require.Equal(t, map[string]string{"only.txt": "payload"}, files) +} + +func TestDumpIncludeListQueryRequiresExecutor(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + writeFile(t, filepath.Join(src, "a.txt"), "payload") + + _, dumpStorage := newStorage(t) + + cfg := &domains.FilestoreDump{ + Enabled: true, + RootPath: src, + IncludeListQuery: "SELECT 'a.txt'", + } + err := Dump(ctx, cfg, dumpStorage, false, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "query executor") +} + +func TestBuildDumpSettingsRejectsConflictingSources(t *testing.T) { + root := t.TempDir() + cfg := &domains.FilestoreDump{ + Enabled: true, + RootPath: root, + FileList: "list.txt", + IncludeListQuery: "SELECT 'a.txt'", + } + _, err := buildDumpSettings(cfg, false) + require.Error(t, err) +} + +func TestNormalizeIncludeListQueryValidation(t *testing.T) { + _, err := normalizeIncludeListQuery("DELETE FROM files") + require.Error(t, err) + + query, err := normalizeIncludeListQuery("SELECT path FROM keep_list ; ") + require.NoError(t, err) + require.Equal(t, "SELECT path FROM keep_list", query) + + _, err = normalizeIncludeListQuery("SELECT path FROM t; SELECT 'x'") + require.Error(t, err) +} + +type stubIncludeListQueryExecutor struct { + t *testing.T + expectedQuery string + result []string + err error +} + +func (s *stubIncludeListQueryExecutor) RunIncludeListQuery(ctx context.Context, query string) ([]string, error) { + if s.expectedQuery != "" { + require.Equal(s.t, s.expectedQuery, query) + } + if s.err != nil { + return nil, s.err + } + return append([]string(nil), s.result...), nil +} + +func newStorage(t *testing.T) (*directory.Storage, storages.Storager) { + t.Helper() + root := t.TempDir() + base, err := directory.NewStorage(&directory.Config{Path: root}) + require.NoError(t, err) + sub := base.SubStorage("dump", true) + dirSub, ok := sub.(*directory.Storage) + require.True(t, ok) + return dirSub, sub +} + +func readMetadata(t *testing.T, st *directory.Storage, name string) metadata { + t.Helper() + fullPath := filepath.Join(st.GetCwd(), "filestore", name) + bytes, err := os.ReadFile(fullPath) + require.NoError(t, err) + var meta metadata + require.NoError(t, json.Unmarshal(bytes, &meta)) + return meta +} + +func readArchive(t *testing.T, st *directory.Storage, name string) map[string]string { + t.Helper() + fullPath := filepath.Join(st.GetCwd(), "filestore", name) + file, err := os.Open(fullPath) + require.NoError(t, err) + defer file.Close() + + gz, err := gzip.NewReader(file) + require.NoError(t, err) + defer gz.Close() + + tr := tar.NewReader(gz) + result := make(map[string]string) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + data, err := io.ReadAll(tr) + require.NoError(t, err) + result[hdr.Name] = string(data) + } + return result +} + +func writeFile(t *testing.T, path, data string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(data), 0o644)) +} + diff --git a/internal/filestore/restore.go b/internal/filestore/restore.go new file mode 100644 index 000000000..ed94b2288 --- /dev/null +++ b/internal/filestore/restore.go @@ -0,0 +1,206 @@ +// Copyright 2023 Greenmask +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filestore + +import ( + "archive/tar" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/rs/zerolog/log" + + "github.com/greenmaskio/greenmask/internal/domains" + "github.com/greenmaskio/greenmask/internal/storages" + "github.com/greenmaskio/greenmask/internal/utils/ioutils" +) + +type restoreSettings struct { + cfg *domains.FilestoreRestore + targetPath string + subdir string + metadataName string + cleanTarget bool + skipExisting bool + usePgzip *bool +} + +// Restore downloads filestore archives from storage and extracts them into target path. +func Restore(ctx context.Context, cfg *domains.FilestoreRestore, st storages.Storager) error { + if cfg == nil || !cfg.Enabled { + return nil + } + settings, err := buildRestoreSettings(cfg) + if err != nil { + return err + } + filestoreStorage := st.SubStorage(settings.subdir, true) + + metaReader, err := filestoreStorage.GetObject(ctx, settings.metadataName) + if err != nil { + return fmt.Errorf("read filestore metadata: %w", err) + } + defer metaReader.Close() + + var meta metadata + if err := json.NewDecoder(metaReader).Decode(&meta); err != nil { + return fmt.Errorf("decode filestore metadata: %w", err) + } + + if settings.usePgzip != nil { + meta.UsePgzip = *settings.usePgzip + } + + if settings.cleanTarget { + if err := os.RemoveAll(settings.targetPath); err != nil { + return fmt.Errorf("clean filestore target: %w", err) + } + } + if err := os.MkdirAll(settings.targetPath, 0o750); err != nil { + return fmt.Errorf("create filestore target: %w", err) + } + + if len(meta.Archives) == 0 { + log.Info().Msg("filestore restore skipped because metadata contains no archives") + return nil + } + + var restoredFiles int + for _, archive := range meta.Archives { + files, err := extractArchive(ctx, filestoreStorage, settings.targetPath, archive.Name, meta.UsePgzip, settings.skipExisting) + if err != nil { + return err + } + restoredFiles += files + } + + log.Info(). + Int("archives", len(meta.Archives)). + Int("files", restoredFiles). + Str("target", settings.targetPath). + Msg("filestore restore completed") + return nil +} + +func buildRestoreSettings(cfg *domains.FilestoreRestore) (*restoreSettings, error) { + if cfg.TargetPath == "" { + return nil, errors.New("restore.filestore.target_path cannot be empty") + } + targetPath := filepath.Clean(cfg.TargetPath) + subdir := cfg.Subdir + if subdir == "" { + subdir = defaultFilestoreSubdir + } + metadataName := cfg.MetadataName + if metadataName == "" { + metadataName = defaultMetadataFileName + } + return &restoreSettings{ + cfg: cfg, + targetPath: targetPath, + subdir: filepath.Clean(subdir), + metadataName: metadataName, + cleanTarget: cfg.CleanTarget, + skipExisting: cfg.SkipExisting, + usePgzip: cfg.UsePgzip, + }, nil +} + +func extractArchive( + ctx context.Context, + st storages.Storager, + targetDir string, + name string, + usePgzip bool, + skipExisting bool, +) (int, error) { + reader, err := st.GetObject(ctx, name) + if err != nil { + return 0, fmt.Errorf("download filestore archive %s: %w", name, err) + } + defer reader.Close() + + gzipReader, err := ioutils.GetGzipReadCloser(reader, usePgzip) + if err != nil { + return 0, fmt.Errorf("open gzip reader for %s: %w", name, err) + } + defer gzipReader.Close() + + tr := tar.NewReader(gzipReader) + var files int + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return files, fmt.Errorf("read tar %s: %w", name, err) + } + if err := restoreEntry(targetDir, hdr, tr, skipExisting); err != nil { + return files, fmt.Errorf("restore entry %s: %w", hdr.Name, err) + } + files++ + } + return files, nil +} + +func restoreEntry(targetDir string, hdr *tar.Header, content io.Reader, skipExisting bool) error { + cleanName := filepath.Clean(hdr.Name) + if strings.Contains(cleanName, "..") { + return fmt.Errorf("unsafe path %s", hdr.Name) + } + fullPath := filepath.Join(targetDir, cleanName) + + switch hdr.Typeflag { + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(fullPath), 0o750); err != nil { + return fmt.Errorf("create parent dirs: %w", err) + } + if skipExisting { + if _, err := os.Stat(fullPath); err == nil { + return nil + } + } + file, err := os.OpenFile(fullPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(hdr.Mode)) + if err != nil { + return fmt.Errorf("create file: %w", err) + } + if _, err := io.Copy(file, content); err != nil { + file.Close() + return fmt.Errorf("write file: %w", err) + } + return file.Close() + case tar.TypeSymlink: + if skipExisting { + if _, err := os.Lstat(fullPath); err == nil { + return nil + } + } + if err := os.MkdirAll(filepath.Dir(fullPath), 0o750); err != nil { + return fmt.Errorf("create parent dirs: %w", err) + } + return os.Symlink(hdr.Linkname, fullPath) + case tar.TypeDir: + return os.MkdirAll(fullPath, os.FileMode(hdr.Mode)) + default: + log.Warn().Str("path", hdr.Name).Msg("skipping unsupported entry type during restore") + return nil + } +} diff --git a/internal/filestore/types.go b/internal/filestore/types.go new file mode 100644 index 000000000..87fb1c453 --- /dev/null +++ b/internal/filestore/types.go @@ -0,0 +1,46 @@ +// Copyright 2023 Greenmask +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filestore + +import "time" + +type archiveMeta struct { + Name string `json:"name"` + Files int `json:"files"` + OriginalBytes int64 `json:"original_bytes"` + CompressedBytes int64 `json:"compressed_bytes"` +} + +type metadata struct { + GeneratedAt time.Time `json:"generated_at"` + RootPath string `json:"root_path,omitempty"` + FileList string `json:"file_list,omitempty"` + IncludeListQuery string `json:"include_list_query,omitempty"` + IncludeListQueryFile string `json:"include_list_query_file,omitempty"` + IncludeListSource string `json:"include_list_source,omitempty"` + Subdir string `json:"subdir"` + ArchiveName string `json:"archive_name"` + UsePgzip bool `json:"use_pgzip"` + TotalFiles int `json:"total_files"` + TotalOriginalBytes int64 `json:"total_original_bytes"` + TotalCompressedBytes int64 `json:"total_compressed_bytes"` + Archives []archiveMeta `json:"archives"` + Split struct { + MaxSizeBytes int64 `json:"max_size_bytes,omitempty"` + MaxFiles int `json:"max_files,omitempty"` + } `json:"split"` + Missing []string `json:"missing,omitempty"` +} + diff --git a/mkdocs.yml b/mkdocs.yml index 8c5ff6cfc..fd0de1b33 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,7 +47,9 @@ nav: - Architecture: architecture.md - Playground: playground.md - Installation: installation.md - - Configuration: configuration.md + - Configuration: + - configuration.md + - Filestore: filestore.md - Commands: - commands/index.md - list-transformers: commands/list-transformers.md