Added support for Odoo Filestore

internal/db/postgres/cmd/dump.go

@@ -39,6 +39,7 @@ import (
 	"github.com/greenmaskio/greenmask/internal/db/postgres/transformers/custom"
 	"github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils"
 	"github.com/greenmaskio/greenmask/internal/domains"
+	"github.com/greenmaskio/greenmask/internal/filestore"
 	"github.com/greenmaskio/greenmask/internal/storages"
 	"github.com/greenmaskio/greenmask/pkg/toolkit"
 )
@@ -538,9 +539,61 @@ func (d *Dump) Run(ctx context.Context) (err error) {
 		return fmt.Errorf("writeMetaData stage dumping error: %w", err)
 	}
 
+	var includeListExecutor filestore.IncludeListQueryExecutor
+	if d.config.Dump.Filestore != nil {
+		includeListExecutor = &filestoreQueryExecutor{dump: d}
+	}
+	if err := filestore.Dump(ctx, d.config.Dump.Filestore, d.st, d.pgDumpOptions.Pgzip, includeListExecutor); err != nil {
+		return fmt.Errorf("filestore dumping error: %w", err)
+	}
+
 	return nil
 }
 
+type filestoreQueryExecutor struct {
+	dump *Dump
+}
+
+func (e *filestoreQueryExecutor) RunIncludeListQuery(ctx context.Context, query string) ([]string, error) {
+	conn, tx, err := e.dump.getWorkerTransaction(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := conn.Close(ctx); err != nil {
+			log.Debug().Err(err).Msg("error closing include list query connection")
+		}
+	}()
+	defer func() {
+		if err := tx.Rollback(ctx); err != nil {
+			log.Debug().Err(err).Msg("unable to rollback include list query transaction")
+		}
+	}()
+
+	rows, err := tx.Query(ctx, query)
+	if err != nil {
+		return nil, fmt.Errorf("run include_list_query: %w", err)
+	}
+	defer rows.Close()
+
+	if len(rows.FieldDescriptions()) != 1 {
+		return nil, fmt.Errorf("include_list_query must return exactly one column, got %d", len(rows.FieldDescriptions()))
+	}
+
+	var values []string
+	for rows.Next() {
+		var rel string
+		if scanErr := rows.Scan(&rel); scanErr != nil {
+			return nil, fmt.Errorf("scan include_list_query result: %w", scanErr)
+		}
+		values = append(values, rel)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate include_list_query result: %w", err)
+	}
+	return values, nil
+}
+
 func (d *Dump) MergeTocEntries(schemaEntries []*toc.Entry, dataEntries []*toc.Entry) (
 	[]*toc.Entry, error,
 ) {

internal/db/postgres/cmd/restore.go

@@ -42,6 +42,7 @@ import (
 	"github.com/greenmaskio/greenmask/internal/db/postgres/toc"
 	"github.com/greenmaskio/greenmask/internal/db/postgres/utils"
 	"github.com/greenmaskio/greenmask/internal/domains"
+	"github.com/greenmaskio/greenmask/internal/filestore"
 	"github.com/greenmaskio/greenmask/internal/storages"
 	"github.com/greenmaskio/greenmask/pkg/toolkit"
 )
@@ -142,6 +143,10 @@ func (r *Restore) Run(ctx context.Context) error {
 		return fmt.Errorf("post-data stage restoration error: %w", err)
 	}
 
+	if err := filestore.Restore(ctx, r.cfg.Filestore, r.st); err != nil {
+		return fmt.Errorf("filestore restoration error: %w", err)
+	}
+
 	return nil
 }
 

internal/domains/config.go

@@ -97,12 +97,14 @@ type Dump struct {
 	PgDumpOptions     pgdump.Options      `mapstructure:"pg_dump_options" yaml:"pg_dump_options" json:"pg_dump_options"`
 	Transformation    []*Table            `mapstructure:"transformation" yaml:"transformation" json:"transformation,omitempty"`
 	VirtualReferences []*VirtualReference `mapstructure:"virtual_references" yaml:"virtual_references" json:"virtual_references,omitempty"`
+	Filestore         *FilestoreDump      `mapstructure:"filestore" yaml:"filestore" json:"filestore,omitempty"`
 }
 
 type Restore struct {
 	PgRestoreOptions pgrestore.Options               `mapstructure:"pg_restore_options" yaml:"pg_restore_options" json:"pg_restore_options"`
 	Scripts          map[string][]pgrestore.Script   `mapstructure:"scripts" yaml:"scripts" json:"scripts,omitempty"`
 	ErrorExclusions  *DataRestorationErrorExclusions `mapstructure:"insert_error_exclusions" yaml:"insert_error_exclusions" json:"insert_error_exclusions,omitempty"`
+	Filestore        *FilestoreRestore               `mapstructure:"filestore" yaml:"filestore" json:"filestore,omitempty"`
 }
 
 type TablesDataRestorationErrorExclusions struct {
@@ -122,6 +124,36 @@ type DataRestorationErrorExclusions struct {
 	Global *GlobalDataRestorationErrorExclusions   `mapstructure:"global" yaml:"global" json:"global,omitempty"`
 }
 
+type FilestoreDump struct {
+	Enabled             bool               `mapstructure:"enabled" yaml:"enabled" json:"enabled,omitempty"`
+	RootPath            string             `mapstructure:"root_path" yaml:"root_path" json:"root_path,omitempty"`
+	FileList            string             `mapstructure:"file_list" yaml:"file_list" json:"file_list,omitempty"`
+	IncludeListFile     string             `mapstructure:"include_list_file" yaml:"include_list_file" json:"include_list_file,omitempty"`
+	IncludeListQuery    string             `mapstructure:"include_list_query" yaml:"include_list_query" json:"include_list_query,omitempty"`
+	IncludeListQueryFile string            `mapstructure:"include_list_query_file" yaml:"include_list_query_file" json:"include_list_query_file,omitempty"`
+	Subdir              string             `mapstructure:"subdir" yaml:"subdir" json:"subdir,omitempty"`
+	ArchiveName         string             `mapstructure:"archive_name" yaml:"archive_name" json:"archive_name,omitempty"`
+	MetadataName        string             `mapstructure:"metadata_name" yaml:"metadata_name" json:"metadata_name,omitempty"`
+	UsePgzip            *bool              `mapstructure:"use_pgzip" yaml:"use_pgzip" json:"use_pgzip,omitempty"`
+	FailOnMissing       bool               `mapstructure:"fail_on_missing" yaml:"fail_on_missing" json:"fail_on_missing,omitempty"`
+	Split               FilestoreDumpSplit `mapstructure:"split" yaml:"split" json:"split,omitempty"`
+}
+
+type FilestoreDumpSplit struct {
+	MaxSizeBytes int64 `mapstructure:"max_size_bytes" yaml:"max_size_bytes" json:"max_size_bytes,omitempty"`
+	MaxFiles     int   `mapstructure:"max_files" yaml:"max_files" json:"max_files,omitempty"`
+}
+
+type FilestoreRestore struct {
+	Enabled      bool   `mapstructure:"enabled" yaml:"enabled" json:"enabled,omitempty"`
+	TargetPath   string `mapstructure:"target_path" yaml:"target_path" json:"target_path,omitempty"`
+	Subdir       string `mapstructure:"subdir" yaml:"subdir" json:"subdir,omitempty"`
+	MetadataName string `mapstructure:"metadata_name" yaml:"metadata_name" json:"metadata_name,omitempty"`
+	UsePgzip     *bool  `mapstructure:"use_pgzip" yaml:"use_pgzip" json:"use_pgzip,omitempty"`
+	CleanTarget  bool   `mapstructure:"clean_target" yaml:"clean_target" json:"clean_target,omitempty"`
+	SkipExisting bool   `mapstructure:"skip_existing" yaml:"skip_existing" json:"skip_existing,omitempty"`
+}
+
 type TransformerConfig struct {
 	Name               string `mapstructure:"name" yaml:"name" json:"name,omitempty"`
 	ApplyForReferences bool   `mapstructure:"apply_for_references" yaml:"apply_for_references" json:"apply_for_references,omitempty"`