From 2936c21a71ca92d1c4da82d45660f075355b029c Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 8 Oct 2025 16:05:52 +0300 Subject: [PATCH 1/2] Add TemplateFS for templated file handling in migrations - Introduced a new TemplateFS struct to wrap embed.FS and apply Go text/template processing to file content. - Updated the runMigrations function to utilize TemplateFS for reading migration files with templating. - Added unit tests for TemplateFS functionality, including conditional templating and error handling. - Created sample SQL template files for testing purposes, demonstrating both valid and invalid template syntax. --- qan-api2/db.go | 13 +- qan-api2/utils/templatefs/templatefs.go | 90 +++++++ qan-api2/utils/templatefs/templatefs_test.go | 230 ++++++++++++++++++ .../utils/templatefs/testdata/conditional.sql | 10 + .../utils/templatefs/testdata/invalid.sql | 4 + qan-api2/utils/templatefs/testdata/simple.sql | 6 + 6 files changed, 351 insertions(+), 2 deletions(-) create mode 100644 qan-api2/utils/templatefs/templatefs.go create mode 100644 qan-api2/utils/templatefs/templatefs_test.go create mode 100644 qan-api2/utils/templatefs/testdata/conditional.sql create mode 100644 qan-api2/utils/templatefs/testdata/invalid.sql create mode 100644 qan-api2/utils/templatefs/testdata/simple.sql diff --git a/qan-api2/db.go b/qan-api2/db.go index 022f7982ada..e4fa9c04165 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -29,6 +29,8 @@ import ( "github.com/jmoiron/sqlx" // TODO: research alternatives. Ex.: https://github.com/go-reform/reform "github.com/jmoiron/sqlx/reflectx" "github.com/pkg/errors" + + "github.com/percona/pmm/qan-api2/utils/templatefs" ) const ( @@ -98,10 +100,17 @@ func createDB(dsn string) error { } //go:embed migrations/sql/*.sql -var fs embed.FS +var migrationFS embed.FS func runMigrations(dsn string) error { - d, err := iofs.New(fs, "migrations/sql") + // Create TemplateFS with simple template data + templateData := map[string]any{ + "DatabaseName": "pmm", + } + tfs := templatefs.NewTemplateFS(migrationFS, templateData) + + // Use TemplateFS directly with golang-migrate + d, err := iofs.New(tfs, "migrations/sql") if err != nil { return err } diff --git a/qan-api2/utils/templatefs/templatefs.go b/qan-api2/utils/templatefs/templatefs.go new file mode 100644 index 00000000000..601960a49e1 --- /dev/null +++ b/qan-api2/utils/templatefs/templatefs.go @@ -0,0 +1,90 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package templatefs + +import ( + "bytes" + "embed" + "text/template" + + iofs "io/fs" +) + +// TemplateFS wraps an embed.FS and applies templating to file content during reads. +// It implements the fs.FS interface and delegates most operations to the underlying embed.FS, +// but applies Go text/template processing when reading file content via ReadFile. +type TemplateFS struct { + // EmbedFS is the underlying embedded filesystem + EmbedFS embed.FS + // Data contains template data that will be used for all files + Data map[string]any +} + +// NewTemplateFS creates a new TemplateFS with the given embedded filesystem and template data. +func NewTemplateFS(embedFS embed.FS, data map[string]any) *TemplateFS { + return &TemplateFS{ + EmbedFS: embedFS, + Data: data, + } +} + +// Open opens the named file for reading and returns the original iofs.File from embed.FS. +// No templating is applied here - use ReadFile for templated content. +func (tfs *TemplateFS) Open(name string) (iofs.File, error) { + return tfs.EmbedFS.Open(name) +} + +// ReadDir reads the named directory and returns a list of directory entries. +// This delegates directly to the underlying embed.FS. +func (tfs *TemplateFS) ReadDir(name string) ([]iofs.DirEntry, error) { + return tfs.EmbedFS.ReadDir(name) +} + +// ReadFile reads the named file and returns its content with templating applied. +// This is where the templating magic happens. +func (tfs *TemplateFS) ReadFile(name string) ([]byte, error) { + // Read original content from embed.FS + content, err := tfs.EmbedFS.ReadFile(name) + if err != nil { + return nil, err + } + + // Apply templating using the same logic as in the user's example + upSQL := string(content) + + // Extract just the filename from the path for template name + filename := name + if idx := len(name) - 1; idx >= 0 { + for i := idx; i >= 0; i-- { + if name[i] == '/' { + filename = name[i+1:] + break + } + } + } + + // Apply template if data exists + if tfs.Data != nil { + if tmpl, err := template.New(filename).Parse(upSQL); err == nil { + var buf bytes.Buffer + if err := tmpl.Execute(&buf, tfs.Data); err == nil { + upSQL = buf.String() + } + } + } + + return []byte(upSQL), nil +} diff --git a/qan-api2/utils/templatefs/templatefs_test.go b/qan-api2/utils/templatefs/templatefs_test.go new file mode 100644 index 00000000000..3bfa01015e8 --- /dev/null +++ b/qan-api2/utils/templatefs/templatefs_test.go @@ -0,0 +1,230 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package templatefs + +import ( + "embed" + "io" + "testing" + + iofs "io/fs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +//go:embed testdata +var testFS embed.FS + +func TestNewTemplateFS(t *testing.T) { + data := map[string]any{ + "TableName": "users", + "DatabaseName": "testdb", + } + + tfs := NewTemplateFS(testFS, data) + + assert.NotNil(t, tfs) + assert.Equal(t, testFS, tfs.EmbedFS) + assert.Equal(t, data, tfs.Data) +} + +func TestTemplateFS_Open(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + // Test opening existing file + file, err := tfs.Open("testdata/simple.sql") + require.NoError(t, err) + require.NotNil(t, file) + defer file.Close() + + // Read content - should be original without templating + content, err := io.ReadAll(file) + require.NoError(t, err) + assert.Contains(t, string(content), "{{.TableName}}") + + // Test opening non-existent file + _, err = tfs.Open("nonexistent.sql") + assert.Error(t, err) +} + +func TestTemplateFS_ReadFile_WithTemplating(t *testing.T) { + data := map[string]any{ + "TableName": "users", + "DatabaseName": "testdb", + } + + tfs := NewTemplateFS(testFS, data) + + content, err := tfs.ReadFile("testdata/simple.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "users") + assert.Contains(t, contentStr, "testdb") + assert.NotContains(t, contentStr, "{{.TableName}}") + assert.NotContains(t, contentStr, "{{.DatabaseName}}") +} + +func TestTemplateFS_ReadFile_WithoutTemplateData(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + content, err := tfs.ReadFile("testdata/simple.sql") + require.NoError(t, err) + + // Should return original content when no template data + contentStr := string(content) + assert.Contains(t, contentStr, "{{.TableName}}") + assert.Contains(t, contentStr, "{{.DatabaseName}}") +} + +func TestTemplateFS_ReadFile_WithEmptyTemplateData(t *testing.T) { + // Empty data map + data := map[string]any{} + + tfs := NewTemplateFS(testFS, data) + + content, err := tfs.ReadFile("testdata/simple.sql") + require.NoError(t, err) + + // Should use empty data, which means template variables will be replaced with zero values + contentStr := string(content) + // Template execution with empty data should remove the variables (they become empty strings) + assert.NotContains(t, contentStr, "{{.TableName}}") + assert.NotContains(t, contentStr, "{{.DatabaseName}}") +} + +func TestTemplateFS_ReadFile_InvalidTemplate(t *testing.T) { + data := map[string]any{ + "TableName": "users", + } + + tfs := NewTemplateFS(testFS, data) + + // Should return original content when template parsing fails + content, err := tfs.ReadFile("testdata/invalid.sql") + require.NoError(t, err) + + contentStr := string(content) + // Should contain the invalid template syntax + assert.Contains(t, contentStr, "{{.TableName") +} + +func TestTemplateFS_ReadFile_NonexistentFile(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + _, err := tfs.ReadFile("nonexistent.sql") + assert.Error(t, err) +} + +func TestTemplateFS_ReadDir(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + entries, err := tfs.ReadDir("testdata") + require.NoError(t, err) + assert.NotEmpty(t, entries) + + // Should contain our test files + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + } + assert.Contains(t, names, "simple.sql") +} + +func TestTemplateFS_ReadDir_NonexistentDir(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + _, err := tfs.ReadDir("nonexistent") + assert.Error(t, err) +} + +func TestTemplateFS_WithStandardLibraryFunctions(t *testing.T) { + data := map[string]any{ + "TableName": "products", + "DatabaseName": "shop", + } + + tfs := NewTemplateFS(testFS, data) + + // Test fs.Sub + subFS, err := iofs.Sub(tfs, "testdata") + require.NoError(t, err) + + // Read from sub filesystem + content, err := iofs.ReadFile(subFS, "simple.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "products") + assert.Contains(t, contentStr, "shop") + + // Test fs.Glob + matches, err := iofs.Glob(tfs, "testdata/*.sql") + require.NoError(t, err) + assert.NotEmpty(t, matches) + assert.Contains(t, matches, "testdata/simple.sql") +} + +func TestTemplateFS_FilenameExtraction(t *testing.T) { + data := map[string]any{ + "TableName": "extracted", + } + + tfs := NewTemplateFS(testFS, data) + + // Test that template data is applied regardless of file path + content, err := tfs.ReadFile("testdata/simple.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "extracted") +} + +func TestTemplateFS_ConditionalTemplating(t *testing.T) { + data := map[string]any{ + "TableName": "users", + "AddIndexes": true, + "IndexName": "idx_users_email", + "ColumnName": "email", + } + + tfs := NewTemplateFS(testFS, data) + + content, err := tfs.ReadFile("testdata/conditional.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "CREATE TABLE users") + assert.Contains(t, contentStr, "CREATE INDEX idx_users_email") + assert.Contains(t, contentStr, "ON users (email)") +} + +func TestTemplateFS_ConditionalTemplating_False(t *testing.T) { + data := map[string]any{ + "TableName": "users", + "AddIndexes": false, + } + + tfs := NewTemplateFS(testFS, data) + + content, err := tfs.ReadFile("testdata/conditional.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "CREATE TABLE users") + assert.NotContains(t, contentStr, "CREATE INDEX") +} diff --git a/qan-api2/utils/templatefs/testdata/conditional.sql b/qan-api2/utils/templatefs/testdata/conditional.sql new file mode 100644 index 00000000000..7dd744165a0 --- /dev/null +++ b/qan-api2/utils/templatefs/testdata/conditional.sql @@ -0,0 +1,10 @@ +-- Template with conditional logic +CREATE TABLE {{.TableName}} ( + id BIGINT PRIMARY KEY, + email VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL +); + +{{if .AddIndexes}} +CREATE INDEX {{.IndexName}} ON {{.TableName}} ({{.ColumnName}}); +{{end}} diff --git a/qan-api2/utils/templatefs/testdata/invalid.sql b/qan-api2/utils/templatefs/testdata/invalid.sql new file mode 100644 index 00000000000..0f18432f04c --- /dev/null +++ b/qan-api2/utils/templatefs/testdata/invalid.sql @@ -0,0 +1,4 @@ +-- Template with invalid syntax +CREATE TABLE {{.TableName ( + id BIGINT PRIMARY KEY +); diff --git a/qan-api2/utils/templatefs/testdata/simple.sql b/qan-api2/utils/templatefs/testdata/simple.sql new file mode 100644 index 00000000000..a8f956d11a6 --- /dev/null +++ b/qan-api2/utils/templatefs/testdata/simple.sql @@ -0,0 +1,6 @@ +-- Simple template file for testing +CREATE TABLE {{.DatabaseName}}.{{.TableName}} ( + id BIGINT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + created_at TIMESTAMP DEFAULT NOW() +); From 7623f6f5cae395dad510f231abcff5e9ab85b074 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 8 Oct 2025 16:15:58 +0300 Subject: [PATCH 2/2] Refactor filename extraction in TemplateFS to use filepath.Base - Updated the ReadFile method in TemplateFS to simplify filename extraction by utilizing filepath.Base instead of manual string manipulation. - This change enhances code readability and maintainability. --- qan-api2/utils/templatefs/templatefs.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/qan-api2/utils/templatefs/templatefs.go b/qan-api2/utils/templatefs/templatefs.go index 601960a49e1..0117eae5aff 100644 --- a/qan-api2/utils/templatefs/templatefs.go +++ b/qan-api2/utils/templatefs/templatefs.go @@ -18,6 +18,7 @@ package templatefs import ( "bytes" "embed" + "path/filepath" "text/template" iofs "io/fs" @@ -64,17 +65,9 @@ func (tfs *TemplateFS) ReadFile(name string) ([]byte, error) { // Apply templating using the same logic as in the user's example upSQL := string(content) - + // Extract just the filename from the path for template name - filename := name - if idx := len(name) - 1; idx >= 0 { - for i := idx; i >= 0; i-- { - if name[i] == '/' { - filename = name[i+1:] - break - } - } - } + filename := filepath.Base(name) // Apply template if data exists if tfs.Data != nil {