diff --git a/qan-api2/db.go b/qan-api2/db.go index 022f7982ada..e4fa9c04165 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -29,6 +29,8 @@ import ( "github.com/jmoiron/sqlx" // TODO: research alternatives. Ex.: https://github.com/go-reform/reform "github.com/jmoiron/sqlx/reflectx" "github.com/pkg/errors" + + "github.com/percona/pmm/qan-api2/utils/templatefs" ) const ( @@ -98,10 +100,17 @@ func createDB(dsn string) error { } //go:embed migrations/sql/*.sql -var fs embed.FS +var migrationFS embed.FS func runMigrations(dsn string) error { - d, err := iofs.New(fs, "migrations/sql") + // Create TemplateFS with simple template data + templateData := map[string]any{ + "DatabaseName": "pmm", + } + tfs := templatefs.NewTemplateFS(migrationFS, templateData) + + // Use TemplateFS directly with golang-migrate + d, err := iofs.New(tfs, "migrations/sql") if err != nil { return err } diff --git a/qan-api2/utils/templatefs/templatefs.go b/qan-api2/utils/templatefs/templatefs.go new file mode 100644 index 00000000000..0117eae5aff --- /dev/null +++ b/qan-api2/utils/templatefs/templatefs.go @@ -0,0 +1,83 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package templatefs + +import ( + "bytes" + "embed" + "path/filepath" + "text/template" + + iofs "io/fs" +) + +// TemplateFS wraps an embed.FS and applies templating to file content during reads. +// It implements the fs.FS interface and delegates most operations to the underlying embed.FS, +// but applies Go text/template processing when reading file content via ReadFile. +type TemplateFS struct { + // EmbedFS is the underlying embedded filesystem + EmbedFS embed.FS + // Data contains template data that will be used for all files + Data map[string]any +} + +// NewTemplateFS creates a new TemplateFS with the given embedded filesystem and template data. +func NewTemplateFS(embedFS embed.FS, data map[string]any) *TemplateFS { + return &TemplateFS{ + EmbedFS: embedFS, + Data: data, + } +} + +// Open opens the named file for reading and returns the original iofs.File from embed.FS. +// No templating is applied here - use ReadFile for templated content. +func (tfs *TemplateFS) Open(name string) (iofs.File, error) { + return tfs.EmbedFS.Open(name) +} + +// ReadDir reads the named directory and returns a list of directory entries. +// This delegates directly to the underlying embed.FS. +func (tfs *TemplateFS) ReadDir(name string) ([]iofs.DirEntry, error) { + return tfs.EmbedFS.ReadDir(name) +} + +// ReadFile reads the named file and returns its content with templating applied. +// This is where the templating magic happens. +func (tfs *TemplateFS) ReadFile(name string) ([]byte, error) { + // Read original content from embed.FS + content, err := tfs.EmbedFS.ReadFile(name) + if err != nil { + return nil, err + } + + // Apply templating using the same logic as in the user's example + upSQL := string(content) + + // Extract just the filename from the path for template name + filename := filepath.Base(name) + + // Apply template if data exists + if tfs.Data != nil { + if tmpl, err := template.New(filename).Parse(upSQL); err == nil { + var buf bytes.Buffer + if err := tmpl.Execute(&buf, tfs.Data); err == nil { + upSQL = buf.String() + } + } + } + + return []byte(upSQL), nil +} diff --git a/qan-api2/utils/templatefs/templatefs_test.go b/qan-api2/utils/templatefs/templatefs_test.go new file mode 100644 index 00000000000..3bfa01015e8 --- /dev/null +++ b/qan-api2/utils/templatefs/templatefs_test.go @@ -0,0 +1,230 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package templatefs + +import ( + "embed" + "io" + "testing" + + iofs "io/fs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +//go:embed testdata +var testFS embed.FS + +func TestNewTemplateFS(t *testing.T) { + data := map[string]any{ + "TableName": "users", + "DatabaseName": "testdb", + } + + tfs := NewTemplateFS(testFS, data) + + assert.NotNil(t, tfs) + assert.Equal(t, testFS, tfs.EmbedFS) + assert.Equal(t, data, tfs.Data) +} + +func TestTemplateFS_Open(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + // Test opening existing file + file, err := tfs.Open("testdata/simple.sql") + require.NoError(t, err) + require.NotNil(t, file) + defer file.Close() + + // Read content - should be original without templating + content, err := io.ReadAll(file) + require.NoError(t, err) + assert.Contains(t, string(content), "{{.TableName}}") + + // Test opening non-existent file + _, err = tfs.Open("nonexistent.sql") + assert.Error(t, err) +} + +func TestTemplateFS_ReadFile_WithTemplating(t *testing.T) { + data := map[string]any{ + "TableName": "users", + "DatabaseName": "testdb", + } + + tfs := NewTemplateFS(testFS, data) + + content, err := tfs.ReadFile("testdata/simple.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "users") + assert.Contains(t, contentStr, "testdb") + assert.NotContains(t, contentStr, "{{.TableName}}") + assert.NotContains(t, contentStr, "{{.DatabaseName}}") +} + +func TestTemplateFS_ReadFile_WithoutTemplateData(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + content, err := tfs.ReadFile("testdata/simple.sql") + require.NoError(t, err) + + // Should return original content when no template data + contentStr := string(content) + assert.Contains(t, contentStr, "{{.TableName}}") + assert.Contains(t, contentStr, "{{.DatabaseName}}") +} + +func TestTemplateFS_ReadFile_WithEmptyTemplateData(t *testing.T) { + // Empty data map + data := map[string]any{} + + tfs := NewTemplateFS(testFS, data) + + content, err := tfs.ReadFile("testdata/simple.sql") + require.NoError(t, err) + + // Should use empty data, which means template variables will be replaced with zero values + contentStr := string(content) + // Template execution with empty data should remove the variables (they become empty strings) + assert.NotContains(t, contentStr, "{{.TableName}}") + assert.NotContains(t, contentStr, "{{.DatabaseName}}") +} + +func TestTemplateFS_ReadFile_InvalidTemplate(t *testing.T) { + data := map[string]any{ + "TableName": "users", + } + + tfs := NewTemplateFS(testFS, data) + + // Should return original content when template parsing fails + content, err := tfs.ReadFile("testdata/invalid.sql") + require.NoError(t, err) + + contentStr := string(content) + // Should contain the invalid template syntax + assert.Contains(t, contentStr, "{{.TableName") +} + +func TestTemplateFS_ReadFile_NonexistentFile(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + _, err := tfs.ReadFile("nonexistent.sql") + assert.Error(t, err) +} + +func TestTemplateFS_ReadDir(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + entries, err := tfs.ReadDir("testdata") + require.NoError(t, err) + assert.NotEmpty(t, entries) + + // Should contain our test files + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + } + assert.Contains(t, names, "simple.sql") +} + +func TestTemplateFS_ReadDir_NonexistentDir(t *testing.T) { + tfs := NewTemplateFS(testFS, nil) + + _, err := tfs.ReadDir("nonexistent") + assert.Error(t, err) +} + +func TestTemplateFS_WithStandardLibraryFunctions(t *testing.T) { + data := map[string]any{ + "TableName": "products", + "DatabaseName": "shop", + } + + tfs := NewTemplateFS(testFS, data) + + // Test fs.Sub + subFS, err := iofs.Sub(tfs, "testdata") + require.NoError(t, err) + + // Read from sub filesystem + content, err := iofs.ReadFile(subFS, "simple.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "products") + assert.Contains(t, contentStr, "shop") + + // Test fs.Glob + matches, err := iofs.Glob(tfs, "testdata/*.sql") + require.NoError(t, err) + assert.NotEmpty(t, matches) + assert.Contains(t, matches, "testdata/simple.sql") +} + +func TestTemplateFS_FilenameExtraction(t *testing.T) { + data := map[string]any{ + "TableName": "extracted", + } + + tfs := NewTemplateFS(testFS, data) + + // Test that template data is applied regardless of file path + content, err := tfs.ReadFile("testdata/simple.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "extracted") +} + +func TestTemplateFS_ConditionalTemplating(t *testing.T) { + data := map[string]any{ + "TableName": "users", + "AddIndexes": true, + "IndexName": "idx_users_email", + "ColumnName": "email", + } + + tfs := NewTemplateFS(testFS, data) + + content, err := tfs.ReadFile("testdata/conditional.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "CREATE TABLE users") + assert.Contains(t, contentStr, "CREATE INDEX idx_users_email") + assert.Contains(t, contentStr, "ON users (email)") +} + +func TestTemplateFS_ConditionalTemplating_False(t *testing.T) { + data := map[string]any{ + "TableName": "users", + "AddIndexes": false, + } + + tfs := NewTemplateFS(testFS, data) + + content, err := tfs.ReadFile("testdata/conditional.sql") + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "CREATE TABLE users") + assert.NotContains(t, contentStr, "CREATE INDEX") +} diff --git a/qan-api2/utils/templatefs/testdata/conditional.sql b/qan-api2/utils/templatefs/testdata/conditional.sql new file mode 100644 index 00000000000..7dd744165a0 --- /dev/null +++ b/qan-api2/utils/templatefs/testdata/conditional.sql @@ -0,0 +1,10 @@ +-- Template with conditional logic +CREATE TABLE {{.TableName}} ( + id BIGINT PRIMARY KEY, + email VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL +); + +{{if .AddIndexes}} +CREATE INDEX {{.IndexName}} ON {{.TableName}} ({{.ColumnName}}); +{{end}} diff --git a/qan-api2/utils/templatefs/testdata/invalid.sql b/qan-api2/utils/templatefs/testdata/invalid.sql new file mode 100644 index 00000000000..0f18432f04c --- /dev/null +++ b/qan-api2/utils/templatefs/testdata/invalid.sql @@ -0,0 +1,4 @@ +-- Template with invalid syntax +CREATE TABLE {{.TableName ( + id BIGINT PRIMARY KEY +); diff --git a/qan-api2/utils/templatefs/testdata/simple.sql b/qan-api2/utils/templatefs/testdata/simple.sql new file mode 100644 index 00000000000..a8f956d11a6 --- /dev/null +++ b/qan-api2/utils/templatefs/testdata/simple.sql @@ -0,0 +1,6 @@ +-- Simple template file for testing +CREATE TABLE {{.DatabaseName}}.{{.TableName}} ( + id BIGINT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + created_at TIMESTAMP DEFAULT NOW() +);