Add TemplateFS for templated file handling in migrations

qan-api2/db.go

@@ -29,6 +29,8 @@ import (
 	"github.com/jmoiron/sqlx" // TODO: research alternatives. Ex.: https://github.com/go-reform/reform
 	"github.com/jmoiron/sqlx/reflectx"
 	"github.com/pkg/errors"
+
+	"github.com/percona/pmm/qan-api2/utils/templatefs"
 )
 
 const (
@@ -98,10 +100,17 @@ func createDB(dsn string) error {
 }
 
 //go:embed migrations/sql/*.sql
-var fs embed.FS
+var migrationFS embed.FS
 
 func runMigrations(dsn string) error {
-	d, err := iofs.New(fs, "migrations/sql")
+	// Create TemplateFS with simple template data
+	templateData := map[string]any{
+		"DatabaseName": "pmm",
+	}
+	tfs := templatefs.NewTemplateFS(migrationFS, templateData)
+
+	// Use TemplateFS directly with golang-migrate
+	d, err := iofs.New(tfs, "migrations/sql")
 	if err != nil {
 		return err
 	}

qan-api2/utils/templatefs/templatefs.go

@@ -0,0 +1,83 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package templatefs
+
+import (
+	"bytes"
+	"embed"
+	"path/filepath"
+	"text/template"
+
+	iofs "io/fs"
+)
+
+// TemplateFS wraps an embed.FS and applies templating to file content during reads.
+// It implements the fs.FS interface and delegates most operations to the underlying embed.FS,
+// but applies Go text/template processing when reading file content via ReadFile.
+type TemplateFS struct {
+	// EmbedFS is the underlying embedded filesystem
+	EmbedFS embed.FS
+	// Data contains template data that will be used for all files
+	Data map[string]any
+}
+
+// NewTemplateFS creates a new TemplateFS with the given embedded filesystem and template data.
+func NewTemplateFS(embedFS embed.FS, data map[string]any) *TemplateFS {
+	return &TemplateFS{
+		EmbedFS: embedFS,
+		Data:    data,
+	}
+}
+
+// Open opens the named file for reading and returns the original iofs.File from embed.FS.
+// No templating is applied here - use ReadFile for templated content.
+func (tfs *TemplateFS) Open(name string) (iofs.File, error) {
+	return tfs.EmbedFS.Open(name)
+}
+
+// ReadDir reads the named directory and returns a list of directory entries.
+// This delegates directly to the underlying embed.FS.
+func (tfs *TemplateFS) ReadDir(name string) ([]iofs.DirEntry, error) {
+	return tfs.EmbedFS.ReadDir(name)
+}
+
+// ReadFile reads the named file and returns its content with templating applied.
+// This is where the templating magic happens.
+func (tfs *TemplateFS) ReadFile(name string) ([]byte, error) {
+	// Read original content from embed.FS
+	content, err := tfs.EmbedFS.ReadFile(name)
+	if err != nil {
+		return nil, err
+	}
+
+	// Apply templating using the same logic as in the user's example
+	upSQL := string(content)
+
+	// Extract just the filename from the path for template name
+	filename := filepath.Base(name)
+
+	// Apply template if data exists
+	if tfs.Data != nil {
+		if tmpl, err := template.New(filename).Parse(upSQL); err == nil {
+			var buf bytes.Buffer
+			if err := tmpl.Execute(&buf, tfs.Data); err == nil {
+				upSQL = buf.String()
+			}
+		}
+	}
+
+	return []byte(upSQL), nil
+}

qan-api2/utils/templatefs/templatefs_test.go

@@ -0,0 +1,230 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package templatefs
+
+import (
+	"embed"
+	"io"
+	"testing"
+
+	iofs "io/fs"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+//go:embed testdata
+var testFS embed.FS
+
+func TestNewTemplateFS(t *testing.T) {
+	data := map[string]any{
+		"TableName":    "users",
+		"DatabaseName": "testdb",
+	}
+
+	tfs := NewTemplateFS(testFS, data)
+
+	assert.NotNil(t, tfs)
+	assert.Equal(t, testFS, tfs.EmbedFS)
+	assert.Equal(t, data, tfs.Data)
+}
+
+func TestTemplateFS_Open(t *testing.T) {
+	tfs := NewTemplateFS(testFS, nil)
+
+	// Test opening existing file
+	file, err := tfs.Open("testdata/simple.sql")
+	require.NoError(t, err)
+	require.NotNil(t, file)
+	defer file.Close()
+
+	// Read content - should be original without templating
+	content, err := io.ReadAll(file)
+	require.NoError(t, err)
+	assert.Contains(t, string(content), "{{.TableName}}")
+
+	// Test opening non-existent file
+	_, err = tfs.Open("nonexistent.sql")
+	assert.Error(t, err)
+}
+
+func TestTemplateFS_ReadFile_WithTemplating(t *testing.T) {
+	data := map[string]any{
+		"TableName":    "users",
+		"DatabaseName": "testdb",
+	}
+
+	tfs := NewTemplateFS(testFS, data)
+
+	content, err := tfs.ReadFile("testdata/simple.sql")
+	require.NoError(t, err)
+
+	contentStr := string(content)
+	assert.Contains(t, contentStr, "users")
+	assert.Contains(t, contentStr, "testdb")
+	assert.NotContains(t, contentStr, "{{.TableName}}")
+	assert.NotContains(t, contentStr, "{{.DatabaseName}}")
+}
+
+func TestTemplateFS_ReadFile_WithoutTemplateData(t *testing.T) {
+	tfs := NewTemplateFS(testFS, nil)
+
+	content, err := tfs.ReadFile("testdata/simple.sql")
+	require.NoError(t, err)
+
+	// Should return original content when no template data
+	contentStr := string(content)
+	assert.Contains(t, contentStr, "{{.TableName}}")
+	assert.Contains(t, contentStr, "{{.DatabaseName}}")
+}
+
+func TestTemplateFS_ReadFile_WithEmptyTemplateData(t *testing.T) {
+	// Empty data map
+	data := map[string]any{}
+
+	tfs := NewTemplateFS(testFS, data)
+
+	content, err := tfs.ReadFile("testdata/simple.sql")
+	require.NoError(t, err)
+
+	// Should use empty data, which means template variables will be replaced with zero values
+	contentStr := string(content)
+	// Template execution with empty data should remove the variables (they become empty strings)
+	assert.NotContains(t, contentStr, "{{.TableName}}")
+	assert.NotContains(t, contentStr, "{{.DatabaseName}}")
+}
+
+func TestTemplateFS_ReadFile_InvalidTemplate(t *testing.T) {
+	data := map[string]any{
+		"TableName": "users",
+	}
+
+	tfs := NewTemplateFS(testFS, data)
+
+	// Should return original content when template parsing fails
+	content, err := tfs.ReadFile("testdata/invalid.sql")
+	require.NoError(t, err)
+
+	contentStr := string(content)
+	// Should contain the invalid template syntax
+	assert.Contains(t, contentStr, "{{.TableName")
+}
+
+func TestTemplateFS_ReadFile_NonexistentFile(t *testing.T) {
+	tfs := NewTemplateFS(testFS, nil)
+
+	_, err := tfs.ReadFile("nonexistent.sql")
+	assert.Error(t, err)
+}
+
+func TestTemplateFS_ReadDir(t *testing.T) {
+	tfs := NewTemplateFS(testFS, nil)
+
+	entries, err := tfs.ReadDir("testdata")
+	require.NoError(t, err)
+	assert.NotEmpty(t, entries)
+
+	// Should contain our test files
+	var names []string
+	for _, entry := range entries {
+		names = append(names, entry.Name())
+	}
+	assert.Contains(t, names, "simple.sql")
+}
+
+func TestTemplateFS_ReadDir_NonexistentDir(t *testing.T) {
+	tfs := NewTemplateFS(testFS, nil)
+
+	_, err := tfs.ReadDir("nonexistent")
+	assert.Error(t, err)
+}
+
+func TestTemplateFS_WithStandardLibraryFunctions(t *testing.T) {
+	data := map[string]any{
+		"TableName":    "products",
+		"DatabaseName": "shop",
+	}
+
+	tfs := NewTemplateFS(testFS, data)
+
+	// Test fs.Sub
+	subFS, err := iofs.Sub(tfs, "testdata")
+	require.NoError(t, err)
+
+	// Read from sub filesystem
+	content, err := iofs.ReadFile(subFS, "simple.sql")
+	require.NoError(t, err)
+
+	contentStr := string(content)
+	assert.Contains(t, contentStr, "products")
+	assert.Contains(t, contentStr, "shop")
+
+	// Test fs.Glob
+	matches, err := iofs.Glob(tfs, "testdata/*.sql")
+	require.NoError(t, err)
+	assert.NotEmpty(t, matches)
+	assert.Contains(t, matches, "testdata/simple.sql")
+}
+
+func TestTemplateFS_FilenameExtraction(t *testing.T) {
+	data := map[string]any{
+		"TableName": "extracted",
+	}
+
+	tfs := NewTemplateFS(testFS, data)
+
+	// Test that template data is applied regardless of file path
+	content, err := tfs.ReadFile("testdata/simple.sql")
+	require.NoError(t, err)
+
+	contentStr := string(content)
+	assert.Contains(t, contentStr, "extracted")
+}
+
+func TestTemplateFS_ConditionalTemplating(t *testing.T) {
+	data := map[string]any{
+		"TableName":  "users",
+		"AddIndexes": true,
+		"IndexName":  "idx_users_email",
+		"ColumnName": "email",
+	}
+
+	tfs := NewTemplateFS(testFS, data)
+
+	content, err := tfs.ReadFile("testdata/conditional.sql")
+	require.NoError(t, err)
+
+	contentStr := string(content)
+	assert.Contains(t, contentStr, "CREATE TABLE users")
+	assert.Contains(t, contentStr, "CREATE INDEX idx_users_email")
+	assert.Contains(t, contentStr, "ON users (email)")
+}
+
+func TestTemplateFS_ConditionalTemplating_False(t *testing.T) {
+	data := map[string]any{
+		"TableName":  "users",
+		"AddIndexes": false,
+	}
+
+	tfs := NewTemplateFS(testFS, data)
+
+	content, err := tfs.ReadFile("testdata/conditional.sql")
+	require.NoError(t, err)
+
+	contentStr := string(content)
+	assert.Contains(t, contentStr, "CREATE TABLE users")
+	assert.NotContains(t, contentStr, "CREATE INDEX")
+}

qan-api2/utils/templatefs/testdata/conditional.sql

@@ -0,0 +1,10 @@
+-- Template with conditional logic
+CREATE TABLE {{.TableName}} (
+    id BIGINT PRIMARY KEY,
+    email VARCHAR(255) NOT NULL,
+    name VARCHAR(255) NOT NULL
+);
+
+{{if .AddIndexes}}
+CREATE INDEX {{.IndexName}} ON {{.TableName}} ({{.ColumnName}});
+{{end}}

qan-api2/utils/templatefs/testdata/invalid.sql

@@ -0,0 +1,4 @@
+-- Template with invalid syntax
+CREATE TABLE {{.TableName (
+    id BIGINT PRIMARY KEY
+);

qan-api2/utils/templatefs/testdata/simple.sql

@@ -0,0 +1,6 @@
+-- Simple template file for testing
+CREATE TABLE {{.DatabaseName}}.{{.TableName}} (
+    id BIGINT PRIMARY KEY,
+    name VARCHAR(255) NOT NULL,
+    created_at TIMESTAMP DEFAULT NOW()
+);