Skip to content

Commit d4f461a

Browse files
pyramationpyramation
authored
Merge pull request #477 from constructive-io/fix-exports/meta
Fix exports/meta
2 parents 0e68789 + c9ba063 commit d4f461a

File tree

3 files changed

+270
-1
lines changed

3 files changed

+270
-1
lines changed

‎pgpm/core/__tests__/export/export-meta.test.ts‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/**
2+
* Tests for export-meta.ts configuration validation
3+
*
4+
* These tests validate that the export-meta config uses correct table names
5+
* and includes all required fields for exporting meta_public and collections_public data.
6+
*/
7+
8+
import { readFileSync } from 'fs';
9+
import { join } from 'path';
10+
11+
describe('Export Meta Config Validation', () => {
12+
let exportMetaSource: string;
13+
14+
beforeAll(() => {
15+
// Read the source file to validate the config
16+
const filePath = join(__dirname, '../../src/export/export-meta.ts');
17+
exportMetaSource = readFileSync(filePath, 'utf-8');
18+
});
19+
20+
describe('table name validation', () => {
21+
it('should use singular table name "database_extension" (not plural "database_extensions")', () => {
22+
// The actual table in collections_public is database_extension (singular)
23+
// This test ensures the config uses the correct table name
24+
25+
// Check that the config defines the table correctly
26+
expect(exportMetaSource).toContain("table: 'database_extension'");
27+
28+
// Ensure we're not using the incorrect plural form in the table definition
29+
// Note: We check specifically in the config section, not the query section
30+
const configSection = exportMetaSource.split('const config:')[1]?.split('interface ExportMetaParams')[0] || '';
31+
expect(configSection).not.toContain("table: 'database_extensions'");
32+
});
33+
34+
it('should query the correct table name in collections_public.database_extension', () => {
35+
// The query should use the correct singular table name
36+
expect(exportMetaSource).toContain('FROM collections_public.database_extension');
37+
});
38+
39+
it('should include field table in queries', () => {
40+
// The field table should be queried (it was missing before)
41+
expect(exportMetaSource).toContain("queryAndParse('field'");
42+
expect(exportMetaSource).toContain('FROM collections_public.field');
43+
});
44+
});
45+
46+
describe('collections_public tables', () => {
47+
it('should include all required collections_public tables in config', () => {
48+
const requiredTables = [
49+
'database',
50+
'database_extension',
51+
'schema',
52+
'table',
53+
'field'
54+
];
55+
56+
for (const table of requiredTables) {
57+
expect(exportMetaSource).toContain(`queryAndParse('${table}'`);
58+
}
59+
});
60+
});
61+
62+
describe('meta_public tables', () => {
63+
it('should include all required meta_public tables in config', () => {
64+
const requiredTables = [
65+
'domains',
66+
'sites',
67+
'apis',
68+
'apps',
69+
'site_modules',
70+
'site_themes',
71+
'api_modules',
72+
'api_extensions',
73+
'api_schemata',
74+
'rls_module',
75+
'user_auth_module'
76+
];
77+
78+
for (const table of requiredTables) {
79+
expect(exportMetaSource).toContain(`queryAndParse('${table}'`);
80+
}
81+
});
82+
});
83+
84+
describe('query order validation', () => {
85+
it('should query database before dependent tables', () => {
86+
// database should be queried first as other tables depend on it
87+
const databaseQueryIndex = exportMetaSource.indexOf("queryAndParse('database',");
88+
const schemaQueryIndex = exportMetaSource.indexOf("queryAndParse('schema',");
89+
const tableQueryIndex = exportMetaSource.indexOf("queryAndParse('table',");
90+
91+
expect(databaseQueryIndex).toBeLessThan(schemaQueryIndex);
92+
expect(schemaQueryIndex).toBeLessThan(tableQueryIndex);
93+
});
94+
});
95+
});
96+
97+
describe('Export Meta Config Drift Detection', () => {
98+
it('should document the expected table names in collections_public', () => {
99+
// This test documents the expected table names that export-meta.ts should use
100+
// If these change, the export-meta.ts config needs to be updated
101+
const expectedCollectionsPublicTables = [
102+
'database',
103+
'database_extension', // NOT 'database_extensions' (plural)
104+
'schema',
105+
'table',
106+
'field'
107+
];
108+
109+
// Document the expected tables
110+
expect(expectedCollectionsPublicTables).toContain('database_extension');
111+
expect(expectedCollectionsPublicTables).not.toContain('database_extensions');
112+
});
113+
114+
it('should document the expected table names in meta_public', () => {
115+
// This test documents the expected table names that export-meta.ts should use
116+
const expectedMetaPublicTables = [
117+
'apis',
118+
'api_extensions',
119+
'api_modules',
120+
'api_schemata',
121+
'apps',
122+
'domains',
123+
'site_modules',
124+
'site_themes',
125+
'sites',
126+
'rls_module',
127+
'user_auth_module'
128+
];
129+
130+
// Document the expected tables
131+
expect(expectedMetaPublicTables.length).toBe(11);
132+
});
133+
134+
it('should document the bug: config uses database_extensions but table is database_extension', () => {
135+
// BUG DOCUMENTATION:
136+
// In export-meta.ts, line 26, the config defines:
137+
// table: 'database_extensions' (plural)
138+
// But the actual table in db-meta-schema is:
139+
// collections_public.database_extension (singular)
140+
//
141+
// This causes the Parser to generate INSERT statements with the wrong table name,
142+
// which will fail when the exported SQL is replayed.
143+
//
144+
// FIX: Change line 26 in export-meta.ts from:
145+
// table: 'database_extensions'
146+
// to:
147+
// table: 'database_extension'
148+
149+
const buggyConfigTableName = 'database_extensions';
150+
const correctTableName = 'database_extension';
151+
152+
expect(buggyConfigTableName).not.toBe(correctTableName);
153+
});
154+
});

‎pgpm/core/src/export/export-meta.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ const config: Record<string, TableConfig> = {
2323
},
2424
database_extension: {
2525
schema: 'collections_public',
26-
table: 'database_extensions',
26+
table: 'database_extension',
2727
fields: {
2828
name: 'text',
2929
database_id: 'uuid'
@@ -243,6 +243,7 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams
243243
await queryAndParse('database', `SELECT * FROM collections_public.database WHERE id = $1`);
244244
await queryAndParse('schema', `SELECT * FROM collections_public.schema WHERE database_id = $1`);
245245
await queryAndParse('table', `SELECT * FROM collections_public.table WHERE database_id = $1`);
246+
await queryAndParse('field', `SELECT * FROM collections_public.field WHERE database_id = $1`);
246247
await queryAndParse('domains', `SELECT * FROM meta_public.domains WHERE database_id = $1`);
247248
await queryAndParse('apis', `SELECT * FROM meta_public.apis WHERE database_id = $1`);
248249
await queryAndParse('sites', `SELECT * FROM meta_public.sites WHERE database_id = $1`);

‎plan.md‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# Export Meta Fix Plan
2+
3+
## Problem Statement
4+
5+
The `pgpm export` command's `meta_public` schema export is broken. The export functionality in `pgpm/core/src/export/export-meta.ts` has several issues that cause the generated SQL to fail when replayed.
6+
7+
## Issues Identified
8+
9+
### Critical Issues
10+
11+
1. **Table name mismatch (`database_extension`)**
12+
- Location: `pgpm/core/src/export/export-meta.ts` line 26-27
13+
- Config defines: `table: 'database_extensions'` (plural)
14+
- Actual table: `collections_public.database_extension` (singular)
15+
- Impact: Generated INSERT statements target non-existent table
16+
17+
2. **Missing columns in `user_auth_module` config**
18+
- Actual table has columns not in export config:
19+
- `audits_table_id`
20+
- `audits_table_name`
21+
- `verify_password_function`
22+
- `check_password_function`
23+
- Impact: Exported data is incomplete
24+
25+
3. **Missing `field` query**
26+
- The `field` table is defined in config (lines 54-65) but never queried
27+
- No `queryAndParse('field', ...)` call exists
28+
- Impact: Field data is never exported
29+
30+
4. **Missing `site_metadata` table**
31+
- Table exists in `meta_public` schema but not in export config
32+
- Impact: Site metadata is never exported
33+
34+
### Secondary Issues
35+
36+
5. **Insert order vs FK constraints**
37+
- Export queries `domains` before `apis`/`sites`
38+
- `domains` has FK references to both `apis` and `sites`
39+
- Code relies on `SET session_replication_role TO replica;` (requires superuser)
40+
- Impact: May fail in non-superuser environments
41+
42+
6. **Type mismatches** (may or may not cause issues depending on csv-to-pg handling)
43+
- `meta_public.sites.favicon`: actual `attachment`, config says `upload`
44+
- `meta_public.domains.subdomain/domain`: actual `hostname`, config says `text`
45+
- `meta_public.api_modules.data`: actual `pg_catalog.json`, config says `jsonb`
46+
- `meta_public.site_modules.data`: actual `pg_catalog.json`, config says `jsonb`
47+
48+
## Proposed Solution
49+
50+
### Phase 1: Create Test Infrastructure
51+
52+
1. **Create a new test module** in `extensions/` directory
53+
- Name: `@pgpm/export-meta-test` or similar
54+
- Purpose: Test the export functionality with seed data
55+
56+
2. **Use `pgsql-test` for database testing**
57+
- Leverage existing test infrastructure
58+
- Transaction-isolated tests with automatic rollback
59+
60+
3. **Test approach**:
61+
- Install `@pgpm/db-meta-schema` and `@pgpm/db-meta-modules`
62+
- Seed representative data into all relevant tables
63+
- Call `exportMeta()` to generate SQL
64+
- Execute generated SQL in a fresh transaction
65+
- Verify data integrity
66+
67+
### Phase 2: Fix Export Issues
68+
69+
1. Fix `database_extensions` -> `database_extension` table name
70+
2. Add missing columns to `user_auth_module` config
71+
3. Add `queryAndParse('field', ...)` call
72+
4. Add `site_metadata` table to config and query
73+
5. Fix insert order to respect FK dependencies
74+
6. Update type mappings if needed
75+
76+
### Phase 3: Add Schema Drift Detection
77+
78+
1. Create a test that compares `information_schema.columns` against the config
79+
2. Fail if config doesn't cover all required columns
80+
3. Prevent future regressions when schema changes
81+
82+
## Test Module Structure
83+
84+
```
85+
extensions/@pgpm/export-meta-test/
86+
├── package.json
87+
├── jest.config.js
88+
├── __tests__/
89+
│ ├── export-meta.test.ts # Main export tests
90+
│ └── schema-drift.test.ts # Schema drift detection
91+
└── seed/
92+
└── seed-data.ts # Seed data helpers
93+
```
94+
95+
## Dependencies
96+
97+
- `@pgpm/db-meta-schema` - Core schema definitions
98+
- `@pgpm/db-meta-modules` - Module table definitions
99+
- `pgsql-test` - Test infrastructure
100+
- `@pgpmjs/core` - Export functionality to test
101+
102+
## Success Criteria
103+
104+
1. Export generates valid SQL that can be replayed
105+
2. All tables in `meta_public` and `collections_public` are properly exported
106+
3. FK constraints are respected (either via proper ordering or explicit handling)
107+
4. Tests catch schema drift automatically
108+
5. CI passes with new tests
109+
110+
## Questions to Resolve
111+
112+
1. Should this be a separate module or part of existing `db-meta-schema` tests?
113+
2. Do we need to support non-superuser deployments (affects `session_replication_role` approach)?
114+
3. Should we add the tests to `launchql-extensions` repo as well for parity?

0 commit comments

Comments
 (0)