From d4452848a6e94d3e20e38041cfad8f66df766738 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 29 Aug 2025 17:13:09 +0300 Subject: [PATCH 1/7] extracted yaml and jinja transpilation in separate functions --- .../src/compiler/DataSchemaCompiler.ts | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index 99bf514977200..2089a030491d0 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -575,14 +575,9 @@ export class DataSchemaCompiler { (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) && file.content.match(JINJA_SYNTAX) ) { - return this.yamlCompiler.compileYamlWithJinjaFile( - file, - errorsReport, - this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext), - this.pythonContext! - ); + return this.transpileJinjaFile(file, errorsReport, options); } else if (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) { - return this.yamlCompiler.transpileYamlFile(file, errorsReport); + return this.transpileYamlFile(file, errorsReport, options); } else { return file; } @@ -705,6 +700,39 @@ export class DataSchemaCompiler { return undefined; } + private async transpileYamlFile( + file: FileContent, + errorsReport: ErrorReporter, + { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions + ): Promise<(FileContent | undefined)> { + // if (getEnv('transpilationNative')) { + // + // } else if (getEnv('transpilationWorkerThreads')) { + // + // } else { + return this.yamlCompiler.transpileYamlFile(file, errorsReport); + // } + } + + private async transpileJinjaFile( + file: FileContent, + errorsReport: ErrorReporter, + { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions + ): Promise<(FileContent | undefined)> { + // if (getEnv('transpilationNative')) { + // + // } else if (getEnv('transpilationWorkerThreads')) { + // + // } else { + return this.yamlCompiler.compileYamlWithJinjaFile( + file, + errorsReport, + this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext), + this.pythonContext! + ); + // } + } + public withQuery(query, fn) { const oldQuery = this.currentQuery; this.currentQuery = query; From d03f4c4002b76cb3729b79f989b7be985fc5c165 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 29 Aug 2025 18:01:48 +0300 Subject: [PATCH 2/7] implement yaml transpilation in worker threads --- .../src/compiler/CubeSymbols.ts | 3 +- .../src/compiler/DataSchemaCompiler.ts | 28 +++++++++++++------ .../src/compiler/YamlCompiler.ts | 15 ++++++---- .../compiler/transpilers/transpiler_worker.ts | 23 +++++++++++++-- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 5330b8f9120ec..1161716d01378 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -7,6 +7,7 @@ import { DynamicReference } from './DynamicReference'; import { camelizeCube } from './utils'; import type { ErrorReporter } from './ErrorReporter'; +import { TranspilerSymbolResolver } from './transpilers'; export type ToString = { toString(): string }; @@ -193,7 +194,7 @@ export const CONTEXT_SYMBOLS = { export const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE']; -export class CubeSymbols { +export class CubeSymbols implements TranspilerSymbolResolver { public symbols: Record; private builtCubes: Record; diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index 2089a030491d0..7f7689cea5a07 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -663,7 +663,7 @@ export class DataSchemaCompiler { cubeSymbols, }; - const res = await this.workerPool!.exec('transpile', [data]); + const res = await this.workerPool!.exec('transpileJs', [data]); errorsReport.addErrors(res.errors); errorsReport.addWarnings(res.warnings); @@ -705,13 +705,25 @@ export class DataSchemaCompiler { errorsReport: ErrorReporter, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions ): Promise<(FileContent | undefined)> { - // if (getEnv('transpilationNative')) { - // - // } else if (getEnv('transpilationWorkerThreads')) { - // - // } else { - return this.yamlCompiler.transpileYamlFile(file, errorsReport); - // } + /* if (getEnv('transpilationNative')) { + + } else */ if (getEnv('transpilationWorkerThreads')) { + const data = { + fileName: file.fileName, + content: file.content, + transpilers: [], + cubeNames, + cubeSymbols, + }; + + const res = await this.workerPool!.exec('transpileYaml', [data]); + errorsReport.addErrors(res.errors); + errorsReport.addWarnings(res.warnings); + + return { ...file, content: res.content }; + } else { + return this.yamlCompiler.transpileYamlFile(file, errorsReport); + } } private async transpileJinjaFile( diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 429debf007249..aa438e4b1295f 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -8,11 +8,14 @@ import { JinjaEngine, NativeInstance, PythonCtx } from '@cubejs-backend/native'; import type { FileContent } from '@cubejs-backend/shared'; import { getEnv } from '@cubejs-backend/shared'; -import { CubePropContextTranspiler, transpiledFields, transpiledFieldsPatterns } from './transpilers'; +import { + CubePropContextTranspiler, + transpiledFields, + transpiledFieldsPatterns, + TranspilerCubeResolver, TranspilerSymbolResolver +} from './transpilers'; import { PythonParser } from '../parser/PythonParser'; -import { CubeSymbols } from './CubeSymbols'; import { nonStringFields } from './CubeValidator'; -import { CubeDictionary } from './CubeDictionary'; import { ErrorReporter } from './ErrorReporter'; import { camelizeCube } from './utils'; import { CompileContext } from './DataSchemaCompiler'; @@ -28,10 +31,10 @@ export class YamlCompiler { protected jinjaEngine: JinjaEngine | null = null; public constructor( - private readonly cubeSymbols: CubeSymbols, - private readonly cubeDictionary: CubeDictionary, + private readonly cubeSymbols: TranspilerSymbolResolver, + private readonly cubeDictionary: TranspilerCubeResolver, private readonly nativeInstance: NativeInstance, - private readonly viewCompiler: CubeSymbols, + private readonly viewCompiler: TranspilerSymbolResolver, ) { } diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts index ffaeba382ec7a..17945263e8d49 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts @@ -3,6 +3,7 @@ import { parse } from '@babel/parser'; import babelGenerator from '@babel/generator'; import babelTraverse from '@babel/traverse'; +import { NativeInstance } from '@cubejs-backend/native'; import { ValidationTranspiler } from './ValidationTranspiler'; import { ImportExportTranspiler } from './ImportExportTranspiler'; import { CubeCheckDuplicatePropTranspiler } from './CubeCheckDuplicatePropTranspiler'; @@ -11,6 +12,7 @@ import { ErrorReporter } from '../ErrorReporter'; import { LightweightSymbolResolver } from './LightweightSymbolResolver'; import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary'; import { IIFETranspiler } from './IIFETranspiler'; +import { YamlCompiler } from '../YamlCompiler'; type TransferContent = { fileName: string; @@ -23,6 +25,7 @@ type TransferContent = { const cubeDictionary = new LightweightNodeCubeDictionary(); const cubeSymbols = new LightweightSymbolResolver(); const errorsReport = new ErrorReporter(null, []); +const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, new NativeInstance(), cubeSymbols); const transpilers = { ValidationTranspiler: new ValidationTranspiler(), @@ -32,7 +35,7 @@ const transpilers = { IIFETranspiler: new IIFETranspiler(), }; -const transpile = (data: TransferContent) => { +const transpileJs = (data: TransferContent) => { cubeDictionary.setCubeNames(data.cubeNames); cubeSymbols.setSymbols(data.cubeSymbols); @@ -64,6 +67,22 @@ const transpile = (data: TransferContent) => { }; }; +const transpileYaml = (data: TransferContent) => { + cubeDictionary.setCubeNames(data.cubeNames); + cubeSymbols.setSymbols(data.cubeSymbols); + + errorsReport.inFile(data); + const transpiledFile = yamlCompiler.transpileYamlFile(data, errorsReport); + errorsReport.exitFile(); + + return { + content: transpiledFile?.content, + errors: errorsReport.getErrors(), + warnings: errorsReport.getWarnings() + }; +}; + workerpool.worker({ - transpile, + transpileJs, + transpileYaml, }); From baf018df3fb8460c0e69a39bd0759a01a4b21f65 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 1 Sep 2025 11:39:36 +0300 Subject: [PATCH 3/7] fix to allow use threads and native in parallel --- .../cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index 7f7689cea5a07..1628aa55d0fe7 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -268,7 +268,7 @@ export class DataSchemaCompiler { const transpilationNativeThreadsCount = getThreadsCount(); const { compilerId } = this; - if (!transpilationNative && transpilationWorkerThreads) { + if (transpilationWorkerThreads) { const wc = getEnv('transpilationWorkerThreadsCount'); this.workerPool = workerpool.pool( path.join(__dirname, 'transpilers/transpiler_worker'), @@ -288,7 +288,7 @@ export class DataSchemaCompiler { if (transpilationNative) { const nonJsFilesTasks = [...jinjaTemplatedFiles, ...yamlFiles] - .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId })); + .map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames, compilerId })); const jsFiles = originalJsFiles; let jsFilesTasks: Promise<(FileContent | undefined)[]>[] = []; From 8af6ed81e9d2dd77fa3fc85516dd641211ef2a42 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 1 Sep 2025 16:32:55 +0300 Subject: [PATCH 4/7] just code polish --- packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index aa438e4b1295f..c63c519d92555 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -128,6 +128,7 @@ export class YamlCompiler { cubeObj.dimensions = this.yamlArrayToObj(cubeObj.dimensions || [], 'dimension', errorsReport); cubeObj.segments = this.yamlArrayToObj(cubeObj.segments || [], 'segment', errorsReport); cubeObj.preAggregations = this.yamlArrayToObj(cubeObj.preAggregations || [], 'preAggregation', errorsReport); + cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport); cubeObj.joins = cubeObj.joins || []; // For edge cases where joins are not defined/null if (!Array.isArray(cubeObj.joins)) { @@ -135,8 +136,6 @@ export class YamlCompiler { cubeObj.joins = []; } - cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport); - return this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport); } From 546965f924bbc069e7075b4e6218fcc9bbcee7c2 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 5 Sep 2025 11:55:08 +0300 Subject: [PATCH 5/7] small fix --- .../src/compiler/transpilers/transpiler_worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts index 17945263e8d49..5344acaef34fa 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts @@ -76,7 +76,7 @@ const transpileYaml = (data: TransferContent) => { errorsReport.exitFile(); return { - content: transpiledFile?.content, + content: transpiledFile?.content || '', errors: errorsReport.getErrors(), warnings: errorsReport.getWarnings() }; From 82dbf8efa6eea1041952ad545987f15bef8182d0 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 4 Sep 2025 23:01:43 +0300 Subject: [PATCH 6/7] remove speed test as it is flacky --- .../postgres/dataschema-compiler.test.ts | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts index faf68af2a3008..c6e1a331486b3 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts @@ -158,49 +158,6 @@ describe('DataSchemaCompiler', () => { compiler.throwIfAnyErrors(); }); }); - - describe('Test perfomance', () => { - const schema = ` - cube('visitors', { - sql: 'select * from visitors', - measures: { - count: { - type: 'count', - sql: 'id' - }, - duration: { - type: 'avg', - sql: 'duration' - }, - }, - dimensions: { - date: { - type: 'string', - sql: 'date' - }, - browser: { - type: 'string', - sql: 'browser' - } - } - }) - `; - - it('Should compile 200 schemas in less than 2500ms * 10', async () => { - const repeats = 200; - - const compilerWith = prepareJsCompiler(schema, { allowJsDuplicatePropsInSchema: false }); - const start = new Date().getTime(); - for (let i = 0; i < repeats; i++) { - delete compilerWith.compiler.compilePromise; // Reset compile result - await compilerWith.compiler.compile(); - } - const end = new Date().getTime(); - const time = end - start; - - expect(time).toBeLessThan(2500 * 10); - }); - }); }); it('calculated metrics', async () => { From 9e42ae37230d1f3b68f91e3d7bcdecf4b06daa09 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 1 Sep 2025 12:53:26 +0300 Subject: [PATCH 7/7] introduce compiledYamlCache --- .../src/compiler/DataSchemaCompiler.ts | 20 ++++++++++++++++++- .../src/compiler/PrepareCompiler.ts | 3 +++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index 1628aa55d0fe7..78a5f3ec99593 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -88,6 +88,7 @@ export type DataSchemaCompilerOptions = { compileContext?: any; allowNodeRequire?: boolean; compiledScriptCache: LRUCache; + compiledYamlCache: LRUCache; }; export type TranspileOptions = { @@ -163,6 +164,8 @@ export class DataSchemaCompiler { private readonly compiledScriptCache: LRUCache; + private readonly compiledYamlCache: LRUCache; + private compileV8ContextCache: vm.Context | null = null; // FIXME: Is public only because of tests, should be private @@ -196,6 +199,7 @@ export class DataSchemaCompiler { this.workerPool = null; this.compilerId = options.compilerId || 'default'; this.compiledScriptCache = options.compiledScriptCache; + this.compiledYamlCache = options.compiledYamlCache; } public compileObjects(compileServices: CompilerInterface[], objects, errorsReport: ErrorReporter) { @@ -705,6 +709,14 @@ export class DataSchemaCompiler { errorsReport: ErrorReporter, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions ): Promise<(FileContent | undefined)> { + const cacheKey = crypto.createHash('md5').update(JSON.stringify(file.content)).digest('hex'); + + if (this.compiledYamlCache.has(cacheKey)) { + const content = this.compiledYamlCache.get(cacheKey)!; + + return { ...file, content }; + } + /* if (getEnv('transpilationNative')) { } else */ if (getEnv('transpilationWorkerThreads')) { @@ -720,9 +732,15 @@ export class DataSchemaCompiler { errorsReport.addErrors(res.errors); errorsReport.addWarnings(res.warnings); + this.compiledYamlCache.set(cacheKey, res.content); + return { ...file, content: res.content }; } else { - return this.yamlCompiler.transpileYamlFile(file, errorsReport); + const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport); + + this.compiledYamlCache.set(cacheKey, transpiledFile?.content || ''); + + return transpiledFile; } } diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index a4cf1a26df922..27a8f159b8eac 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -37,6 +37,7 @@ export type PrepareCompilerOptions = { headCommitId?: string; adapter?: string; compiledScriptCache?: LRUCache; + compiledYamlCache?: LRUCache; }; export interface CompilerInterface { @@ -59,6 +60,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler); const compiledScriptCache = options.compiledScriptCache || new LRUCache({ max: 250 }); + const compiledYamlCache = options.compiledYamlCache || new LRUCache({ max: 250 }); const transpilers: TranspilerInterface[] = [ new ValidationTranspiler(), @@ -79,6 +81,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp transpilers, viewCompilationGate, compiledScriptCache, + compiledYamlCache, viewCompilers: [viewCompiler], cubeCompilers: [cubeEvaluator, joinGraph, metaTransformer], contextCompilers: [contextEvaluator],