From 13b50d1a7e249e83c2c1366c6a44c5d98794018e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alja=C5=BE=20Mur=20Er=C5=BEen?= Date: Mon, 2 Sep 2024 09:21:27 +0200 Subject: [PATCH] feat-types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed commit of the following: commit cbdda4ff7614bee711d785036544b867f981c0bc Merge: b898560f 6fa675ee Author: Aljaž Mur Eržen Date: Mon Jul 22 16:43:16 2024 +0200 Merge branch 'main' into feat-indirections commit b898560f3e93f13be9c452f85fac36e5e1ae4a7b Merge: 96844669 755811b0 Author: Aljaž Mur Eržen Date: Wed Jun 26 11:42:24 2024 +0200 merge main into feat-types commit 755811b0f56e3d359a97f67881998c88523dad2e Author: Aljaž Mur Eržen Date: Wed Jun 26 10:52:47 2024 +0200 Revert "chore: Move `db.` to a new branch (#4349)" This reverts commit 76c4f767c4312976dba0fbadfa6c27b84e31faa9. commit 96844669f9c834fe43d18b79a79ceebe588ce6f0 Author: Aljaž Mur Eržen Date: Fri Jun 14 09:31:57 2024 +0200 fix: infer types for empty arrays commit 70fd898c8c231c895cab9f3e033b9ef42127a9c4 Merge: e71ae6dd 80e4a1d7 Author: Aljaž Mur Eržen Date: Fri Jun 14 09:20:11 2024 +0200 merge main into feat-types commit 80e4a1d7f50e95fecc51100353919d33e754cf2a Author: Aljaž Mur Eržen Date: Sat Jun 1 18:59:48 2024 +0200 Revert "chore: Move `db.` to a new branch (#4349)" This reverts commit 76c4f767c4312976dba0fbadfa6c27b84e31faa9. commit e71ae6dd4f5eda78472d6ad5fe65555c6bce4597 Author: Aljaž Mur Eržen Date: Sun Jun 2 20:28:46 2024 +0200 fix: tuple type checking commit f0627d91ecea24d6c5bebce640cdb16fd1d1bff5 Merge: 041310ea f0fd93b3 Author: Aljaž Mur Eržen Date: Sat Jun 1 19:29:53 2024 +0200 merge main into feat-types commit f0fd93b31bc451aaf4455d51a615bad5de60b820 Author: Aljaž Mur Eržen Date: Sat Jun 1 18:59:48 2024 +0200 Revert "chore: Move `db.` to a new branch (#4349)" This reverts commit 76c4f767c4312976dba0fbadfa6c27b84e31faa9. commit 041310ea3cfc15f17017b0b76f0523916047910a Author: Aljaž Mur Eržen Date: Sun May 19 16:39:12 2024 +0200 refactor: minor cleanup commit 5867f625d54009840d72c48fd3b802778b5b4e86 Author: Aljaž Mur Eržen Date: Sun May 19 16:36:24 2024 +0200 fix: lowering aggregate commit 2e67021433106155b6b5ff6b7cd1c96e47717e9f Author: Aljaž Mur Eržen Date: Sun May 19 13:51:02 2024 +0200 refactor: scope commit fff7a75dcd79f84a520e8f37abe530492b23ac50 Author: Aljaž Mur Eržen Date: Sun May 19 12:14:32 2024 +0200 feat: infer type as array commit 87df34febf3aad264a504f587d24f0a46d426ecc Author: Aljaž Mur Eržen Date: Wed May 15 21:43:20 2024 +0200 fix: organize tests, minor fixes commit d855aa9cb739851976e9863e87f0303aad8a52fa Author: Aljaž Mur Eržen Date: Wed May 15 20:38:40 2024 +0200 fix: excluding fields commit 1980e03e2e2623f16ba13f5f29e6b6639a2b5d15 Author: Aljaž Mur Eržen Date: Wed May 15 19:31:48 2024 +0200 refactor: cleanup TyFunc commit 39cb797b27b188174bbd1a97a3a6895a5bd1a933 Author: Aljaž Mur Eržen Date: Wed May 15 19:17:47 2024 +0200 feat: nested tuple indirection commit f5fa4f356a3d5f5d560f691a76a079689bf4a061 Author: Aljaž Mur Eržen Date: Tue May 14 14:27:45 2024 +0200 refactor: resolving tuple indirection commit 043047ed438efd845d26bb388b70f8ceb3f5f6b9 Author: Aljaž Mur Eržen Date: Tue May 14 14:04:22 2024 +0200 feat: lowering for rq operators and relational literals commit 35eeb8498414fbb76524d873cbcebcabaaee780f Author: Aljaž Mur Eržen Date: Tue May 14 12:13:06 2024 +0200 feat: finalize global generics after each stmt commit 983c08551945512014f2a9c50e2a4c16ff309b8b Author: Aljaž Mur Eržen Date: Tue May 14 11:58:50 2024 +0200 refactor: cleanup LayeredModules commit 4b060ef46c2aa80bf65e918b8b5605ddb6053eba Author: Aljaž Mur Eržen Date: Tue May 14 11:42:26 2024 +0200 refactor: lowering commit 9d378884374fb3b2bed0ea543370e3afe577f9c4 Author: Aljaž Mur Eržen Date: Tue May 14 10:39:04 2024 +0200 feat: create generics for all unknown types commit f13c93d9e83833375904312bd14ae9fc4b719048 Author: Aljaž Mur Eržen Date: Mon May 13 20:19:25 2024 +0200 refactor: remove DeclKind::TableDecl in favor DeclKind::Expr commit f5d2588c3342f1d5c328ff3ddeceb224ac0c1c66 Author: Aljaž Mur Eržen Date: Mon May 13 19:52:06 2024 +0200 fix: relational functions commit 6990ff8e373ac52b8de915197764a7f97d684742 Author: Aljaž Mur Eržen Date: Mon May 13 18:40:17 2024 +0200 fix: lowering and special_functions commit 5120b9a6e631e50e87a6a1b0cda3eb8d484e0b37 Author: Aljaž Mur Eržen Date: Mon May 13 18:17:25 2024 +0200 feat: local scopes instead of modules commit 16315280b4fe2ee21ea7a47c8ddf2913581fb088 Author: Aljaž Mur Eržen Date: Mon May 13 13:52:47 2024 +0200 refactor: move table inference into name resolver commit db274090db398068247d0dbc8c86b7eaa7a4e345 Author: Aljaž Mur Eržen Date: Mon May 13 12:47:25 2024 +0200 feat: unpack must come last commit 43255ecddaf0998f4ae8d9e35036f9f258b195e7 Author: Aljaž Mur Eržen Date: Mon May 13 12:10:04 2024 +0200 feat!: forbid functions without paramaters commit 85da3495f97ee22ba2241fac210b2bdecd795dcc Author: Aljaž Mur Eržen Date: Sun May 12 12:08:05 2024 +0200 ignore failed tests commit 17af0727e938ecc217e6543df4fae50d4aa620d6 Author: Aljaž Mur Eržen Date: Mon Apr 15 19:20:12 2024 +0200 feat: resolve functions without inlining commit e1ebdc4184a862c971839ae17bd0ede311577479 Author: Aljaž Mur Eržen Date: Mon Apr 15 15:35:27 2024 +0200 feat: split pl::FuncApplication from pl::Func commit 98e26f07c899357f4dc4ab0171876908e04d86de Author: Aljaž Mur Eržen Date: Mon Apr 15 13:19:48 2024 +0200 refactor: cosmetic tweaks for types commit 4319c045466b2bc8636e03da40583b8abf55fa31 Author: Aljaž Mur Eržen Date: Sun Apr 14 20:50:14 2024 +0200 feat: inference between generics commit e4f4e40d79cafa4f4c37850917c4b1e9847615e9 Author: Aljaž Mur Eržen Date: Sun Apr 14 19:49:57 2024 +0200 refactor: merge local and global _generic modules commit 5e5c0157d3b0906e4e72da13ba6d604e8c26730d Author: Aljaž Mur Eržen Date: Sun Apr 14 10:45:36 2024 +0200 refactor: validate_type commit b8f9895310904501b1852643d0f67cc96d2fbe41 Author: Aljaž Mur Eržen Date: Thu Apr 4 13:24:04 2024 +0200 feat: move flatten and special functions into lowering commit 6d11a771db11affd31f800c3e38fe9a650646dbb Author: Aljaž Mur Eržen Date: Thu Apr 4 11:00:25 2024 +0200 feat: ty tuple exclude commit 33415cefdb0b7ea86a4ffdda614fed07dfbc31dd Author: Aljaž Mur Eržen Date: Wed Apr 3 12:40:59 2024 +0200 refactors and fixes commit 03c6f512eb233fd893cd91c7b71b90b1ca85552d Author: Aljaž Mur Eržen Date: Wed Apr 3 10:04:01 2024 +0200 fmt commit ac36f23c50754b463d17ef6592248bc0d2daeaf8 Author: Aljaž Mur Eržen Date: Tue Apr 2 13:51:09 2024 +0200 fix: a few tests commit f6c29cf17b88412b33ce197f7dbaf7e3ebefd877 Author: Aljaž Mur Eržen Date: Sun Mar 31 12:23:42 2024 +0200 feat: global generic arguments for table inference commit 2d47aebd766571ea0c6941cf032a28d364c65b8d Author: Aljaž Mur Eržen Date: Fri Mar 29 16:05:11 2024 +0100 refactor: generic type parameters commit 3eac2761d5b34e0cb51c0e22d1d2bdf88c239a30 Author: Aljaž Mur Eržen Date: Fri Mar 29 15:42:47 2024 +0100 cleanup inference commit 2de728630ed989a7246f0fb50c5806baa709f4a1 Author: Aljaž Mur Eržen Date: Fri Mar 29 14:06:38 2024 +0100 feat: tuple unpacking commit b33153f42fba0e50c8c7afea40d9f02c5cab345e Author: Aljaž Mur Eržen Date: Fri Mar 29 13:14:22 2024 +0100 feat: wildcard includes and excludes commit 9aa3397413e2de90c7efe20ac0d26ca43614f2d8 Author: Aljaž Mur Eržen Date: Fri Mar 29 11:54:19 2024 +0100 feat: implicit closures and joins commit 2b3233a1749eaa833f6995332829f01089c56ac6 Author: Aljaž Mur Eržen Date: Thu Mar 28 16:53:56 2024 +0100 feat: remove obsolete type machinery This includes: - TypeKind::Singleton, - TypeKind::Union, - type normalization, - subtyping, This commit also: - changes syntax for generic type arguments, - adds type annotations to transforms, Many tests now fail. commit 107c22e88730cd748a03a2fc1882187e8ba47d5e Author: Aljaž Mur Eržen Date: Thu Mar 28 12:24:03 2024 +0100 more lowering & test cases commit a8040ce8af5d918d64743779ba53141ac530fd7f Author: Aljaž Mur Eržen Date: Thu Mar 28 11:49:05 2024 +0100 feat: rewire lowering to work with indirections commit 4c537298a977e6e16eba3188c8916625b6f41536 Author: Aljaž Mur Eržen Date: Fri Feb 16 19:24:55 2024 +0100 feat: indirection commit 43bb7a7f229efee231d3beaabf49d2a7e4166c9d Author: Aljaž Mur Eržen Date: Thu Apr 4 13:27:22 2024 +0200 refactor: remove `debug eval` command commit 2e9795d57d4259065c7c6977e9a86deabfef6bcd Author: Aljaž Mur Eržen Date: Thu Mar 28 14:27:20 2024 +0100 refactor: minor changes to lowering (#4364) commit 125aafb8d4e1f96959dd585f8b009d38123d632f Author: Aljaž Mur Eržen Date: Mon Mar 25 17:19:58 2024 +0100 feat!: resolve declaration names before the resolver (#4353) commit 7dbe057353985cdaf85373a7b97ef5e0e6c06535 Author: Aljaž Mur Eržen Date: Mon Mar 25 14:21:41 2024 +0100 build: test for feat-types commit ee6b2ce2d7a6a7d1e58b47380560d8daaf4b14ea Author: Aljaž Mur Eržen Date: Mon Mar 25 12:30:38 2024 +0100 feat: indirection parsing (#4356) --- Cargo.lock | 12 +- Cargo.toml | 1 - prqlc/Taskfile.yaml | 25 +- prqlc/prqlc-parser/src/parser/expr.rs | 10 +- prqlc/prqlc-parser/src/parser/pr/expr.rs | 2 +- prqlc/prqlc-parser/src/parser/pr/types.rs | 50 +- prqlc/prqlc-parser/src/parser/types.rs | 108 +- prqlc/prqlc-parser/src/span.rs | 20 + prqlc/prqlc/Cargo.toml | 1 + prqlc/prqlc/src/cli/docs_generator.rs | 20 - prqlc/prqlc/src/cli/mod.rs | 30 +- prqlc/prqlc/src/codegen/ast.rs | 37 +- prqlc/prqlc/src/codegen/types.rs | 70 +- prqlc/prqlc/src/ir/decl.rs | 82 +- prqlc/prqlc/src/ir/pl/expr.rs | 50 +- prqlc/prqlc/src/ir/pl/extra.rs | 21 +- prqlc/prqlc/src/ir/pl/fold.rs | 104 +- prqlc/prqlc/src/ir/pl/lineage.rs | 96 -- prqlc/prqlc/src/ir/pl/mod.rs | 6 - prqlc/prqlc/src/ir/pl/stmt.rs | 20 +- prqlc/prqlc/src/ir/pl/utils.rs | 7 - prqlc/prqlc/src/lib.rs | 3 +- prqlc/prqlc/src/semantic/ast_expand.rs | 154 +- prqlc/prqlc/src/semantic/eval.rs | 541 ------- prqlc/prqlc/src/semantic/lowering.rs | 1125 +++++++------- .../{resolver => lowering}/flatten.rs | 48 +- prqlc/prqlc/src/semantic/lowering/inline.rs | 135 ++ .../semantic/lowering/special_functions.rs | 697 +++++++++ prqlc/prqlc/src/semantic/mod.rs | 55 +- prqlc/prqlc/src/semantic/module.rs | 400 +---- prqlc/prqlc/src/semantic/reporting.rs | 39 +- .../semantic/resolve_decls/init_modules.rs | 71 + prqlc/prqlc/src/semantic/resolve_decls/mod.rs | 5 + .../prqlc/src/semantic/resolve_decls/names.rs | 457 ++++++ prqlc/prqlc/src/semantic/resolver/expr.rs | 364 ++--- .../prqlc/src/semantic/resolver/functions.rs | 747 +++++---- .../prqlc/src/semantic/resolver/inference.rs | 254 +-- prqlc/prqlc/src/semantic/resolver/mod.rs | 79 +- prqlc/prqlc/src/semantic/resolver/names.rs | 264 ---- prqlc/prqlc/src/semantic/resolver/scope.rs | 151 ++ prqlc/prqlc/src/semantic/resolver/stmt.rs | 247 ++- .../prqlc/src/semantic/resolver/transforms.rs | 1263 --------------- prqlc/prqlc/src/semantic/resolver/tuple.rs | 617 ++++++++ prqlc/prqlc/src/semantic/resolver/types.rs | 1374 +++++++---------- prqlc/prqlc/src/semantic/std.prql | 245 +-- prqlc/prqlc/src/sql/gen_projection.rs | 2 +- prqlc/prqlc/src/sql/gen_query.rs | 6 +- prqlc/prqlc/src/sql/mod.rs | 2 +- prqlc/prqlc/src/sql/operators.rs | 64 +- prqlc/prqlc/src/sql/pq/mod.rs | 8 +- prqlc/prqlc/src/sql/std.sql.prql | 24 +- prqlc/prqlc/src/utils/id_gen.rs | 2 +- prqlc/prqlc/tests/integration/resolving.rs | 302 +++- prqlc/prqlc/tests/integration/sql.rs | 1361 ++++++++++------ 54 files changed, 6063 insertions(+), 5815 deletions(-) delete mode 100644 prqlc/prqlc/src/ir/pl/lineage.rs delete mode 100644 prqlc/prqlc/src/semantic/eval.rs rename prqlc/prqlc/src/semantic/{resolver => lowering}/flatten.rs (76%) create mode 100644 prqlc/prqlc/src/semantic/lowering/inline.rs create mode 100644 prqlc/prqlc/src/semantic/lowering/special_functions.rs create mode 100644 prqlc/prqlc/src/semantic/resolve_decls/init_modules.rs create mode 100644 prqlc/prqlc/src/semantic/resolve_decls/mod.rs create mode 100644 prqlc/prqlc/src/semantic/resolve_decls/names.rs delete mode 100644 prqlc/prqlc/src/semantic/resolver/names.rs create mode 100644 prqlc/prqlc/src/semantic/resolver/scope.rs delete mode 100644 prqlc/prqlc/src/semantic/resolver/transforms.rs create mode 100644 prqlc/prqlc/src/semantic/resolver/tuple.rs diff --git a/Cargo.lock b/Cargo.lock index 978d5ddc0c1b..999b46534d8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -856,13 +856,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "compile-files" -version = "0.13.1" -dependencies = [ - "prqlc", -] - [[package]] name = "connection-string" version = "0.2.0" @@ -1749,9 +1742,9 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indexmap" -version = "2.2.6" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2875,6 +2868,7 @@ dependencies = [ "duckdb", "enum-as-inner", "glob", + "indexmap", "insta", "insta-cmd", "is-terminal", diff --git a/Cargo.toml b/Cargo.toml index 0595f8fb1e5f..19373f687c70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,6 @@ members = [ "prqlc/prqlc-macros", "prqlc/prqlc-parser", "prqlc/prqlc", - "prqlc/prqlc/examples/compile-files", # An example "lutra/lutra", "lutra/bindings/python", "web/book", diff --git a/prqlc/Taskfile.yaml b/prqlc/Taskfile.yaml index 41fe161ae013..3ae9c60881dd 100644 --- a/prqlc/Taskfile.yaml +++ b/prqlc/Taskfile.yaml @@ -37,21 +37,44 @@ tasks: --ignore-unknown \ --log-level=warn + fmt-check: + desc: Validate that source files are formatted + cmds: + - cargo fmt --check {{.packages_core}} {{.packages_addon}} + {{.packages_bindings}} + test-fast: desc: A fast test used for feedback during compiler development cmds: - cmd: | - INSTA_FORCE_PASS=1 cargo nextest run {{.packages_core}} --no-fail-fast + INSTA_FORCE_PASS=1 cargo nextest run {{.packages_core}} -- {{.CLI_ARGS}} - cmd: cargo insta review - cmd: cargo clippy --all-targets {{.packages_core}} + test-types: + desc: + A subset of tests that should work on feat-types + cmds: + - cmd: | + INSTA_FORCE_PASS=1 cargo nextest run --package prqlc --test=integration -- sql:: resolving:: + + - cmd: cargo insta review + + - cmd: cargo clippy --all-targets {{.packages_core}} + + - cmd: | + echo -e '\n=======\nrunning just ignored tests, to see if any of them passes' + cargo nextest run --package prqlc --test=integration --run-ignored=ignored-only --success-output=never --failure-output=never --status-level=pass --final-status-level=none -- sql:: resolving:: + test: desc: | A full test of prqlc (excluding --test-dbs-external). Generates coverage report. cmds: + - task: fmt-check + - cmd: | cargo \ llvm-cov --lcov --output-path lcov.info \ diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index 46a5d4b024e7..2925441311ea 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -481,9 +481,8 @@ where .then(ctrl(':').ignore_then(expr.clone().map(Box::new)).or_not()); let generic_args = ident_part() - .then_ignore(ctrl(':')) - .then(type_expr().separated_by(ctrl('|'))) - .map(|(name, domain)| GenericTypeParam { name, domain }) + .then(ctrl(':').ignore_then(type_expr()).or_not()) + .map(|(name, bound)| GenericTypeParam { name, bound }) .separated_by(ctrl(',')) .at_least(1) .delimited_by(ctrl('<'), ctrl('>')) @@ -500,7 +499,10 @@ where .allow_trailing(), ), // plain - param.repeated().map(|params| (Vec::new(), params)), + param + .repeated() + .at_least(1) + .map(|params| (Vec::new(), params)), )) .then_ignore(just(TokenKind::ArrowThin)) // return type diff --git a/prqlc/prqlc-parser/src/parser/pr/expr.rs b/prqlc/prqlc-parser/src/parser/pr/expr.rs index 5dfabb3a28f1..a2eefade89f5 100644 --- a/prqlc/prqlc-parser/src/parser/pr/expr.rs +++ b/prqlc/prqlc-parser/src/parser/pr/expr.rs @@ -165,7 +165,7 @@ pub struct GenericTypeParam { /// Assigned name of this generic type argument. pub name: String, - pub domain: Vec, + pub bound: Option, } /// A value and a series of functions that are to be applied to that value one after another. diff --git a/prqlc/prqlc-parser/src/parser/pr/types.rs b/prqlc/prqlc-parser/src/parser/pr/types.rs index 98bda0ba6f25..12eed3aaa84d 100644 --- a/prqlc/prqlc-parser/src/parser/pr/types.rs +++ b/prqlc/prqlc-parser/src/parser/pr/types.rs @@ -3,11 +3,11 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use strum::AsRefStr; -use crate::lexer::lr::Literal; use crate::parser::pr::ident::Ident; +use crate::parser::pr::expr::GenericTypeParam; use crate::span::Span; -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct Ty { pub kind: TyKind, @@ -25,12 +25,6 @@ pub enum TyKind { /// Type of a built-in primitive type Primitive(PrimitiveSet), - /// Type that contains only a one value - Singleton(Literal), - - /// Union of sets (sum) - Union(Vec<(Option, Ty)>), - /// Type of tuples (product) Tuple(Vec), @@ -40,15 +34,9 @@ pub enum TyKind { /// Type of functions with defined params and return types. Function(Option), - /// Type of every possible value. Super type of all other types. - /// The breaker of chains. Mother of types. - Any, - - /// Type that is the largest subtype of `base` while not a subtype of `exclude`. - Difference { base: Box, exclude: Box }, - - /// A generic argument. Contains id of the function call node and generic type param name. - GenericArg((usize, String)), + /// Tuples that have fields of `base` tuple, but don't have fields of `except` tuple. + /// Implies that `base` has all fields of `except`. + Exclude { base: Box, except: Box }, } impl TyKind { @@ -66,9 +54,11 @@ pub enum TyTupleField { /// Named tuple element. Single(Option, Option), - /// Placeholder for possibly many elements. - /// Means "and other unmentioned columns". Does not mean "all columns". - Wildcard(Option), + /// Many tuple elements contained in a type that must eventually resolve to a tuple. + /// In most cases, this starts as a generic type argument. + // TODO: make this non-optional Ty + // TODO: merge this into TyTuple (that does not exist at the moment) + Unpack(Option), } /// Built-in sets. @@ -103,9 +93,11 @@ pub enum PrimitiveSet { // Type of a function #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct TyFunc { - pub name_hint: Option, pub params: Vec>, + pub return_ty: Option>, + + pub generic_type_params: Vec, } impl Ty { @@ -122,14 +114,6 @@ impl Ty { Ty::new(TyKind::Array(Box::new(tuple))) } - pub fn never() -> Self { - Ty::new(TyKind::Union(Vec::new())) - } - - pub fn is_never(&self) -> bool { - self.kind.as_union().map_or(false, |x| x.is_empty()) - } - pub fn as_relation(&self) -> Option<&Vec> { self.kind.as_array()?.kind.as_tuple() } @@ -156,7 +140,7 @@ impl TyTupleField { pub fn ty(&self) -> Option<&Ty> { match self { TyTupleField::Single(_, ty) => ty.as_ref(), - TyTupleField::Wildcard(ty) => ty.as_ref(), + TyTupleField::Unpack(ty) => ty.as_ref(), } } } @@ -173,8 +157,8 @@ impl From for TyKind { } } -impl From for TyKind { - fn from(value: Literal) -> Self { - TyKind::Singleton(value) +impl PartialEq for Ty { + fn eq(&self, other: &Self) -> bool { + self.kind == other.kind && self.name == other.name } } diff --git a/prqlc/prqlc-parser/src/parser/types.rs b/prqlc/prqlc-parser/src/parser/types.rs index 5f34d6212430..4b09fc805d07 100644 --- a/prqlc/prqlc-parser/src/parser/types.rs +++ b/prqlc/prqlc-parser/src/parser/types.rs @@ -9,7 +9,7 @@ use crate::lexer::lr::TokenKind; pub(crate) fn type_expr() -> impl Parser + Clone { recursive(|nested_type_expr| { let basic = select! { - TokenKind::Literal(lit) => TyKind::Singleton(lit), + // TokenKind::Literal(lit) => TyKind::Singleton(lit), TokenKind::Ident(i) if i == "int"=> TyKind::Primitive(PrimitiveSet::Int), TokenKind::Ident(i) if i == "float"=> TyKind::Primitive(PrimitiveSet::Float), TokenKind::Ident(i) if i == "bool"=> TyKind::Primitive(PrimitiveSet::Bool), @@ -17,7 +17,7 @@ pub(crate) fn type_expr() -> impl Parser + Clone TokenKind::Ident(i) if i == "date"=> TyKind::Primitive(PrimitiveSet::Date), TokenKind::Ident(i) if i == "time"=> TyKind::Primitive(PrimitiveSet::Time), TokenKind::Ident(i) if i == "timestamp"=> TyKind::Primitive(PrimitiveSet::Timestamp), - TokenKind::Ident(i) if i == "anytype"=> TyKind::Any, + // TokenKind::Ident(i) if i == "anytype"=> TyKind::Any, }; let ident = ident().map(TyKind::Ident); @@ -31,9 +31,9 @@ pub(crate) fn type_expr() -> impl Parser + Clone .then_ignore(just(TokenKind::ArrowThin)) .then(nested_type_expr.clone().map(Box::new).map(Some)) .map(|(params, return_ty)| TyFunc { - name_hint: None, params, return_ty, + generic_type_params: vec![], }) .or_not(), ) @@ -42,7 +42,7 @@ pub(crate) fn type_expr() -> impl Parser + Clone let tuple = sequence(choice(( select! { TokenKind::Range { bind_right: true, bind_left: _ } => () } .ignore_then(nested_type_expr.clone()) - .map(|ty| TyTupleField::Wildcard(Some(ty))), + .map(|ty| TyTupleField::Unpack(Some(ty))), ident_part() .then_ignore(ctrl('=')) .or_not() @@ -63,7 +63,7 @@ pub(crate) fn type_expr() -> impl Parser + Clone .try_map(|fields, span| { let without_last = &fields[0..fields.len().saturating_sub(1)]; - if let Some(unpack) = without_last.iter().find_map(|f| f.as_wildcard()) { + if let Some(unpack) = without_last.iter().find_map(|f| f.as_unpack()) { let span = unpack.as_ref().and_then(|s| s.span).unwrap_or(span); return Err(PError::custom( span, @@ -76,32 +76,32 @@ pub(crate) fn type_expr() -> impl Parser + Clone .map(TyKind::Tuple) .labelled("tuple"); - let enum_ = keyword("enum") - .ignore_then( - sequence( - ident_part() - .then(ctrl('=').ignore_then(nested_type_expr.clone()).or_not()) - .map(|(name, ty)| { - ( - Some(name), - ty.unwrap_or_else(|| Ty::new(TyKind::Tuple(vec![]))), - ) - }), - ) - .delimited_by(ctrl('{'), ctrl('}')) - .recover_with(nested_delimiters( - TokenKind::Control('{'), - TokenKind::Control('}'), - [ - (TokenKind::Control('{'), TokenKind::Control('}')), - (TokenKind::Control('('), TokenKind::Control(')')), - (TokenKind::Control('['), TokenKind::Control(']')), - ], - |_| vec![], - )), - ) - .map(TyKind::Union) - .labelled("union"); + // let enum_ = keyword("enum") + // .ignore_then( + // sequence( + // ident_part() + // .then(ctrl('=').ignore_then(nested_type_expr.clone()).or_not()) + // .map(|(name, ty)| { + // ( + // Some(name), + // ty.unwrap_or_else(|| Ty::new(TyKind::Tuple(vec![]))), + // ) + // }), + // ) + // .delimited_by(ctrl('{'), ctrl('}')) + // .recover_with(nested_delimiters( + // TokenKind::Control('{'), + // TokenKind::Control('}'), + // [ + // (TokenKind::Control('{'), TokenKind::Control('}')), + // (TokenKind::Control('('), TokenKind::Control(')')), + // (TokenKind::Control('['), TokenKind::Control(']')), + // ], + // |_| vec![], + // )), + // ) + // .map(TyKind::Union) + // .labelled("union"); let array = nested_type_expr .map(Box::new) @@ -120,41 +120,27 @@ pub(crate) fn type_expr() -> impl Parser + Clone .map(TyKind::Array) .labelled("array"); - let term = choice((basic, ident, func, tuple, array, enum_)) + let term = choice((basic, ident, func, tuple, array)) .map_with_span(TyKind::into_ty) .boxed(); // exclude - // term.clone() - // .then(ctrl('-').ignore_then(term).repeated()) - // .foldl(|left, right| { - // let left_span = left.span.as_ref().unwrap(); - // let right_span = right.span.as_ref().unwrap(); - // let span = Span { - // start: left_span.start, - // end: right_span.end, - // source_id: left_span.source_id, - // }; - - // let kind = TyKind::Exclude { - // base: Box::new(left), - // except: Box::new(right), - // }; - // into_ty(kind, span) - // }); - - // union term.clone() - .then(just(TokenKind::Or).ignore_then(term).repeated()) - .map_with_span(|(first, following), span| { - if following.is_empty() { - first - } else { - let mut all = Vec::with_capacity(following.len() + 1); - all.push((None, first)); - all.extend(following.into_iter().map(|x| (None, x))); - TyKind::Union(all).into_ty(span) - } + .then(ctrl('-').ignore_then(term).repeated()) + .foldl(|left, right| { + let left_span = left.span.as_ref().unwrap(); + let right_span = right.span.as_ref().unwrap(); + let span = Span { + start: left_span.start, + end: right_span.end, + source_id: left_span.source_id, + }; + + let kind = TyKind::Exclude { + base: Box::new(left), + except: Box::new(right), + }; + TyKind::into_ty(kind, span) }) }) .labelled("type expression") diff --git a/prqlc/prqlc-parser/src/span.rs b/prqlc/prqlc-parser/src/span.rs index 03c36a746638..b8aef35d74f6 100644 --- a/prqlc/prqlc-parser/src/span.rs +++ b/prqlc/prqlc-parser/src/span.rs @@ -15,6 +15,26 @@ pub struct Span { pub source_id: u16, } +impl Span { + pub fn merge(a: Span, b: Span) -> Span { + assert_eq!(a.source_id, b.source_id); + Span { + start: usize::min(a.start, b.start), + end: usize::max(a.end, b.end), + + source_id: a.source_id, + } + } + + pub fn merge_opt(a: Option, b: Option) -> Option { + match (a, b) { + (Some(a), Some(b)) => Some(Self::merge(a, b)), + (Some(s), None) | (None, Some(s)) => Some(s), + (None, None) => None, + } + } +} + impl From for Range { fn from(a: Span) -> Self { a.start..a.end diff --git a/prqlc/prqlc/Cargo.toml b/prqlc/prqlc/Cargo.toml index 388472820e5b..62454b46f96c 100644 --- a/prqlc/prqlc/Cargo.toml +++ b/prqlc/prqlc/Cargo.toml @@ -46,6 +46,7 @@ ariadne = "0.4.1" chrono = "0.4.38" csv = "1.3.0" enum-as-inner = {workspace = true} +indexmap = "2.5.0" itertools = {workspace = true} log = {workspace = true} regex = "1.10.6" diff --git a/prqlc/prqlc/src/cli/docs_generator.rs b/prqlc/prqlc/src/cli/docs_generator.rs index bc789b0bea4c..075882374d31 100644 --- a/prqlc/prqlc/src/cli/docs_generator.rs +++ b/prqlc/prqlc/src/cli/docs_generator.rs @@ -160,23 +160,12 @@ pub fn generate_html_docs(stmts: Vec) -> String { if let Some(return_ty) = &func.return_ty { docs.push_str("

Returns

\n"); match &return_ty.kind { - TyKind::Any => docs.push_str("

Any

\n"), TyKind::Ident(ident) => { docs.push_str(&format!("

{}

\n", ident.name)); } TyKind::Primitive(primitive) => { docs.push_str(&format!("

{primitive}

\n")); } - TyKind::Singleton(literal) => { - docs.push_str(&format!("

{literal}

\n")); - } - TyKind::Union(vec) => { - docs.push_str("
    \n"); - for (_, ty) in vec { - docs.push_str(&format!("
  • {:?}
    • \n", ty.kind)); - } - docs.push_str("
\n"); - } _ => docs.push_str("

Not implemented

\n"), } } @@ -319,21 +308,12 @@ Generated with [prqlc](https://prql-lang.org/) {}. if let Some(return_ty) = &func.return_ty { docs.push_str("#### Returns\n"); match &return_ty.kind { - TyKind::Any => docs.push_str("Any\n"), TyKind::Ident(ident) => { docs.push_str(&format!("`{}`\n", ident.name)); } TyKind::Primitive(primitive) => { docs.push_str(&format!("`{primitive}`\n")); } - TyKind::Singleton(literal) => { - docs.push_str(&format!("`{literal}`\n")); - } - TyKind::Union(vec) => { - for (_, ty) in vec { - docs.push_str(&format!("* {:?}\n", ty.kind)); - } - } _ => docs.push_str("Not implemented\n"), } } diff --git a/prqlc/prqlc/src/cli/mod.rs b/prqlc/prqlc/src/cli/mod.rs index 85d93afa9b19..adb6b69a127a 100644 --- a/prqlc/prqlc/src/cli/mod.rs +++ b/prqlc/prqlc/src/cli/mod.rs @@ -416,6 +416,7 @@ impl Command { let mut root_module_def = prql_to_pl_tree(sources)?; drop_module_def(&mut root_module_def.stmts, "std"); + drop_module_def(&mut root_module_def.stmts, "_local"); pl_to_prql(&root_module_def)?.into_bytes() } @@ -440,8 +441,7 @@ impl Command { let ctx = semantic::resolve(root_mod)?; let frames = if let Ok((main, _)) = ctx.find_main_rel(&[]) { - semantic::reporting::collect_frames(*main.clone().into_relation_var().unwrap()) - .frames + semantic::reporting::collect_frames(main.clone()).frames } else { vec![] }; @@ -616,7 +616,7 @@ fn read_files(input: &mut clio::ClioPath) -> Result { Ok(SourceTree::new(sources, Some(root.to_path_buf()))) } -fn combine_prql_and_frames(source: &str, frames: Vec<(Option, pl::Lineage)>) -> String { +fn combine_prql_and_frames(source: &str, frames: Vec<(Option, pr::Ty)>) -> String { let source = Source::from(source); let lines = source.lines().collect_vec(); let width = lines.iter().map(|l| l.len()).max().unwrap_or(0); @@ -656,8 +656,18 @@ fn combine_prql_and_frames(source: &str, frames: Vec<(Option, pl::Line .to_string(); printed_lines_count += 1; - result.push(format!("{chars:width$} # {frame}")); + result.push(format!("{chars:width$} # {frame:?}")); } + let chars: String = source + .get_line_text(source.line(printed_lines_count).unwrap()) + .unwrap() + // Ariadne 0.4.1 added a line break at the end of the line, so we + // trim it. + .trim_end() + .to_string(); + printed_lines_count += 1; + + result.push(format!("{chars:width$} # {frame:?}")); } for line in lines.iter().skip(printed_lines_count) { result.push(source.get_line_text(line.to_owned()).unwrap().to_string()); @@ -679,7 +689,7 @@ mod tests { let output = Command::execute( &Command::Debug(DebugCommand::Annotate(IoArgs::default())), &mut r#" -from initial_table +from db.initial_table select {f = first_name, l = last_name, gender} derive full_name = f"{f} {l}" take 23 @@ -692,7 +702,7 @@ sort full .unwrap(); assert_snapshot!(String::from_utf8(output).unwrap().trim(), @r###" - from initial_table + from db.initial_table select {f = first_name, l = last_name, gender} # [f, l, initial_table.gender] derive full_name = f"{f} {l}" # [f, l, initial_table.gender, full_name] take 23 # [f, l, initial_table.gender, full_name] @@ -741,10 +751,13 @@ sort full }, &mut SourceTree::new( [ - ("Project.prql".into(), "orders.x | select y".to_string()), + ( + "Project.prql".into(), + "project.orders.x | select y".to_string(), + ), ( "orders.prql".into(), - "let x = (from z | select {y, u})".to_string(), + "let x = (from db.z | select {y, u})".to_string(), ), ], None, @@ -808,6 +821,7 @@ sort full span: 1:0-17 "###); } + #[test] fn lex() { let output = Command::execute( diff --git a/prqlc/prqlc/src/codegen/ast.rs b/prqlc/prqlc/src/codegen/ast.rs index 51517f0ea904..38c8f0f8f8c1 100644 --- a/prqlc/prqlc/src/codegen/ast.rs +++ b/prqlc/prqlc/src/codegen/ast.rs @@ -198,15 +198,11 @@ impl WriteSource for pr::ExprKind { r += opt.consume("<")?; for generic_param in &c.generic_type_params { r += opt.consume(&maybe_escape_ident_part(&generic_param.name))?; - r += opt.consume(": ")?; - r += &opt.consume( - SeparatedExprs { - exprs: &generic_param.domain, - inline: " | ", - line_end: "|", - } - .write(opt.clone())?, - )?; + + if let Some(bound) = &generic_param.bound { + r += opt.consume(": ")?; + r += opt.consume(&bound.write(opt.clone())?)?; + } } r += opt.consume("> ")?; } @@ -263,6 +259,18 @@ impl WriteSource for pr::ExprKind { } } +impl WriteSource for pr::GenericTypeParam { + fn write(&self, mut opt: WriteOpt) -> Option { + let mut r = opt.consume(maybe_escape_ident_part(&self.name))?; + + if let Some(bound) = &self.bound { + r += opt.consume(": ")?; + r += &opt.consume(bound.write(opt.clone())?)?; + } + Some(r) + } +} + fn break_line_within_parenthesis(expr: &T, mut opt: WriteOpt) -> Option { let mut r = "(\n".to_string(); opt.indent += 1; @@ -351,7 +359,12 @@ impl WriteSource for pr::Ident { opt.consume_width(width as u16)?; let mut r = String::new(); - for part in &self.path { + + let mut path = &self.path[..]; + if path.first().map_or(false, |f| f == "_local") { + path = &path[1..]; + } + for part in path { r += &maybe_escape_ident_part(part); r += "."; } @@ -364,7 +377,9 @@ fn keywords() -> &'static HashSet<&'static str> { static KEYWORDS: OnceLock> = OnceLock::new(); KEYWORDS.get_or_init(|| { HashSet::from_iter([ - "let", "into", "case", "prql", "type", "module", "internal", "func", + "let", "into", "case", "prql", "type", "internal", + "func", + // "module" can be both keyword and ident ]) }) } diff --git a/prqlc/prqlc/src/codegen/types.rs b/prqlc/prqlc/src/codegen/types.rs index bca6ca0aa201..029eee386633 100644 --- a/prqlc/prqlc/src/codegen/types.rs +++ b/prqlc/prqlc/src/codegen/types.rs @@ -26,7 +26,7 @@ impl WriteSource for Option<&pr::Ty> { fn write(&self, opt: WriteOpt) -> Option { match self { Some(ty) => ty.write(opt), - None => Some("infer".to_string()), + None => Some("?".to_string()), } } } @@ -38,27 +38,26 @@ impl WriteSource for pr::TyKind { match &self { Ident(ident) => ident.write(opt), Primitive(prim) => Some(prim.to_string()), - Union(variants) => { - let parenthesize = - // never must be parenthesized - variants.is_empty() || - // named union must be parenthesized - variants.iter().any(|(n, _)| n.is_some()); + // Union(variants) => { + // let parenthesize = + // // never must be parenthesized + // variants.is_empty() || + // // named union must be parenthesized + // variants.iter().any(|(n, _)| n.is_some()); - let variants: Vec<_> = variants.iter().map(|(n, t)| UnionVariant(n, t)).collect(); - let sep_exprs = SeparatedExprs { - exprs: &variants, - inline: " || ", - line_end: " ||", - }; + // let variants: Vec<_> = variants.iter().map(|(n, t)| UnionVariant(n, t)).collect(); + // let sep_exprs = SeparatedExprs { + // exprs: &variants, + // inline: " || ", + // line_end: " ||", + // }; - if parenthesize { - sep_exprs.write_between("(", ")", opt) - } else { - sep_exprs.write(opt) - } - } - Singleton(lit) => Some(lit.to_string()), + // if parenthesize { + // sep_exprs.write_between("(", ")", opt) + // } else { + // sep_exprs.write(opt) + // } + // } Tuple(elements) => SeparatedExprs { exprs: elements, inline: ", ", @@ -75,16 +74,14 @@ impl WriteSource for pr::TyKind { r += " "; } r += "-> "; - r += &func.return_ty.as_deref().write(opt)?; + r += &func.return_ty.as_ref().map(|x| x.as_ref()).write(opt)?; Some(r) } - Any => Some("anytype".to_string()), - Difference { base, exclude } => { + Exclude { base, except } => { let base = base.write(opt.clone())?; - let exclude = exclude.write(opt.clone())?; - Some(format!("{base} - {exclude}")) + let except = except.write(opt.clone())?; + Some(format!("{base} - {except}")) } - GenericArg(_) => Some("?".to_string()), } } } @@ -92,9 +89,9 @@ impl WriteSource for pr::TyKind { impl WriteSource for pr::TyTupleField { fn write(&self, opt: WriteOpt) -> Option { match self { - Self::Wildcard(generic_el) => match generic_el { - Some(el) => Some(format!("{}..", el.write(opt)?)), - None => Some("*..".to_string()), + Self::Unpack(generic_el) => match generic_el { + Some(el) => Some(format!("..{}", el.write(opt)?)), + None => Some("..".to_string()), }, Self::Single(name, expr) => { let mut r = String::new(); @@ -113,18 +110,3 @@ impl WriteSource for pr::TyTupleField { } } } - -struct UnionVariant<'a>(&'a Option, &'a pr::Ty); - -impl WriteSource for UnionVariant<'_> { - fn write(&self, mut opt: WriteOpt) -> Option { - let mut r = String::new(); - if let Some(name) = &self.0 { - r += name; - r += " = "; - } - opt.consume_width(r.len() as u16); - r += &self.1.write(opt)?; - Some(r) - } -} diff --git a/prqlc/prqlc/src/ir/decl.rs b/prqlc/prqlc/src/ir/decl.rs index fc6c940862fe..f1220ee081f3 100644 --- a/prqlc/prqlc/src/ir/decl.rs +++ b/prqlc/prqlc/src/ir/decl.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::codegen::write_ty; use crate::ir::pl; -use crate::pr::{Span, Ty}; +use crate::pr::{self, Span, Ty}; use crate::semantic::write_pl; /// Context of the pipeline. @@ -40,9 +40,11 @@ pub struct Module { pub shadowed: Option>, } -/// A struct containing information about a single declaration. +/// A struct containing information about a single declaration +/// within a PRQL module. #[derive(Debug, PartialEq, Default, Serialize, Deserialize, Clone)] pub struct Decl { + // TODO: make this plain usize, it is populated at creation anyway #[serde(skip_serializing_if = "Option::is_none")] pub declared_at: Option, @@ -57,26 +59,31 @@ pub struct Decl { pub annotations: Vec, } -/// The Declaration itself. +/// Declaration kind. #[derive(Debug, PartialEq, Serialize, Deserialize, Clone, EnumAsInner)] pub enum DeclKind { /// A nested namespace Module(Module), - /// Nested namespaces that do lookup in layers from top to bottom, stopping at first match. - LayeredModules(Vec), + /// A function parameter (usually the implicit `this` param) + // TODO: make this type non-optional + Variable(Option), - TableDecl(TableDecl), - - InstanceOf(pl::Ident, Option), - - /// A single column. Contains id of target which is either: - /// - an input relation that is source of this column or - /// - a column expression. - Column(usize), + TupleField, /// Contains a default value to be created in parent namespace when NS_INFER is matched. - Infer(Box), + Infer(InferTarget), + + /// A generic type argument. + /// It contains the candidate for this generic type that has been inferred during + /// type validation. If the candidate is, for example, an `int` this means that + /// this generic must be `int` or one of the previous type check would have failed. + /// If the candidate is, for example, tuple `{a = int, b = bool}`, this means that + /// previous type checks require the tuple to have fields `a` and `b`. It might contain + /// other fields as well. + /// + /// Span describes the node that proposed the candidate. + GenericParam(Option<(Ty, Option)>), Expr(Box), @@ -85,7 +92,18 @@ pub enum DeclKind { QueryDef(pl::QueryDef), /// Equivalent to the declaration pointed to by the fully qualified ident - Import(pl::Ident), + Import(pr::Ident), + + /// A declaration that has not yet been resolved. + /// Created during the first pass of the AST, must not be present in + /// a fully resolved module structure. + Unresolved(pl::StmtKind), +} + +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +pub enum InferTarget { + Table, + TupleField, } #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] @@ -99,19 +117,7 @@ pub struct TableDecl { } #[derive(Debug, PartialEq, Serialize, Deserialize, Clone, EnumAsInner)] -pub enum TableExpr { - /// In SQL, this is a CTE - RelationVar(Box), - - /// Actual table in a database. In SQL it can be referred to by name. - LocalTable, - - /// No expression (this decl just tracks a relation literal). - None, - - /// A placeholder for a relation that will be provided later. - Param(String), -} +pub enum TableExpr {} #[derive(Clone, Eq, Debug, PartialEq, Serialize, Deserialize)] pub enum TableColumn { @@ -164,6 +170,7 @@ impl Default for DeclKind { } } +// TODO: convert to Decl::new impl From for Decl { fn from(kind: DeclKind) -> Self { Decl { @@ -185,21 +192,20 @@ impl std::fmt::Display for DeclKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Module(arg0) => f.debug_tuple("Module").field(arg0).finish(), - Self::LayeredModules(arg0) => f.debug_tuple("LayeredModules").field(arg0).finish(), - Self::TableDecl(TableDecl { ty, expr }) => { - write!( - f, - "TableDecl: {} {expr:?}", - ty.as_ref().map(write_ty).unwrap_or_default() - ) + Self::Variable(Some(arg0)) => { + write!(f, "Variable of type {}", write_ty(arg0)) + } + Self::Variable(None) => { + write!(f, "Variable of unknown type") } - Self::InstanceOf(arg0, _) => write!(f, "InstanceOf: {arg0}"), - Self::Column(arg0) => write!(f, "Column (target {arg0})"), - Self::Infer(arg0) => write!(f, "Infer (default: {arg0})"), + Self::TupleField => write!(f, "TupleField"), + Self::Infer(arg0) => write!(f, "Infer {arg0:?}"), Self::Expr(arg0) => write!(f, "Expr: {}", write_pl(*arg0.clone())), Self::Ty(arg0) => write!(f, "Ty: {}", write_ty(arg0)), + Self::GenericParam(_) => write!(f, "GenericParam"), Self::QueryDef(_) => write!(f, "QueryDef"), Self::Import(arg0) => write!(f, "Import {arg0}"), + Self::Unresolved(_) => write!(f, "Unresolved"), } } } diff --git a/prqlc/prqlc/src/ir/pl/expr.rs b/prqlc/prqlc/src/ir/pl/expr.rs index e0b8eb7bd1b4..e2f9b9075f62 100644 --- a/prqlc/prqlc/src/ir/pl/expr.rs +++ b/prqlc/prqlc/src/ir/pl/expr.rs @@ -1,11 +1,10 @@ -use std::collections::HashMap; - use enum_as_inner::EnumAsInner; use prqlc_parser::generic; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; -use super::{Lineage, TransformCall}; +use super::TransformCall; use crate::codegen::write_ty; use crate::pr::{GenericTypeParam, Ident, Literal, Span, Ty}; @@ -35,20 +34,13 @@ pub struct Expr { #[serde(skip_serializing_if = "Option::is_none")] pub ty: Option, - /// Information about where data of this expression will come from. - /// - /// Currently, this is used to infer relational pipeline frames. - /// Must always exists if ty is a relation. - #[serde(skip_serializing_if = "Option::is_none")] - pub lineage: Option, - #[serde(skip)] pub needs_window: bool, /// When true on [ExprKind::Tuple], this list will be flattened when placed /// in some other list. // TODO: maybe we should have a special ExprKind instead of this flag? - #[serde(skip)] + #[serde(skip_serializing_if = "is_false")] pub flatten: bool, } @@ -61,12 +53,18 @@ pub enum ExprKind { within: Box, except: Box, }, + Indirection { + base: Box, + field: IndirectionKind, + }, Literal(Literal), Tuple(Vec), Array(Vec), FuncCall(FuncCall), Func(Box), + FuncApplication(FuncApplication), + TransformCall(TransformCall), SString(Vec), FString(Vec), @@ -84,6 +82,12 @@ pub enum ExprKind { Internal(String), } +#[derive(Debug, EnumAsInner, PartialEq, Clone, Serialize, Deserialize, JsonSchema)] +pub enum IndirectionKind { + Name(String), + Position(i64), +} + /// Function call. #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, JsonSchema)] pub struct FuncCall { @@ -97,9 +101,6 @@ pub struct FuncCall { /// May also contain environment that is needed to evaluate the body. #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, JsonSchema)] pub struct Func { - /// Name of the function. Used for user-facing messages only. - pub name_hint: Option, - /// Type requirement for the function body expression. pub return_ty: Option, @@ -114,13 +115,6 @@ pub struct Func { /// Generic type arguments within this function. pub generic_type_params: Vec, - - /// Arguments that have already been provided. - pub args: Vec, - - /// Additional variables that the body of the function may need to be - /// evaluated. - pub env: HashMap, } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, JsonSchema)] @@ -133,6 +127,13 @@ pub struct FuncParam { pub default_value: Option>, } +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, JsonSchema)] +pub struct FuncApplication { + pub func: Box, // TODO: change this to Expr + + pub args: Vec, +} + pub type Range = generic::Range>; pub type InterpolateItem = generic::InterpolateItem; pub type SwitchCase = generic::SwitchCase>; @@ -189,9 +190,10 @@ impl std::fmt::Debug for Expr { } ds.field("ty", &DebugTy(x)); } - if let Some(x) = &self.lineage { - ds.field("lineage", x); - } ds.finish() } } + +fn is_false(b: &bool) -> bool { + !b +} diff --git a/prqlc/prqlc/src/ir/pl/extra.rs b/prqlc/prqlc/src/ir/pl/extra.rs index 2fd5cd265738..03f7f36962aa 100644 --- a/prqlc/prqlc/src/ir/pl/extra.rs +++ b/prqlc/prqlc/src/ir/pl/extra.rs @@ -3,7 +3,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::ir::generic::WindowKind; -use crate::ir::pl::{Expr, ExprKind, Func, FuncCall, Ident, Range}; +use crate::ir::pl::{Expr, ExprKind, FuncCall, Ident, Range}; use crate::pr::Ty; impl FuncCall { @@ -23,7 +23,23 @@ pub enum TyOrExpr { Expr(Box), } -impl Func { +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Default)] +pub struct FuncMetadata { + /// Name of the function. Used for user-facing messages only. + pub name_hint: Option, + + pub implicit_closure: Option>, + pub coerce_tuple: Option, +} + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct ImplicitClosureConfig { + pub param: u8, + pub this: Option, + pub that: Option, +} + +impl FuncMetadata { pub(crate) fn as_debug_name(&self) -> &str { let ident = self.name_hint.as_ref(); @@ -123,7 +139,6 @@ impl Expr { span: None, target_id: None, ty: None, - lineage: None, needs_window: false, alias: None, flatten: false, diff --git a/prqlc/prqlc/src/ir/pl/fold.rs b/prqlc/prqlc/src/ir/pl/fold.rs index 8714d52b1f2f..2e60c530850b 100644 --- a/prqlc/prqlc/src/ir/pl/fold.rs +++ b/prqlc/prqlc/src/ir/pl/fold.rs @@ -77,6 +77,10 @@ pub fn fold_expr_kind(fold: &mut T, expr_kind: ExprKind) -> within: Box::new(fold.fold_expr(*within)?), except: Box::new(fold.fold_expr(*except)?), }, + Indirection { base, field } => Indirection { + base: Box::new(fold.fold_expr(*base)?), + field, + }, Tuple(items) => Tuple(fold.fold_exprs(items)?), Array(items) => Array(fold.fold_exprs(items)?), SString(items) => SString( @@ -94,7 +98,11 @@ pub fn fold_expr_kind(fold: &mut T, expr_kind: ExprKind) -> Case(cases) => Case(fold_cases(fold, cases)?), FuncCall(func_call) => FuncCall(fold.fold_func_call(func_call)?), - Func(closure) => Func(Box::new(fold.fold_func(*closure)?)), + Func(func) => Func(Box::new(fold.fold_func(*func)?)), + FuncApplication(func_app) => FuncApplication(super::expr::FuncApplication { + func: Box::new(fold.fold_expr(*func_app.func)?), + args: fold.fold_exprs(func_app.args)?, + }), TransformCall(transform) => TransformCall(fold.fold_transform_call(transform)?), RqOperator { name, args } => RqOperator { @@ -277,12 +285,10 @@ pub fn fold_transform_kind( pub fn fold_func(fold: &mut T, func: Func) -> Result { Ok(Func { body: Box::new(fold.fold_expr(*func.body)?), - args: func - .args - .into_iter() - .map(|item| fold.fold_expr(item)) - .try_collect()?, - ..func + return_ty: fold_type_opt(fold, func.return_ty)?, + params: fold_func_param(fold, func.params)?, + named_params: fold_func_param(fold, func.named_params)?, + generic_type_params: func.generic_type_params, // recurse into this too? }) } @@ -294,8 +300,9 @@ pub fn fold_func_param( .into_iter() .map(|param| { Ok(FuncParam { + name: param.name, + ty: fold_type_opt(fold, param.ty)?, default_value: fold_optional_box(fold, param.default_value)?, - ..param }) }) .try_collect() @@ -309,53 +316,50 @@ pub fn fold_type_opt(fold: &mut T, ty: Option) -> Result pub fn fold_type(fold: &mut T, ty: Ty) -> Result { Ok(Ty { kind: match ty.kind { - TyKind::Union(variants) => TyKind::Union( - variants - .into_iter() - .map(|(name, ty)| -> Result<_> { Ok((name, fold.fold_type(ty)?)) }) - .try_collect()?, - ), - TyKind::Tuple(fields) => TyKind::Tuple( - fields - .into_iter() - .map(|field| -> Result<_> { - Ok(match field { - TyTupleField::Single(name, ty) => { - TyTupleField::Single(name, fold_type_opt(fold, ty)?) - } - TyTupleField::Wildcard(ty) => { - TyTupleField::Wildcard(fold_type_opt(fold, ty)?) - } - }) - }) - .try_collect()?, - ), + TyKind::Tuple(fields) => TyKind::Tuple(fold_ty_tuple_fields(fold, fields)?), TyKind::Array(ty) => TyKind::Array(Box::new(fold.fold_type(*ty)?)), - TyKind::Function(func) => TyKind::Function( - func.map(|f| -> Result<_> { - Ok(TyFunc { - params: f - .params - .into_iter() - .map(|a| fold_type_opt(fold, a)) - .try_collect()?, - return_ty: fold_type_opt(fold, f.return_ty.map(|x| *x))?.map(Box::new), - name_hint: f.name_hint, - }) - }) - .transpose()?, - ), - TyKind::Difference { base, exclude } => TyKind::Difference { + TyKind::Function(func) => { + TyKind::Function(func.map(|f| fold_ty_func(fold, f)).transpose()?) + } + TyKind::Exclude { base, except } => TyKind::Exclude { base: Box::new(fold.fold_type(*base)?), - exclude: Box::new(fold.fold_type(*exclude)?), + except: Box::new(fold.fold_type(*except)?), }, - TyKind::Any - | TyKind::Ident(_) - | TyKind::Primitive(_) - | TyKind::Singleton(_) - | TyKind::GenericArg(_) => ty.kind, + TyKind::Ident(_) | TyKind::Primitive(_) => ty.kind, }, span: ty.span, name: ty.name, }) } + +pub fn fold_ty_func(fold: &mut F, f: TyFunc) -> Result { + Ok(TyFunc { + params: f + .params + .into_iter() + .map(|a| fold_type_opt(fold, a)) + .try_collect()?, + return_ty: f + .return_ty + .map(|t| fold.fold_type(*t).map(Box::new)) + .transpose()?, + generic_type_params: f.generic_type_params, + }) +} + +pub fn fold_ty_tuple_fields( + fold: &mut F, + fields: Vec, +) -> Result> { + fields + .into_iter() + .map(|field| -> Result<_> { + Ok(match field { + TyTupleField::Single(name, ty) => { + TyTupleField::Single(name, fold_type_opt(fold, ty)?) + } + TyTupleField::Unpack(ty) => TyTupleField::Unpack(fold_type_opt(fold, ty)?), + }) + }) + .try_collect() +} diff --git a/prqlc/prqlc/src/ir/pl/lineage.rs b/prqlc/prqlc/src/ir/pl/lineage.rs deleted file mode 100644 index 21a64ebfb3d4..000000000000 --- a/prqlc/prqlc/src/ir/pl/lineage.rs +++ /dev/null @@ -1,96 +0,0 @@ -use std::collections::HashSet; -use std::fmt::{Debug, Display, Formatter}; - -use enum_as_inner::EnumAsInner; -use itertools::{Itertools, Position}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use super::Ident; - -/// Represents the object that is manipulated by the pipeline transforms. -/// Similar to a view in a database or a data frame. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Lineage { - pub columns: Vec, - - pub inputs: Vec, - - // A hack that allows name retention when applying `ExprKind::All { except }` - #[serde(skip)] - pub prev_columns: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct LineageInput { - /// Id of the node in AST that declares this input. - pub id: usize, - - /// Local name of this input within a query. - pub name: String, - - /// Fully qualified name of the table that provides the data for this input. - pub table: Ident, -} - -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, EnumAsInner, JsonSchema)] -pub enum LineageColumn { - Single { - name: Option, - - // id of the defining expr (which can be actual expr or lineage input expr) - target_id: usize, - - // if target is a relation, this is the name within the relation - target_name: Option, - }, - - /// All columns (including unknown ones) from an input (i.e. `foo_table.*`) - All { - input_id: usize, - except: HashSet, - }, -} - -impl Display for Lineage { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - display_lineage(self, f, false) - } -} - -fn display_lineage(lineage: &Lineage, f: &mut Formatter, display_ids: bool) -> std::fmt::Result { - write!(f, "[")?; - for (pos, col) in lineage.columns.iter().with_position() { - let is_last = matches!(pos, Position::Last | Position::Only); - display_lineage_column(col, f, display_ids)?; - if !is_last { - write!(f, ", ")?; - } - } - write!(f, "]") -} - -fn display_lineage_column( - col: &LineageColumn, - f: &mut Formatter, - display_ids: bool, -) -> std::fmt::Result { - match col { - LineageColumn::All { input_id, .. } => { - write!(f, "{input_id}.*")?; - } - LineageColumn::Single { - name, target_id, .. - } => { - if let Some(name) = name { - write!(f, "{name}")? - } else { - write!(f, "?")? - } - if display_ids { - write!(f, ":{target_id}")? - } - } - } - Ok(()) -} diff --git a/prqlc/prqlc/src/ir/pl/mod.rs b/prqlc/prqlc/src/ir/pl/mod.rs index 00b18ed930d3..e2963e120e31 100644 --- a/prqlc/prqlc/src/ir/pl/mod.rs +++ b/prqlc/prqlc/src/ir/pl/mod.rs @@ -15,14 +15,12 @@ pub use crate::pr::{BinOp, BinaryExpr, Ident, UnOp, UnaryExpr}; pub use self::expr::*; pub use self::extra::*; pub use self::fold::*; -pub use self::lineage::*; pub use self::stmt::*; pub use self::utils::*; mod expr; mod extra; mod fold; -mod lineage; mod stmt; mod utils; @@ -41,7 +39,6 @@ pub fn print_mem_sizes() { println!("{:16}= {}", "decl::DeclKind", size_of::()); println!("{:16}= {}", "decl::Module", size_of::()); println!("{:16}= {}", "decl::TableDecl", size_of::()); - println!("{:16}= {}", "decl::TableExpr", size_of::()); println!("{:16}= {}", "ErrorMessage", size_of::()); println!("{:16}= {}", "ErrorMessages", size_of::()); println!("{:16}= {}", "ExprKind", size_of::()); @@ -60,9 +57,6 @@ pub fn print_mem_sizes() { ); println!("{:16}= {}", "InterpolateItem", size_of::()); println!("{:16}= {}", "JoinSide", size_of::()); - println!("{:16}= {}", "Lineage", size_of::()); - println!("{:16}= {}", "LineageColumn", size_of::()); - println!("{:16}= {}", "LineageInput", size_of::()); println!("{:16}= {}", "ModuleDef", size_of::()); println!("{:16}= {}", "pl::Expr", size_of::()); println!("{:16}= {}", "PrimitiveSet", size_of::()); diff --git a/prqlc/prqlc/src/ir/pl/stmt.rs b/prqlc/prqlc/src/ir/pl/stmt.rs index f128f1798170..60b27af49863 100644 --- a/prqlc/prqlc/src/ir/pl/stmt.rs +++ b/prqlc/prqlc/src/ir/pl/stmt.rs @@ -2,11 +2,10 @@ use enum_as_inner::EnumAsInner; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::pr::Ident; -use crate::pr::QueryDef; -use crate::pr::{Span, Ty}; +use crate::pr::{Ident, QueryDef, Ty}; +use crate::Span; -use super::expr::Expr; +use super::{Expr, FuncCall}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct Stmt { @@ -61,3 +60,16 @@ pub struct ImportDef { pub struct Annotation { pub expr: Box, } + +impl Annotation { + /// Utility to match function calls by name and unpack its arguments. + pub fn as_func_call(&self, name: &str) -> Option<&FuncCall> { + let call = self.expr.kind.as_func_call()?; + + let func_name = call.name.kind.as_ident()?; + if func_name.len() != 1 || func_name.name != name { + return None; + } + Some(call) + } +} diff --git a/prqlc/prqlc/src/ir/pl/utils.rs b/prqlc/prqlc/src/ir/pl/utils.rs index ad59fc6a4296..8038a32cd7a7 100644 --- a/prqlc/prqlc/src/ir/pl/utils.rs +++ b/prqlc/prqlc/src/ir/pl/utils.rs @@ -1,13 +1,6 @@ use super::{Expr, ExprKind, FuncCall}; use crate::pr::Ident; -pub fn maybe_binop(left: Option, op_name: &[&str], right: Option) -> Option { - match (left, right) { - (Some(left), Some(right)) => Some(new_binop(left, op_name, right)), - (left, right) => left.or(right), - } -} - pub fn new_binop(left: Expr, op_name: &[&str], right: Expr) -> Expr { Expr::new(ExprKind::FuncCall(FuncCall { name: Box::new(Expr::new(Ident::from_path(op_name.to_vec()))), diff --git a/prqlc/prqlc/src/lib.rs b/prqlc/prqlc/src/lib.rs index 40ead6683031..e2073c73fb52 100644 --- a/prqlc/prqlc/src/lib.rs +++ b/prqlc/prqlc/src/lib.rs @@ -518,8 +518,7 @@ pub mod internal { let root_module = semantic::resolve(pl).map_err(ErrorMessages::from)?; let (main, _) = root_module.find_main_rel(&[]).unwrap(); - let mut fc = - semantic::reporting::collect_frames(*main.clone().into_relation_var().unwrap()); + let mut fc = semantic::reporting::collect_frames(main.clone()); fc.ast = ast; Ok(fc) diff --git a/prqlc/prqlc/src/semantic/ast_expand.rs b/prqlc/prqlc/src/semantic/ast_expand.rs index 8394a41fdbba..a87ccc9e17f9 100644 --- a/prqlc/prqlc/src/semantic/ast_expand.rs +++ b/prqlc/prqlc/src/semantic/ast_expand.rs @@ -1,47 +1,65 @@ -use std::collections::HashMap; - use itertools::Itertools; -use prqlc_parser::error::WithErrorInfo; -use prqlc_parser::generic; use crate::ir::decl; use crate::ir::pl::{self, new_binop}; use crate::pr; use crate::semantic::{NS_THAT, NS_THIS}; use crate::{Error, Result}; +use prqlc_parser::generic; + +use super::NS_LOCAL; /// An AST pass that maps AST to PL. pub fn expand_expr(expr: pr::Expr) -> Result { let kind = match expr.kind { pr::ExprKind::Ident(v) => pl::ExprKind::Ident(pr::Ident::from_name(v)), - pr::ExprKind::Indirection { base, field } => { - let field_as_name = match field { - pr::IndirectionKind::Name(n) => n, - pr::IndirectionKind::Position(_) => Err(Error::new_simple( - "Positional indirection not supported yet", - ) - .with_span(expr.span))?, - pr::IndirectionKind::Star => "*".to_string(), - }; - - // convert lookups into ident - // (in the future, resolve will support proper lookup handling) - let base = expand_expr_box(base)?; - let base_ident = base.kind.into_ident().map_err(|_| { - Error::new_simple("lookup (the dot) is supported only on names.") - .with_span(expr.span) - })?; + pr::ExprKind::Indirection { + base, + field: pr::IndirectionKind::Name(field), + } => pl::ExprKind::Indirection { + base: expand_expr_box(base)?, + field: pl::IndirectionKind::Name(field), + }, + pr::ExprKind::Indirection { + base, + field: pr::IndirectionKind::Position(field), + } => pl::ExprKind::Indirection { + base: expand_expr_box(base)?, + field: pl::IndirectionKind::Position(field), + }, + pr::ExprKind::Indirection { + base, + field: pr::IndirectionKind::Star, + } => pl::ExprKind::All { + within: expand_expr_box(base)?, + except: Box::new(pl::Expr::new(pl::ExprKind::Tuple(vec![]))), + }, - let ident = base_ident + pr::Ident::from_name(field_as_name); - pl::ExprKind::Ident(ident) - } pr::ExprKind::Literal(v) => pl::ExprKind::Literal(v), pr::ExprKind::Pipeline(v) => { let mut e = desugar_pipeline(v)?; e.alias = expr.alias.or(e.alias); return Ok(e); } - pr::ExprKind::Tuple(v) => pl::ExprKind::Tuple(expand_exprs(v)?), + pr::ExprKind::Tuple(mut v) => { + // maybe extract last element for unpacking + let mut last_unpacking = None; + if v.last() + .and_then(|v| v.kind.as_range()) + .map_or(false, |x| x.start.is_none() && x.end.is_some()) + { + last_unpacking = Some(v.pop().unwrap().kind.into_range().unwrap().end.unwrap()); + } + + let mut fields = expand_exprs(v)?; + + if let Some(last) = last_unpacking { + let mut last = expand_expr(*last)?; + last.flatten = true; + fields.push(last); + } + pl::ExprKind::Tuple(fields) + } pr::ExprKind::Array(v) => pl::ExprKind::Array(expand_exprs(v)?), pr::ExprKind::Range(v) => expands_range(v)?, @@ -64,9 +82,6 @@ pub fn expand_expr(expr: pr::Expr) -> Result { body: expand_expr_box(v.body)?, params: expand_func_params(v.params)?, named_params: expand_func_params(v.named_params)?, - name_hint: None, - args: Vec::new(), - env: HashMap::new(), generic_type_params: v.generic_type_params, } .into(), @@ -102,7 +117,6 @@ pub fn expand_expr(expr: pr::Expr) -> Result { id: None, target_id: None, ty: None, - lineage: None, needs_window: false, flatten: false, }) @@ -320,7 +334,13 @@ fn restrict_exprs(exprs: Vec) -> Vec { fn restrict_expr_kind(value: pl::ExprKind) -> pr::ExprKind { match value { pl::ExprKind::Ident(v) => { + // HACK: remove the '_local' prefix + let skip_first = v.starts_with_part(NS_LOCAL); + let mut parts = v.into_iter(); + if skip_first { + parts.next(); + } let mut base = Box::new(pr::Expr::new(pr::ExprKind::Ident(parts.next().unwrap()))); for part in parts { let field = pr::IndirectionKind::Name(part); @@ -328,6 +348,13 @@ fn restrict_expr_kind(value: pl::ExprKind) -> pr::ExprKind { } base.kind } + pl::ExprKind::Indirection { base, field } => pr::ExprKind::Indirection { + base: restrict_expr_box(base), + field: match field { + pl::IndirectionKind::Name(name) => pr::IndirectionKind::Name(name), + pl::IndirectionKind::Position(pos) => pr::IndirectionKind::Position(pos), + }, + }, pl::ExprKind::Literal(v) => pr::ExprKind::Literal(v), pl::ExprKind::Tuple(v) => pr::ExprKind::Tuple(restrict_exprs(v)), pl::ExprKind::Array(v) => pr::ExprKind::Array(restrict_exprs(v)), @@ -340,27 +367,21 @@ fn restrict_expr_kind(value: pl::ExprKind) -> pr::ExprKind { .map(|(k, v)| (k, restrict_expr(v))) .collect(), }), - pl::ExprKind::Func(v) => { - let func = pr::ExprKind::Func( - pr::Func { - return_ty: v.return_ty, - body: restrict_expr_box(v.body), - params: restrict_func_params(v.params), - named_params: restrict_func_params(v.named_params), - generic_type_params: v.generic_type_params, - } - .into(), - ); - if v.args.is_empty() { - func - } else { - pr::ExprKind::FuncCall(pr::FuncCall { - name: Box::new(pr::Expr::new(func)), - args: restrict_exprs(v.args), - named_args: Default::default(), - }) + pl::ExprKind::Func(v) => pr::ExprKind::Func( + pr::Func { + return_ty: v.return_ty, + body: restrict_expr_box(v.body), + params: restrict_func_params(v.params), + named_params: restrict_func_params(v.named_params), + generic_type_params: v.generic_type_params, } - } + .into(), + ), + pl::ExprKind::FuncApplication(v) => pr::ExprKind::FuncCall(pr::FuncCall { + name: restrict_expr_box(v.func), + args: restrict_exprs(v.args), + named_args: Default::default(), + }), pl::ExprKind::SString(v) => { pr::ExprKind::SString(v.into_iter().map(|v| v.map(restrict_expr)).collect()) } @@ -496,35 +517,16 @@ fn restrict_decl(name: String, value: decl::Decl) -> Option { name, stmts: restrict_module(module).stmts, }), - decl::DeclKind::LayeredModules(mut stack) => { - let module = stack.pop()?; - pr::StmtKind::ModuleDef(pr::ModuleDef { - name, - stmts: restrict_module(module).stmts, - }) - } - decl::DeclKind::TableDecl(table_decl) => pr::StmtKind::VarDef(pr::VarDef { - kind: pr::VarDefKind::Let, - name: name.clone(), - value: Some(Box::new(match table_decl.expr { - decl::TableExpr::RelationVar(expr) => restrict_expr(*expr), - decl::TableExpr::LocalTable => { - pr::Expr::new(pr::ExprKind::Internal("local_table".into())) - } - decl::TableExpr::None => { - pr::Expr::new(pr::ExprKind::Internal("literal_tracker".to_string())) - } - decl::TableExpr::Param(id) => pr::Expr::new(pr::ExprKind::Param(id)), - })), - ty: table_decl.ty, - }), + decl::DeclKind::Variable(_) => new_internal_stmt(name, "_variable".into()), + decl::DeclKind::TupleField => new_internal_stmt(name, "_tuple_field".into()), + decl::DeclKind::Infer(_) => new_internal_stmt(name, "_infer".to_string()), + decl::DeclKind::Unresolved(_) => new_internal_stmt(name, "_unresolved".to_string()), - decl::DeclKind::InstanceOf(ident, _) => { - new_internal_stmt(name, format!("instance_of.{ident}")) - } - decl::DeclKind::Column(id) => new_internal_stmt(name, format!("column.{id}")), - decl::DeclKind::Infer(_) => new_internal_stmt(name, "infer".to_string()), + decl::DeclKind::GenericParam(arg) => pr::StmtKind::TypeDef(pr::TypeDef { + name, + value: arg.map(|a| a.0), + }), decl::DeclKind::Expr(mut expr) => pr::StmtKind::VarDef(pr::VarDef { kind: pr::VarDefKind::Let, diff --git a/prqlc/prqlc/src/semantic/eval.rs b/prqlc/prqlc/src/semantic/eval.rs deleted file mode 100644 index 531140c8f9fb..000000000000 --- a/prqlc/prqlc/src/semantic/eval.rs +++ /dev/null @@ -1,541 +0,0 @@ -use std::iter::zip; - -use itertools::Itertools; -use prqlc_parser::lexer::lr::Literal; - -use super::ast_expand; -use crate::ir::pl::{Expr, ExprKind, Func, FuncParam, Ident, PlFold}; -use crate::{Error, Result, Span, WithErrorInfo}; - -pub fn eval(expr: crate::pr::Expr) -> Result { - let expr = ast_expand::expand_expr(expr)?; - - Evaluator::new().fold_expr(expr) -} - -/// Converts an expression to a value -/// -/// Serves as a working draft of PRQL semantics definition. -struct Evaluator { - context: Option, -} - -impl Evaluator { - fn new() -> Self { - Evaluator { context: None } - } -} - -impl PlFold for Evaluator { - fn fold_expr(&mut self, expr: Expr) -> Result { - let mut expr = expr; - - expr.kind = match expr.kind { - // these are values already - ExprKind::Literal(l) => ExprKind::Literal(l), - - // these are values, iff their contents are values too - ExprKind::Array(_) | ExprKind::Tuple(_) => self.fold_expr_kind(expr.kind)?, - - // functions are values - ExprKind::Func(f) => ExprKind::Func(f), - - // ident are not values - ExprKind::Ident(ident) => { - // here we'd have to implement the whole name resolution, but for now, - // let's do something simple - - // this is very crude, but for simple cases, it's enough - let mut ident = ident; - let mut base = self.context.clone(); - loop { - let (first, remaining) = ident.pop_front(); - let res = lookup(base.as_ref(), &first).with_span(expr.span)?; - - if let Some(remaining) = remaining { - ident = remaining; - base = Some(res); - } else { - return Ok(res); - } - } - } - - // the beef happens here - ExprKind::FuncCall(func_call) => { - let func = self.fold_expr(*func_call.name)?; - let mut func = func.try_cast(|x| x.into_func(), Some("func call"), "function")?; - - func.args.extend(func_call.args); - - if func.args.len() < func.params.len() { - ExprKind::Func(func) - } else { - self.eval_function(*func, expr.span)? - } - } - - ExprKind::All { .. } - | ExprKind::TransformCall(_) - | ExprKind::SString(_) - | ExprKind::FString(_) - | ExprKind::Case(_) - | ExprKind::RqOperator { .. } - | ExprKind::Param(_) - | ExprKind::Internal(_) => { - return Err(Error::new_simple("not a value").with_span(expr.span)) - } - }; - Ok(expr) - } -} - -fn lookup(base: Option<&Expr>, name: &str) -> Result { - if let Some(base) = base { - if let ExprKind::Tuple(items) = &base.kind { - if let Some(item) = items.iter().find(|i| i.alias.as_deref() == Some(name)) { - return Ok(item.clone()); - } - } - } - if name == "std" { - return Ok(std_module()); - } - Err(Error::new_simple(format!( - "cannot find `{}` in {:?}", - name, base - ))) -} - -impl Evaluator { - fn eval_function(&mut self, func: Func, span: Option) -> Result { - let func_name = func.name_hint.unwrap().to_string(); - - // eval args - let closure = (func.params.iter()).find_position(|x| x.name == "closure"); - - let args = if let Some((closure_position, _)) = closure { - let mut args = Vec::new(); - - for (pos, arg) in func.args.into_iter().enumerate() { - if pos == closure_position { - // no evaluation - args.push(arg); - } else { - // eval - args.push(self.fold_expr(arg)?); - } - } - args - } else { - self.fold_exprs(func.args)? - }; - - // eval body - Ok(match func_name.as_str() { - "std.add" => { - let [l, r]: [_; 2] = args.try_into().unwrap(); - - let l = l.kind.into_literal().unwrap(); - let r = r.kind.into_literal().unwrap(); - - let res = match (l, r) { - (Literal::Integer(l), Literal::Integer(r)) => (l + r) as f64, - (Literal::Float(l), Literal::Integer(r)) => l + (r as f64), - (Literal::Integer(l), Literal::Float(r)) => (l as f64) + r, - (Literal::Float(l), Literal::Float(r)) => l + r, - - _ => return Err(Error::new_simple("bad arg types").with_span(span)), - }; - - ExprKind::Literal(Literal::Float(res)) - } - - "std.floor" => { - let [x]: [_; 1] = args.try_into().unwrap(); - - let res = match x.kind { - ExprKind::Literal(Literal::Integer(i)) => i, - ExprKind::Literal(Literal::Float(f)) => f.floor() as i64, - _ => return Err(Error::new_simple("bad arg types").with_span(x.span)), - }; - - ExprKind::Literal(Literal::Integer(res)) - } - - "std.neg" => { - let [x]: [_; 1] = args.try_into().unwrap(); - - match x.kind { - ExprKind::Literal(Literal::Integer(i)) => { - ExprKind::Literal(Literal::Integer(-i)) - } - ExprKind::Literal(Literal::Float(f)) => ExprKind::Literal(Literal::Float(-f)), - _ => return Err(Error::new_simple("bad arg types").with_span(x.span)), - } - } - - "std.select" => { - let [tuple_closure, relation]: [_; 2] = args.try_into().unwrap(); - - self.eval_for_each_row(relation, tuple_closure)?.kind - } - - "std.derive" => { - let [tuple_closure, relation]: [_; 2] = args.try_into().unwrap(); - - let new = self.eval_for_each_row(relation.clone(), tuple_closure)?; - - zip_relations(relation, new) - } - - "std.filter" => { - let [condition_closure, relation]: [_; 2] = args.try_into().unwrap(); - - let condition = self.eval_for_each_row(relation.clone(), condition_closure)?; - - let condition = condition.kind.into_array().unwrap(); - let relation = relation.kind.into_array().unwrap(); - - let mut res = Vec::new(); - for (cond, tuple) in zip(condition, relation) { - let f = cond.kind.into_literal().unwrap().into_boolean().unwrap(); - - if f { - res.push(tuple); - } - } - - ExprKind::Array(res) - } - - "std.aggregate" => { - let [tuple_closure, relation]: [_; 2] = args.try_into().unwrap(); - - let relation = rows_to_cols(relation)?; - let tuple = self.eval_within_context(tuple_closure, relation)?; - - ExprKind::Array(vec![tuple]) - } - - "std.window" => { - let [tuple_closure, relation]: [_; 2] = args.try_into().unwrap(); - let relation_size = relation.kind.as_array().unwrap().len(); - let relation = rows_to_cols(relation)?; - - let mut res = Vec::new(); - - const FRAME_ROWS: std::ops::Range = -1..1; - - for row_index in 0..relation_size { - let rel = windowed(relation.clone(), row_index, FRAME_ROWS, relation_size); - - let row_value = self.eval_within_context(tuple_closure.clone(), rel)?; - - res.push(row_value); - } - - ExprKind::Array(res) - } - - "std.columnar" => { - let [relation_closure, relation]: [_; 2] = args.try_into().unwrap(); - let relation = rows_to_cols(relation)?; - - let res = self.eval_within_context(relation_closure, relation)?; - - cols_to_rows(res)?.kind - } - - "std.sum" => { - let [array]: [_; 1] = args.try_into().unwrap(); - - let mut sum = 0.0; - for item in array.kind.into_array().unwrap() { - let lit = item.kind.into_literal().unwrap(); - match lit { - Literal::Integer(x) => sum += x as f64, - Literal::Float(x) => sum += x, - _ => panic!("bad type"), - } - } - - ExprKind::Literal(Literal::Float(sum)) - } - - "std.lag" => { - let [array]: [_; 1] = args.try_into().unwrap(); - - let mut array = array.try_cast(|x| x.into_array(), Some("lag"), "an array")?; - - if !array.is_empty() { - array.pop(); - array.insert(0, Expr::new(Literal::Null)); - } - - ExprKind::Array(array) - } - - _ => { - return Err( - Error::new_simple(format!("unknown function {func_name}")).with_span(span) - ) - } - }) - } - - fn eval_for_each_row(&mut self, relation: Expr, closure: Expr) -> Result { - // save relation from outer calls - let prev_relation = self.context.take(); - - let relation_rows = relation.try_cast(|x| x.into_array(), None, "an array")?; - - // for every item in relation array, evaluate args - let mut output_array = Vec::new(); - for relation_row in relation_rows { - let row_value = self.eval_within_context(closure.clone(), relation_row)?; - output_array.push(row_value); - } - - // restore relation for outer calls - self.context = prev_relation; - - Ok(Expr::new(ExprKind::Array(output_array))) - } - - fn eval_within_context(&mut self, expr: Expr, context: Expr) -> Result { - // save relation from outer calls - let prev_relation = self.context.take(); - - self.context = Some(context); - let res = self.fold_expr(expr)?; - - // restore relation for outer calls - self.context = prev_relation; - - Ok(res) - } -} - -fn windowed( - mut relation: Expr, - row_index: usize, - frame: std::ops::Range, - relation_size: usize, -) -> Expr { - let row = row_index as i64; - let end = (row + frame.end).clamp(0, relation_size as i64) as usize; - let start = (row + frame.start).clamp(0, end as i64) as usize; - - for field in relation.kind.as_tuple_mut().unwrap() { - let column = field.kind.as_array_mut().unwrap(); - - column.drain(end..); - column.drain(0..start); - } - relation -} - -/// Converts `[{a = 1, b = false}, {a = 2, b = true}]` -/// into `{a = [1, 2], b = [false, true]}` -fn rows_to_cols(expr: Expr) -> Result { - let relation_rows = expr.try_cast(|x| x.into_array(), None, "an array")?; - - // prepare output - let mut arg_tuple = Vec::new(); - for field in relation_rows.first().unwrap().kind.as_tuple().unwrap() { - arg_tuple.push(Expr { - alias: field.alias.clone(), - ..Expr::new(ExprKind::Array(Vec::new())) - }); - } - - // place entries - for relation_row in relation_rows { - let fields = relation_row.try_cast(|x| x.into_tuple(), None, "a tuple")?; - - for (index, field) in fields.into_iter().enumerate() { - arg_tuple[index].kind.as_array_mut().unwrap().push(field); - } - } - Ok(Expr::new(ExprKind::Tuple(arg_tuple))) -} - -/// Converts `{a = [1, 2], b = [false, true]}` -/// into `[{a = 1, b = false}, {a = 2, b = true}]` -fn cols_to_rows(expr: Expr) -> Result { - let fields = expr.try_cast(|x| x.into_tuple(), None, "an tuple")?; - - let len = fields.first().unwrap().kind.as_array().unwrap().len(); - - let mut rows = Vec::new(); - for index in 0..len { - let mut row = Vec::new(); - for field in &fields { - row.push(Expr { - alias: field.alias.clone(), - ..field.kind.as_array().unwrap()[index].clone() - }) - } - - rows.push(Expr::new(ExprKind::Tuple(row))); - } - - Ok(Expr::new(ExprKind::Array(rows))) -} - -fn std_module() -> Expr { - Expr::new(ExprKind::Tuple( - [ - new_func("floor", &["x"]), - new_func("add", &["x", "y"]), - new_func("neg", &["x"]), - new_func("select", &["closure", "relation"]), - new_func("derive", &["closure", "relation"]), - new_func("filter", &["closure", "relation"]), - new_func("aggregate", &["closure", "relation"]), - new_func("window", &["closure", "relation"]), - new_func("columnar", &["closure", "relation"]), - new_func("sum", &["x"]), - new_func("lag", &["x"]), - ] - .to_vec(), - )) -} - -fn new_func(name: &str, params: &[&str]) -> Expr { - let params = params - .iter() - .map(|name| FuncParam { - name: name.to_string(), - default_value: None, - ty: None, - }) - .collect(); - - let kind = ExprKind::Func(Box::new(Func { - name_hint: Some(Ident { - path: vec!["std".to_string()], - name: name.to_string(), - }), - - // these don't matter - return_ty: Default::default(), - body: Box::new(Expr::new(Literal::Null)), - params, - named_params: Default::default(), - args: Default::default(), - env: Default::default(), - generic_type_params: Default::default(), - })); - Expr { - alias: Some(name.to_string()), - ..Expr::new(kind) - } -} - -fn zip_relations(l: Expr, r: Expr) -> ExprKind { - let l = l.kind.into_array().unwrap(); - let r = r.kind.into_array().unwrap(); - - let mut res = Vec::new(); - for (l, r) in zip(l, r) { - let l_fields = l.kind.into_tuple().unwrap(); - let r_fields = r.kind.into_tuple().unwrap(); - - res.push(Expr::new(ExprKind::Tuple([l_fields, r_fields].concat()))); - } - - ExprKind::Array(res) -} - -#[cfg(test)] -mod test { - - use insta::assert_snapshot; - - use super::*; - use crate::semantic::write_pl; - - #[track_caller] - fn eval(source: &str) -> Result { - let stmts = crate::prql_to_pl(source).unwrap().stmts.into_iter(); - let stmt = stmts.exactly_one().unwrap(); - let expr = stmt.kind.into_var_def().unwrap().value.unwrap(); - - let value = super::eval(*expr)?; - - Ok(write_pl(value)) - } - - #[test] - fn basic() { - assert_snapshot!(eval(r" - [std.floor (3.5 + 2.9) + 3, 3] - ").unwrap(), - @"[9, 3]" - ); - } - - #[test] - fn tuples() { - assert_snapshot!(eval(r" - {{a_a = 4, a_b = false}, b = 2.1 + 3.6, c = [false, true, false]} - ").unwrap(), - @"{{a_a = 4, a_b = false}, b = 5.7, c = [false, true, false]}" - ); - } - - #[test] - fn pipelines() { - assert_snapshot!(eval(r" - (4.5 | std.floor | std.neg) - ").unwrap(), - @"-4" - ); - } - - #[test] - fn transforms() { - assert_snapshot!(eval(r" - [ - { b = 4, c = false }, - { b = 5, c = true }, - { b = 12, c = true }, - ] - std.select {c, b + 2} - std.derive {d = 42} - std.filter c - ").unwrap(), - @"[{c = true, 7, d = 42}, {c = true, 14, d = 42}]" - ); - } - - #[test] - fn window() { - assert_snapshot!(eval(r" - [ - { b = 4, c = false }, - { b = 5, c = true }, - { b = 12, c = true }, - ] - std.window {d = std.sum b} - ").unwrap(), - @"[{d = 4}, {d = 9}, {d = 17}]" - ); - } - - #[test] - fn columnar() { - assert_snapshot!(eval(r" - [ - { b = 4, c = false }, - { b = 5, c = true }, - { b = 12, c = true }, - ] - std.columnar {g = std.lag b} - ").unwrap(), - @"[{g = null}, {g = 4}, {g = 5}]" - ); - } -} diff --git a/prqlc/prqlc/src/semantic/lowering.rs b/prqlc/prqlc/src/semantic/lowering.rs index fc089100927e..bad3dfae74c9 100644 --- a/prqlc/prqlc/src/semantic/lowering.rs +++ b/prqlc/prqlc/src/semantic/lowering.rs @@ -1,6 +1,9 @@ +mod flatten; +mod inline; +mod special_functions; + use std::collections::hash_map::RandomState; use std::collections::{HashMap, HashSet}; -use std::iter::zip; use enum_as_inner::EnumAsInner; use itertools::Itertools; @@ -8,19 +11,17 @@ use prqlc_parser::generic::{InterpolateItem, Range, SwitchCase}; use prqlc_parser::lexer::lr::Literal; use semver::{Prerelease, Version}; -use crate::compiler_version; -use crate::ir::decl::{self, DeclKind, Module, RootModule, TableExpr}; +use crate::ir::decl::{DeclKind, Module, RootModule}; use crate::ir::generic::{ColumnSort, WindowFrame}; -use crate::ir::pl::TableExternRef::LocalTable; -use crate::ir::pl::{self, Ident, Lineage, LineageColumn, PlFold, QueryDef}; -use crate::ir::rq::{ - self, CId, RelationColumn, RelationLiteral, RelationalQuery, TId, TableDecl, Transform, -}; -use crate::pr::TyTupleField; +use crate::ir::pl::{self, FuncApplication, Ident, PlFold, QueryDef}; +use crate::ir::rq::{self, CId, RelationColumn, RelationalQuery, TId, TableDecl, Transform}; +use crate::pr::{Ty, TyKind, TyTupleField}; use crate::semantic::write_pl; use crate::utils::{toposort, IdGenerator}; use crate::{Error, Reason, Result, Span, WithErrorInfo}; +use super::{NS_LOCAL, NS_THAT, NS_THIS}; + /// Convert a resolved expression at path `main_path` relative to `root_mod` /// into RQ and make sure that: /// - transforms are not nested, @@ -53,7 +54,7 @@ pub fn lower_to_ir( validate_query_def(&def)?; // find all tables in the root module - let tables = TableExtractor::extract(&root_mod.module); + let tables = TableExtractor::extract(&root_mod); // prune and toposort let tables = toposort_tables(tables, &main_ident); @@ -81,52 +82,9 @@ pub fn lower_to_ir( Ok((query, l.root_mod)) } -fn extern_ref_to_relation( - mut columns: Vec, - fq_ident: &Ident, - database_module_path: &[String], -) -> Result<(rq::Relation, Option), Error> { - let extern_name = if fq_ident.starts_with_path(database_module_path) { - let relative_to_database: Vec<&String> = - fq_ident.iter().skip(database_module_path.len()).collect(); - if relative_to_database.is_empty() { - None - } else { - Some(Ident::from_path(relative_to_database)) - } - } else { - None - }; - - let Some(extern_name) = extern_name else { - let database_module = Ident::from_path(database_module_path.to_vec()); - return Err(Error::new_simple("this table is not in the current database") - .push_hint(format!("If this is a table in the current database, move its declaration into module {database_module}"))); - }; - - // put wildcards last - columns.sort_by_key(|a| matches!(a, TyTupleField::Wildcard(_))); - - let relation = rq::Relation { - kind: rq::RelationKind::ExternRef(LocalTable(extern_name)), - columns: tuple_fields_to_relation_columns(columns), - }; - Ok((relation, None)) -} - -fn tuple_fields_to_relation_columns(columns: Vec) -> Vec { - columns - .into_iter() - .map(|field| match field { - TyTupleField::Single(name, _) => RelationColumn::Single(name), - TyTupleField::Wildcard(_) => RelationColumn::Wildcard, - }) - .collect_vec() -} - fn validate_query_def(query_def: &QueryDef) -> Result<()> { if let Some(requirement) = &query_def.version { - let current_version = compiler_version(); + let current_version = crate::compiler_version(); // We need to remove the pre-release part of the version, because // otherwise those will fail the match. @@ -149,6 +107,7 @@ fn validate_query_def(query_def: &QueryDef) -> Result<()> { struct Lowerer { cid: IdGenerator, tid: IdGenerator, + id: IdGenerator, root_mod: RootModule, database_module_path: Vec, @@ -159,24 +118,28 @@ struct Lowerer { /// mapping from [Ident] of [crate::pr::TableDef] into [TId]s table_mapping: HashMap, - // current window for any new column defs - window: Option, + /// A buffer to be added into query tables + table_buffer: Vec, + // --- Fields after here make sense only in context of "current pipeline". + // (they should maybe be moved into a separate struct to make this clear) /// A buffer to be added into current pipeline pipeline: Vec, - /// A buffer to be added into query tables - table_buffer: Vec, + /// current window for any new column defs + window: Option<(Vec, rq::Window)>, + + local_this_id: Option, + local_that_id: Option, } #[derive(Clone, EnumAsInner, Debug)] enum LoweredTarget { /// Lowered node was a computed expression. - Compute(CId), + Column(CId), - /// Lowered node was a pipeline input. - /// Contains mapping from column names to CIds, along with order in frame. - Input(HashMap), + /// Lowered node was a tuple with following columns. + Relation(Vec), } impl Lowerer { @@ -187,6 +150,13 @@ impl Lowerer { cid: IdGenerator::new(), tid: IdGenerator::new(), + id: { + // HACK: create id generator start starts at really large numbers + // because we need to invent new ids after the resolver has finished. + let mut gen = IdGenerator::new(); + gen.skip(100000000); + gen + }, node_mapping: HashMap::new(), table_mapping: HashMap::new(), @@ -194,25 +164,24 @@ impl Lowerer { window: None, pipeline: Vec::new(), table_buffer: Vec::new(), + + local_this_id: None, + local_that_id: None, } } - fn lower_table_decl(&mut self, table: decl::TableDecl, fq_ident: Ident) -> Result<()> { - let decl::TableDecl { ty, expr } = table; + fn lower_table_decl(&mut self, expr: pl::Expr, fq_ident: Ident) -> Result<()> { + let columns = expr.ty.clone().unwrap().into_relation().unwrap(); - // TODO: can this panic? - let columns = ty.unwrap().into_relation().unwrap(); + let (relation, name) = if let pl::ExprKind::Param(_) = &expr.kind { + self.extern_ref_to_relation(columns, &fq_ident)? + } else { + let expr = inline::Inliner::run(&self.root_mod, expr); + let expr = flatten::Flattener::run(expr)?; - let (relation, name) = match expr { - TableExpr::RelationVar(expr) => { - // a CTE - (self.lower_relation(*expr)?, Some(fq_ident.name.clone())) - } - TableExpr::LocalTable => { - extern_ref_to_relation(columns, &fq_ident, &self.database_module_path)? - } - TableExpr::Param(_) => unreachable!(), - TableExpr::None => return Ok(()), + log::debug!("lowering: {:#?}", expr); + + (self.lower_relation(expr)?, Some(fq_ident.name.clone())) }; let id = *self @@ -220,173 +189,125 @@ impl Lowerer { .entry(fq_ident) .or_insert_with(|| self.tid.gen()); - log::debug!("lowering table {name:?}, columns = {:?}", relation.columns); + log::debug!("lowered table {name:?}, columns = {:?}", relation.columns); let table = TableDecl { id, name, relation }; self.table_buffer.push(table); Ok(()) } - /// Lower an expression into a instance of a table in the query + fn lower_relation(&mut self, mut expr: pl::Expr) -> Result { + let id = self.get_id(&mut expr); + let expr = expr; + + // look at the type of the expr and determine what will be the columns of the output relation + let relation_fields = expr.ty.as_ref().and_then(|t| t.as_relation()).unwrap(); + let columns = self.ty_tuple_to_relation_columns(relation_fields.clone(), None)?; + + // take out the pipeline that we might have been previously working on + let prev_pipeline = self.pipeline.drain(..).collect_vec(); + + self.lower_relational_expr(expr, None)?; + + // retrieve resulting pipeline and replace the previous one + let mut transforms = self.pipeline.drain(..).collect_vec(); + self.pipeline = prev_pipeline; + + // push a select to the end of the pipeline + transforms.push(Transform::Select( + self.flatten_tuple_fields_into_cids(&[id])?, + )); + Ok(rq::Relation { + kind: rq::RelationKind::Pipeline(transforms), + columns, + }) + } + + /// Lower an expression into a new instance of a table in the query fn lower_table_ref(&mut self, expr: pl::Expr) -> Result { - let mut expr = expr; - if expr.lineage.is_none() { - // make sure that type of this expr has been inferred to be a table - expr.lineage = Some(Lineage::default()); - } + let id = expr.id.unwrap(); - Ok(match expr.kind { + // find the tid (table id) of the table that we will create a new instance of + let tid = match expr.kind { pl::ExprKind::Ident(fq_table_name) => { - // ident that refer to table: create an instance of the table - let id = expr.id.unwrap(); - let tid = *self - .table_mapping - .get(&fq_table_name) - .ok_or_else(|| Error::new_bug(4474))?; + // ident that refers to table: lookup the existing table by name + // We know that table exists, because it has been previously extracted + // and lowered in topological order (if it hasn't, that would be a bug). log::debug!("lowering an instance of table {fq_table_name} (id={id})..."); - let input_name = expr - .lineage - .as_ref() - .and_then(|f| f.inputs.first()) - .map(|i| i.name.clone()); - let name = input_name.or(Some(fq_table_name.name)); - - self.create_a_table_instance(id, name, tid) + self.table_mapping.get(&fq_table_name).cloned().unwrap() } pl::ExprKind::TransformCall(_) => { - // pipeline that has to be pulled out into a table - let id = expr.id.unwrap(); - - // create a new table - let tid = self.tid.gen(); + // this function is requesting a table new table instance, but we got a pipeline + // -> we need to pull the pipeline out into a standalone table + // lower the relation let relation = self.lower_relation(expr)?; - let last_transform = &relation.kind.as_pipeline().unwrap().last().unwrap(); - let cids = last_transform.as_select().unwrap().clone(); - log::debug!("lowering inline table, columns = {:?}", relation.columns); - self.table_buffer.push(TableDecl { - id: tid, - name: None, - relation, - }); - - // return an instance of this new table - let table_ref = self.create_a_table_instance(id, None, tid); - - let redirects = zip(cids, table_ref.columns.iter().map(|(_, c)| *c)).collect(); - self.redirect_mappings(redirects); - table_ref + // define the relation as a new table + self.create_table(relation) } pl::ExprKind::SString(items) => { - let id = expr.id.unwrap(); - - // create a new table - let tid = self.tid.gen(); - // pull columns from the table decl - let frame = expr.lineage.as_ref().unwrap(); - let input = frame.inputs.first().unwrap(); - - let table_decl = self.root_mod.module.get(&input.table).unwrap(); - let table_decl = table_decl.kind.as_table_decl().unwrap(); - let ty = table_decl.ty.as_ref(); - // TODO: can this panic? - let columns = ty.unwrap().as_relation().unwrap().clone(); - - log::debug!("lowering sstring table, columns = {columns:?}"); // lower the expr let items = self.lower_interpolations(items)?; + + let relation_fields = expr.ty.unwrap().into_relation().unwrap(); + let columns = self.ty_tuple_to_relation_columns(relation_fields, None)?; let relation = rq::Relation { kind: rq::RelationKind::SString(items), - columns: tuple_fields_to_relation_columns(columns), + columns, }; - self.table_buffer.push(TableDecl { - id: tid, - name: None, - relation, - }); - - // return an instance of this new table - self.create_a_table_instance(id, None, tid) + // define the relation as a new table + self.create_table(relation) } pl::ExprKind::RqOperator { name, args } => { - let id = expr.id.unwrap(); - - // create a new table - let tid = self.tid.gen(); - - // pull columns from the table decl - let frame = expr.lineage.as_ref().unwrap(); - let input = frame.inputs.first().unwrap(); - - let table_decl = self.root_mod.module.get(&input.table).unwrap(); - let table_decl = table_decl.kind.as_table_decl().unwrap(); - let ty = table_decl.ty.as_ref(); - // TODO: can this panic? - let columns = ty.unwrap().as_relation().unwrap().clone(); - - log::debug!("lowering function table, columns = {columns:?}"); - // lower the expr let args = args.into_iter().map(|a| self.lower_expr(a)).try_collect()?; + + let relation_fields = expr.ty.unwrap().into_relation().unwrap(); + let columns = self.ty_tuple_to_relation_columns(relation_fields, None)?; let relation = rq::Relation { kind: rq::RelationKind::BuiltInFunction { name, args }, - columns: tuple_fields_to_relation_columns(columns), + columns, }; - self.table_buffer.push(TableDecl { - id: tid, - name: None, - relation, - }); - - // return an instance of this new table - self.create_a_table_instance(id, None, tid) + self.create_table(relation) } - pl::ExprKind::Array(elements) => { - let id = expr.id.unwrap(); - - // create a new table - let tid = self.tid.gen(); - + pl::ExprKind::Array(items) => { // pull columns from the table decl - let frame = expr.lineage.as_ref().unwrap(); - let columns = (frame.columns.iter()) - .map(|c| { - RelationColumn::Single( - c.as_single().unwrap().0.as_ref().map(|i| i.name.clone()), - ) - }) - .collect_vec(); - let lit = RelationLiteral { + let relation_fields = expr.ty.unwrap().into_relation().unwrap(); + let columns = self.ty_tuple_to_relation_columns(relation_fields, None)?; + + let lit = rq::RelationLiteral { columns: columns .iter() - .map(|c| c.as_single().unwrap().clone().unwrap()) + .map(|c| c.as_single().cloned().unwrap().unwrap_or_else(String::new)) .collect_vec(), - rows: elements + rows: items .into_iter() - .map(|row| { - row.kind - .into_tuple() - .unwrap() + .map(|row| match row.kind { + pl::ExprKind::Tuple(fields) => fields .into_iter() - .map(|element| { - element.try_cast( - |x| x.into_literal(), - Some("relation literal"), - "literals", + .map(|element| match element.kind { + pl::ExprKind::Literal(lit) => Ok(lit), + _ => Err(Error::new_simple( + "relation literals currently support only literals", ) + .with_span(element.span)), }) - .try_collect() + .try_collect(), + _ => Err(Error::new_simple( + "relation literals currently support only plain tuples", + ) + .with_span(row.span)), }) .try_collect()?, }; @@ -397,14 +318,8 @@ impl Lowerer { columns, }; - self.table_buffer.push(TableDecl { - id: tid, - name: None, - relation, - }); - - // return an instance of this new table - self.create_a_table_instance(id, None, tid) + // create a new table + self.create_table(relation) } _ => { @@ -416,34 +331,23 @@ impl Lowerer { .push_hint("are you missing `from` statement?") .with_span(expr.span)) } - }) + }; + Ok(self.create_table_instance(id, tid)) } - fn redirect_mappings(&mut self, redirects: HashMap) { - for target in self.node_mapping.values_mut() { - match target { - LoweredTarget::Compute(cid) => { - if let Some(new) = redirects.get(cid) { - *cid = *new; - } - } - LoweredTarget::Input(mapping) => { - for (cid, _) in mapping.values_mut() { - if let Some(new) = redirects.get(cid) { - *cid = *new; - } - } - } - } - } + /// Declare a new table as the supplied relation. + /// Generates and returns the new table id. + fn create_table(&mut self, relation: rq::Relation) -> TId { + let tid = self.tid.gen(); + self.table_buffer.push(TableDecl { + id: tid, + name: None, + relation, + }); + tid } - fn create_a_table_instance( - &mut self, - id: usize, - name: Option, - tid: TId, - ) -> rq::TableRef { + fn create_table_instance(&mut self, id: usize, tid: TId) -> rq::TableRef { // create instance columns from table columns let table = self.table_buffer.iter().find(|t| t.id == tid).unwrap(); @@ -455,124 +359,251 @@ impl Lowerer { log::debug!("... columns = {:?}", columns); - let input_cids: HashMap<_, _> = columns - .iter() - .cloned() - .enumerate() - .map(|(index, (col, cid))| (col, (cid, index))) - .collect(); + let mut rel_columns = Vec::new(); + for (_rel_col, cid) in &columns { + let id = self.id.gen(); + self.node_mapping.insert(id, LoweredTarget::Column(*cid)); + + rel_columns.push(id); + } self.node_mapping - .insert(id, LoweredTarget::Input(input_cids)); + .insert(id, LoweredTarget::Relation(rel_columns)); + rq::TableRef { source: tid, - name, + name: None, columns, } } - fn lower_relation(&mut self, expr: pl::Expr) -> Result { - let span = expr.span; - let lineage = expr.lineage.clone(); - let prev_pipeline = self.pipeline.drain(..).collect_vec(); - - self.lower_pipeline(expr, None)?; + fn extern_ref_to_relation( + &self, + ty_tuple_fields: Vec, + fq_ident: &Ident, + ) -> Result<(rq::Relation, Option), Error> { + let extern_name = if fq_ident.starts_with_path(&self.database_module_path) { + let relative_to_database: Vec<&String> = fq_ident + .iter() + .skip(self.database_module_path.len()) + .collect(); + if relative_to_database.is_empty() { + None + } else { + Some(Ident::from_path(relative_to_database)) + } + } else { + None + }; - let mut transforms = self.pipeline.drain(..).collect_vec(); - let columns = self.push_select(lineage, &mut transforms).with_span(span)?; + let Some(extern_name) = extern_name else { + let database_module = Ident::from_path(self.database_module_path.clone()); + return Err(Error::new_simple("this table is not in the current database") + .push_hint(format!("If this is a table in the current database, move its declaration into module {database_module}"))); + }; - self.pipeline = prev_pipeline; + // put unpack last + let mut ty_tuple_fields = ty_tuple_fields; + ty_tuple_fields.sort_by_key(|a| matches!(a, TyTupleField::Unpack(_))); let relation = rq::Relation { - kind: rq::RelationKind::Pipeline(transforms), - columns, + kind: rq::RelationKind::ExternRef(pl::TableExternRef::LocalTable(extern_name)), + columns: self.ty_tuple_to_relation_columns(ty_tuple_fields, None)?, }; - Ok(relation) + Ok((relation, None)) } - // Result is stored in self.pipeline - fn lower_pipeline(&mut self, ast: pl::Expr, closure_param: Option) -> Result<()> { - let transform_call = match ast.kind { - pl::ExprKind::TransformCall(transform) => transform, - pl::ExprKind::Func(closure) => { - let param = closure.params.first(); + fn ty_tuple_to_relation_columns( + &self, + fields: Vec, + prefix: Option, + ) -> Result> { + let mut new_fields = Vec::with_capacity(fields.len()); + + for field in fields { + match field { + TyTupleField::Single(mut name, ty) => { + if let Some(p) = &prefix { + if let Some(n) = &mut name { + *n = format!("{p}.{n}"); + } else { + name = Some(p.clone()); + } + } + + if ty.as_ref().map_or(false, |t| t.kind.is_tuple()) { + // flatten tuples + let inner = ty.unwrap().kind.into_tuple().unwrap(); + new_fields.extend(self.ty_tuple_to_relation_columns(inner, name)?); + } else { + // base case: + new_fields.push(RelationColumn::Single(name)); + } + } + TyTupleField::Unpack(Some(ty)) => { + let TyKind::Ident(fq_ident) = ty.kind else { + return Err(Error::new_assert( + "unpack should contain only ident of a generic, probably", + )); + }; + let decl = self.root_mod.module.get(&fq_ident).unwrap(); + let DeclKind::GenericParam(inferred_ty) = &decl.kind else { + return Err(Error::new_assert( + "unpack should contain only ident of a generic, probably", + )); + }; + + let Some((ty, _)) = inferred_ty else { + // no info about the type + new_fields.push(RelationColumn::Wildcard); + continue; + }; + + let TyKind::Tuple(ty_fields) = &ty.kind else { + return Err(Error::new_assert("unpack can only contain a tuple type")); + }; + + for field in ty_fields { + let (name, _ty) = field.as_single().unwrap(); // generic cannot contain unpacks, right? + new_fields.push(RelationColumn::Single(name.clone())); + } + + // we are not sure about this type (because it is still a generic) + // so we must append "all other unmentioned columns" + new_fields.push(RelationColumn::Wildcard); + } + TyTupleField::Unpack(None) => todo!("make Unpack contain a non Option-al Ty"), + } + } + Ok(new_fields) + } + + /// Lower a relational expression (or a function that returns a relational expression) to a pipeline. + /// + /// **Result is stored in self.pipeline** + fn lower_relational_expr(&mut self, ast: pl::Expr, closure_param: Option) -> Result<()> { + // find the actual transform that we want to compile to relational pipeline + // this is non trivial, because sometimes the transforms will be wrapped into + // functions that are still waiting for arguments + // for example: this would happen when lowering loop's pipeline + match ast.kind { + // base case + pl::ExprKind::TransformCall(transform) => { + let tuple_fields = self.lower_transform_call(transform, closure_param, ast.span)?; + + self.node_mapping + .insert(ast.id.unwrap(), LoweredTarget::Relation(tuple_fields)); + } + + // actually operate on func's body + pl::ExprKind::Func(func) => { + let param = func.params.first(); let param = param.and_then(|p| p.name.parse::().ok()); - return self.lower_pipeline(*closure.body, param); + self.lower_relational_expr(*func.body, param)?; } + + // this relational expr is not a transform _ => { if let Some(target) = ast.target_id { if Some(target) == closure_param { - // ast is a closure param, so we can skip pushing From + // ast is a closure param, so don't need to push From return Ok(()); } } let table_ref = self.lower_table_ref(ast)?; self.pipeline.push(Transform::From(table_ref)); - return Ok(()); } }; + Ok(()) + } + /// **Result is stored in self.pipeline** + fn lower_transform_call( + &mut self, + transform_call: pl::TransformCall, + closure_param: Option, + span: Option, + ) -> Result> { // lower input table - self.lower_pipeline(*transform_call.input, closure_param)?; + let input_id = transform_call.input.id.unwrap(); + self.lower_relational_expr(*transform_call.input, closure_param)?; // ... and continues with transforms created in this function + self.local_this_id = Some(input_id); + // prepare window + let (partition_ids, partition) = if let Some(partition) = transform_call.partition { + let ids = self.lower_and_flatten_tuple(*partition, false)?; + let cids = self.flatten_tuple_fields_into_cids(&ids)?; + (ids, cids) + } else { + (vec![], vec![]) + }; let window = rq::Window { frame: WindowFrame { kind: transform_call.frame.kind, range: self.lower_range(transform_call.frame.range)?, }, - partition: if let Some(partition) = transform_call.partition { - self.declare_as_columns(*partition, false)? - } else { - vec![] - }, + partition, sort: self.lower_sorts(transform_call.sort)?, }; - self.window = Some(window); + self.window = Some((partition_ids, window)); - match *transform_call.kind { + // main thing + let new_fields: Option> = match *transform_call.kind { pl::TransformKind::Derive { assigns, .. } => { - self.declare_as_columns(*assigns, false)?; + let ids = self.lower_and_flatten_tuple(*assigns, false)?; + Some([vec![input_id], ids].concat()) } pl::TransformKind::Select { assigns, .. } => { - let cids = self.declare_as_columns(*assigns, false)?; - self.pipeline.push(Transform::Select(cids)); + let ids = self.lower_and_flatten_tuple(*assigns, false)?; + Some(ids) } pl::TransformKind::Filter { filter, .. } => { let filter = self.lower_expr(*filter)?; self.pipeline.push(Transform::Filter(filter)); + + None } pl::TransformKind::Aggregate { assigns, .. } => { - let window = self.window.take(); + let (partition_ids, window) = self.window.take().unwrap(); - let compute = self.declare_as_columns(*assigns, true)?; + let ids = self.lower_and_flatten_tuple(*assigns, true)?; - let partition = window.unwrap().partition; - self.pipeline - .push(Transform::Aggregate { partition, compute }); + self.pipeline.push(Transform::Aggregate { + partition: window.partition, + compute: self.flatten_tuple_fields_into_cids(&ids)?, + }); + + Some([partition_ids, ids].concat()) } pl::TransformKind::Sort { by, .. } => { let sorts = self.lower_sorts(by)?; self.pipeline.push(Transform::Sort(sorts)); + + None } pl::TransformKind::Take { range, .. } => { - let window = self.window.take().unwrap_or_default(); + let (_, window) = self.window.take().unwrap_or_default(); let range = self.lower_range(range)?; - validate_take_range(&range, ast.span)?; + validate_take_range(&range, span)?; self.pipeline.push(Transform::Take(rq::Take { range, partition: window.partition, sort: window.sort, })); + + None } pl::TransformKind::Join { side, with, filter, .. } => { + let with_id = with.id.unwrap(); let with = self.lower_table_ref(*with)?; + self.local_that_id = Some(with_id); let transform = Transform::Join { side, @@ -580,11 +611,15 @@ impl Lowerer { filter: self.lower_expr(*filter)?, }; self.pipeline.push(transform); + + Some(vec![input_id, with_id]) } pl::TransformKind::Append(bottom) => { let bottom = self.lower_table_ref(*bottom)?; self.pipeline.push(Transform::Append(bottom)); + + todo!() } pl::TransformKind::Loop(pipeline) => { let relation = self.lower_relation(*pipeline)?; @@ -594,16 +629,24 @@ impl Lowerer { pipeline.pop(); self.pipeline.push(Transform::Loop(pipeline)); + + todo!() } pl::TransformKind::Group { .. } | pl::TransformKind::Window { .. } => unreachable!( "transform `{}` cannot be lowered.", (*transform_call.kind).as_ref() ), - } + }; self.window = None; - // result is stored in self.pipeline - Ok(()) + if let Some(new_fields) = new_fields { + Ok(new_fields) + } else { + let input_target = self.node_mapping.get(&input_id).unwrap(); + Ok(input_target.as_relation().unwrap().clone()) + } + + // resulting transforms are stored in self.pipeline } fn lower_range(&mut self, range: Range>) -> Result> { @@ -616,263 +659,154 @@ impl Lowerer { fn lower_sorts(&mut self, by: Vec>>) -> Result>> { by.into_iter() .map(|ColumnSort { column, direction }| { - let column = self.declare_as_column(*column, false)?; + let id = column.id.unwrap(); + self.ensure_lowered(*column, false)?; + let column = *self.node_mapping.get(&id).unwrap().as_column().unwrap(); Ok(ColumnSort { direction, column }) }) .try_collect() } - /// Append a Select of final table columns derived from frame - fn push_select( + /// Lowers an expression node. + /// If expr is a tuple, this tuple will flattened into a column of a relations, arbitrarily deep. + /// For example: + /// expr={a = 1, b = {c = 2, d = {e = 3}}, f = 4} + /// ... will be converted into: + /// ids=[a, b, f], cids=[a, b.c, b.d.e, f] + fn lower_and_flatten_tuple( &mut self, - lineage: Option, - transforms: &mut Vec, - ) -> Result> { - let lineage = lineage.unwrap_or_default(); - - log::debug!("push_select of a frame: {:?}", lineage); + exprs: pl::Expr, + is_aggregation: bool, + ) -> Result> { + if exprs.ty.as_ref().unwrap().kind.is_tuple() { + let id = exprs.id.unwrap(); + self.ensure_lowered(exprs, is_aggregation)?; - let mut columns = Vec::new(); + let ids = self.node_mapping.get(&id).unwrap().as_relation().unwrap(); + Ok(ids.clone()) + } else { + todo!() + } + } - // normal columns - for col in &lineage.columns { - match col { - LineageColumn::Single { - name, - target_id, - target_name, - } => { - let cid = self.lookup_cid(*target_id, target_name.as_ref())?; + fn flatten_tuple_fields_into_cids(&self, ids: &[usize]) -> Result> { + let mut cids = Vec::new(); + let mut ids_rev = ids.to_vec(); + ids_rev.reverse(); - let name = name.as_ref().map(|i| i.name.clone()); - columns.push((RelationColumn::Single(name), cid)); - } - LineageColumn::All { input_id, except } => { - let input = lineage.find_input(*input_id).unwrap(); - - match &self.node_mapping[&input.id] { - LoweredTarget::Compute(_cid) => unreachable!(), - LoweredTarget::Input(input_cols) => { - let mut input_cols = input_cols - .iter() - .filter(|(c, _)| match c { - RelationColumn::Single(Some(name)) => !except.contains(name), - _ => true, - }) - .collect_vec(); - input_cols.sort_by_key(|e| e.1 .1); + while let Some(id) = ids_rev.pop() { + let target = self.node_mapping.get(&id).ok_or_else(|| { + Error::new_assert("not lowered yet").push_hint(format!("id={id}")) + })?; - for (col, (cid, _)) in input_cols { - columns.push((col.clone(), *cid)); - } - } - } + match target { + LoweredTarget::Column(cid) => cids.push(*cid), + LoweredTarget::Relation(column_ids) => { + ids_rev.extend(column_ids.iter().rev()); } } } - let (cols, cids) = columns.into_iter().unzip(); - - log::debug!("... cids={:?}", cids); - transforms.push(Transform::Select(cids)); - - Ok(cols) + Ok(cids) } - fn declare_as_columns(&mut self, exprs: pl::Expr, is_aggregation: bool) -> Result> { - // special case: reference to a tuple that is a relational input - if exprs.ty.as_ref().map_or(false, |x| x.kind.is_tuple()) && exprs.kind.is_ident() { - // return all contained columns - let input_id = exprs.target_id.as_ref().unwrap(); - let id_mapping = self.node_mapping.get(input_id).unwrap(); - let input_columns = id_mapping.as_input().unwrap(); - return Ok(input_columns - .iter() - .sorted_by_key(|c| c.1 .1) - .map(|(_, (cid, _))| *cid) - .collect_vec()); + fn ensure_lowered(&mut self, mut expr_ast: pl::Expr, is_aggregation: bool) -> Result<()> { + let id = self.get_id(&mut expr_ast); + let expr_ast = expr_ast; + + // short-circuit if this node has already been lowered + if self.node_mapping.contains_key(&id) { + return Ok(()); } - let mut r = Vec::new(); + let target = match expr_ast.kind { + pl::ExprKind::Ident(ident) => self.lookup_ident(ident).with_span(expr_ast.span)?, + pl::ExprKind::Indirection { base, field } => { + let base_id = base.id.unwrap(); + self.ensure_lowered(*base, is_aggregation)?; - match exprs.kind { - pl::ExprKind::All { within, except } => { - // special case: ExprKind::All - r.extend(self.find_selected_all(*within, Some(*except))?); + self.lookup_indirection(base_id, &field) + .with_span(expr_ast.span)? + .clone() } pl::ExprKind::Tuple(fields) => { // tuple unpacking - for expr in fields { - r.extend(self.declare_as_columns(expr, is_aggregation)?); + let mut ids = Vec::new(); + for mut field in fields { + ids.push(self.get_id(&mut field)); + self.ensure_lowered(field, is_aggregation)?; } + LoweredTarget::Relation(ids) } - _ => { - // base case - r.push(self.declare_as_column(exprs, is_aggregation)?); - } - } - Ok(r) - } - - fn find_selected_all( - &mut self, - within: pl::Expr, - except: Option, - ) -> Result> { - let mut selected = self.declare_as_columns(within, false)?; - if let Some(except) = except { - let except: HashSet<_> = self.find_except_ids(except)?; - selected.retain(|t| !except.contains(t)); - } - Ok(selected) - } - - fn find_except_ids(&mut self, except: pl::Expr) -> Result> { - let pl::ExprKind::Tuple(fields) = except.kind else { - return Ok(HashSet::new()); - }; - - let mut res = HashSet::new(); - for e in fields { - if e.target_id.is_none() { - continue; - } - - let id = e.target_id.unwrap(); - match e.kind { - pl::ExprKind::Ident(_) if e.ty.as_ref().map_or(false, |x| x.kind.is_tuple()) => { - res.extend(self.find_selected_all(e, None).with_span(except.span)?); - } - pl::ExprKind::Ident(ident) => { - res.insert( - self.lookup_cid(id, Some(&ident.name)) - .with_span(except.span)?, - ); - } - pl::ExprKind::All { within, except } => { - res.extend(self.find_selected_all(*within, Some(*except))?) - } - _ => { - return Err(Error::new(Reason::Expected { - who: None, - expected: "an identifier".to_string(), - found: write_pl(e), - })); - } + pl::ExprKind::All { within, except } => { + // this should never fail since it succeeded during resolution + let base_ty = within.ty.as_ref().unwrap(); + let except_ty = except.ty.as_ref().unwrap(); + let field_mask = self.ty_tuple_exclusion_mask(base_ty, except_ty); + + // lower within + let within_id = within.id.unwrap(); + self.ensure_lowered(*within, is_aggregation)?; + let within_target = self.node_mapping.get(&within_id).unwrap(); + let within_ids = within_target.as_relation().ok_or_else(|| { + Error::new_assert("indirection on non-relation") + .push_hint(format!("within={within_target:?}")) + })?; + + // apply mask + let ids = itertools::zip_eq(within_ids, field_mask) + .filter(|(_, p)| *p) + .map(|(x, _)| *x) + .collect_vec(); + LoweredTarget::Relation(ids) } - } - Ok(res) - } - - fn declare_as_column( - &mut self, - mut expr_ast: pl::Expr, - is_aggregation: bool, - ) -> Result { - // short-circuit if this node has already been lowered - if let Some(LoweredTarget::Compute(lowered)) = self.node_mapping.get(&expr_ast.id.unwrap()) - { - return Ok(*lowered); - } - - // copy metadata before lowering - let alias = expr_ast.alias.clone(); - let has_alias = alias.is_some(); - let needs_window = expr_ast.needs_window; - expr_ast.needs_window = false; - let alias_for = if has_alias { - expr_ast.kind.as_ident().map(|x| x.name.clone()) - } else { - None - }; - let id = expr_ast.id.unwrap(); - - // lower - let expr = self.lower_expr(expr_ast)?; + _ => { + // lower expr and define a Compute + let expr = self.lower_expr(expr_ast)?; + + // construct ColumnDef + let cid = self.cid.gen(); + let compute = rq::Compute { + id: cid, + expr, + window: None, + is_aggregation, + }; + self.pipeline.push(Transform::Compute(compute)); - // don't create new ColumnDef if expr is just a ColumnRef with no renaming - if let rq::ExprKind::ColumnRef(cid) = &expr.kind { - if !needs_window && (!has_alias || alias == alias_for) { - self.node_mapping.insert(id, LoweredTarget::Compute(*cid)); - return Ok(*cid); + LoweredTarget::Column(cid) } - } - - // determine window - let window = if needs_window { - self.window.clone() - } else { - None - }; - - // construct ColumnDef - let cid = self.cid.gen(); - let compute = rq::Compute { - id: cid, - expr, - window, - is_aggregation, }; - self.node_mapping.insert(id, LoweredTarget::Compute(cid)); - - self.pipeline.push(Transform::Compute(compute)); - Ok(cid) + self.node_mapping.insert(id, target); + Ok(()) } fn lower_expr(&mut self, expr: pl::Expr) -> Result { let span = expr.span; - if expr.needs_window { - let span = expr.span; - let cid = self.declare_as_column(expr, false)?; + let kind = match expr.kind { + pl::ExprKind::Ident(_) | pl::ExprKind::All { .. } => { + return Err(Error::new_assert( + "unreachable code: should have been lowered earlier", + ) + .with_span(span)); + } - let kind = rq::ExprKind::ColumnRef(cid); - return Ok(rq::Expr { kind, span }); - } + pl::ExprKind::Indirection { base, field } => { + let base_id = base.id.unwrap(); + self.ensure_lowered(*base, false)?; - let kind = match expr.kind { - pl::ExprKind::Ident(ident) => { - log::debug!("lowering ident {ident} (target {:?})", expr.target_id); - - if expr.ty.as_ref().map_or(false, |x| x.kind.is_tuple()) { - // special case: tuple ref - let expr = pl::Expr { - kind: pl::ExprKind::Ident(ident), - ..expr - }; - let selected = self.find_selected_all(expr, None)?; + let target = self + .lookup_indirection(base_id, &field) + .with_span(expr.span)? + .clone(); - if selected.len() == 1 { - rq::ExprKind::ColumnRef(selected[0]) - } else { - return Err( - Error::new_simple("This wildcard usage is not yet supported.") - .with_span(span), - ); - } - } else if let Some(id) = expr.target_id { - // base case: column ref - let cid = self.lookup_cid(id, Some(&ident.name)).with_span(span)?; - - rq::ExprKind::ColumnRef(cid) - } else { - // fallback: unresolved ident - // Let's hope that the database engine can resolve it. - rq::ExprKind::SString(vec![InterpolateItem::String(ident.name)]) - } - } - pl::ExprKind::All { within, except } => { - let selected = self.find_selected_all(*within, Some(*except))?; - - if selected.len() == 1 { - rq::ExprKind::ColumnRef(selected[0]) - } else { - return Err( - Error::new_simple("This wildcard usage is not yet supported.") - .with_span(span), - ); - } + let cid = target.into_column().map_err(|_| { + Error::new_assert("lower_expr to refer to columns only").with_span(span) + })?; + rq::ExprKind::ColumnRef(cid) } + pl::ExprKind::Literal(literal) => rq::ExprKind::Literal(literal), pl::ExprKind::SString(items) => { @@ -924,7 +858,10 @@ impl Lowerer { .try_collect()?, ), - pl::ExprKind::FuncCall(_) | pl::ExprKind::Func(_) | pl::ExprKind::TransformCall(_) => { + pl::ExprKind::FuncCall(_) + | pl::ExprKind::Func(_) + | pl::ExprKind::FuncApplication(_) + | pl::ExprKind::TransformCall(_) => { log::debug!("cannot lower {expr:?}"); return Err(Error::new(Reason::Unexpected { found: format!("`{}`", write_pl(expr.clone())), @@ -962,32 +899,135 @@ impl Lowerer { .try_collect() } - fn lookup_cid(&mut self, id: usize, name: Option<&String>) -> Result { - let cid = match self.node_mapping.get(&id) { - Some(LoweredTarget::Compute(cid)) => *cid, - Some(LoweredTarget::Input(input_columns)) => { - let name = match name { - Some(v) => RelationColumn::Single(Some(v.clone())), - None => return Err(Error::new_simple( - "This table contains unnamed columns that need to be referenced by name", - ) - .with_span(self.root_mod.span_map.get(&id).cloned()) - .push_hint("the name may have been overridden later in the pipeline.")), - }; - log::trace!("lookup cid of name={name:?} in input {input_columns:?}"); - - if let Some((cid, _)) = input_columns.get(&name) { - *cid - } else { - panic!("cannot find cid by id={id} and name={name:?}"); - } - } - None => { - return Err(Error::new_bug(3870))?; + fn lookup_ident(&self, ident: Ident) -> Result { + if ident.path != [NS_LOCAL] { + return Err(Error::new_assert("non-local unresolved reference") + .push_hint(format!("ident={ident:?}"))); + } + let target_id = match ident.name.as_str() { + NS_THIS => self.local_this_id.as_ref(), + NS_THAT => self.local_that_id.as_ref(), + _ => { + return Err(Error::new_assert(format!( + "unhandled local reference: {}", + ident.name + ))); } }; + let Some(target_id) = target_id else { + return Err(Error::new_assert("local reference from non-local context") + .push_hint(format!("ident={ident}"))); + }; + let Some(target) = self.node_mapping.get(target_id) else { + return Err( + Error::new_assert("node not lowered yet").push_hint(format!("ident={ident}")) + ); + }; - Ok(cid) + Ok(target.clone()) + } + + fn lookup_indirection( + &self, + base_id: usize, + field: &pl::IndirectionKind, + ) -> Result<&LoweredTarget> { + let base_target = self.node_mapping.get(&base_id).unwrap(); + + let base_relation = base_target.as_relation().ok_or_else(|| { + Error::new_assert("indirection on non-relation") + .push_hint(format!("base={base_target:?}")) + .push_hint(format!("field={field:?}")) + })?; + + let pos = field + .as_position() + .expect("indirections to be resolved into positional"); + + let target_id = base_relation.get(*pos as usize).ok_or_else(|| { + Error::new_assert("bad lowering: tuple field position out of bounds") + .push_hint(format!("base relation={base_relation:?}")) + .push_hint(format!("pos={pos}")) + })?; + + let target = self.node_mapping.get(target_id).ok_or_else(|| { + Error::new_assert("node not lowered yet") + .push_hint(format!("base_target={base_target:?}")) + .push_hint(format!("field={field:?}")) + })?; + + Ok(target) + } + + fn get_id(&mut self, expr: &mut pl::Expr) -> usize { + // This *should* throw an error, because resolver *should not* emit exprs without ids. + // But we do create new exprs in special_functions, so I guess it is fine to generate + // new ids here? + // + // Error::new_assert("expression not resolved during lowering") + // .push_hint(format!("expr = {expr:?}")) + // + + if expr.id.is_none() { + let id = self.id.gen(); + log::debug!("generated id {id}"); + expr.id = Some(id); + } + expr.id.unwrap() + } + + /// Computes the "field mask", which is a vector of booleans indicating if a field of + /// base tuple type should appear in the resulting type. + fn ty_tuple_exclusion_mask(&self, base: &Ty, except: &Ty) -> Vec { + let within_fields = self.get_fields_of_ty(base); + let except_fields = self.get_fields_of_ty(except); + + let except_fields: HashSet<&String> = except_fields + .iter() + .filter_map(|field| match field { + TyTupleField::Single(Some(name), _) => Some(name), + _ => None, + }) + .collect(); + + let mut mask = Vec::new(); + for field in within_fields { + mask.push(match &field { + TyTupleField::Single(Some(name), _) => !except_fields.contains(&name), + TyTupleField::Single(None, _) => true, + TyTupleField::Unpack(_) => true, + }); + } + mask + } + + fn get_fields_of_ty<'a>(&'a self, ty: &'a Ty) -> Vec<&TyTupleField> { + match &ty.kind { + TyKind::Tuple(f) => f + .iter() + .flat_map(|f| match f { + TyTupleField::Single(_, _) => vec![f], + TyTupleField::Unpack(Some(unpack_ty)) => { + let mut r = self.get_fields_of_ty(unpack_ty); + if unpack_ty.kind.is_ident() { + r.push(f); // the wildcard created from the generic + } + r + } + TyTupleField::Unpack(None) => todo!(), + }) + .collect(), + + TyKind::Ident(ident) => { + let decl = self.root_mod.module.get(ident).unwrap(); + let DeclKind::GenericParam(Some(candidate)) = &decl.kind else { + return vec![]; + }; + + self.get_fields_of_ty(&candidate.0) + } + _ => unreachable!(), + } } } @@ -1005,12 +1045,6 @@ fn validate_take_range(range: &Range, span: Option) -> Result<() .map(|e| e.kind.as_literal().and_then(|l| l.as_integer())) } - fn bound_display(bound: Option>) -> String { - bound - .map(|x| x.map(|l| l.to_string()).unwrap_or_else(|| "?".to_string())) - .unwrap_or_default() - } - let start = bound_as_int(&range.start); let end = bound_as_int(&range.end); @@ -1027,13 +1061,7 @@ fn validate_take_range(range: &Range, span: Option) -> Result<() }; if !start_ok || !end_ok { - let range_display = format!("{}..{}", bound_display(start), bound_display(end)); - Err(Error::new(Reason::Expected { - who: Some("take".to_string()), - expected: "a positive int range".to_string(), - found: range_display, - }) - .with_span(span)) + Err(Error::new_simple("take expected a positive int range").with_span(span)) } else { Ok(()) } @@ -1043,14 +1071,14 @@ fn validate_take_range(range: &Range, span: Option) -> Result<() struct TableExtractor { path: Vec, - tables: Vec<(Ident, (decl::TableDecl, Option))>, + tables: Vec<(Ident, (pl::Expr, Option))>, } impl TableExtractor { /// Finds table declarations in a module, recursively. - fn extract(root_module: &Module) -> Vec<(Ident, (decl::TableDecl, Option))> { + fn extract(root: &RootModule) -> Vec<(Ident, (pl::Expr, Option))> { let mut te = TableExtractor::default(); - te.extract_from_module(root_module); + te.extract_from_module(&root.module); te.tables } @@ -1063,10 +1091,10 @@ impl TableExtractor { DeclKind::Module(ns) => { self.extract_from_module(ns); } - DeclKind::TableDecl(table) => { + DeclKind::Expr(expr) if expr.ty.as_ref().unwrap().is_relation() => { let fq_ident = Ident::from_path(self.path.clone()); self.tables - .push((fq_ident, (table.clone(), entry.declared_at))); + .push((fq_ident, (*expr.clone(), entry.declared_at))); } _ => {} } @@ -1079,18 +1107,14 @@ impl TableExtractor { /// are not needed for the main pipeline. To do this, it needs to collect references /// between pipelines. fn toposort_tables( - tables: Vec<(Ident, (decl::TableDecl, Option))>, + tables: Vec<(Ident, (pl::Expr, Option))>, main_table: &Ident, -) -> Vec<(Ident, (decl::TableDecl, Option))> { +) -> Vec<(Ident, (pl::Expr, Option))> { let tables: HashMap<_, _, RandomState> = HashMap::from_iter(tables); let mut dependencies: Vec<(Ident, Vec)> = Vec::new(); for (ident, table) in &tables { - let deps = if let TableExpr::RelationVar(e) = &table.0.expr { - TableDepsCollector::collect(*e.clone()) - } else { - vec![] - }; + let deps = TableDepsCollector::collect(table.0.clone()); dependencies.push((ident.clone(), deps)); } @@ -1130,12 +1154,15 @@ impl PlFold for TableDepsCollector { } expr.kind } - pl::ExprKind::TransformCall(tc) => { - pl::ExprKind::TransformCall(self.fold_transform_call(tc)?) + pl::ExprKind::FuncApplication(FuncApplication { func, args }) => { + pl::ExprKind::FuncApplication(FuncApplication { + func: Box::new(self.fold_expr(*func)?), + args: self.fold_exprs(args)?, + }) } pl::ExprKind::Func(func) => pl::ExprKind::Func(Box::new(self.fold_func(*func)?)), - // optimization: don't recurse into anything else than TransformCalls and Func + // optimization: don't recurse into anything else than RqOperator and Func _ => expr.kind, }; Ok(expr) diff --git a/prqlc/prqlc/src/semantic/resolver/flatten.rs b/prqlc/prqlc/src/semantic/lowering/flatten.rs similarity index 76% rename from prqlc/prqlc/src/semantic/resolver/flatten.rs rename to prqlc/prqlc/src/semantic/lowering/flatten.rs index 7c9afb00a4f9..8b47d7e75638 100644 --- a/prqlc/prqlc/src/semantic/resolver/flatten.rs +++ b/prqlc/prqlc/src/semantic/lowering/flatten.rs @@ -1,14 +1,14 @@ use std::collections::HashMap; +use crate::ir::pl::{fold_column_sorts, fold_transform_kind}; use crate::ir::pl::{ - fold_column_sorts, fold_transform_kind, ColumnSort, Expr, ExprKind, PlFold, TransformCall, - TransformKind, WindowFrame, + ColumnSort, Expr, ExprKind, PlFold, TransformCall, TransformKind, WindowFrame, }; +use crate::semantic::NS_LOCAL; use crate::Result; /// Flattens group and window [TransformCall]s into a single pipeline. /// Sets partition, window and sort of [TransformCall]. -#[derive(Default, Debug)] pub struct Flattener { /// Sort affects downstream transforms in a pipeline. /// Because transform pipelines are represented by nested [TransformCall]s, @@ -39,21 +39,35 @@ pub struct Flattener { /// preceding the group/window transform. /// /// That's what `replace_map` is for. - replace_map: HashMap, + replace_map: HashMap, } impl Flattener { - pub fn fold(expr: Expr) -> Expr { - let mut f = Flattener::default(); - f.fold_expr(expr).unwrap() + pub fn run(expr: Expr) -> Result { + let mut f = Flattener { + sort: Default::default(), + sort_undone: Default::default(), + partition: Default::default(), + window: Default::default(), + replace_map: Default::default(), + }; + f.fold_expr(expr) } } impl PlFold for Flattener { fn fold_expr(&mut self, mut expr: Expr) -> Result { - if let Some(target) = &expr.target_id { - if let Some(replacement) = self.replace_map.remove(target) { - return Ok(replacement); + if let ExprKind::Ident(fq_ident) = &expr.kind { + if fq_ident.starts_with_part(NS_LOCAL) && fq_ident.len() == 2 { + if let Some(replacement) = self.replace_map.remove(&fq_ident.name) { + return Ok(replacement); + } + } + } + + if let ExprKind::RqOperator { name, .. } = &expr.kind { + if !name.starts_with("std.") { + expr = super::special_functions::resolve_special_func(expr)? } } @@ -67,7 +81,7 @@ impl PlFold for Flattener { let by = fold_column_sorts(self, by)?; let input = self.fold_expr(*t.input)?; - self.sort.clone_from(&by); + self.sort = by.clone(); if self.sort_undone { return Ok(input); @@ -84,22 +98,20 @@ impl PlFold for Flattener { let pipeline = pipeline.kind.into_func().unwrap(); let table_param = &pipeline.params[0]; - let param_id = table_param.name.parse::().unwrap(); - self.replace_map.insert(param_id, input); + self.replace_map.insert(table_param.name.clone(), input); self.partition = Some(by); self.sort.clear(); let pipeline = self.fold_expr(*pipeline.body)?; - self.replace_map.remove(¶m_id); + self.replace_map.remove(&table_param.name); self.partition = None; self.sort.clear(); self.sort_undone = sort_undone; return Ok(Expr { ty: expr.ty, - lineage: expr.lineage, ..pipeline }); } @@ -112,19 +124,17 @@ impl PlFold for Flattener { let pipeline = pipeline.kind.into_func().unwrap(); let table_param = &pipeline.params[0]; - let param_id = table_param.name.parse::().unwrap(); - self.replace_map.insert(param_id, tbl); + self.replace_map.insert(table_param.name.clone(), tbl); self.window = WindowFrame { kind, range }; let pipeline = self.fold_expr(*pipeline.body)?; self.window = WindowFrame::default(); - self.replace_map.remove(¶m_id); + self.replace_map.remove(&table_param.name); return Ok(Expr { ty: expr.ty, - lineage: expr.lineage, ..pipeline }); } diff --git a/prqlc/prqlc/src/semantic/lowering/inline.rs b/prqlc/prqlc/src/semantic/lowering/inline.rs new file mode 100644 index 000000000000..7b21fef1f78c --- /dev/null +++ b/prqlc/prqlc/src/semantic/lowering/inline.rs @@ -0,0 +1,135 @@ +use std::collections::HashMap; + +use crate::ir::decl::{DeclKind, RootModule}; +use crate::ir::pl::*; +use crate::semantic::NS_LOCAL; +use crate::{Error, Result, WithErrorInfo}; + +pub struct Inliner<'a> { + root_mod: &'a RootModule, +} + +impl<'a> Inliner<'a> { + pub fn run(root_mod: &'a RootModule, expr: Expr) -> Expr { + let mut i = Inliner { root_mod }; + i.fold_expr(expr).unwrap() + } + + fn lookup_expr(&self, fq_ident: &Ident) -> Option<&Expr> { + let mut ident = fq_ident; + loop { + let decl = self.root_mod.module.get(ident)?; + + match &decl.kind { + DeclKind::Expr(expr) => { + if let ExprKind::Ident(i) = &expr.kind { + ident = i; + } else { + break Some(expr); + } + } + DeclKind::Import(i) => ident = i, + _ => break None, + } + } + } + + fn lookup_func(&self, ident: &Expr) -> Option<(Ident, &Func)> { + let fq_ident = ident.kind.as_ident()?; + let func_decl = self.lookup_expr(fq_ident)?; + let func = func_decl.kind.as_func()?; + Some((fq_ident.clone(), func)) + } +} + +impl<'a> PlFold for Inliner<'a> { + fn fold_expr(&mut self, mut expr: Expr) -> crate::Result { + expr.kind = match expr.kind { + ExprKind::FuncApplication(FuncApplication { func, args }) => { + if let Some((fn_ident, fn_func)) = self.lookup_func(&func) { + if let ExprKind::Internal(internal) = &fn_func.body.kind { + // rq operator + ExprKind::RqOperator { + name: internal.clone(), + args: self.fold_exprs(args)?, + } + } else { + // inline + FuncInliner::run(fn_ident, fn_func, args)?.kind + } + } else { + // potentially throw an error here, since we don't know how to translate this + // function a relational expression? + // it is gonna error out in lowering so we might as well do it earlier + ExprKind::FuncApplication(FuncApplication { func, args }) + } + } + ExprKind::Ident(fq_ident) => { + if let Some(expr) = self.lookup_expr(&fq_ident) { + match &expr.kind { + ExprKind::Internal(internal) => ExprKind::RqOperator { + name: internal.clone(), + args: vec![], + }, + ExprKind::Param(key) if !expr.ty.as_ref().unwrap().is_relation() => { + ExprKind::Param(key.clone()) + } + ExprKind::Literal(lit) => ExprKind::Literal(lit.clone()), + _ => ExprKind::Ident(fq_ident), + } + } else { + ExprKind::Ident(fq_ident) + } + } + k => fold_expr_kind(self, k)?, + }; + Ok(expr) + } +} + +struct FuncInliner<'a> { + // fq ident of the functions we are inlining + fn_ident: Ident, + + param_args: HashMap<&'a str, Expr>, +} + +impl<'a> FuncInliner<'a> { + fn run(fn_ident: Ident, fn_func: &Func, args: Vec) -> Result { + let mut i = FuncInliner { + fn_ident, + param_args: HashMap::with_capacity(fn_func.params.len()), + }; + + for (param, arg) in itertools::zip_eq(&fn_func.params, args) { + i.param_args.insert(param.name.as_str(), arg); + } + i.fold_expr(*fn_func.body.clone()) + } +} + +impl PlFold for FuncInliner<'_> { + fn fold_expr(&mut self, mut expr: Expr) -> crate::Result { + expr.kind = match expr.kind { + ExprKind::Ident(fq_ident) => { + if fq_ident == self.fn_ident { + return Err( + Error::new_simple("recursive functions not supported").with_span(expr.span) + ); + } + + if fq_ident.starts_with_path(&[NS_LOCAL]) { + assert_eq!(fq_ident.len(), 2); + let param_name = fq_ident.name; + + let param = self.param_args.get(param_name.as_str()).unwrap(); + param.kind.clone() + } else { + ExprKind::Ident(fq_ident) + } + } + k => fold_expr_kind(self, k)?, + }; + Ok(expr) + } +} diff --git a/prqlc/prqlc/src/semantic/lowering/special_functions.rs b/prqlc/prqlc/src/semantic/lowering/special_functions.rs new file mode 100644 index 000000000000..f1ca58118166 --- /dev/null +++ b/prqlc/prqlc/src/semantic/lowering/special_functions.rs @@ -0,0 +1,697 @@ +use std::collections::HashMap; + +use itertools::Itertools; +use serde::Deserialize; + +use crate::ir::generic::{SortDirection, WindowKind}; +use crate::ir::pl::*; + +use crate::semantic::ast_expand::{restrict_null_literal, try_restrict_range}; +use crate::semantic::write_pl; +use crate::{Error, Reason, Result, WithErrorInfo}; + +/// try to convert function call with enough args into transform +#[allow(clippy::boxed_local)] +pub fn resolve_special_func(expr: Expr) -> Result { + let ExprKind::RqOperator { name, args } = expr.kind else { + unreachable!() + }; + + let (kind, input) = match name.as_str() { + "select" => { + let [assigns, tbl] = unpack::<2>(args); + (TransformKind::Select { assigns }, tbl) + } + "filter" => { + let [filter, tbl] = unpack::<2>(args); + (TransformKind::Filter { filter }, tbl) + } + "derive" => { + let [assigns, tbl] = unpack::<2>(args); + (TransformKind::Derive { assigns }, tbl) + } + "aggregate" => { + let [assigns, tbl] = unpack::<2>(args); + (TransformKind::Aggregate { assigns }, tbl) + } + "sort" => { + let [by, tbl] = unpack::<2>(args); + + let by_fields = by.try_cast(|x| x.into_tuple(), Some("sort"), "tuple")?; + let by = by_fields + .into_iter() + .map(|expr| { + let (column, direction) = match expr.kind { + ExprKind::RqOperator { name, mut args } if name == "std.neg" => { + (args.remove(0), SortDirection::Desc) + } + _ => (expr, SortDirection::default()), + }; + let column = Box::new(column); + + ColumnSort { direction, column } + }) + .collect(); + + (TransformKind::Sort { by }, tbl) + } + "take" => { + let [expr, tbl] = unpack::<2>(args); + + let range = if let ExprKind::Literal(Literal::Integer(n)) = expr.kind { + range_from_ints(None, Some(n)) + } else { + match try_restrict_range(*expr) { + Ok((start, end)) => Range { + start: restrict_null_literal(start).map(Box::new), + end: restrict_null_literal(end).map(Box::new), + }, + Err(expr) => { + return Err(Error::new(Reason::Expected { + who: Some("`take`".to_string()), + expected: "int or range".to_string(), + found: write_pl(expr.clone()), + }) + // Possibly this should refer to the item after the `take` where + // one exists? + .with_span(expr.span)); + } + } + }; + + (TransformKind::Take { range }, tbl) + } + "join" => { + let [with, filter, tbl] = unpack::<3>(args); + + let side = { + JoinSide::Inner + // let span = side.span; + // let ident = side.try_cast(ExprKind::into_ident, Some("side"), "ident")?; + // match ident.name.as_str() { + // "inner" => JoinSide::Inner, + // "left" => JoinSide::Left, + // "right" => JoinSide::Right, + // "full" => JoinSide::Full, + + // found => { + // return Err(Error::new(Reason::Expected { + // who: Some("`side`".to_string()), + // expected: "inner, left, right or full".to_string(), + // found: found.to_string(), + // }) + // .with_span(span)) + // } + // } + }; + + (TransformKind::Join { side, with, filter }, tbl) + } + "group" => { + let [by, pipeline, tbl] = unpack::<3>(args); + (TransformKind::Group { by, pipeline }, tbl) + } + "window" => { + let [rows, range, expanding, rolling, pipeline, tbl] = unpack::<6>(args); + + let expanding = { + let as_bool = expanding.kind.as_literal().and_then(|l| l.as_boolean()); + + *as_bool.ok_or_else(|| { + Error::new(Reason::Expected { + who: Some("parameter `expanding`".to_string()), + expected: "a boolean".to_string(), + found: write_pl(*expanding.clone()), + }) + .with_span(expanding.span) + })? + }; + + let rolling = { + let as_int = rolling.kind.as_literal().and_then(|x| x.as_integer()); + + *as_int.ok_or_else(|| { + Error::new(Reason::Expected { + who: Some("parameter `rolling`".to_string()), + expected: "a number".to_string(), + found: write_pl(*rolling.clone()), + }) + .with_span(rolling.span) + })? + }; + + let rows = into_literal_range(try_restrict_range(*rows).unwrap())?; + + let range = into_literal_range(try_restrict_range(*range).unwrap())?; + + let (kind, start, end) = if expanding { + (WindowKind::Rows, None, Some(0)) + } else if rolling > 0 { + (WindowKind::Rows, Some(-rolling + 1), Some(0)) + } else if !range_is_empty(&rows) { + (WindowKind::Rows, rows.0, rows.1) + } else if !range_is_empty(&range) { + (WindowKind::Range, range.0, range.1) + } else { + (WindowKind::Rows, None, None) + }; + // let start = Expr::new(start.map_or(Literal::Null, Literal::Integer)); + // let end = Expr::new(end.map_or(Literal::Null, Literal::Integer)); + let range = Range { + start: start.map(Literal::Integer).map(Expr::new).map(Box::new), + end: end.map(Literal::Integer).map(Expr::new).map(Box::new), + }; + + let transform_kind = TransformKind::Window { + kind, + range, + pipeline, + }; + (transform_kind, tbl) + } + "append" => { + let [bottom, top] = unpack::<2>(args); + + (TransformKind::Append(bottom), top) + } + "loop" => { + let [pipeline, tbl] = unpack::<2>(args); + (TransformKind::Loop(pipeline), tbl) + } + + "in" => { + let [pattern, value] = unpack::<2>(args); + + if pattern.ty.as_ref().map_or(false, |x| x.kind.is_array()) { + return Ok(Expr { + kind: ExprKind::RqOperator { + name: "std.array_in".to_string(), + args: vec![*value, *pattern], + }, + ..expr + }); + } + + let pattern = match try_restrict_range(*pattern) { + Ok((start, end)) => { + let start = restrict_null_literal(start); + let end = restrict_null_literal(end); + + let start = start.map(|s| new_binop(*value.clone(), "std.gte", s)); + let end = end.map(|e| new_binop(*value, "std.lte", e)); + + let res = maybe_binop(start, "std.and", end); + let res = + res.unwrap_or_else(|| Expr::new(ExprKind::Literal(Literal::Boolean(true)))); + return Ok(res); + } + Err(expr) => expr, + }; + let pattern = Expr { + kind: pattern.kind, + ..expr + }; + + return Err(Error::new(Reason::Expected { + who: Some("std.in".to_string()), + expected: "a pattern".to_string(), + found: write_pl(pattern.clone()), + }) + .with_span(pattern.span)); + } + + "tuple_every" => { + let [list] = unpack::<1>(args); + let list = list.kind.into_tuple().unwrap(); + + let mut res = None; + for item in list { + res = maybe_binop(res, "std.and", Some(item)); + } + let res = res.unwrap_or_else(|| Expr::new(ExprKind::Literal(Literal::Boolean(true)))); + + return Ok(res); + } + + "tuple_map" => { + let [func, list] = unpack::<2>(args); + let list_items = list.kind.into_tuple().unwrap(); + + let list_items = list_items + .into_iter() + .map(|item| { + Expr::new(ExprKind::FuncCall(FuncCall::new_simple( + *func.clone(), + vec![item], + ))) + }) + .collect_vec(); + + return Ok(Expr { + kind: ExprKind::Tuple(list_items), + ..*list + }); + } + + "tuple_zip" => { + let [a, b] = unpack::<2>(args); + let a = a.kind.into_tuple().unwrap(); + let b = b.kind.into_tuple().unwrap(); + + let mut res = Vec::new(); + for (a, b) in std::iter::zip(a, b) { + res.push(Expr::new(ExprKind::Tuple(vec![a, b]))); + } + + return Ok(Expr::new(ExprKind::Tuple(res))); + } + + "_eq" => { + let [list] = unpack::<1>(args); + let list = list.kind.into_tuple().unwrap(); + let [a, b]: [Expr; 2] = list.try_into().unwrap(); + + let res = maybe_binop(Some(a), "std.eq", Some(b)).unwrap(); + return Ok(res); + } + + "from_text" => { + let [format, text_expr] = unpack::<2>(args); + + let text = match text_expr.kind { + ExprKind::Literal(Literal::String(text)) => text, + _ => { + return Err(Error::new(Reason::Expected { + who: Some("std.from_text".to_string()), + expected: "a string literal".to_string(), + found: format!("`{}`", write_pl(*text_expr.clone())), + }) + .with_span(text_expr.span)); + } + }; + + let res = { + let span = format.span; + let format = format + .try_cast(ExprKind::into_ident, Some("format"), "ident")? + .name; + match format.as_str() { + "csv" => from_text::parse_csv(&text) + .map_err(|r| Error::new_simple(r).with_span(span))?, + "json" => from_text::parse_json(&text) + .map_err(|r| Error::new_simple(r).with_span(span))?, + + _ => { + return Err(Error::new(Reason::Expected { + who: Some("`format`".to_string()), + expected: "csv or json".to_string(), + found: format, + }) + .with_span(span)) + } + } + }; + + // let ty = self.declare_table_for_literal(expr_id, Some(columns)); + + let res = Expr::new(ExprKind::Array( + res.rows + .into_iter() + .map(|row| { + Expr::new(ExprKind::Tuple( + row.into_iter() + .map(|lit| Expr::new(ExprKind::Literal(lit))) + .collect(), + )) + }) + .collect(), + )); + let res = Expr { + ty: None, + id: text_expr.id, + ..res + }; + return Ok(res); + } + + "prql_version" => { + let ver = crate::compiler_version().to_string(); + return Ok(Expr { + kind: ExprKind::Literal(Literal::String(ver)), + ..expr + }); + } + + "count" | "row_number" => { + // HACK: these functions get `this`, resolved to `{x = {_self}}`, which + // throws an error during lowering. + // But because these functions don't *really* need an arg, we can just pass + // a null instead. + return Ok(Expr { + needs_window: expr.needs_window, + ..Expr::new(ExprKind::RqOperator { + name: format!("std.{name}"), + args: vec![Expr::new(Literal::Null)], + }) + }); + } + + _ => return Err(Error::new_assert(format!("unknown operator {name}")).with_span(expr.span)), + }; + + let transform_call = TransformCall { + kind: Box::new(kind), + input, + partition: None, + frame: WindowFrame::default(), + sort: Vec::new(), + }; + Ok(Expr { + kind: ExprKind::TransformCall(transform_call), + ..expr + }) +} + +fn range_is_empty(range: &(Option, Option)) -> bool { + match (&range.0, &range.1) { + (Some(s), Some(e)) => s > e, + _ => false, + } +} + +fn range_from_ints(start: Option, end: Option) -> Range { + let start = start.map(|x| Box::new(Expr::new(ExprKind::Literal(Literal::Integer(x))))); + let end = end.map(|x| Box::new(Expr::new(ExprKind::Literal(Literal::Integer(x))))); + Range { start, end } +} + +fn into_literal_range(range: (Expr, Expr)) -> Result<(Option, Option)> { + fn into_int(bound: Expr) -> Result> { + match bound.kind { + ExprKind::Literal(Literal::Null) => Ok(None), + ExprKind::Literal(Literal::Integer(i)) => Ok(Some(i)), + _ => Err(Error::new_simple("expected an int literal").with_span(bound.span)), + } + } + Ok((into_int(range.0)?, into_int(range.1)?)) +} + +/// Expects closure's args to be resolved. +/// Note that named args are before positional args, in order of declaration. +fn unpack(func_args: Vec) -> [Box; P] { + let boxed = func_args.into_iter().map(Box::new).collect_vec(); + boxed.try_into().expect("bad special function cast") +} + +fn maybe_binop(left: Option, op_name: &str, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(new_binop(left, op_name, right)), + (left, right) => left.or(right), + } +} + +fn new_binop(left: Expr, op_name: &str, right: Expr) -> Expr { + Expr::new(ExprKind::RqOperator { + name: op_name.to_string(), + args: vec![left, right], + }) +} + +mod from_text { + use crate::ir::rq::RelationLiteral; + + use super::*; + + // TODO: Can we dynamically get the types, like in pandas? We need to put + // quotes around strings and not around numbers. + // https://stackoverflow.com/questions/64369887/how-do-i-read-csv-data-without-knowing-the-structure-at-compile-time + pub fn parse_csv(text: &str) -> Result { + let text = text.trim(); + let mut rdr = csv::Reader::from_reader(text.as_bytes()); + + fn parse_header(row: &csv::StringRecord) -> Vec { + row.into_iter().map(|x| x.to_string()).collect() + } + + fn parse_row(row: csv::StringRecord) -> Vec { + row.into_iter() + .map(|x| Literal::String(x.to_string())) + .collect() + } + + Ok(RelationLiteral { + columns: parse_header(rdr.headers().map_err(|e| e.to_string())?), + rows: rdr + .records() + .map(|row_result| row_result.map(parse_row)) + .try_collect() + .map_err(|e| e.to_string())?, + }) + } + + type JsonFormat1Row = HashMap; + + #[derive(Deserialize)] + struct JsonFormat2 { + columns: Vec, + data: Vec>, + } + + fn map_json_primitive(primitive: serde_json::Value) -> Literal { + use serde_json::Value::*; + match primitive { + Null => Literal::Null, + Bool(bool) => Literal::Boolean(bool), + Number(number) if number.is_i64() => Literal::Integer(number.as_i64().unwrap()), + Number(number) if number.is_f64() => Literal::Float(number.as_f64().unwrap()), + Number(_) => Literal::Null, + String(string) => Literal::String(string), + Array(_) => Literal::Null, + Object(_) => Literal::Null, + } + } + + fn object_to_vec( + mut row_map: HashMap, + columns: &[String], + ) -> Vec { + columns + .iter() + .map(|c| { + row_map + .remove(c) + .map(map_json_primitive) + .unwrap_or(Literal::Null) + }) + .collect_vec() + } + + pub fn parse_json(text: &str) -> Result { + parse_json1(text).or_else(|err1| { + parse_json2(text) + .map_err(|err2| format!("While parsing rows: {err1}\nWhile parsing object: {err2}")) + }) + } + + fn parse_json1(text: &str) -> Result { + let data: Vec = serde_json::from_str(text).map_err(|e| e.to_string())?; + let mut columns = data + .first() + .ok_or("json: no rows")? + .keys() + .cloned() + .collect_vec(); + + // JSON object keys are not ordered, so have to apply some order to produce + // deterministic results + columns.sort(); + + let rows = data + .into_iter() + .map(|row_map| object_to_vec(row_map, &columns)) + .collect_vec(); + Ok(RelationLiteral { columns, rows }) + } + + fn parse_json2(text: &str) -> Result { + let JsonFormat2 { columns, data } = + serde_json::from_str(text).map_err(|x| x.to_string())?; + + Ok(RelationLiteral { + columns, + rows: data + .into_iter() + .map(|row| row.into_iter().map(map_json_primitive).collect_vec()) + .collect_vec(), + }) + } +} + +#[cfg(test)] +mod tests { + use insta::assert_yaml_snapshot; + + use crate::semantic::test::parse_resolve_and_lower; + + #[test] + fn test_aggregate_positional_arg() { + // distinct query #292 + + assert_yaml_snapshot!(parse_resolve_and_lower(" + from db.c_invoice + select invoice_no + group invoice_no ( + take 1 + ) + ").unwrap(), @r###" + --- + def: + version: ~ + other: {} + tables: + - id: 0 + name: ~ + relation: + kind: + ExternRef: + - c_invoice + columns: + - Single: invoice_no + - Wildcard + relation: + kind: + Pipeline: + - From: + source: 0 + columns: + - - Single: invoice_no + - 0 + - - Wildcard + - 1 + name: c_invoice + - Select: + - 0 + - Take: + range: + start: ~ + end: + kind: + Literal: + Integer: 1 + span: ~ + partition: + - 0 + sort: [] + - Select: + - 0 + columns: + - Single: invoice_no + "###); + + // oops, two arguments #339 + let result = parse_resolve_and_lower( + " + from db.c_invoice + aggregate average amount + ", + ); + assert!(result.is_err()); + + // oops, two arguments + let result = parse_resolve_and_lower( + " + from db.c_invoice + group issued_at (aggregate average amount) + ", + ); + assert!(result.is_err()); + + // correct function call + let ctx = crate::semantic::test::parse_and_resolve( + " + from db.c_invoice + group issued_at ( + aggregate (average amount) + ) + ", + ) + .unwrap(); + let (res, _) = ctx.find_main_rel(&[]).unwrap().clone(); + let expr = res.clone(); + let expr = crate::semantic::resolver::test::erase_ids(expr); + assert_yaml_snapshot!(expr); + } + + #[test] + fn test_transform_sort() { + assert_yaml_snapshot!(parse_resolve_and_lower(" + from db.invoices + sort {issued_at, -amount, +num_of_articles} + sort issued_at + sort (-issued_at) + sort {issued_at} + sort {-issued_at} + ").unwrap(), @r###" + --- + def: + version: ~ + other: {} + tables: + - id: 0 + name: ~ + relation: + kind: + ExternRef: + - invoices + columns: + - Single: issued_at + - Single: amount + - Single: num_of_articles + - Wildcard + relation: + kind: + Pipeline: + - From: + source: 0 + columns: + - - Single: issued_at + - 0 + - - Single: amount + - 1 + - - Single: num_of_articles + - 2 + - - Wildcard + - 3 + name: invoices + - Sort: + - direction: Asc + column: 0 + - direction: Desc + column: 1 + - direction: Asc + column: 2 + - Sort: + - direction: Asc + column: 0 + - Sort: + - direction: Desc + column: 0 + - Sort: + - direction: Asc + column: 0 + - Sort: + - direction: Desc + column: 0 + - Select: + - 0 + - 1 + - 2 + - 3 + columns: + - Single: issued_at + - Single: amount + - Single: num_of_articles + - Wildcard + "###); + } +} diff --git a/prqlc/prqlc/src/semantic/mod.rs b/prqlc/prqlc/src/semantic/mod.rs index 6e2ca359dd92..cbdcc6853125 100644 --- a/prqlc/prqlc/src/semantic/mod.rs +++ b/prqlc/prqlc/src/semantic/mod.rs @@ -1,19 +1,18 @@ //! Semantic resolver (name resolution, type checking and lowering to RQ) pub mod ast_expand; -mod eval; mod lowering; mod module; pub mod reporting; +mod resolve_decls; mod resolver; -pub use eval::eval; -pub use lowering::lower_to_ir; - use self::resolver::Resolver; pub use self::resolver::ResolverOptions; +pub use lowering::lower_to_ir; + use crate::ir::constant::ConstExpr; -use crate::ir::decl::{Module, RootModule}; +use crate::ir::decl::RootModule; use crate::ir::pl::{self, Expr, ImportDef, ModuleDef, Stmt, StmtKind, TypeDef, VarDef}; use crate::ir::rq::RelationalQuery; use crate::parser::is_mod_def_for; @@ -48,16 +47,19 @@ pub fn resolve(mut module_tree: pr::ModuleDef) -> Result { let root_module_def = ast_expand::expand_module_def(module_tree)?; debug::log_entry(|| debug::DebugEntryKind::ReprPl(root_module_def.clone())); - // init new root module - let mut root_module = RootModule { - module: Module::new_root(), - ..Default::default() - }; + // init the module structure + debug::log_stage(debug::Stage::Semantic(debug::StageSemantic::Resolver)); + let mut root_module = resolve_decls::init_module_tree(root_module_def); + + // resolve name references between declarations + let resolution_order = resolve_decls::resolve_decl_refs(&mut root_module)?; + + // resolve let mut resolver = Resolver::new(&mut root_module); - // resolve the module def into the root module - debug::log_stage(debug::Stage::Semantic(debug::StageSemantic::Resolver)); - resolver.fold_statements(root_module_def.stmts)?; + for decl_fq in resolution_order { + resolver.resolve_decl(decl_fq)?; + } debug::log_entry(|| debug::DebugEntryKind::ReprDecl(root_module.clone())); Ok(root_module) @@ -107,19 +109,18 @@ pub const NS_STD: &str = "std"; pub const NS_THIS: &str = "this"; pub const NS_THAT: &str = "that"; pub const NS_PARAM: &str = "_param"; -pub const NS_DEFAULT_DB: &str = "default_db"; +pub const NS_DEFAULT_DB: &str = "db"; pub const NS_QUERY_DEF: &str = "prql"; pub const NS_MAIN: &str = "main"; +pub const NS_LOCAL: &str = "_local"; // refers to the containing module (direct parent) pub const NS_SELF: &str = "_self"; +// TODO: convert this to module annotation // implies we can infer new non-module declarations in the containing module pub const NS_INFER: &str = "_infer"; -// implies we can infer new module declarations in the containing module -pub const NS_INFER_MODULE: &str = "_infer_module"; - pub const NS_GENERIC: &str = "_generic"; impl Stmt { @@ -171,11 +172,12 @@ pub fn write_pl(expr: pl::Expr) -> String { pub mod test { use insta::assert_yaml_snapshot; - use super::{resolve, resolve_and_lower, RootModule}; use crate::ir::rq::RelationalQuery; use crate::parser::parse; use crate::Errors; + use super::{resolve, resolve_and_lower, RootModule}; + pub fn parse_resolve_and_lower(query: &str) -> Result { let source_tree = query.into(); Ok(resolve_and_lower(parse(&source_tree)?, &[], None)?) @@ -189,7 +191,7 @@ pub mod test { #[test] fn test_resolve_01() { assert_yaml_snapshot!(parse_resolve_and_lower(r###" - from employees + from db.employees select !{foo} "###).unwrap().relation.columns, @r###" --- @@ -200,7 +202,7 @@ pub mod test { #[test] fn test_resolve_02() { assert_yaml_snapshot!(parse_resolve_and_lower(r###" - from foo + from db.foo sort day window range:-4..4 ( derive {next_four_days = sum b} @@ -217,7 +219,8 @@ pub mod test { #[test] fn test_resolve_03() { assert_yaml_snapshot!(parse_resolve_and_lower(r###" - from a=albums + from db.albums + select {a = this} filter is_sponsored select {a.*} "###).unwrap().relation.columns, @r###" @@ -230,7 +233,7 @@ pub mod test { #[test] fn test_resolve_04() { assert_yaml_snapshot!(parse_resolve_and_lower(r###" - from x + from db.x select {a, a, a = a + 1} "###).unwrap().relation.columns, @r###" --- @@ -245,7 +248,7 @@ pub mod test { assert_yaml_snapshot!(parse_resolve_and_lower(r#" prql target:sql.mssql version:"0" - from employees + from db.employees "#).unwrap(), @r###" --- def: @@ -280,7 +283,7 @@ pub mod test { assert!(parse_resolve_and_lower( r###" prql target:sql.bigquery version:foo - from employees + from db.employees "###, ) .is_err()); @@ -288,7 +291,7 @@ pub mod test { assert!(parse_resolve_and_lower( r#" prql target:sql.bigquery version:"25" - from employees + from db.employees "#, ) .is_err()); @@ -296,7 +299,7 @@ pub mod test { assert!(parse_resolve_and_lower( r###" prql target:sql.yah version:foo - from employees + from db.employees "###, ) .is_err()); diff --git a/prqlc/prqlc/src/semantic/module.rs b/prqlc/prqlc/src/semantic/module.rs index 56719d8a3d98..b9361e774628 100644 --- a/prqlc/prqlc/src/semantic/module.rs +++ b/prqlc/prqlc/src/semantic/module.rs @@ -1,15 +1,11 @@ -use std::collections::{HashMap, HashSet}; - -use super::{ - NS_DEFAULT_DB, NS_GENERIC, NS_INFER, NS_INFER_MODULE, NS_MAIN, NS_PARAM, NS_QUERY_DEF, NS_SELF, - NS_STD, NS_THAT, NS_THIS, -}; -use crate::ir::decl::{Decl, DeclKind, Module, RootModule, TableDecl, TableExpr}; -use crate::ir::pl::{Annotation, Expr, Ident, Lineage, LineageColumn}; -use crate::pr::QueryDef; -use crate::pr::{Literal, Span, Ty, TyKind, TyTupleField}; -use crate::Error; -use crate::Result; +use std::collections::HashMap; + +use crate::ir::decl::{Decl, DeclKind, InferTarget, Module, RootModule}; +use crate::ir::pl; +use crate::pr; +use crate::{Error, Result, Span}; + +use super::{NS_DEFAULT_DB, NS_INFER, NS_MAIN, NS_QUERY_DEF, NS_STD}; impl Module { pub fn singleton(name: S, entry: Decl) -> Module { @@ -31,42 +27,22 @@ impl Module { (NS_STD.to_string(), Decl::from(DeclKind::default())), ]), shadowed: None, - redirects: vec![ - Ident::from_name(NS_THIS), - Ident::from_name(NS_THAT), - Ident::from_name(NS_PARAM), - Ident::from_name(NS_STD), - Ident::from_name(NS_GENERIC), - ], + redirects: vec![], } } pub fn new_database() -> Module { - let names = HashMap::from([ - ( - NS_INFER.to_string(), - Decl::from(DeclKind::Infer(Box::new(DeclKind::TableDecl(TableDecl { - ty: Some(Ty::relation(vec![TyTupleField::Wildcard(None)])), - expr: TableExpr::LocalTable, - })))), - ), - ( - NS_INFER_MODULE.to_string(), - Decl::from(DeclKind::Infer(Box::new(DeclKind::Module(Module { - names: HashMap::new(), - redirects: vec![], - shadowed: None, - })))), - ), - ]); + let names = HashMap::from([( + NS_INFER.to_string(), + Decl::from(DeclKind::Infer(InferTarget::Table)), + )]); Module { names, - shadowed: None, - redirects: vec![], + ..Default::default() } } - pub fn insert(&mut self, fq_ident: Ident, decl: Decl) -> Result, Error> { + pub fn insert(&mut self, fq_ident: pr::Ident, decl: Decl) -> Result, Error> { if fq_ident.path.is_empty() { Ok(self.names.insert(fq_ident.name, decl)) } else { @@ -83,7 +59,7 @@ impl Module { } } - pub fn get_mut(&mut self, ident: &Ident) -> Option<&mut Decl> { + pub fn get_mut(&mut self, ident: &pr::Ident) -> Option<&mut Decl> { let mut ns = self; for part in &ident.path { @@ -104,199 +80,49 @@ impl Module { } /// Get namespace entry using a fully qualified ident. - pub fn get(&self, fq_ident: &Ident) -> Option<&Decl> { + pub fn get(&self, fq_ident: &pr::Ident) -> Option<&Decl> { let mut ns = self; - for (index, part) in fq_ident.path.iter().enumerate() { + for part in fq_ident.path.iter() { let decl = ns.names.get(part)?; - match &decl.kind { - DeclKind::Module(inner) => { - ns = inner; - } - DeclKind::LayeredModules(stack) => { - let next = fq_ident.path.get(index + 1).unwrap_or(&fq_ident.name); - let mut found = false; - for n in stack.iter().rev() { - if n.names.contains_key(next) { - ns = n; - found = true; - break; - } - } - if !found { - return None; - } - } - _ => return None, + if let DeclKind::Module(inner) = &decl.kind { + ns = inner; + } else { + return None; } } ns.names.get(&fq_ident.name) } - pub fn lookup(&self, ident: &Ident) -> HashSet { - fn lookup_in(module: &Module, ident: Ident) -> HashSet { - let (prefix, ident) = ident.pop_front(); - - if let Some(ident) = ident { - if let Some(entry) = module.names.get(&prefix) { - let redirected = match &entry.kind { - DeclKind::Module(ns) => ns.lookup(&ident), - DeclKind::LayeredModules(stack) => { - let mut r = HashSet::new(); - for ns in stack.iter().rev() { - r = ns.lookup(&ident); - - if !r.is_empty() { - break; - } - } - r - } - _ => HashSet::new(), - }; - - return redirected - .into_iter() - .map(|i| Ident::from_name(&prefix) + i) - .collect(); - } - } else if let Some(decl) = module.names.get(&prefix) { - if let DeclKind::Module(inner) = &decl.kind { - if inner.names.contains_key(NS_SELF) { - return HashSet::from([Ident::from_path(vec![ - prefix, - NS_SELF.to_string(), - ])]); - } - } - - return HashSet::from([Ident::from_name(prefix)]); - } - HashSet::new() - } - - log::trace!("lookup: {ident}"); - - let mut res = HashSet::new(); - - res.extend(lookup_in(self, ident.clone())); - - for redirect in &self.redirects { - log::trace!("... following redirect {redirect}"); - let r = lookup_in(self, redirect.clone() + ident.clone()); - log::trace!("... result of redirect {redirect}: {r:?}"); - res.extend(r); + pub fn get_submodule(&self, path: &[String]) -> Option<&Module> { + let mut curr_mod = self; + for step in path { + let decl = curr_mod.names.get(step)?; + curr_mod = decl.kind.as_module()?; } - res + Some(curr_mod) } - pub(super) fn insert_frame(&mut self, lineage: &Lineage, namespace: &str) { - let namespace = self.names.entry(namespace.to_string()).or_default(); - let namespace = namespace.kind.as_module_mut().unwrap(); - - let lin_ty = *ty_of_lineage(lineage).kind.into_array().unwrap(); - - for (col_index, column) in lineage.columns.iter().enumerate() { - // determine input name - let input_name = match column { - LineageColumn::All { input_id, .. } => { - lineage.find_input(*input_id).map(|i| &i.name) - } - LineageColumn::Single { name, .. } => name.as_ref().and_then(|n| n.path.first()), - }; - - // get or create input namespace - let ns; - if let Some(input_name) = input_name { - let entry = match namespace.names.get_mut(input_name) { - Some(x) => x, - None => { - namespace.redirects.push(Ident::from_name(input_name)); - - let input = lineage.find_input_by_name(input_name).unwrap(); - let order = lineage.inputs.iter().position(|i| i.id == input.id); - let order = order.unwrap(); - - let mut sub_ns = Module::default(); - - let self_ty = lin_ty.clone().kind.into_tuple().unwrap(); - let self_ty = self_ty - .into_iter() - .flat_map(|x| x.into_single()) - .find(|(name, _)| name.as_ref() == Some(input_name)) - .and_then(|(_, ty)| ty) - .or(Some(Ty::new(TyKind::Tuple(vec![TyTupleField::Wildcard( - None, - )])))); - - let self_decl = Decl { - declared_at: Some(input.id), - kind: DeclKind::InstanceOf(input.table.clone(), self_ty), - ..Default::default() - }; - sub_ns.names.insert(NS_SELF.to_string(), self_decl); - - let sub_ns = Decl { - declared_at: Some(input.id), - order, - kind: DeclKind::Module(sub_ns), - ..Default::default() - }; - - namespace.names.entry(input_name.clone()).or_insert(sub_ns) - } - }; - ns = entry.kind.as_module_mut().unwrap() - } else { - ns = namespace; - } - - // insert column decl - match column { - LineageColumn::All { input_id, .. } => { - let input = lineage.find_input(*input_id).unwrap(); - - let kind = DeclKind::Infer(Box::new(DeclKind::Column(input.id))); - let declared_at = Some(input.id); - let decl = Decl { - kind, - declared_at, - order: col_index + 1, - ..Default::default() - }; - ns.names.insert(NS_INFER.to_string(), decl); - } - LineageColumn::Single { - name: Some(name), - target_id, - .. - } => { - let decl = Decl { - kind: DeclKind::Column(*target_id), - declared_at: None, - order: col_index + 1, - ..Default::default() - }; - ns.names.insert(name.name.clone(), decl); - } - _ => {} - } + pub fn get_submodule_mut(&mut self, path: &[String]) -> Option<&mut Module> { + let mut curr_mod = self; + for step in path { + let decl = curr_mod.names.get_mut(step)?; + curr_mod = decl.kind.as_module_mut()?; } - - // insert namespace._self with correct type - namespace.names.insert( - NS_SELF.to_string(), - Decl::from(DeclKind::InstanceOf(Ident::from_name(""), Some(lin_ty))), - ); + Some(curr_mod) } - pub(super) fn insert_frame_col(&mut self, namespace: &str, name: String, id: usize) { - let namespace = self.names.entry(namespace.to_string()).or_default(); - let namespace = namespace.kind.as_module_mut().unwrap(); + pub fn get_module_path(&self, path: &[String]) -> Option> { + let mut res = vec![self]; + for step in path { + let decl = res.last().unwrap().names.get(step)?; + let module = decl.kind.as_module()?; + res.push(module); + } - namespace.names.insert(name, DeclKind::Column(id).into()); + Some(res) } pub fn shadow(&mut self, ident: &str) { @@ -318,46 +144,7 @@ impl Module { } } - pub fn stack_push(&mut self, ident: &str, namespace: Module) { - let entry = self - .names - .entry(ident.to_string()) - .or_insert_with(|| DeclKind::LayeredModules(Vec::new()).into()); - let stack = entry.kind.as_layered_modules_mut().unwrap(); - - stack.push(namespace); - } - - pub fn stack_pop(&mut self, ident: &str) -> Option { - (self.names.get_mut(ident)) - .and_then(|e| e.kind.as_layered_modules_mut()) - .and_then(|stack| stack.pop()) - } - - pub(crate) fn into_exprs(self) -> HashMap { - self.names - .into_iter() - .map(|(k, v)| (k, *v.kind.into_expr().unwrap())) - .collect() - } - - pub(crate) fn from_exprs(exprs: HashMap) -> Module { - Module { - names: exprs - .into_iter() - .map(|(key, expr)| { - let decl = Decl { - kind: DeclKind::Expr(Box::new(expr)), - ..Default::default() - }; - (key, decl) - }) - .collect(), - ..Default::default() - } - } - - pub fn as_decls(&self) -> Vec<(Ident, &Decl)> { + pub fn as_decls(&self) -> Vec<(pr::Ident, &Decl)> { let mut r = Vec::new(); for (name, decl) in &self.names { match &decl.kind { @@ -365,16 +152,16 @@ impl Module { module .as_decls() .into_iter() - .map(|(inner, decl)| (Ident::from_name(name) + inner, decl)), + .map(|(inner, decl)| (pr::Ident::from_name(name) + inner, decl)), ), - _ => r.push((Ident::from_name(name), decl)), + _ => r.push((pr::Ident::from_name(name), decl)), } } r } /// Recursively finds all declarations that end in suffix. - pub fn find_by_suffix(&self, suffix: &str) -> Vec { + pub fn find_by_suffix(&self, suffix: &str) -> Vec { let mut res = Vec::new(); for (name, decl) in &self.names { @@ -385,7 +172,7 @@ impl Module { } if name == suffix { - res.push(Ident::from_name(name)); + res.push(pr::Ident::from_name(name)); } } @@ -393,7 +180,7 @@ impl Module { } /// Recursively finds all declarations with an annotation that has a specific name. - pub fn find_by_annotation_name(&self, annotation_name: &Ident) -> Vec { + pub fn find_by_annotation_name(&self, annotation_name: &pr::Ident) -> Vec { let mut res = Vec::new(); for (name, decl) in &self.names { @@ -404,14 +191,14 @@ impl Module { let has_annotation = decl_has_annotation(decl, annotation_name); if has_annotation { - res.push(Ident::from_name(name)); + res.push(pr::Ident::from_name(name)); } } res } } -fn decl_has_annotation(decl: &Decl, annotation_name: &Ident) -> bool { +fn decl_has_annotation(decl: &Decl, annotation_name: &pr::Ident) -> bool { for ann in &decl.annotations { if super::is_ident_or_func_call(&ann.expr, annotation_name) { return true; @@ -423,33 +210,9 @@ fn decl_has_annotation(decl: &Decl, annotation_name: &Ident) -> bool { type HintAndSpan = (Option, Option); impl RootModule { - pub(super) fn declare( - &mut self, - ident: Ident, - decl: DeclKind, - id: Option, - annotations: Vec, - ) -> Result<()> { - let existing = self.module.get(&ident); - if existing.is_some() { - return Err(Error::new_simple(format!( - "duplicate declarations of {ident}" - ))); - } - - let decl = Decl { - kind: decl, - declared_at: id, - order: 0, - annotations, - }; - self.module.insert(ident, decl).unwrap(); - Ok(()) - } - /// Finds that main pipeline given a path to either main itself or its parent module. /// Returns main expr and fq ident of the decl. - pub fn find_main_rel(&self, path: &[String]) -> Result<(&TableExpr, Ident), HintAndSpan> { + pub fn find_main_rel(&self, path: &[String]) -> Result<(&pl::Expr, pr::Ident), HintAndSpan> { let (decl, ident) = self.find_main(path).map_err(|x| (x, None))?; let span = decl @@ -457,18 +220,19 @@ impl RootModule { .and_then(|id| self.span_map.get(&id)) .cloned(); - let decl = (decl.kind.as_table_decl()) + let decl = (decl.kind.as_expr()) + .filter(|e| e.ty.as_ref().unwrap().is_relation()) .ok_or((Some(format!("{ident} is not a relational variable")), span))?; - Ok((&decl.expr, ident)) + Ok((decl.as_ref(), ident)) } - pub fn find_main(&self, path: &[String]) -> Result<(&Decl, Ident), Option> { + pub fn find_main(&self, path: &[String]) -> Result<(&Decl, pr::Ident), Option> { let mut tried_idents = Vec::new(); // is path referencing the relational var directly? if !path.is_empty() { - let ident = Ident::from_path(path.to_vec()); + let ident = pr::Ident::from_path(path.to_vec()); let decl = self.module.get(&ident); if let Some(decl) = decl { @@ -483,7 +247,7 @@ impl RootModule { let mut path = path.to_vec(); path.push(NS_MAIN.to_string()); - let ident = Ident::from_path(path); + let ident = pr::Ident::from_path(path); let decl = self.module.get(&ident); if let Some(decl) = decl { @@ -499,8 +263,8 @@ impl RootModule { ))) } - pub fn find_query_def(&self, main: &Ident) -> Option<&QueryDef> { - let ident = Ident { + pub fn find_query_def(&self, main: &pr::Ident) -> Option<&pr::QueryDef> { + let ident = pr::Ident { path: main.path.clone(), name: NS_QUERY_DEF.to_string(), }; @@ -510,62 +274,28 @@ impl RootModule { } /// Finds all main pipelines. - pub fn find_mains(&self) -> Vec { + pub fn find_mains(&self) -> Vec { self.module.find_by_suffix(NS_MAIN) } /// Finds declarations that are annotated with a specific name. - pub fn find_by_annotation_name(&self, annotation_name: &Ident) -> Vec { + pub fn find_by_annotation_name(&self, annotation_name: &pr::Ident) -> Vec { self.module.find_by_annotation_name(annotation_name) } } -pub fn ty_of_lineage(lineage: &Lineage) -> Ty { - Ty::relation( - lineage - .columns - .iter() - .map(|col| match col { - LineageColumn::All { .. } => TyTupleField::Wildcard(None), - LineageColumn::Single { name, .. } => TyTupleField::Single( - name.as_ref().map(|i| i.name.clone()), - Some(Ty::new(Literal::Null)), - ), - }) - .collect(), - ) -} - #[cfg(test)] mod tests { use super::*; - use crate::ir::pl::ExprKind; - - // TODO: tests / docstrings for `stack_pop` & `stack_push` & `insert_frame` - #[test] - fn test_module() { - let mut module = Module::default(); - - let ident = Ident::from_name("test_name"); - let expr: Expr = Expr::new(ExprKind::Literal(Literal::Integer(42))); - let decl: Decl = DeclKind::Expr(Box::new(expr)).into(); - - assert!(module.insert(ident.clone(), decl.clone()).is_ok()); - assert_eq!(module.get(&ident).unwrap(), &decl); - assert_eq!(module.get_mut(&ident).unwrap(), &decl); - - // Lookup - let lookup_result = module.lookup(&ident); - assert_eq!(lookup_result.len(), 1); - assert!(lookup_result.contains(&ident)); - } + use crate::ir::pl; + use crate::pr::Literal; #[test] fn test_module_shadow_unshadow() { let mut module = Module::default(); - let ident = Ident::from_name("test_name"); - let expr: Expr = Expr::new(ExprKind::Literal(Literal::Integer(42))); + let ident = pr::Ident::from_name("test_name"); + let expr: pl::Expr = pl::Expr::new(pl::ExprKind::Literal(Literal::Integer(42))); let decl: Decl = DeclKind::Expr(Box::new(expr)).into(); module.insert(ident.clone(), decl.clone()).unwrap(); diff --git a/prqlc/prqlc/src/semantic/reporting.rs b/prqlc/prqlc/src/semantic/reporting.rs index 472bb883c179..f544a890db2c 100644 --- a/prqlc/prqlc/src/semantic/reporting.rs +++ b/prqlc/prqlc/src/semantic/reporting.rs @@ -5,10 +5,9 @@ use ariadne::{Color, Label, Report, ReportBuilder, ReportKind, Source}; use schemars::JsonSchema; use serde::Serialize; -use crate::ir::decl::{DeclKind, Module, RootModule, TableDecl, TableExpr}; -use crate::ir::pl; -use crate::ir::pl::PlFold; use crate::pr; +use crate::ir::decl::{DeclKind, Module, RootModule}; +use crate::ir::pl::{*, self}; use crate::{Result, Span}; pub fn label_references(root_mod: &RootModule, source_id: String, source: String) -> Vec { @@ -44,11 +43,7 @@ struct Labeler<'a> { impl<'a> Labeler<'a> { fn label_module(&mut self, module: &Module) { for (_, decl) in module.names.iter() { - if let DeclKind::TableDecl(TableDecl { - expr: TableExpr::RelationVar(expr), - .. - }) = &decl.kind - { + if let DeclKind::Expr(expr) = &decl.kind { self.fold_expr(*expr.clone()).unwrap(); } } @@ -79,18 +74,18 @@ impl<'a> pl::PlFold for Labeler<'a> { let color = match &decl.kind { DeclKind::Expr(_) => Color::Blue, DeclKind::Ty(_) => Color::Green, - DeclKind::Column { .. } => Color::Yellow, - DeclKind::InstanceOf(_, _) => Color::Yellow, - DeclKind::TableDecl { .. } => Color::Red, + DeclKind::GenericParam(_) => Color::Green, + DeclKind::Variable { .. } => Color::Yellow, + DeclKind::TupleField => Color::Yellow, DeclKind::Module(module) => { self.label_module(module); Color::Cyan } - DeclKind::LayeredModules(_) => Color::Cyan, DeclKind::Infer(_) => Color::White, DeclKind::QueryDef(_) => Color::White, DeclKind::Import(_) => Color::White, + DeclKind::Unresolved(_) => Color::White, }; let location = decl @@ -98,16 +93,7 @@ impl<'a> pl::PlFold for Labeler<'a> { .and_then(|id| self.get_span_lines(id)) .unwrap_or_default(); - let decl = match &decl.kind { - DeclKind::TableDecl(TableDecl { ty, .. }) => { - format!( - "table {}", - ty.as_ref().and_then(|t| t.name.clone()).unwrap_or_default() - ) - } - _ => decl.to_string(), - }; - + let decl = decl.to_string(); (format!("{decl}{location}"), color) } else if let Some(decl_id) = node.target_id { let lines = self.get_span_lines(decl_id).unwrap_or_default(); @@ -201,7 +187,7 @@ pub struct FrameCollector { /// Each transformation step in the main pipeline corresponds to a single /// frame. This holds the output columns at each frame, as well as the span /// position of the frame. - pub frames: Vec<(Option, pl::Lineage)>, + pub frames: Vec<(Option, pr::Ty)>, /// A mapping of expression graph node IDs to their node definitions. pub nodes: Vec, @@ -321,10 +307,9 @@ impl PlFold for FrameCollector { self.nodes.sort_by(|a, b| a.id.cmp(&b.id)); self.nodes.dedup(); - if matches!(expr.kind, pl::ExprKind::TransformCall(_)) { - let lineage = expr.lineage.clone(); - if let Some(lineage) = lineage { - self.frames.push((expr.span, lineage)); + if matches!(expr.kind, ExprKind::TransformCall(_)) { + if let Some(ty) = &expr.ty { + self.frames.push((expr.span, ty.clone())); } } diff --git a/prqlc/prqlc/src/semantic/resolve_decls/init_modules.rs b/prqlc/prqlc/src/semantic/resolve_decls/init_modules.rs new file mode 100644 index 000000000000..c2f815b82bd8 --- /dev/null +++ b/prqlc/prqlc/src/semantic/resolve_decls/init_modules.rs @@ -0,0 +1,71 @@ +use std::collections::HashMap; + +use crate::ir::decl; +use crate::ir::pl; +use crate::utils::IdGenerator; +use crate::Span; + +pub fn init_module_tree(root_module_def: pl::ModuleDef) -> decl::RootModule { + let mut root = decl::Module::new_root(); + + let mut ctx = Context { + span_map: Default::default(), + id: IdGenerator::new(), + }; + + ctx.populate_module(&mut root, root_module_def.stmts); + + decl::RootModule { + module: root, + span_map: ctx.span_map, + } +} + +struct Context { + span_map: HashMap, + id: IdGenerator, +} + +impl Context { + fn populate_module(&mut self, module: &mut decl::Module, stmts: Vec) { + for (index, stmt) in stmts.into_iter().enumerate() { + let id = self.id.gen(); + if let Some(span) = stmt.span { + self.span_map.insert(id, span); + } + + let name = stmt.name().to_string(); + + let kind = match stmt.kind { + pl::StmtKind::ModuleDef(module_def) => { + // init new module and recurse + let mut new_mod = decl::Module::default(); + self.populate_module(&mut new_mod, module_def.stmts); + + decl::DeclKind::Module(new_mod) + } + mut kind => { + // insert "DeclKind::Unresolved" + + // hack: add type annotation to `main` var defs + if let pl::StmtKind::VarDef(def) = &mut kind { + if def.name == "main" && def.ty.is_none() { + // def.ty = Some(crate::ast::Ty::new(crate::ast::TyKind::Ident(crate::ast::Ident::from_path( + // vec!["std", "relation"], + // )))); + } + } + + decl::DeclKind::Unresolved(kind) + } + }; + let decl = decl::Decl { + declared_at: Some(id), + kind, + order: index + 1, + annotations: stmt.annotations, + }; + module.names.insert(name, decl); + } + } +} diff --git a/prqlc/prqlc/src/semantic/resolve_decls/mod.rs b/prqlc/prqlc/src/semantic/resolve_decls/mod.rs new file mode 100644 index 000000000000..585037bc7848 --- /dev/null +++ b/prqlc/prqlc/src/semantic/resolve_decls/mod.rs @@ -0,0 +1,5 @@ +mod init_modules; +mod names; + +pub use init_modules::init_module_tree; +pub use names::resolve_decl_refs; diff --git a/prqlc/prqlc/src/semantic/resolve_decls/names.rs b/prqlc/prqlc/src/semantic/resolve_decls/names.rs new file mode 100644 index 000000000000..efe9ef6de60c --- /dev/null +++ b/prqlc/prqlc/src/semantic/resolve_decls/names.rs @@ -0,0 +1,457 @@ +use itertools::Itertools; + +use crate::ir::decl::{self, Decl, DeclKind, InferTarget}; +use crate::ir::pl::{self, PlFold}; +use crate::semantic::{NS_DEFAULT_DB, NS_GENERIC, NS_INFER, NS_LOCAL, NS_STD, NS_THIS}; +use crate::utils::IdGenerator; +use crate::{pr, utils}; +use crate::{Error, Result, WithErrorInfo}; + +/// Runs name resolution for global names - names that refer to declarations. +/// +/// Keeps track of all inter-declaration references. +/// Returns a resolution order. +pub fn resolve_decl_refs(root: &mut decl::RootModule) -> Result> { + // resolve inter-declaration references + let refs = { + let mut r = ModuleRefResolver { + root, + generic_name: IdGenerator::new(), + refs: Default::default(), + current_path: Vec::new(), + }; + r.resolve_refs()?; + r.refs + }; + + // HACK: put std.* declarations first + // this is needed because during compilation of transforms, we inject refs to "std.lte" and a few others + // sorting here makes std decls appear first in the final ordering + let mut refs = refs; + refs.sort_by_key(|(a, _)| !a.path.first().map_or(false, |p| p == "std")); + + // toposort the declarations + // TODO: we might not need to compile all declarations if they are not used + // to prevent that, this start should be something else than None + // a list of all public declarations? + // let main = pl::Ident::from_name("main"); + let order = utils::toposort::(&refs, None); + + if let Some(order) = order { + Ok(order.into_iter().cloned().collect_vec()) + } else { + todo!("error for a cyclic references between expressions") + } +} + +/// Traverses module tree and runs name resolution on each of the declarations. +/// Collects references of each declaration. +struct ModuleRefResolver<'a> { + root: &'a mut decl::RootModule, + generic_name: IdGenerator, + current_path: Vec, + + // TODO: maybe make these ids, instead of Ident? + refs: Vec<(pr::Ident, Vec)>, +} + +impl ModuleRefResolver<'_> { + fn resolve_refs(&mut self) -> Result<()> { + let path = &mut self.current_path; + let module = self.root.module.get_submodule_mut(path).unwrap(); + + let mut submodules = Vec::new(); + let mut unresolved_decls = Vec::new(); + for (name, decl) in &module.names { + match &decl.kind { + decl::DeclKind::Module(_) => { + submodules.push(name.clone()); + } + decl::DeclKind::Unresolved(_) => { + unresolved_decls.push(name.clone()); + } + _ => {} + } + } + + for name in unresolved_decls { + // take the decl out of the module tree + let mut decl = { + let submodule = self.root.module.get_submodule_mut(path).unwrap(); + submodule.names.remove(&name).unwrap() + }; + let span = decl + .declared_at + .and_then(|x| self.root.span_map.get(&x)) + .cloned(); + + // resolve the decl + path.push(name); + let mut r = NameResolver { + root: self.root, + generic_name: &mut self.generic_name, + decl_module_path: &path[0..(path.len() - 1)], + refs: Vec::new(), + }; + + let stmt = decl.kind.into_unresolved().unwrap(); + let stmt = r.fold_stmt_kind(stmt).with_span_fallback(span)?; + decl.kind = decl::DeclKind::Unresolved(stmt); + + let decl_ident = pl::Ident::from_path(path.clone()); + self.refs.push((decl_ident, r.refs)); + + let name = path.pop().unwrap(); + + // put the decl back in + { + let submodule = self.root.module.get_submodule_mut(path).unwrap(); + submodule.names.insert(name, decl); + }; + } + + for name in submodules { + self.current_path.push(name); + self.resolve_refs()?; + self.current_path.pop(); + } + Ok(()) + } +} + +/// Traverses AST and resolves all global (non-local) identifiers. +struct NameResolver<'a> { + root: &'a mut decl::RootModule, + generic_name: &'a mut IdGenerator, + decl_module_path: &'a [String], + refs: Vec, +} + +impl NameResolver<'_> { + fn fold_stmt_kind(&mut self, stmt: pl::StmtKind) -> Result { + Ok(match stmt { + pl::StmtKind::QueryDef(_) => stmt, + pl::StmtKind::VarDef(var_def) => pl::StmtKind::VarDef(self.fold_var_def(var_def)?), + pl::StmtKind::TypeDef(ty_def) => pl::StmtKind::TypeDef(self.fold_type_def(ty_def)?), + pl::StmtKind::ImportDef(import_def) => { + pl::StmtKind::ImportDef(self.fold_import_def(import_def)?) + } + pl::StmtKind::ModuleDef(_) => unreachable!(), + }) + } + + fn fold_import_def(&mut self, import_def: pl::ImportDef) -> Result { + let (fq_ident, indirections) = self.resolve_ident(import_def.name)?; + if !indirections.is_empty() { + return Err(Error::new_simple( + "Import can only reference modules and declarations", + )); + } + if fq_ident.is_empty() { + log::debug!("resolved type ident to : {fq_ident:?} + {indirections:?}"); + return Err(Error::new_simple("invalid type name")); + } + Ok(pl::ImportDef { + name: pr::Ident::from_path(fq_ident), + alias: import_def.alias, + }) + } +} + +impl pl::PlFold for NameResolver<'_> { + fn fold_expr(&mut self, expr: pl::Expr) -> Result { + // Convert indirections into ident, since the algo below works with + // full idents and not indirections. + // We could change that to work with indirections, but then we'd + // need to change how idents in types and imports are resolved. + let expr = push_indirections_into_ident(expr); + + Ok(match expr.kind { + pl::ExprKind::Ident(ident) => { + let (ident, indirections) = self.resolve_ident(ident).with_span(expr.span)?; + // TODO: can this ident have length 0? + + let mut kind = pl::ExprKind::Ident(pr::Ident::from_path(ident)); + for indirection in indirections { + let mut e = pl::Expr::new(kind); + e.span = expr.span; + kind = pl::ExprKind::Indirection { + base: Box::new(e), + field: pl::IndirectionKind::Name(indirection), + }; + } + + pl::Expr { kind, ..expr } + } + _ => pl::Expr { + kind: pl::fold_expr_kind(self, expr.kind)?, + ..expr + }, + }) + } + + fn fold_type(&mut self, ty: pr::Ty) -> Result { + Ok(match ty.kind { + pr::TyKind::Ident(ident) => { + let (ident, indirections) = self.resolve_ident(ident).with_span(ty.span)?; + + if !indirections.is_empty() { + log::debug!("resolved type ident to : {ident:?} + {indirections:?}"); + return Err( + Error::new_simple("types are not allowed indirections").with_span(ty.span) + ); + } + + if ident.is_empty() { + log::debug!("resolved type ident to : {ident:?} + {indirections:?}"); + return Err(Error::new_simple("invalid type name").with_span(ty.span)); + } + + pr::Ty { + kind: pr::TyKind::Ident(pr::Ident::from_path(ident)), + ..ty + } + } + _ => pl::fold_type(self, ty)?, + }) + } +} + +/// Converts `Indirection { base: Ident(x), field: y }` into `Ident(x.y)`. +fn push_indirections_into_ident(mut expr: pl::Expr) -> pl::Expr { + let mut indirections = Vec::new(); + while let pl::ExprKind::Indirection { + base, + field: pl::IndirectionKind::Name(name), + } = expr.kind + { + indirections.push((name, expr.span, expr.alias, expr.flatten)); + expr = *base; + } + + if let pl::ExprKind::Ident(ident) = &mut expr.kind { + for (part, span, alias, flatten) in indirections.into_iter().rev() { + ident.push(part); + expr.span = pr::Span::merge_opt(expr.span, span); + expr.alias = alias.or(expr.alias); + expr.flatten = flatten; + } + } else { + // this is not on an ident - we have to revert it + for (name, span, alias, flatten) in indirections { + expr = pl::Expr::new(pl::ExprKind::Indirection { + base: Box::new(expr), + field: pl::IndirectionKind::Name(name), + }); + expr.span = span; + expr.alias = alias; + expr.flatten = flatten; + } + } + expr +} + +impl NameResolver<'_> { + /// Returns resolved fully-qualified ident and a list of indirections + fn resolve_ident(&mut self, mut ident: pr::Ident) -> Result<(Vec, Vec)> { + // this is the name we are looking for + let first = ident.iter().next().unwrap(); + let mod_path = match first.as_str() { + "project" => Some(vec![]), + "module" => Some(self.decl_module_path.to_vec()), + "super" => { + let mut path = self.decl_module_path.to_vec(); + path.pop(); + Some(path) + } + + NS_STD => Some(vec![NS_STD.to_string()]), + NS_DEFAULT_DB => Some(vec![NS_DEFAULT_DB.to_string()]), + NS_THIS => Some(vec![NS_LOCAL.to_string(), NS_THIS.to_string()]), + "prql" => Some(vec![NS_STD.to_string(), "prql".to_string()]), + + // transforms + "from" | + "select" | + "filter" | + "derive" | + "aggregate" | + "sort" | + "take" | + "join" | + "group" | + "window" | + "append" | + "intersect" | + "remove" | + "loop" | + // agg + "min" | + "max" | + "sum" | + "average" | + "stddev" | + "all" | + "any" | + "concat_array" | + "count" | + "count_distinct" | + "lag" | + "lead" | + "first" | + "last" | + "rank" | + "rank_dense" | + "row_number" | + // utils + "in" | + "as" => { + ident = ident.prepend(vec![NS_STD.to_string()]); + Some(vec![NS_STD.to_string()]) + } + + _ => None, + }; + let mod_decl = mod_path + .as_ref() + .and_then(|p| self.root.module.get_submodule_mut(p)); + + // let decl = find_lookup_base(&self.root.module, self.decl_module_path, name); + Ok(if let Some(module) = mod_decl { + let mod_path = mod_path.unwrap(); + // module found + + // now find the decl within that module + if let Some(ident_within) = ident.pop_front().1 { + let mut module_lookup = ModuleLookup::new(self.generic_name); + + let (path, indirections) = module_lookup.run(module, ident_within)?; + + // prepend the ident with the module path + // this will make this ident a fully-qualified ident + let mut fq_ident = mod_path; + fq_ident.extend(path); + + self.refs.push(pr::Ident::from_path(fq_ident.clone())); + + module_lookup.finish(self.root); + (fq_ident, indirections) + } else { + // there is no inner ident - we return the fq path to the module + (mod_path, vec![]) + } + } else { + // cannot find module, so this must be a ref to a local var + indirections + let mut steps = ident.into_iter(); + let first = steps.next().unwrap(); + let indirections = steps.collect_vec(); + (vec![NS_LOCAL.to_string(), first], indirections) + }) + } +} + +struct ModuleLookup<'a> { + generic_name: &'a mut IdGenerator, + + generated_generics: Vec<(String, Decl)>, +} + +impl<'a> ModuleLookup<'a> { + fn new(generic_name: &'a mut IdGenerator) -> Self { + ModuleLookup { + generic_name, + generated_generics: Vec::new(), + } + } + + fn run( + &mut self, + module: &mut decl::Module, + ident_within: pr::Ident, + ) -> Result<(Vec, Vec)> { + let mut steps = ident_within.into_iter().collect_vec(); + + let mut module = module; + for i in 0..steps.len() { + let is_last = i == steps.len() - 1; + + let decl = self.run_step(module, &steps[i], is_last)?; + if let decl::DeclKind::Module(inner) = &mut decl.kind { + module = inner; + continue; + } else { + // we've found a declaration that is not a module: + // this and preceding steps are identifier, steps following are indirections + let indirections = steps.drain((i + 1)..).collect_vec(); + return Ok((steps, indirections)); + } + } + + Err(Error::new_simple("direct references modules not allowed")) + } + + fn run_step<'m>( + &mut self, + module: &'m mut decl::Module, + step: &str, + is_last: bool, + ) -> Result<&'m mut decl::Decl> { + if module.names.contains_key(step) { + return Ok(module.names.get_mut(step).unwrap()); + } + + let infer_decl = module.names.get(NS_INFER); + let can_infer_tables = infer_decl + .and_then(|i| i.kind.as_infer()) + .map_or(false, |i| matches!(i, InferTarget::Table)); + if !can_infer_tables { + return Err(Error::new_simple(format!("`{}` does not exist", step))); + } + + let decl = if is_last { + // infer a table + + // generate a new global generic type argument + let ident = self.init_new_global_generic(); + + // prepare the table type + let generic_param = pr::Ty::new(pr::TyKind::Ident(ident)); + let relation = pr::Ty::relation(vec![pr::TyTupleField::Unpack(Some(generic_param))]); + + // create the table decl + decl::Decl::from(decl::DeclKind::Expr(Box::new(pl::Expr { + ty: Some(relation), + ..pl::Expr::new(pl::ExprKind::Param("".to_string())) + }))) + } else { + // infer a database module + Decl::from(DeclKind::Module(decl::Module::new_database())) + }; + + module.names.insert(step.to_string(), decl); + Ok(module.names.get_mut(step).unwrap()) + } + + fn init_new_global_generic(&mut self) -> pr::Ident { + let a_unique_number = self.generic_name.gen(); + let param_name = format!("T{a_unique_number}"); + let ident = pr::Ident::from_path(vec![NS_GENERIC, ¶m_name]); + let decl = Decl::from(DeclKind::GenericParam(None)); + + self.generated_generics.push((param_name, decl)); + ident + } + + fn finish(self, root: &mut decl::RootModule) { + let generic_mod = root + .module + .names + .entry(NS_GENERIC.to_string()) + .or_insert_with(|| decl::Decl::from(decl::DeclKind::Module(decl::Module::default()))); + let generic_mod = generic_mod.kind.as_module_mut().unwrap(); + + for (name, decl) in self.generated_generics { + generic_mod.names.insert(name, decl); + } + } +} diff --git a/prqlc/prqlc/src/semantic/resolver/expr.rs b/prqlc/prqlc/src/semantic/resolver/expr.rs index 6377b192ebed..f79dcdf0dac9 100644 --- a/prqlc/prqlc/src/semantic/resolver/expr.rs +++ b/prqlc/prqlc/src/semantic/resolver/expr.rs @@ -1,58 +1,30 @@ use itertools::Itertools; -use crate::ir::decl::{DeclKind, Module}; -use crate::ir::pl; -use crate::ir::pl::PlFold; -use crate::pr::{Ty, TyKind, TyTupleField}; -use crate::semantic::resolver::{flatten, types, Resolver}; -use crate::semantic::{NS_INFER, NS_SELF, NS_THAT, NS_THIS}; -use crate::utils::IdGenerator; -use crate::Result; -use crate::{Error, Reason, Span, WithErrorInfo}; - -impl pl::PlFold for Resolver<'_> { - fn fold_stmts(&mut self, _: Vec) -> Result> { +use crate::pr::{Ty, TyKind}; +use crate::ir::decl::DeclKind; +use crate::ir::pl::{*, self}; +use crate::semantic::resolver::scope::LookupResult; +use crate::semantic::{NS_LOCAL, NS_STD, NS_THIS}; +use crate::{Error, Result, Span, WithErrorInfo}; + +use super::tuple::StepOwned; + +impl PlFold for super::Resolver<'_> { + fn fold_stmts(&mut self, _: Vec) -> Result> { unreachable!() } fn fold_type(&mut self, ty: Ty) -> Result { - Ok(match ty.kind { - TyKind::Ident(ident) => { - self.root_mod.module.shadow(NS_THIS); - self.root_mod.module.shadow(NS_THAT); - - let fq_ident = self.resolve_ident(&ident)?; - - let decl = self.root_mod.module.get(&fq_ident).unwrap(); - let decl_ty = decl.kind.as_ty().ok_or_else(|| { - Error::new(Reason::Expected { - who: None, - expected: "a type".to_string(), - found: decl.to_string(), - }) - })?; - let mut ty = decl_ty.clone(); - ty.name = ty.name.or(Some(fq_ident.name)); - - self.root_mod.module.unshadow(NS_THIS); - self.root_mod.module.unshadow(NS_THAT); - - ty - } - _ => pl::fold_type(self, ty)?, - }) + self.fold_type_actual(ty) } - fn fold_var_def(&mut self, var_def: pl::VarDef) -> Result { - let value = match var_def.value { - Some(value) if matches!(value.kind, pl::ExprKind::Func(_)) => Some(value), - Some(value) => Some(Box::new(flatten::Flattener::fold(self.fold_expr(*value)?))), - None => None, - }; - - Ok(pl::VarDef { + fn fold_var_def(&mut self, var_def: VarDef) -> Result { + Ok(VarDef { name: var_def.name, - value, + value: match var_def.value { + Some(value) => Some(Box::new(self.fold_expr(*value)?)), + None => None, + }, ty: var_def.ty.map(|x| self.fold_type(x)).transpose()?, }) } @@ -70,82 +42,107 @@ impl pl::PlFold for Resolver<'_> { self.root_mod.span_map.insert(id, span); } - log::trace!("folding expr [{id:?}] {node:?}"); + log::trace!("folding expr {node:?}"); let r = match node.kind { - pl::ExprKind::Ident(ident) => { - log::debug!("resolving ident {ident}..."); - let fq_ident = self.resolve_ident(&ident).with_span(node.span)?; - log::debug!("... resolved to {fq_ident}"); - let entry = self.root_mod.module.get(&fq_ident).unwrap(); - log::debug!("... which is {entry}"); - - match &entry.kind { - DeclKind::Infer(_) => pl::Expr { - kind: pl::ExprKind::Ident(fq_ident), - target_id: entry.declared_at, - ..node - }, - DeclKind::Column(target_id) => pl::Expr { - kind: pl::ExprKind::Ident(fq_ident), - target_id: Some(*target_id), - ..node - }, - - DeclKind::TableDecl(_) => { - let input_name = ident.name.clone(); - - let lineage = self.lineage_of_table_decl(&fq_ident, input_name, id); - - pl::Expr { - kind: pl::ExprKind::Ident(fq_ident), - ty: Some(ty_of_lineage(&lineage)), - lineage: Some(lineage), - alias: None, + ExprKind::Ident(ident) => { + log::debug!("resolving ident {ident:?}..."); + + let result = self.lookup_ident(&ident).with_span(node.span)?; + + let (ident, indirections) = match result { + LookupResult::Direct => (ident, vec![]), + LookupResult::Indirect { + real_name, + indirections, + } => { + let mut ident = ident; + ident.name = real_name; + (ident, indirections) + } + }; + + let mut expr = { + let decl = self.get_ident(&ident).unwrap(); + + let log_debug = !ident.starts_with_part(NS_STD); + if log_debug { + log::debug!("... resolved to {decl}"); + } + + match &decl.kind { + DeclKind::Variable(ty) => Expr { + kind: ExprKind::Ident(ident), + ty: ty.clone(), ..node + }, + + DeclKind::TupleField => { + unimplemented!(); + // indirections.push(IndirectionKind::Name(ident.name)); + // Expr::new(ExprKind::Ident(Ident::from_path(ident.path))) } - } - DeclKind::Expr(expr) => match &expr.kind { - pl::ExprKind::Func(closure) => { - let closure = self.fold_function_types(closure.clone(), id)?; + DeclKind::Expr(expr) => { + // keep as ident, but pull in the type + let ty = expr.ty.clone().unwrap(); - let expr = pl::Expr::new(pl::ExprKind::Func(closure)); + // if the type contains generics, we need to instantiate those + // generics into current function scope + let ty = self.instantiate_type(ty, id); - if self.in_func_call_name { - expr - } else { - self.fold_expr(expr)? + Expr { + kind: ExprKind::Ident(ident), + ty: Some(ty), + ..node } } - _ => self.fold_expr(expr.as_ref().clone())?, - }, - DeclKind::InstanceOf(_, ty) => { - let ty = ty.clone(); + DeclKind::Ty(_) => { + return Err(Error::new_simple("expected a value, but found a type") + .with_span(*span)); + } - let fields = self.construct_wildcard_include(&fq_ident); + DeclKind::Infer(_) => unreachable!(), + DeclKind::Unresolved(_) => { + return Err(Error::new_assert(format!( + "bad resolution order: unresolved {ident} while resolving {}", + self.debug_current_decl + ))); + } - pl::Expr { - kind: pl::ExprKind::Tuple(fields), - ty, + _ => Expr { + kind: ExprKind::Ident(ident), ..node - } + }, } + }; - DeclKind::Ty(_) => { - return Err(Error::new(Reason::Expected { - who: None, - expected: "a value".to_string(), - found: "a type".to_string(), - }) - .with_span(*span)); - } + expr.id = expr.id.or(Some(id)); + let flatten = expr.flatten; + expr.flatten = false; + let alias = expr.alias.take(); + + let mut expr = self.apply_indirections(expr, indirections); + + expr.flatten = flatten; + expr.alias = alias; + expr + } + + ExprKind::Indirection { base, field } => { + let base = self.fold_expr(*base)?; - _ => pl::Expr { - kind: pl::ExprKind::Ident(fq_ident), - ..node - }, + let ty = base.ty.as_ref().unwrap(); + + let steps = self.resolve_indirection(ty, &field).with_span(*span)?; + + let expr = self.apply_indirections(base, steps); + Expr { + id: expr.id, + kind: expr.kind, + ty: expr.ty, + ..node } } @@ -163,20 +160,23 @@ impl pl::PlFold for Resolver<'_> { named_args, }) => { // fold function name - self.default_namespace = None; let old = self.in_func_call_name; self.in_func_call_name = true; - let name = Box::new(self.fold_expr(*name)?); + let func = Box::new(self.fold_expr(*name)?); self.in_func_call_name = old; - let func = name.try_cast(|n| n.into_func(), None, "a function")?; - - // fold function - let func = self.apply_args_to_closure(func, args, named_args)?; - self.fold_function(func, id, *span)? + // convert to function application + let fn_app = self.apply_args_to_function(func, args, named_args)?; + self.resolve_func_application(fn_app, *span)? } - pl::ExprKind::Func(closure) => self.fold_function(closure, id, *span)?, + ExprKind::Func(func) => { + let func = self.resolve_func(func)?; + Expr { + kind: ExprKind::Func(func), + ..node + } + } pl::ExprKind::Tuple(exprs) => { let exprs = self.fold_exprs(exprs)?; @@ -205,7 +205,7 @@ impl pl::PlFold for Resolver<'_> { } } -impl Resolver<'_> { +impl super::Resolver<'_> { fn finish_expr_resolve( &mut self, expr: pl::Expr, @@ -220,30 +220,39 @@ impl Resolver<'_> { r.span = r.span.or(span); if r.ty.is_none() { - r.ty = Resolver::infer_type(&r)?; + r.ty = self.infer_type(&r)?; } - if r.lineage.is_none() { - if let pl::ExprKind::TransformCall(call) = &r.kind { - r.lineage = Some(call.infer_lineage()?); - } else if let Some(relation_columns) = r.ty.as_ref().and_then(|t| t.as_relation()) { - // lineage from ty - let columns = Some(relation_columns.clone()); - - let name = r.alias.clone(); - let frame = self.declare_table_for_literal(id, columns, name); - - r.lineage = Some(frame); - } + if r.ty.is_none() { + let generic = self.init_new_global_generic("E"); + r.ty = Some(Ty::new(TyKind::Ident(generic))); } - if let Some(lineage) = &mut r.lineage { - if let Some(alias) = r.alias.take() { - lineage.rename(alias.clone()); - - if let Some(ty) = &mut r.ty { - types::rename_relation(&mut ty.kind, alias); + if let Some(ty) = &mut r.ty { + if ty.is_relation() { + if let Some(alias) = r.alias.take() { + // This is relation wrapping operation. + // Convert: + // alias = r + // into: + // _local.select {alias = _local.this} r + + let expr = Expr::new(ExprKind::FuncCall(FuncCall { + name: Box::new(Expr::new(ExprKind::Ident(Ident::from_path(vec![ + NS_STD, "select", + ])))), + args: vec![ + Expr::new(ExprKind::Tuple(vec![Expr { + alias: Some(alias), + ..Expr::new(Ident::from_path(vec![NS_LOCAL, NS_THIS])) + }])), + *r, + ], + named_args: Default::default(), + })); + return self.fold_expr(expr); } } } + Ok(*r) } @@ -251,76 +260,31 @@ impl Resolver<'_> { let expr = self.fold_expr(expr)?; let except = self.coerce_into_tuple(expr)?; - self.fold_expr(pl::Expr::new(pl::ExprKind::All { - within: Box::new(pl::Expr::new(pl::Ident::from_name(NS_THIS))), + self.fold_expr(Expr::new(ExprKind::All { + within: Box::new(Expr::new(Ident::from_path(vec![NS_LOCAL, NS_THIS]))), except: Box::new(except), })) } - pub fn construct_wildcard_include(&mut self, module_fq_self: &pl::Ident) -> Vec { - let module_fq = module_fq_self.clone().pop().unwrap(); - - let decl = self.root_mod.module.get(&module_fq).unwrap(); - let module = decl.kind.as_module().unwrap(); - - let prefix = module_fq.iter().collect_vec(); - Self::construct_tuple_from_module(&mut self.id, &prefix, module) - } - - pub fn construct_tuple_from_module( - id: &mut IdGenerator, - prefix: &[&String], - module: &Module, - ) -> Vec { - let mut res = Vec::new(); - - if let Some(decl) = module.names.get(NS_INFER) { - let wildcard_field = pl::Expr { - id: Some(id.gen()), - target_id: decl.declared_at, - flatten: true, - ty: Some(Ty::new(TyKind::Tuple(vec![TyTupleField::Wildcard(None)]))), - ..pl::Expr::new(pl::Ident::from_name(NS_SELF)) - }; - return vec![wildcard_field]; - } - - for (name, decl) in module.names.iter().sorted_by_key(|(_, d)| d.order) { - res.push(match &decl.kind { - DeclKind::Module(submodule) => { - let prefix = [prefix.to_vec(), vec![name]].concat(); - let sub_fields = Self::construct_tuple_from_module(id, &prefix, submodule); - pl::Expr { - id: Some(id.gen()), - alias: Some(name.clone()), - ..pl::Expr::new(pl::ExprKind::Tuple(sub_fields)) - } - } - DeclKind::Column(target_id) => pl::Expr { - id: Some(id.gen()), - target_id: Some(*target_id), - // alias: Some(name.clone()), - ..pl::Expr::new(pl::Ident::from_path([prefix.to_vec(), vec![name]].concat())) - }, - _ => continue, - }); + /// Resolve tuple indirections. + /// For example, `base.indirection` where `base` has a tuple type. + /// + /// Returns the position of the tuple field within the base tuple. + pub fn resolve_indirection( + &mut self, + base: &Ty, + indirection: &IndirectionKind, + ) -> Result> { + match indirection { + IndirectionKind::Name(name) => self.lookup_name_in_tuple(base, name).and_then(|res| { + res.ok_or_else(|| Error::new_simple(format!("Unknown name {name}"))) + }), + IndirectionKind::Position(pos) => { + let step = super::tuple::lookup_position_in_tuple(base, *pos as usize)? + .ok_or_else(|| Error::new_simple("Out of bounds"))?; + + Ok(vec![step]) + } } - res } } - -fn ty_of_lineage(lineage: &pl::Lineage) -> Ty { - Ty::relation( - lineage - .columns - .iter() - .map(|col| match col { - pl::LineageColumn::All { .. } => TyTupleField::Wildcard(None), - pl::LineageColumn::Single { name, .. } => TyTupleField::Single( - name.as_ref().map(|i| i.name.clone()), - Some(Ty::new(pl::Literal::Null)), - ), - }) - .collect(), - ) -} diff --git a/prqlc/prqlc/src/semantic/resolver/functions.rs b/prqlc/prqlc/src/semantic/resolver/functions.rs index bbb632763c00..cb83149daf1f 100644 --- a/prqlc/prqlc/src/semantic/resolver/functions.rs +++ b/prqlc/prqlc/src/semantic/resolver/functions.rs @@ -1,396 +1,487 @@ +use itertools::Itertools; use std::collections::HashMap; -use std::iter::zip; -use itertools::{Itertools, Position}; +use crate::codegen::write_ty; +use crate::ir::decl::{Decl, DeclKind}; +use crate::ir::pl::*; +use crate::pr::{GenericTypeParam, Ty, TyFunc, TyKind, TyTupleField}; +use crate::semantic::{write_pl, NS_GENERIC, NS_LOCAL, NS_THAT, NS_THIS}; +use crate::{Error, Result, Span, WithErrorInfo}; +use super::scope::Scope; +use super::types::TypeReplacer; use super::Resolver; -use crate::ir::decl::{Decl, DeclKind, Module}; -use crate::ir::pl::*; -use crate::pr::{Ty, TyFunc, TyKind}; -use crate::semantic::resolver::types; -use crate::semantic::{NS_GENERIC, NS_PARAM, NS_THAT, NS_THIS}; -use crate::Result; -use crate::{Error, Span, WithErrorInfo}; impl Resolver<'_> { - pub fn fold_function( - &mut self, - closure: Box, - id: usize, - span: Option, - ) -> Result { - let closure = self.fold_function_types(closure, id)?; + /// Folds function types, so they are resolved to material types, ready for type checking. + /// Requires id of the function call node, so it can be used to generic type arguments. + pub fn resolve_func(&mut self, mut func: Box) -> Result> { + let mut scope = Scope::new(); - log::debug!( - "func {} {}/{} params", - closure.as_debug_name(), - closure.args.len(), - closure.params.len() - ); + // prepare generic arguments + for generic_param in &func.generic_type_params { + let bound: Option = generic_param + .bound + .clone() + .map(|b| self.fold_type(b)) + .transpose()?; - if closure.args.len() > closure.params.len() { - return Err(Error::new_simple(format!( - "Too many arguments to function `{}`", - closure.as_debug_name() - )) - .with_span(span)); + // register the generic type param in the resolver + let generic = Decl::from(DeclKind::GenericParam(bound.map(|b| (b, None)))); + scope.types.insert(generic_param.name.clone(), generic); } + self.scopes.push(scope); - let enough_args = closure.args.len() == closure.params.len(); - if !enough_args { - return Ok(*expr_of_func(closure, span)); - } + // fold types + func.params = func + .params + .into_iter() + .map(|p| -> Result<_> { + Ok(FuncParam { + ty: fold_type_opt(self, p.ty)?, + ..p + }) + }) + .try_collect()?; + func.return_ty = fold_type_opt(self, func.return_ty)?; - // make sure named args are pushed into params - let closure = if !closure.named_params.is_empty() { - self.apply_args_to_closure(closure, [].into(), [].into())? - } else { - closure - }; + // put params into scope + prepare_scope_of_func(self.scopes.last_mut().unwrap(), &func); - // push the env - let closure_env = Module::from_exprs(closure.env); - self.root_mod.module.stack_push(NS_PARAM, closure_env); - let closure = Box::new(Func { - env: HashMap::new(), - ..*closure - }); - - if log::log_enabled!(log::Level::Debug) { - let name = closure - .name_hint - .clone() - .unwrap_or_else(|| Ident::from_name("")); - log::debug!("resolving args of function {}", name); - } - let res = self.resolve_function_args(closure)?; + func.body = Box::new(self.fold_expr(*func.body)?); - let mut closure = match res { - Ok(func) => func, - Err(func) => { - return Ok(*expr_of_func(func, span)); - } - }; + // validate that the body has correct type + self.validate_expr_type(&mut func.body, func.return_ty.as_ref(), &|| None)?; - closure.return_ty = self.resolve_generic_args_opt(closure.return_ty)?; + // pop the scope + let mut scope = self.scopes.pop().unwrap(); - let needs_window = (closure.params.last()) - .and_then(|p| p.ty.as_ref()) - .map(types::is_sub_type_of_array) - .unwrap_or_default(); + // pop generic types + if !func.generic_type_params.is_empty() { + let mut new_generic_type_params = Vec::new(); + let mut finalized_args = HashMap::new(); + for gtp in func.generic_type_params { + let inferred_generic = scope.types.swap_remove(>p.name).unwrap(); + let inferred_type = inferred_generic.kind.into_generic_param().unwrap(); + + match inferred_type { + Some((inferred_type, _)) if !inferred_type.kind.is_tuple() => { + // The bounds of this generic type param restrict it to a single type. + // In other words: we have enough information to conclude that this param can only be one specific type. + // So we can finalize it to that type and inline any references to the param. + log::debug!("finalizing generic param {}", gtp.name); - // evaluate - let res = if let ExprKind::Internal(operator_name) = &closure.body.kind { - // special case: functions that have internal body - - if operator_name.starts_with("std.") { - Expr { - ty: closure.return_ty, - needs_window, - ..Expr::new(ExprKind::RqOperator { - name: operator_name.clone(), - args: closure.args, - }) + finalized_args.insert( + Ident::from_path(vec![NS_LOCAL, NS_GENERIC, >p.name]), + inferred_type, + ); + } + _ => { + let bound = inferred_type.map(|(t, _)| t); + new_generic_type_params.push(GenericTypeParam { + name: gtp.name, + bound, + }) + } } - } else { - let expr = self.resolve_special_func(closure, needs_window)?; - self.fold_expr(expr)? } - } else { - // base case: materialize - self.materialize_function(closure)? - }; + func.generic_type_params = new_generic_type_params; - // pop the env - self.root_mod.module.stack_pop(NS_PARAM).unwrap(); + func = Box::new(TypeReplacer::on_func(*func, finalized_args)); + } - Ok(Expr { span, ..res }) + Ok(func) } - #[allow(clippy::boxed_local)] - fn materialize_function(&mut self, closure: Box) -> Result { - log::debug!("stack_push for {}", closure.as_debug_name()); + pub fn apply_args_to_function( + &mut self, + func: Box, + args: Vec, + mut _named_args: HashMap, + ) -> Result { + let mut fn_app = if let ExprKind::FuncApplication(fn_app) = func.kind { + fn_app + } else { + FuncApplication { + func, + args: Vec::new(), + } + }; - let (func_env, body, return_ty) = env_of_closure(*closure); + // named + // let fn_ty = fn_app.func.ty.as_ref().unwrap(); + // let fn_ty = fn_ty.kind.as_function().unwrap(); + // let fn_ty = fn_ty.as_ref().unwrap().clone(); + // for mut param in fn_ty.named_params.drain(..) { + // let param_name = param.name.split('.').last().unwrap_or(¶m.name); + // let default = param.default_value.take().unwrap(); + // let arg = named_args.remove(param_name).unwrap_or(*default); + // fn_app.args.push(arg); + // fn_app.func.params.insert(fn_app.args.len() - 1, param); + // } + // if let Some((name, _)) = named_args.into_iter().next() { + // // TODO: report all remaining named_args as separate errors + // return Err(Error::new_simple(format!( + // "unknown named argument `{name}` to closure {:?}", + // fn_app.func.name_hint + // ))); + // } - self.root_mod.module.stack_push(NS_PARAM, func_env); + // positional + fn_app.args.extend(args); + Ok(fn_app) + } - // fold again, to resolve inner variables & functions - let body = self.fold_expr(body)?; + pub fn resolve_func_application( + &mut self, + fn_app: FuncApplication, + span: Option, + ) -> Result { + let metadata = self.gather_func_metadata(&fn_app.func); - // remove param decls - log::debug!("stack_pop: {:?}", body.id); - let func_env = self.root_mod.module.stack_pop(NS_PARAM).unwrap(); + let fn_ty = fn_app.func.ty.as_ref().unwrap(); + let fn_ty = fn_ty.kind.as_function().unwrap(); + let fn_ty = fn_ty.as_ref().unwrap().clone(); - Ok(if let ExprKind::Func(mut inner_closure) = body.kind { - // body couldn't been resolved - construct a closure to be evaluated later + log::debug!( + "func {} {}/{} params", + metadata.as_debug_name(), + fn_app.args.len(), + fn_ty.params.len() + ); - inner_closure.env = func_env.into_exprs(); + if fn_app.args.len() > fn_ty.params.len() { + return Err(Error::new_simple(format!( + "Too many arguments to function `{}`", + metadata.as_debug_name() + )) + .with_span(span)); + } - let (got, missing) = inner_closure.params.split_at(inner_closure.args.len()); - let missing = missing.to_vec(); - inner_closure.params = got.to_vec(); + let enough_args = fn_app.args.len() == fn_ty.params.len(); + if !enough_args { + return Ok(*expr_of_func_application( + fn_app, + fn_ty.return_ty.map(|x| *x), + span, + )); + } - Expr::new(ExprKind::Func(Box::new(Func { - name_hint: None, - args: vec![], - params: missing, - body: Box::new(Expr::new(ExprKind::Func(inner_closure))), + self.init_func_app_generic_args(&fn_ty, fn_app.func.id.unwrap()); - // these don't matter - named_params: Default::default(), - return_ty: Default::default(), - env: Default::default(), - generic_type_params: Default::default(), - }))) - } else { - // resolved, return result + log::debug!("resolving args of function {}", metadata.as_debug_name()); + let res = self.resolve_func_app_args(fn_app, &metadata)?; - // make sure to use the resolved type - let mut body = body; - if let Some(ret_ty) = return_ty.map(|x| *x) { - body.ty = Some(ret_ty.clone()); + let app = match res { + Ok(func) => func, + Err(func) => { + return Ok(Expr::new(ExprKind::Func(func))); } + }; - body - }) + self.finalize_func_app_generic_args(&fn_ty, app.func.id.unwrap()) + .with_span_fallback(span)?; + + // run fold again, so idents that used to point to generics get inlined + let return_ty = fn_ty + .return_ty + .clone() + .map(|ty| self.fold_type(*ty)) + .transpose()?; + + Ok(*expr_of_func_application(app, return_ty, span)) } - /// Folds function types, so they are resolved to material types, ready for type checking. - /// Requires id of the function call node, so it can be used to generic type arguments. - pub fn fold_function_types(&mut self, mut func: Box, id: usize) -> Result> { - // prepare generic arguments - for generic_param in &func.generic_type_params { - // fold the domain - let domain: Vec = generic_param - .domain - .iter() - .map(|t| self.fold_type(t.clone())) - .try_collect()?; + /// In PRQL, func is just an expression and does not have a name (the same way + /// as literals don't have a name). Regardless, we want to provide name hints for functions + /// in error messages (i.e. `std.count requires 2 arguments, found 1`), so here we infer name + /// and annotations for functions from its declaration. + fn gather_func_metadata(&self, func: &Expr) -> FuncMetadata { + let mut res = FuncMetadata::default(); - // register the generic type param in the resolver - let generic_id = (id, generic_param.name.clone()); - self.generics.insert(generic_id.clone(), domain); + let ExprKind::Ident(fq_ident) = &func.kind else { + return res; + }; + // let fq_ident = loop { + // match &func.kind { + // ExprKind::Ident(i) => break i, + // ExprKind::FuncApplication(FuncApplication { func: f, .. }) => { + // func = f.as_ref(); + // } + // _ => return res, + // } + // }; + + // populate name hint + res.name_hint = Some(fq_ident.clone()); + + let decl = self.root_mod.module.get(fq_ident).unwrap(); + + fn literal_as_u8(expr: Option<&Expr>) -> Option { + Some(*expr?.kind.as_literal()?.as_integer()? as u8) + } - // insert _generic.name declaration - let ident = Ident::from_path(vec![NS_GENERIC, generic_param.name.as_str()]); - let decl = Decl::from(DeclKind::Ty(Ty::new(TyKind::GenericArg(generic_id)))); - self.root_mod.module.insert(ident, decl).unwrap(); + // populate implicit_closure config + if let Some(im_clos) = decl + .annotations + .iter() + .find_map(|a| a.as_func_call("implicit_closure")) + { + res.implicit_closure = Some(Box::new(ImplicitClosureConfig { + param: literal_as_u8(im_clos.args.first()).unwrap(), + this: literal_as_u8(im_clos.named_args.get("this")), + that: literal_as_u8(im_clos.named_args.get("that")), + })); } - func.params = func - .params - .into_iter() - .map(|p| -> Result<_> { - Ok(FuncParam { - ty: fold_type_opt(self, p.ty)?, - ..p - }) - }) - .try_collect()?; - func.return_ty = fold_type_opt(self, func.return_ty)?; + // populate coerce_tuple config + if let Some(coerce_tuple) = decl + .annotations + .iter() + .find_map(|a| a.as_func_call("coerce_tuple")) + { + res.coerce_tuple = Some(literal_as_u8(coerce_tuple.args.first()).unwrap()); + } - self.root_mod.module.names.remove(NS_GENERIC); - Ok(func) + res } - pub fn apply_args_to_closure( - &mut self, - mut closure: Box, - args: Vec, - mut named_args: HashMap, - ) -> Result> { - // named arguments are consumed only by the first function - - // named - for mut param in closure.named_params.drain(..) { - let param_name = param.name.split('.').last().unwrap_or(¶m.name); - let default = param.default_value.take().unwrap(); + fn init_func_app_generic_args(&mut self, fn_ty: &TyFunc, func_id: usize) { + for generic_param in &fn_ty.generic_type_params { + // register the generic type param in the resolver + let generic_ident = Ident::from_path(vec![ + NS_GENERIC.to_string(), + func_id.to_string(), + generic_param.name.clone(), + ]); + + let candidate = generic_param.bound.clone().map(|mut b| { + if let TyKind::Tuple(fields) = &mut b.kind { + // bounds that are tuples mean "a tuple with at least these fields" + // so we need a global generic to track information about the other fields + + let generic = self.init_new_global_generic("A"); + let generic = Ty::new(TyKind::Ident(generic)); + fields.push(TyTupleField::Unpack(Some(generic))); + } - let arg = named_args.remove(param_name).unwrap_or(*default); + (b, None) + }); - closure.args.push(arg); - closure.params.insert(closure.args.len() - 1, param); - } - if let Some((name, _)) = named_args.into_iter().next() { - // TODO: report all remaining named_args as separate errors - return Err(Error::new_simple(format!( - "unknown named argument `{name}` to closure {:?}", - closure.name_hint - ))); + let generic = Decl::from(DeclKind::GenericParam(candidate)); + self.root_mod.module.insert(generic_ident, generic).unwrap(); } + } - // positional - closure.args.extend(args); - Ok(closure) + fn finalize_func_app_generic_args(&mut self, fn_ty: &TyFunc, func_id: usize) -> Result<()> { + for generic_param in &fn_ty.generic_type_params { + let ident = Ident::from_path(vec![ + NS_GENERIC.to_string(), + func_id.to_string(), + generic_param.name.clone(), + ]); + + let decl = self.root_mod.module.get_mut(&ident).unwrap(); + + let DeclKind::GenericParam(inferred_type) = &mut decl.kind else { + // this case means that we have already finalized this generic arg and should never happen + // hack: this case does happen, because our resolution order is all over the place, + // so I had to add "finalize_function_generic_args" into "resolve_function_arg". + // This only sorta makes sense, so I want to mark this case as "will remove in the future". + panic!() + }; + + let Some((ty, _span)) = inferred_type.take() else { + return Err(Error::new_simple(format!( + "cannot determine the type {}", + generic_param.name + ))); + }; + log::debug!("finalizing {ident} into {}", write_ty(&ty)); + decl.kind = DeclKind::Ty(ty); + } + Ok(()) } /// Resolves function arguments. Will return `Err(func)` is partial application is required. - fn resolve_function_args( + fn resolve_func_app_args( &mut self, - #[allow(clippy::boxed_local)] to_resolve: Box, - ) -> Result, Box>> { - let mut closure = Box::new(Func { + to_resolve: FuncApplication, + metadata: &FuncMetadata, + ) -> Result>> { + let mut app = FuncApplication { + func: to_resolve.func, args: vec![Expr::new(Literal::Null); to_resolve.args.len()], - ..*to_resolve - }); + }; let mut partial_application_position = None; - let func_name = &closure.name_hint; - - let (relations, other): (Vec<_>, Vec<_>) = zip(&closure.params, to_resolve.args) - .enumerate() - .partition(|(_, (param, _))| { - let is_relation = param - .ty - .as_ref() - .map(|t| t.is_relation()) - .unwrap_or_default(); + let func_name = &metadata.name_hint; - is_relation - }); - - let has_relations = !relations.is_empty(); - - // resolve relational args - if has_relations { - self.root_mod.module.shadow(NS_THIS); - self.root_mod.module.shadow(NS_THAT); + let func_ty = app.func.ty.as_ref().unwrap(); + let func_ty = func_ty.kind.as_function().unwrap(); + let func_ty = func_ty.as_ref().unwrap(); + let mut param_args = itertools::zip_eq(&func_ty.params, to_resolve.args) + .map(Box::new) + .map(Some) + .collect_vec(); - for (pos, (index, (param, mut arg))) in relations.into_iter().with_position() { - let is_last = matches!(pos, Position::Last | Position::Only); + // pull out this and that + let impl_cl_pos = metadata.implicit_closure.as_ref().map(|i| i.param as usize); + let this_pos = metadata.implicit_closure.as_ref().and_then(|i| i.this); + let that_pos = metadata.implicit_closure.as_ref().and_then(|i| i.that); - // just fold the argument alone - if partial_application_position.is_none() { - arg = self - .fold_and_type_check(arg, param, func_name)? - .unwrap_or_else(|a| { - partial_application_position = Some(index); - a - }); - } - log::debug!("resolved arg to {}", arg.kind.as_ref()); - - // add relation frame into scope - if partial_application_position.is_none() { - let frame = arg - .lineage - .as_ref() - .ok_or_else(|| Error::new_bug(4317).with_span(closure.body.span))?; - if is_last { - self.root_mod.module.insert_frame(frame, NS_THIS); - } else { - self.root_mod.module.insert_frame(frame, NS_THAT); - } - } + // prepare order + let order = this_pos + .into_iter() + .chain(that_pos) + .map(|x| x as usize) + .chain(0..param_args.len()) + .unique() + .collect_vec(); - closure.args[index] = arg; - } - } + for index in order { + let (param, mut arg) = *param_args[index].take().unwrap(); + let should_coerce_tuple = metadata.coerce_tuple.map_or(false, |i| i as usize == index); - // resolve other positional - for (index, (param, mut arg)) in other { if partial_application_position.is_none() { - if let ExprKind::Tuple(fields) = arg.kind { - // if this is a tuple, resolve elements separately, - // so they can be added to scope, before resolving subsequent elements. - - let mut fields_new = Vec::with_capacity(fields.len()); - for field in fields { - let field = self.fold_within_namespace(field, ¶m.name)?; - - // add aliased columns into scope - if let Some(alias) = field.alias.clone() { - let id = field.id.unwrap(); - self.root_mod.module.insert_frame_col(NS_THIS, alias, id); - } - fields_new.push(field); + if impl_cl_pos.map_or(false, |pos| pos == index) { + let mut scope = Scope::new(); + if let Some(pos) = this_pos { + let arg = &app.args[pos as usize]; + self.prepare_scope_of_implicit_closure_arg(&mut scope, NS_THIS, arg)?; } - - // note that this tuple node has to be resolved itself - // (it's elements are already resolved and so their resolving - // should be skipped) - arg.kind = ExprKind::Tuple(fields_new); + if let Some(pos) = that_pos { + let arg = &app.args[pos as usize]; + self.prepare_scope_of_implicit_closure_arg(&mut scope, NS_THAT, arg)?; + } + self.scopes.push(scope); } arg = self - .fold_and_type_check(arg, param, func_name)? + .resolve_func_app_arg(arg, param, func_name, should_coerce_tuple)? .unwrap_or_else(|a| { partial_application_position = Some(index); a }); - } - closure.args[index] = arg; - } - - if has_relations { - self.root_mod.module.unshadow(NS_THIS); - self.root_mod.module.unshadow(NS_THAT); + if impl_cl_pos.map_or(false, |pos| pos == index) { + self.scopes.pop(); + } + } + app.args[index] = arg; } Ok(if let Some(position) = partial_application_position { log::debug!( "partial application of {} at arg {position}", - closure.as_debug_name() + metadata.as_debug_name() ); - Err(extract_partial_application(closure, position)) + Err(extract_partial_application(app, position)?) } else { - Ok(closure) + Ok(app) }) } - fn fold_and_type_check( + fn resolve_func_app_arg( &mut self, arg: Expr, - param: &FuncParam, + param: &Option, func_name: &Option, + coerce_tuple: bool, ) -> Result> { - let mut arg = self.fold_within_namespace(arg, ¶m.name)?; + // fold + // if param.name.starts_with("noresolve.") { + // return Ok(Ok(arg)); + // }; - // don't validate types of unresolved exprs - if arg.id.is_some() { - // validate type + let mut arg = self.fold_expr(arg)?; - let expects_func = param - .ty - .as_ref() - .map(|t| t.kind.is_function()) - .unwrap_or_default(); - if !expects_func && arg.kind.is_func() { - return Ok(Err(arg)); - } + if coerce_tuple { + arg = self.coerce_into_tuple(arg)?; + } - let who = || { - func_name - .as_ref() - .map(|n| format!("function {n}, param `{}`", param.name)) - }; - self.validate_expr_type(&mut arg, param.ty.as_ref(), &who)?; + // special case: (I forgot why this is needed) + let expects_func = param + .as_ref() + .map(|t| t.kind.is_function()) + .unwrap_or_default(); + if !expects_func && arg.kind.is_func() { + return Ok(Err(arg)); } + // validate type + let who = || { + func_name + .as_ref() + .map(|n| format!("function {n}, one of the params")) // TODO: param name + }; + self.validate_expr_type(&mut arg, param.as_ref(), &who)?; + + // special case: the arg is a func, finalize it generic arguments + // (this is somewhat of a hack that is needed because of our weird resolution order) + // if let ExprKind::FuncApplication(func) = &arg.kind { + // self.finalize_function_generic_args(func) + // .with_span_fallback(arg.span)?; + // } + Ok(Ok(arg)) } - fn fold_within_namespace(&mut self, expr: Expr, param_name: &str) -> Result { - let prev_namespace = self.default_namespace.take(); + /// Wraps non-tuple Exprs into a singleton Tuple. + pub(super) fn coerce_into_tuple(&mut self, expr: Expr) -> Result { + let is_tuple_ty = expr.ty.as_ref().unwrap().kind.is_tuple() && !expr.kind.is_all(); + Ok(if is_tuple_ty { + // a helpful check for a common anti-pattern + if let Some(alias) = expr.alias { + return Err(Error::new_simple(format!("unexpected assign to `{alias}`")) + .push_hint(format!("move assign into the tuple: `{{{alias} = ...}}`")) + .with_span(expr.span)); + } - if param_name.starts_with("noresolve.") { - return Ok(expr); - } else if let Some((ns, _)) = param_name.split_once('.') { - self.default_namespace = Some(ns.to_string()); + expr } else { - self.default_namespace = None; - }; + let span = expr.span; + let mut expr = Expr::new(ExprKind::Tuple(vec![expr])); + expr.span = span; - let res = self.fold_expr(expr); - self.default_namespace = prev_namespace; - res + self.fold_expr(expr)? + }) + } + + fn prepare_scope_of_implicit_closure_arg( + &mut self, + scope: &mut Scope, + namespace: &str, + expr: &Expr, + ) -> Result<()> { + let ty = expr.ty.as_ref().unwrap(); + + // we expect the param to be an array of tuples, but have the type of this to be a tuple + // here we unwrap the array and keep only the inner tuple + let tuple_ty = match &ty.kind { + TyKind::Array(tuple_ty) => *tuple_ty.clone(), + TyKind::Ident(ident_of_generic) => { + self.infer_generic_as_array(ident_of_generic, expr.span)? + } + _ => { + return Err( + Error::new_simple("implict closure param was expected to be an array") + .push_hint(format!("got ty: {}", write_ty(ty))), + ); + } + }; + scope.values.insert( + namespace.to_string(), + Decl::from(DeclKind::Variable(Some(tuple_ty))), + ); + Ok(()) } } -fn extract_partial_application(mut func: Box, position: usize) -> Box { +fn extract_partial_application(mut func: FuncApplication, position: usize) -> Result> { + dbg!(&func); + // Input: // Func { // params: [x, y, z], @@ -431,67 +522,67 @@ fn extract_partial_application(mut func: Box, position: usize) -> Box (Module, Expr, Option>) { - let mut func_env = Module::default(); - - for (param, arg) in zip(closure.params, closure.args) { +fn prepare_scope_of_func(scope: &mut Scope, func: &Func) { + for param in &func.params { let v = Decl { - declared_at: arg.id, - kind: DeclKind::Expr(Box::new(arg)), + kind: DeclKind::Variable(param.ty.clone()), ..Default::default() }; let param_name = param.name.split('.').last().unwrap(); - func_env.names.insert(param_name.to_string(), v); + scope.values.insert(param_name.to_string(), v); } - - (func_env, *closure.body, closure.return_ty.map(Box::new)) } -pub fn expr_of_func(func: Box, span: Option) -> Box { - let ty = TyFunc { - params: func - .params - .iter() - .skip(func.args.len()) - .map(|a| a.ty.clone()) - .collect(), - return_ty: func - .return_ty - .clone() - .or_else(|| func.clone().body.ty) - .map(Box::new), - name_hint: func.name_hint.clone(), +pub fn expr_of_func_application( + func_app: FuncApplication, + body_ty: Option, + span: Option, +) -> Box { + let fn_ty = func_app.func.ty.as_ref().unwrap(); + let fn_ty = fn_ty.kind.as_function().unwrap(); + let fn_ty = fn_ty.as_ref().unwrap(); + + let ty_func_params: Vec<_> = fn_ty.params[func_app.args.len()..].to_vec(); + + let ty = if ty_func_params.is_empty() { + body_ty + } else { + Some(Ty::new(TyFunc { + params: ty_func_params, + return_ty: body_ty.map(Box::new), + generic_type_params: vec![], + })) }; Box::new(Expr { - ty: Some(Ty::new(ty)), + ty, span, - ..Expr::new(ExprKind::Func(func)) + ..Expr::new(ExprKind::FuncApplication(func_app)) }) } diff --git a/prqlc/prqlc/src/semantic/resolver/inference.rs b/prqlc/prqlc/src/semantic/resolver/inference.rs index fa8b55678fac..1d662a874db0 100644 --- a/prqlc/prqlc/src/semantic/resolver/inference.rs +++ b/prqlc/prqlc/src/semantic/resolver/inference.rs @@ -1,148 +1,152 @@ -use itertools::Itertools; +use crate::pr::{Ident, Ty, TyKind, TyTupleField}; +use crate::codegen::write_ty; +use crate::ir::decl::{Decl, DeclKind}; +use crate::ir::pl::IndirectionKind; +use crate::semantic::NS_GENERIC; +use crate::{Error, Result, Span, WithErrorInfo}; use super::Resolver; -use crate::ir::decl::{Decl, TableDecl, TableExpr}; -use crate::ir::pl::{Lineage, LineageColumn, LineageInput}; -use crate::pr::{Ident, Ty, TyTupleField}; -use crate::semantic::{NS_DEFAULT_DB, NS_INFER}; -use crate::Result; impl Resolver<'_> { - pub fn infer_table_column( + pub fn init_new_global_generic(&mut self, prefix: &str) -> Ident { + let a_unique_number = self.id.gen(); + let param_name = format!("{prefix}{a_unique_number}"); + let ident = Ident::from_path(vec![NS_GENERIC.to_string(), param_name]); + let decl = Decl::from(DeclKind::GenericParam(None)); + + self.root_mod.module.insert(ident.clone(), decl).unwrap(); + ident + } + + /// For a given generic, infer that it must be of type `ty`. + pub fn infer_generic_as_ty( &mut self, - table_ident: &Ident, - col_name: &str, - ) -> Result<(), String> { - let table = self.root_mod.module.get_mut(table_ident).unwrap(); - let table_decl = table.kind.as_table_decl_mut().unwrap(); - - let Some(columns) = table_decl.ty.as_mut().and_then(|t| t.as_relation_mut()) else { - return Err(format!("Variable {table_ident:?} is not a relation.")); + ident_of_generic: &Ident, + ty: Ty, + span: Option, + ) -> Result<()> { + if let TyKind::Ident(ty_ident) = &ty.kind { + if ty_ident == ident_of_generic { + // don't infer that T is T + return Ok(()); + } + } + + log::debug!("inferring that {ident_of_generic:?} is {}", write_ty(&ty)); + + let Some(decl) = self.get_ident_mut(ident_of_generic) else { + return Err(Error::new_assert("type not found")); + }; + let DeclKind::GenericParam(candidate) = &mut decl.kind else { + return Err(Error::new_assert("expected a generic type param") + .push_hint(format!("found {:?}", decl.kind))); }; - let has_wildcard = columns - .iter() - .any(|c| matches!(c, TyTupleField::Wildcard(_))); - if !has_wildcard { - return Err(format!("Table {table_ident:?} does not have wildcard.")); - } + if let Some((candidate, _)) = candidate { + // validate that ty has all fields of the candidate + let candidate = candidate.clone(); + self.validate_type(&ty, &candidate, span, &|| None)?; - let exists = columns.iter().any(|c| match c { - TyTupleField::Single(Some(n), _) => n == col_name, - _ => false, - }); - if exists { - return Ok(()); - } + // ty has all fields of the candidate, but it might have additional ones + // so we need to add all of them to the candidate + // (we need to get the candidate ref again, since we need &mut self for validate_type) + let Some(decl) = self.get_ident_mut(ident_of_generic) else { + unreachable!() + }; + let DeclKind::GenericParam(Some(candidate)) = &mut decl.kind else { + unreachable!() + }; - columns.push(TyTupleField::Single(Some(col_name.to_string()), None)); - - // also add into input tables of this table expression - if let TableExpr::RelationVar(expr) = &table_decl.expr { - if let Some(frame) = &expr.lineage { - let wildcard_inputs = (frame.columns.iter()) - .filter_map(|c| c.as_all()) - .collect_vec(); - - match wildcard_inputs.len() { - 0 => return Err(format!("Cannot infer where {table_ident}.{col_name} is from")), - 1 => { - let (input_id, _) = wildcard_inputs.into_iter().next().unwrap(); - - let input = frame.find_input(*input_id).unwrap(); - let table_ident = input.table.clone(); - self.infer_table_column(&table_ident, col_name)?; - } - _ => { - return Err(format!("Cannot infer where {table_ident}.{col_name} is from. It could be any of {wildcard_inputs:?}")) - } - } - } + candidate.0.kind = ty.kind; // maybe merge the fields here? + return Ok(()); } + *candidate = Some((ty, span)); Ok(()) } - /// Converts a identifier that points to a table declaration to lineage of that table. - pub fn lineage_of_table_decl( + /// When we refer to `Generic.my_field`, this function pushes information that `Generic` + /// is a tuple with a field `my_field` into the generic type argument. + /// + /// Contract: + /// - ident must be fq ident of a generic type param, + /// - generic candidate either must not exist yet or be a tuple, + /// - if it is a tuple, it must not yet contain the indirection target. + pub fn infer_generic_as_tuple( &mut self, - table_fq: &Ident, - input_name: String, - input_id: usize, - ) -> Lineage { - let table_decl = self.root_mod.module.get(table_fq).unwrap(); - let TableDecl { ty, .. } = table_decl.kind.as_table_decl().unwrap(); - - // TODO: can this panic? - let columns = ty.as_ref().unwrap().as_relation().unwrap(); - - let mut instance_frame = Lineage { - inputs: vec![LineageInput { - id: input_id, - name: input_name.clone(), - table: table_fq.clone(), - }], - columns: Vec::new(), - ..Default::default() - }; - - for col in columns { - let col = match col { - TyTupleField::Wildcard(_) => LineageColumn::All { - input_id, - except: columns - .iter() - .flat_map(|c| c.as_single().map(|x| x.0).cloned().flatten()) - .collect(), - }, - TyTupleField::Single(col_name, _) => LineageColumn::Single { - name: col_name - .clone() - .map(|col_name| Ident::from_path(vec![input_name.clone(), col_name])), - target_id: input_id, - target_name: col_name.clone(), - }, - }; - instance_frame.columns.push(col); + ident_of_generic: &Ident, + indirection: IndirectionKind, + ) -> (usize, Ty) { + // generate the type of inferred field (to be an unknown type - a new generic) + // (this has to be done early in this function since we borrow self later) + let ty_of_field = self.init_new_global_generic("F"); + let ty = Ty::new(TyKind::Ident(ty_of_field)); + + let ident = ident_of_generic; + let generic_decl = self.root_mod.module.get_mut(ident).unwrap(); + let candidate = generic_decl.kind.as_generic_param_mut().unwrap(); + + // if there is no candidate yet, propose a new tuple type + if candidate.is_none() { + *candidate = Some((Ty::new(TyKind::Tuple(vec![])), None)); } + let (candidate_ty, _) = candidate.as_mut().unwrap(); + let candidate_fields = candidate_ty.kind.as_tuple_mut().unwrap(); + + // create new field(s) + match indirection { + IndirectionKind::Name(field_name) => { + candidate_fields.push(TyTupleField::Single(Some(field_name), Some(ty.clone()))); + + let pos_within_candidate = candidate_fields.len() - 1; + (pos_within_candidate, ty) + } + IndirectionKind::Position(pos) => { + let pos = pos as usize; - log::debug!("instanced table {table_fq} as {instance_frame:?}"); - instance_frame + // fill-in padding fields + for _ in 0..(pos - candidate_fields.len()) { + // TODO: these should all be generics + candidate_fields.push(TyTupleField::Single(None, None)); + } + + // push the actual field + candidate_fields.push(TyTupleField::Single(None, Some(ty.clone()))); + (pos, ty) + } + } } - /// Declares a new table for a relation literal. - /// This is needed for column inference to work properly. - pub(super) fn declare_table_for_literal( + pub fn infer_generic_as_array( &mut self, - input_id: usize, - columns: Option>, - name_hint: Option, - ) -> Lineage { - let id = input_id; - let global_name = format!("_literal_{}", id); - - // declare a new table in the `default_db` module - let default_db_ident = Ident::from_name(NS_DEFAULT_DB); - let default_db = self.root_mod.module.get_mut(&default_db_ident).unwrap(); - let default_db = default_db.kind.as_module_mut().unwrap(); - - let infer_default = default_db.get(&Ident::from_name(NS_INFER)).unwrap().clone(); - let mut infer_default = *infer_default.kind.into_infer().unwrap(); - - let table_decl = infer_default.as_table_decl_mut().unwrap(); - table_decl.expr = TableExpr::None; - - if let Some(columns) = columns { - table_decl.ty = Some(Ty::relation(columns)); + ident_of_generic: &Ident, + span: Option, + ) -> Result { + // generate the type of array items (to be an unknown type - a new generic) + // (this has to be done early in this function since we borrow self later) + let items_ty = self.init_new_global_generic("A"); + let items_ty = Ty::new(TyKind::Ident(items_ty)); + + let ident = ident_of_generic; + let generic_decl = self.root_mod.module.get_mut(ident).unwrap(); + let candidate = generic_decl.kind.as_generic_param_mut().unwrap(); + + // if there is no candidate yet, propose a new tuple type + if let Some((candidate, _)) = candidate.as_mut() { + if let TyKind::Array(items_ty) = &candidate.kind { + // ok, we already know it is an array + Ok(*items_ty.clone()) + } else { + // nope + Err(Error::new_simple(format!( + "generic type argument {} needs to be an array", + ident_of_generic + )) + .push_hint(format!("existing candidate: {}", write_ty(candidate)))) + } + } else { + *candidate = Some((Ty::new(TyKind::Array(Box::new(items_ty.clone()))), span)); + Ok(items_ty) } - - default_db - .names - .insert(global_name.clone(), Decl::from(infer_default)); - - // produce a frame of that table - let input_name = name_hint.unwrap_or_else(|| global_name.clone()); - let table_fq = default_db_ident + Ident::from_name(global_name); - self.lineage_of_table_decl(&table_fq, input_name, id) } } diff --git a/prqlc/prqlc/src/semantic/resolver/mod.rs b/prqlc/prqlc/src/semantic/resolver/mod.rs index f692082b9953..3563d028cd94 100644 --- a/prqlc/prqlc/src/semantic/resolver/mod.rs +++ b/prqlc/prqlc/src/semantic/resolver/mod.rs @@ -1,32 +1,27 @@ -use std::collections::HashMap; - use crate::ir::decl::RootModule; use crate::utils::IdGenerator; mod expr; -mod flatten; mod functions; mod inference; -mod names; +mod scope; mod static_eval; mod stmt; -mod transforms; +mod tuple; mod types; /// Can fold (walk) over AST and for each function call or variable find what they are referencing. pub struct Resolver<'a> { root_mod: &'a mut RootModule, - current_module_path: Vec, - - default_namespace: Option, + pub debug_current_decl: crate::pr::Ident, /// Sometimes ident closures must be resolved and sometimes not. See [test::test_func_call_resolve]. in_func_call_name: bool, pub id: IdGenerator, - pub generics: HashMap<(usize, String), Vec>, + scopes: Vec, } #[derive(Default, Clone)] @@ -34,23 +29,35 @@ pub struct ResolverOptions {} impl Resolver<'_> { pub fn new(root_mod: &mut RootModule) -> Resolver { + let mut id = IdGenerator::new(); + let max_id = root_mod.span_map.keys().max().cloned().unwrap_or(0); + id.skip(max_id); + Resolver { root_mod, - current_module_path: Vec::new(), - default_namespace: None, + debug_current_decl: crate::pr::Ident::from_name("?"), in_func_call_name: false, - id: IdGenerator::new(), - generics: Default::default(), + id, + scopes: Vec::new(), + } + } + + #[allow(dead_code)] + fn scope_mut(&mut self) -> &mut scope::Scope { + if self.scopes.is_empty() { + self.scopes.push(scope::Scope::new()); } + self.scopes.last_mut().unwrap() } } #[cfg(test)] -pub(super) mod test { +pub(in crate::semantic) mod test { + use crate::{Errors, Result}; use insta::assert_yaml_snapshot; - use crate::ir::pl::{Expr, Lineage, PlFold}; - use crate::{Errors, Result}; + use crate::pr::Ty; + use crate::ir::pl::{Expr, PlFold}; pub fn erase_ids(expr: Expr) -> Expr { IdEraser {}.fold_expr(expr).unwrap() @@ -70,11 +77,11 @@ pub(super) mod test { fn parse_and_resolve(query: &str) -> Result { let ctx = crate::semantic::test::parse_and_resolve(query)?; let (main, _) = ctx.find_main_rel(&[]).unwrap(); - Ok(*main.clone().into_relation_var().unwrap()) + Ok(main.clone()) } - fn resolve_lineage(query: &str) -> Result { - Ok(parse_and_resolve(query)?.lineage.unwrap()) + fn resolve_lineage(query: &str) -> Result { + Ok(parse_and_resolve(query)?.ty.unwrap()) } fn resolve_derive(query: &str) -> Result, Errors> { @@ -96,7 +103,7 @@ pub(super) mod test { fn test_variables_1() { assert_yaml_snapshot!(resolve_derive( r#" - from employees + from db.employees derive { gross_salary = salary + payroll_tax, gross_cost = gross_salary + benefits_cost @@ -121,9 +128,9 @@ pub(super) mod test { r#" let subtract = a b -> a - b - from employees + from db.employees derive { - net_salary = subtract gross_salary tax + net_salary = module.subtract gross_salary tax } "# ) @@ -135,10 +142,10 @@ pub(super) mod test { assert_yaml_snapshot!(resolve_derive( r#" let lag_day = x -> s"lag_day_todo({x})" - let ret = x dividend_return -> x / (lag_day x) - 1 + dividend_return + let ret = x dividend_return -> x / (module.lag_day x) - 1 + dividend_return - from a - derive (ret b c) + from db.a + derive (module.ret b c) "# ) .unwrap()); @@ -148,7 +155,7 @@ pub(super) mod test { fn test_functions_pipeline() { assert_yaml_snapshot!(resolve_derive( r#" - from a + from db.a derive one = (foo | sum) "# ) @@ -159,8 +166,8 @@ pub(super) mod test { let plus_one = x -> x + 1 let plus = x y -> x + y - from a - derive {b = (sum foo | plus_one | plus 2)} + from db.a + derive {b = (sum foo | module.plus_one | module.plus 2)} "# ) .unwrap()); @@ -171,10 +178,10 @@ pub(super) mod test { r#" let add_one = x to:1 -> x + to - from foo_table + from db.foo_table derive { - added = add_one bar to:3, - added_default = add_one bar + added = module.add_one bar to:3, + added_default = module.add_one bar } "# ) @@ -185,7 +192,7 @@ pub(super) mod test { fn test_frames_and_names() { assert_yaml_snapshot!(resolve_lineage( r#" - from orders + from db.orders select {customer_no, gross, tax, gross - tax} take 20 "# @@ -194,16 +201,16 @@ pub(super) mod test { assert_yaml_snapshot!(resolve_lineage( r#" - from table_1 - join customers (==customer_no) + from db.table_1 + join db.customers (==customer_no) "# ) .unwrap()); assert_yaml_snapshot!(resolve_lineage( r#" - from e = employees - join salaries (==emp_no) + from e = db.employees + join db.salaries (==emp_no) group {e.emp_no, e.gender} ( aggregate { emp_salary = average salaries.salary diff --git a/prqlc/prqlc/src/semantic/resolver/names.rs b/prqlc/prqlc/src/semantic/resolver/names.rs deleted file mode 100644 index a44c0008edaa..000000000000 --- a/prqlc/prqlc/src/semantic/resolver/names.rs +++ /dev/null @@ -1,264 +0,0 @@ -use std::collections::HashSet; - -use itertools::Itertools; - -use super::Resolver; -use crate::ir::decl::{Decl, DeclKind, Module}; -use crate::ir::pl::{Expr, ExprKind}; -use crate::pr::Ident; -use crate::semantic::{NS_INFER, NS_INFER_MODULE, NS_SELF, NS_THAT, NS_THIS}; -use crate::Error; -use crate::Result; -use crate::WithErrorInfo; - -impl Resolver<'_> { - pub(super) fn resolve_ident(&mut self, ident: &Ident) -> Result { - let mut res = if let Some(default_namespace) = self.default_namespace.clone() { - self.resolve_ident_core(ident, Some(&default_namespace)) - } else { - let mut ident = ident.clone().prepend(self.current_module_path.clone()); - - let mut res = self.resolve_ident_core(&ident, None); - for _ in 0..self.current_module_path.len() { - if res.is_ok() { - break; - } - ident = ident.pop_front().1.unwrap(); - res = self.resolve_ident_core(&ident, None); - } - res - }; - - match &res { - Ok(fq_ident) => { - let decl = self.root_mod.module.get(fq_ident).unwrap(); - if let DeclKind::Import(target) = &decl.kind { - let target = target.clone(); - return self.resolve_ident(&target); - } - } - Err(e) => { - log::debug!( - "cannot resolve `{ident}`: `{e:?}`, root_mod={:#?}", - self.root_mod - ); - - // attach available names - let mut available_names = Vec::new(); - available_names.extend(self.collect_columns_in_module(NS_THIS)); - available_names.extend(self.collect_columns_in_module(NS_THAT)); - if !available_names.is_empty() { - let available_names = available_names.iter().map(Ident::to_string).join(", "); - res = res.push_hint(format!("available columns: {available_names}")); - } - } - } - res - } - - fn collect_columns_in_module(&mut self, mod_name: &str) -> Vec { - let mut cols = Vec::new(); - - let Some(module) = self.root_mod.module.names.get(mod_name) else { - return cols; - }; - - let DeclKind::Module(this) = &module.kind else { - return cols; - }; - - for (ident, decl) in this.as_decls().into_iter().sorted_by_key(|x| x.1.order) { - if let DeclKind::Column(_) = decl.kind { - cols.push(ident); - } - } - cols - } - - pub(super) fn resolve_ident_core( - &mut self, - ident: &Ident, - default_namespace: Option<&String>, - ) -> Result { - // special case: wildcard - if ident.name == "*" { - // TODO: we may want to raise an error if someone has passed `download*` in - // an attempt to query for all `download` columns and expects to be able - // to select a `download_2020_01_01` column later in the query. But - // sometimes we want to query for `*.parquet` files, and give them an - // alias. So we don't raise an error here, but if there's a way of - // differentiating the cases, we can implement that. - // if ident.name != "*" { - // return Err("Unsupported feature: advanced wildcard column matching".to_string()); - // } - return self - .resolve_ident_wildcard(ident) - .map_err(Error::new_simple); - } - - // base case: direct lookup - let decls = self.root_mod.module.lookup(ident); - match decls.len() { - // no match: try match * - 0 => {} - - // single match, great! - 1 => return Ok(decls.into_iter().next().unwrap()), - - // ambiguous - _ => return Err(ambiguous_error(decls, None)), - } - - let ident = if let Some(default_namespace) = default_namespace { - let ident = ident.clone().prepend(vec![default_namespace.clone()]); - - let decls = self.root_mod.module.lookup(&ident); - match decls.len() { - // no match: try match * - 0 => ident, - - // single match, great! - 1 => return Ok(decls.into_iter().next().unwrap()), - - // ambiguous - _ => return Err(ambiguous_error(decls, None)), - } - } else { - ident.clone() - }; - - // fallback case: try to match with NS_INFER and infer the declaration - // from the original ident. - match self.resolve_ident_fallback(&ident, NS_INFER) { - // The declaration and all needed parent modules were created - // -> just return the fq ident - Ok(inferred_ident) => Ok(inferred_ident), - - // Was not able to infer. - Err(None) => Err(Error::new_simple( - format!("Unknown name `{}`", &ident).to_string(), - )), - Err(Some(msg)) => Err(msg), - } - } - - /// Try lookup of the ident with name replaced. If unsuccessful, recursively retry parent ident. - fn resolve_ident_fallback( - &mut self, - ident: &Ident, - name_replacement: &'static str, - ) -> Result> { - let infer_ident = ident.clone().with_name(name_replacement); - - // lookup of infer_ident - let mut decls = self.root_mod.module.lookup(&infer_ident); - - if decls.is_empty() { - if let Some(parent) = infer_ident.clone().pop() { - // try to infer parent - let _ = self.resolve_ident_fallback(&parent, NS_INFER_MODULE)?; - - // module was successfully inferred, retry the lookup - decls = self.root_mod.module.lookup(&infer_ident) - } - } - - match decls.len() { - 1 => { - // single match, great! - let infer_ident = decls.into_iter().next().unwrap(); - self.infer_decl(infer_ident, ident) - .map_err(|x| Some(Error::new_simple(x))) - } - 0 => Err(None), - _ => Err(Some(ambiguous_error(decls, Some(&ident.name)))), - } - } - - /// Create a declaration of [original] from template provided by declaration of [infer_ident]. - fn infer_decl(&mut self, infer_ident: Ident, original: &Ident) -> Result { - let infer = self.root_mod.module.get(&infer_ident).unwrap(); - let mut infer_default = *infer.kind.as_infer().cloned().unwrap(); - - if let DeclKind::Module(new_module) = &mut infer_default { - // Modules are inferred only for database inference. - // Because we want to infer database modules that nested arbitrarily deep, - // we cannot store the template in DeclKind::Infer, but we override it here. - *new_module = Module::new_database(); - } - - let module_ident = infer_ident.pop().unwrap(); - let module = self.root_mod.module.get_mut(&module_ident).unwrap(); - let module = module.kind.as_module_mut().unwrap(); - - // insert default - module - .names - .insert(original.name.clone(), Decl::from(infer_default)); - - // infer table columns - if let Some(decl) = module.names.get(NS_SELF).cloned() { - if let DeclKind::InstanceOf(table_ident, _) = decl.kind { - log::debug!("inferring {original} to be from table {table_ident}"); - self.infer_table_column(&table_ident, &original.name)?; - } - } - - Ok(module_ident + Ident::from_name(original.name.clone())) - } - - fn resolve_ident_wildcard(&mut self, ident: &Ident) -> Result { - let ident_self = ident.clone().pop().unwrap() + Ident::from_name(NS_SELF); - let mut res = self.root_mod.module.lookup(&ident_self); - if res.contains(&ident_self) { - res = HashSet::from_iter([ident_self]); - } - if res.len() != 1 { - return Err(format!("Unknown relation {ident}")); - } - let module_fq_self = res.into_iter().next().unwrap(); - - // Materialize into a tuple literal, containing idents. - let fields = self.construct_wildcard_include(&module_fq_self); - - // This is just a workaround to return an Expr from this function. - // We wrap the expr into DeclKind::Expr and save it into the root module. - let cols_expr = Expr { - flatten: true, - ..Expr::new(ExprKind::Tuple(fields)) - }; - let cols_expr = DeclKind::Expr(Box::new(cols_expr)); - let save_as = "_wildcard_match"; - self.root_mod - .module - .names - .insert(save_as.to_string(), cols_expr.into()); - - // Then we can return ident to that decl. - Ok(Ident::from_name(save_as)) - } -} - -fn ambiguous_error(idents: HashSet, replace_name: Option<&String>) -> Error { - let all_this = idents.iter().all(|d| d.starts_with_part(NS_THIS)); - - let mut chunks = Vec::new(); - for mut ident in idents { - if all_this { - let (_, rem) = ident.pop_front(); - if let Some(rem) = rem { - ident = rem; - } else { - continue; - } - } - - if let Some(name) = replace_name { - ident.name.clone_from(name); - } - chunks.push(ident.to_string()); - } - chunks.sort(); - let hint = format!("could be any of: {}", chunks.join(", ")); - Error::new_simple("Ambiguous name").push_hint(hint) -} diff --git a/prqlc/prqlc/src/semantic/resolver/scope.rs b/prqlc/prqlc/src/semantic/resolver/scope.rs new file mode 100644 index 000000000000..4648bd0ac64b --- /dev/null +++ b/prqlc/prqlc/src/semantic/resolver/scope.rs @@ -0,0 +1,151 @@ +use indexmap::IndexMap; + +use crate::pr::{Ident, Ty}; +use crate::codegen; +use crate::ir::decl::{Decl, DeclKind}; +use crate::semantic::{NS_LOCAL, NS_THAT, NS_THIS}; +use crate::{Error, Result, WithErrorInfo}; + +use super::tuple::StepOwned; +use super::Resolver; + +#[derive(Debug)] +pub(super) struct Scope { + pub types: IndexMap, + + pub values: IndexMap, +} + +impl Scope { + pub fn new() -> Self { + Self { + types: IndexMap::new(), + values: IndexMap::new(), + } + } + + pub fn get(&self, name: &str) -> Option<&Decl> { + if let Some(decl) = self.types.get(name) { + return Some(decl); + } + + self.values.get(name) + } + + pub fn get_mut(&mut self, name: &str) -> Option<&mut Decl> { + if let Some(decl) = self.types.get_mut(name) { + return Some(decl); + } + + self.values.get_mut(name) + } +} + +impl Resolver<'_> { + /// Get declaration from within the current scope. + /// + /// Does not mutate the current scope or module structure. + pub(super) fn get_ident(&self, ident: &Ident) -> Option<&Decl> { + if ident.starts_with_part(NS_LOCAL) { + assert!(ident.len() == 2); + self.scopes.last()?.get(&ident.name) + } else { + self.root_mod.module.get(ident) + } + } + + /// Get mutable reference to a declaration from within the current scope. + /// + /// Does not mutate the current scope or module structure. + pub(super) fn get_ident_mut(&mut self, ident: &Ident) -> Option<&mut Decl> { + if ident.starts_with_part(NS_LOCAL) { + assert!(ident.len() == 2); + self.scopes.last_mut()?.get_mut(&ident.name) + } else { + self.root_mod.module.get_mut(ident) + } + } + + /// Performs an identifer lookup, possibly infering type information or + /// even new declarations. + pub(super) fn lookup_ident(&mut self, ident: &Ident) -> Result { + if !ident.starts_with_part(NS_LOCAL) { + // if ident is not local, it must have been resolved eariler + // so we can just do a direct lookup + return Ok(LookupResult::Direct); + } + assert!(ident.len() == 2); + + let res = if let Some(scope) = self.scopes.pop() { + let r = self.lookup_in_scope(&scope, &ident.name); + self.scopes.push(scope); + r + } else { + Ok(None) + }; + let mut res = res.and_then(|x| { + x.ok_or_else(|| Error::new_simple(format!("Unknown name `{}`", &ident.name))) + }); + + if let Err(e) = &res { + log::debug!( + "cannot resolve `{}`: `{e:?}`,\nscope={:#?}", + ident.name, + self.scopes.last(), + ); + + // attach available names + if let Some(this_ty) = self.get_ty_of_scoped_name(NS_THIS) { + let this_ty = super::types::TypePreviewer::run(self, this_ty.clone()); + res = res.push_hint(format!("this = {}", codegen::write_ty(&this_ty))); + } + if let Some(that_ty) = self.get_ty_of_scoped_name(NS_THAT) { + let that_ty = super::types::TypePreviewer::run(self, that_ty.clone()); + res = res.push_hint(format!("that = {}", codegen::write_ty(&that_ty))); + } + } + res + } + + fn lookup_in_scope(&mut self, scope: &Scope, name: &str) -> Result> { + if scope.get(name).is_some() { + return Ok(Some(LookupResult::Direct)); + } + + for (param_name, decl) in &scope.values { + let DeclKind::Variable(Some(var_ty)) = &decl.kind else { + continue; + }; + + let Some(steps) = self.lookup_name_in_tuple(var_ty, name)? else { + continue; + }; + + return Ok(Some(LookupResult::Indirect { + real_name: param_name.clone(), + indirections: steps, + })); + } + Ok(None) + } + + fn get_ty_of_scoped_name(&self, name: &str) -> Option<&Ty> { + let scope = self.scopes.last()?; + + let self_decl = scope.values.get(name)?; + let self_ty = self_decl.kind.as_variable()?; + self_ty.as_ref() + } +} + +/// When doing a lookup of, for example, `a` it might turn out that what we are +/// looking for is under fully-qualified path `this.b.a`. In such cases, lookup will +/// return an "indirect result". In this example, it would be +/// `Indirect { real_name: "this", indirections: vec!["b", "a"] }`. +pub enum LookupResult { + Direct, + Indirect { + real_name: String, + indirections: Vec, + }, +} diff --git a/prqlc/prqlc/src/semantic/resolver/stmt.rs b/prqlc/prqlc/src/semantic/resolver/stmt.rs index 7287c64e7c9b..559129475e7a 100644 --- a/prqlc/prqlc/src/semantic/resolver/stmt.rs +++ b/prqlc/prqlc/src/semantic/resolver/stmt.rs @@ -1,159 +1,130 @@ use std::collections::HashMap; -use crate::ir::decl::{Decl, DeclKind, Module, TableDecl, TableExpr}; +use crate::pr::{Ty, TyKind}; +use crate::ir::decl::{Decl, DeclKind}; use crate::ir::pl::*; -use crate::pr::{Ty, TyKind, TyTupleField}; +use crate::semantic::{NS_GENERIC, NS_STD}; use crate::Result; -use crate::WithErrorInfo; -impl super::Resolver<'_> { - // entry point to the resolver - pub fn fold_statements(&mut self, stmts: Vec) -> Result<()> { - for mut stmt in stmts { - stmt.id = Some(self.id.gen()); - if let Some(span) = stmt.span { - self.root_mod.span_map.insert(stmt.id.unwrap(), span); - } +use super::types::TypeReplacer; - let ident = Ident { - path: self.current_module_path.clone(), - name: stmt.name().to_string(), - }; - - let mut def = match stmt.kind { - StmtKind::QueryDef(d) => { - let decl = DeclKind::QueryDef(*d); - self.root_mod - .declare(ident, decl, stmt.id, Vec::new()) - .with_span(stmt.span)?; - continue; - } - StmtKind::VarDef(var_def) => self.fold_var_def(var_def)?, - StmtKind::TypeDef(ty_def) => { - let value = if let Some(value) = ty_def.value { - value - } else { - Ty::new(Literal::Null) - }; - - let ty = fold_type_opt(self, Some(value))?.unwrap(); - let mut ty = super::types::normalize_type(ty); - ty.name = Some(ident.name.clone()); - - let decl = DeclKind::Ty(ty); - - self.root_mod - .declare(ident, decl, stmt.id, stmt.annotations) - .with_span(stmt.span)?; - continue; - } - StmtKind::ModuleDef(module_def) => { - self.current_module_path.push(ident.name); - - let decl = Decl { - declared_at: stmt.id, - kind: DeclKind::Module(Module { - names: HashMap::new(), - redirects: Vec::new(), - shadowed: None, - }), - annotations: stmt.annotations, - ..Default::default() - }; - let ident = Ident::from_path(self.current_module_path.clone()); - self.root_mod - .module - .insert(ident, decl) - .with_span(stmt.span)?; - - self.fold_statements(module_def.stmts)?; - self.current_module_path.pop(); - continue; - } - StmtKind::ImportDef(target) => { - let decl = Decl { - declared_at: stmt.id, - kind: DeclKind::Import(target.name), - annotations: stmt.annotations, - ..Default::default() - }; - - self.root_mod - .module - .insert(ident, decl) - .with_span(stmt.span)?; - continue; - } - }; +impl super::Resolver<'_> { + /// Entry point to the resolver. + /// fq_ident must point to an unresolved declaration. + pub fn resolve_decl(&mut self, fq_ident: Ident) -> Result<()> { + if !fq_ident.starts_with_part(NS_STD) { + log::debug!("resolving decl {fq_ident}"); + } - if def.name == "main" { - def.ty = Some(Ty::new(TyKind::Ident(Ident::from_path(vec![ - "std", "relation", - ])))); + // take decl out of the module + let mut decl = { + let module = self.root_mod.module.get_submodule_mut(&fq_ident.path); + module.unwrap().names.remove(&fq_ident.name).unwrap() + }; + let stmt = decl.kind.into_unresolved().unwrap(); + self.debug_current_decl = fq_ident.clone(); + + // resolve + match stmt { + StmtKind::QueryDef(d) => { + decl.kind = DeclKind::QueryDef(*d); } - - if let Some(ExprKind::Func(closure)) = def.value.as_mut().map(|x| &mut x.kind) { - if closure.name_hint.is_none() { - closure.name_hint = Some(ident.clone()); - } + StmtKind::ModuleDef(_) => { + unreachable!("module def cannot be unresolved at this point") + // it should have been converted into Module in resolve_decls::init_module_tree } - - let expected_ty = fold_type_opt(self, def.ty)?; - - let decl = match def.value { - Some(mut def_value) => { - // var value is provided - - // validate type - if expected_ty.is_some() { - let who = || Some(def.name.clone()); - self.validate_expr_type(&mut def_value, expected_ty.as_ref(), &who)?; + StmtKind::VarDef(var_def) => { + let def = self.fold_var_def(var_def)?; + let expected_ty = def.ty; + + decl.kind = match def.value { + Some(mut def_value) => { + // var value is provided + + // validate type + if expected_ty.is_some() { + let who = || Some(fq_ident.name.clone()); + self.validate_expr_type(&mut def_value, expected_ty.as_ref(), &who)?; + } + + // finalize global generics + if let Some(mapping) = self.finalize_global_generics() { + let ty = def_value.ty.unwrap(); + def_value.ty = Some(TypeReplacer::on_ty(ty, mapping)); + } + + DeclKind::Expr(def_value) } - - prepare_expr_decl(def_value) - } - None => { - // var value is not provided - - // is this a relation? - if expected_ty.as_ref().map_or(false, |t| t.is_relation()) { - // treat this var as a TableDecl - DeclKind::TableDecl(TableDecl { - ty: expected_ty, - expr: TableExpr::LocalTable, - }) - } else { - // treat this var as a param - let mut expr = Box::new(Expr::new(ExprKind::Param(def.name))); + None => { + // var value is not provided: treat this var as a param + let mut expr = Box::new(Expr::new(ExprKind::Param(fq_ident.name.clone()))); expr.ty = expected_ty; DeclKind::Expr(expr) } - } - }; - self.root_mod - .declare(ident, decl, stmt.id, stmt.annotations) - .with_span(stmt.span)?; + }; + } + StmtKind::TypeDef(ty_def) => { + let value = if let Some(value) = ty_def.value { + value + } else { + Ty::new(TyKind::Tuple(vec![])) + }; + + let mut ty = fold_type_opt(self, Some(value))?.unwrap(); + ty.name = Some(fq_ident.name.clone()); + + decl.kind = DeclKind::Ty(ty); + } + StmtKind::ImportDef(target) => { + decl.kind = DeclKind::Import(target.name); + } + }; + + // put decl back in + { + let module = self.root_mod.module.get_submodule_mut(&fq_ident.path); + module.unwrap().names.insert(fq_ident.name, decl); } Ok(()) } -} -fn prepare_expr_decl(value: Box) -> DeclKind { - match &value.lineage { - Some(frame) => { - let columns = (frame.columns.iter()) - .map(|col| match col { - LineageColumn::All { .. } => TyTupleField::Wildcard(None), - LineageColumn::Single { name, .. } => { - TyTupleField::Single(name.as_ref().map(|n| n.name.clone()), None) - } - }) - .collect(); - let ty = Some(Ty::relation(columns)); + pub fn finalize_global_generics(&mut self) -> Option> { + let generics = self.root_mod.module.names.get_mut(NS_GENERIC)?; + let generics = generics.kind.as_module_mut()?; + + let mut type_mapping = HashMap::new(); + + let mut new_generics = Vec::new(); + for (name, decl) in generics.names.drain() { + if let DeclKind::GenericParam(Some((candidate, span))) = decl.kind { + // TODO: reject GenericParam(None) with 'cannot infer type, add annotations' + + if candidate.kind.is_tuple() { + // don't finalize tuples because they might not be complete yet + new_generics.push(( + name, + Decl { + kind: DeclKind::GenericParam(Some((candidate, span))), + ..decl + }, + )); + } else { + // finalize this generic + type_mapping.insert( + Ident::from_path(vec![NS_GENERIC.to_string(), name]), + candidate, + ); + } + } else { + new_generics.push((name, decl)); + } + } + generics.names.extend(new_generics); - let expr = TableExpr::RelationVar(value); - DeclKind::TableDecl(TableDecl { ty, expr }) + if type_mapping.is_empty() { + None + } else { + Some(type_mapping) } - _ => DeclKind::Expr(value), } } diff --git a/prqlc/prqlc/src/semantic/resolver/transforms.rs b/prqlc/prqlc/src/semantic/resolver/transforms.rs deleted file mode 100644 index 4eb8f43a7e69..000000000000 --- a/prqlc/prqlc/src/semantic/resolver/transforms.rs +++ /dev/null @@ -1,1263 +0,0 @@ -use std::collections::HashMap; -use std::iter::zip; - -use itertools::Itertools; -use serde::Deserialize; - -use super::types::{ty_tuple_kind, type_intersection}; -use super::Resolver; -use crate::ir::decl::{Decl, DeclKind, Module}; -use crate::ir::generic::{SortDirection, WindowKind}; -use crate::ir::pl::*; -use crate::pr::{Ty, TyKind, TyTupleField}; -use crate::semantic::ast_expand::{restrict_null_literal, try_restrict_range}; -use crate::semantic::resolver::functions::expr_of_func; -use crate::semantic::{write_pl, NS_PARAM, NS_THIS}; -use crate::{compiler_version, Error, Reason, Result, WithErrorInfo}; - -impl Resolver<'_> { - /// try to convert function call with enough args into transform - #[allow(clippy::boxed_local)] - pub fn resolve_special_func(&mut self, func: Box, needs_window: bool) -> Result { - let internal_name = func.body.kind.into_internal().unwrap(); - - let (kind, input) = match internal_name.as_str() { - "select" => { - let [assigns, tbl] = unpack::<2>(func.args); - - let assigns = Box::new(self.coerce_into_tuple(assigns)?); - (TransformKind::Select { assigns }, tbl) - } - "filter" => { - let [filter, tbl] = unpack::<2>(func.args); - - let filter = Box::new(filter); - (TransformKind::Filter { filter }, tbl) - } - "derive" => { - let [assigns, tbl] = unpack::<2>(func.args); - - let assigns = Box::new(self.coerce_into_tuple(assigns)?); - (TransformKind::Derive { assigns }, tbl) - } - "aggregate" => { - let [assigns, tbl] = unpack::<2>(func.args); - - let assigns = Box::new(self.coerce_into_tuple(assigns)?); - (TransformKind::Aggregate { assigns }, tbl) - } - "sort" => { - let [by, tbl] = unpack::<2>(func.args); - - let by = self - .coerce_into_tuple(by)? - .try_cast(|x| x.into_tuple(), Some("sort"), "tuple")? - .into_iter() - .map(|expr| { - let (column, direction) = match expr.kind { - ExprKind::RqOperator { name, mut args } if name == "std.neg" => { - (args.remove(0), SortDirection::Desc) - } - _ => (expr, SortDirection::default()), - }; - let column = Box::new(column); - - ColumnSort { direction, column } - }) - .collect(); - - (TransformKind::Sort { by }, tbl) - } - "take" => { - let [expr, tbl] = unpack::<2>(func.args); - - let range = if let ExprKind::Literal(Literal::Integer(n)) = expr.kind { - range_from_ints(None, Some(n)) - } else { - match try_restrict_range(expr) { - Ok((start, end)) => Range { - start: restrict_null_literal(start).map(Box::new), - end: restrict_null_literal(end).map(Box::new), - }, - Err(expr) => { - return Err(Error::new(Reason::Expected { - who: Some("`take`".to_string()), - expected: "int or range".to_string(), - found: write_pl(expr.clone()), - }) - // Possibly this should refer to the item after the `take` where - // one exists? - .with_span(expr.span)); - } - } - }; - - (TransformKind::Take { range }, tbl) - } - "join" => { - let [side, with, filter, tbl] = unpack::<4>(func.args); - - let side = { - let span = side.span; - let ident = - side.clone() - .try_cast(ExprKind::into_ident, Some("side"), "ident")?; - - // first try to match the raw ident string as a bare word - match ident.to_string().as_str() { - "inner" => JoinSide::Inner, - "left" => JoinSide::Left, - "right" => JoinSide::Right, - "full" => JoinSide::Full, - - _ => { - // if that fails, fold the ident and try treating the result as a literal - // this allows the join side to be passed as a function parameter - // NOTE: this is temporary, pending discussions and implementation, tracked in #4501 - let folded = self.fold_expr(side)?.try_cast( - ExprKind::into_literal, - Some("side"), - "string literal", - )?; - - match folded.to_string().as_str() { - "\"inner\"" => JoinSide::Inner, - "\"left\"" => JoinSide::Left, - "\"right\"" => JoinSide::Right, - "\"full\"" => JoinSide::Full, - - _ => { - return Err(Error::new(Reason::Expected { - who: Some("`side`".to_string()), - expected: "inner, left, right or full".to_string(), - found: folded.to_string(), - }) - .with_span(span)) - } - } - } - } - }; - - let filter = Box::new(filter); - let with = Box::new(with); - (TransformKind::Join { side, with, filter }, tbl) - } - "group" => { - let [by, pipeline, tbl] = unpack::<3>(func.args); - - let by = Box::new(self.coerce_into_tuple(by)?); - - // construct the relation that is passed into the pipeline - // (when generics are a thing, this can be removed) - let partition = { - let partition = Expr::new(ExprKind::All { - within: Box::new(Expr::new(Ident::from_name(NS_THIS))), - except: by.clone(), - }); - // wrap into select, so the names are resolved correctly - let partition = FuncCall { - name: Box::new(Expr::new(Ident::from_path(vec!["std", "select"]))), - args: vec![partition, tbl], - named_args: Default::default(), - }; - let partition = Expr::new(ExprKind::FuncCall(partition)); - // fold, so lineage and types are inferred - self.fold_expr(partition)? - }; - let pipeline = self.fold_by_simulating_eval(pipeline, &partition)?; - - // unpack tbl back out - let tbl = *partition.kind.into_transform_call().unwrap().input; - - let pipeline = Box::new(pipeline); - (TransformKind::Group { by, pipeline }, tbl) - } - "window" => { - let [rows, range, expanding, rolling, pipeline, tbl] = unpack::<6>(func.args); - - let expanding = { - let as_bool = expanding.kind.as_literal().and_then(|l| l.as_boolean()); - - *as_bool.ok_or_else(|| { - Error::new(Reason::Expected { - who: Some("parameter `expanding`".to_string()), - expected: "a boolean".to_string(), - found: write_pl(expanding.clone()), - }) - .with_span(expanding.span) - })? - }; - - let rolling = { - let as_int = rolling.kind.as_literal().and_then(|x| x.as_integer()); - - *as_int.ok_or_else(|| { - Error::new(Reason::Expected { - who: Some("parameter `rolling`".to_string()), - expected: "a number".to_string(), - found: write_pl(rolling.clone()), - }) - .with_span(rolling.span) - })? - }; - - let rows = into_literal_range(try_restrict_range(rows).unwrap())?; - - let range = into_literal_range(try_restrict_range(range).unwrap())?; - - let (kind, start, end) = if expanding { - (WindowKind::Rows, None, Some(0)) - } else if rolling > 0 { - (WindowKind::Rows, Some(-rolling + 1), Some(0)) - } else if !range_is_empty(&rows) { - (WindowKind::Rows, rows.0, rows.1) - } else if !range_is_empty(&range) { - (WindowKind::Range, range.0, range.1) - } else { - (WindowKind::Rows, None, None) - }; - // let start = Expr::new(start.map_or(Literal::Null, Literal::Integer)); - // let end = Expr::new(end.map_or(Literal::Null, Literal::Integer)); - let range = Range { - start: start.map(Literal::Integer).map(Expr::new).map(Box::new), - end: end.map(Literal::Integer).map(Expr::new).map(Box::new), - }; - - let pipeline = self.fold_by_simulating_eval(pipeline, &tbl)?; - - let transform_kind = TransformKind::Window { - kind, - range, - pipeline: Box::new(pipeline), - }; - (transform_kind, tbl) - } - "append" => { - let [bottom, top] = unpack::<2>(func.args); - - (TransformKind::Append(Box::new(bottom)), top) - } - "loop" => { - let [pipeline, tbl] = unpack::<2>(func.args); - - let pipeline = self.fold_by_simulating_eval(pipeline, &tbl)?; - - (TransformKind::Loop(Box::new(pipeline)), tbl) - } - - "in" => { - // yes, this is not a transform, but this is the most appropriate place for it - - let [pattern, value] = unpack::<2>(func.args); - - if pattern.ty.as_ref().map_or(false, |x| x.kind.is_array()) { - return Ok(Expr::new(ExprKind::RqOperator { - name: "std.array_in".to_string(), - args: vec![value, pattern], - })); - } - - let pattern = match try_restrict_range(pattern) { - Ok((start, end)) => { - let start = restrict_null_literal(start); - let end = restrict_null_literal(end); - - let start = start.map(|s| new_binop(value.clone(), &["std", "gte"], s)); - let end = end.map(|e| new_binop(value, &["std", "lte"], e)); - - let res = maybe_binop(start, &["std", "and"], end); - let res = res.unwrap_or_else(|| { - Expr::new(ExprKind::Literal(Literal::Boolean(true))) - }); - return Ok(res); - } - Err(expr) => expr, - }; - - return Err(Error::new(Reason::Expected { - who: Some("std.in".to_string()), - expected: "a pattern".to_string(), - found: write_pl(pattern.clone()), - }) - .with_span(pattern.span)); - } - - "tuple_every" => { - // yes, this is not a transform, but this is the most appropriate place for it - - let [list] = unpack::<1>(func.args); - let list = list.kind.into_tuple().unwrap(); - - let mut res = None; - for item in list { - res = maybe_binop(res, &["std", "and"], Some(item)); - } - let res = - res.unwrap_or_else(|| Expr::new(ExprKind::Literal(Literal::Boolean(true)))); - - return Ok(res); - } - - "tuple_map" => { - // yes, this is not a transform, but this is the most appropriate place for it - - let [func, list] = unpack::<2>(func.args); - let list_items = list.kind.into_tuple().unwrap(); - - let list_items = list_items - .into_iter() - .map(|item| { - Expr::new(ExprKind::FuncCall(FuncCall::new_simple( - func.clone(), - vec![item], - ))) - }) - .collect_vec(); - - return Ok(Expr { - kind: ExprKind::Tuple(list_items), - ..list - }); - } - - "tuple_zip" => { - // yes, this is not a transform, but this is the most appropriate place for it - - let [a, b] = unpack::<2>(func.args); - let a = a.kind.into_tuple().unwrap(); - let b = b.kind.into_tuple().unwrap(); - - let mut res = Vec::new(); - for (a, b) in std::iter::zip(a, b) { - res.push(Expr::new(ExprKind::Tuple(vec![a, b]))); - } - - return Ok(Expr::new(ExprKind::Tuple(res))); - } - - "_eq" => { - // yes, this is not a transform, but this is the most appropriate place for it - - let [list] = unpack::<1>(func.args); - let list = list.kind.into_tuple().unwrap(); - let [a, b]: [Expr; 2] = list.try_into().unwrap(); - - let res = maybe_binop(Some(a), &["std", "eq"], Some(b)).unwrap(); - return Ok(res); - } - - "from_text" => { - // yes, this is not a transform, but this is the most appropriate place for it - - let [format, text_expr] = unpack::<2>(func.args); - - let text = match text_expr.kind { - ExprKind::Literal(Literal::String(text)) => text, - _ => { - return Err(Error::new(Reason::Expected { - who: Some("std.from_text".to_string()), - expected: "a string literal".to_string(), - found: format!("`{}`", write_pl(text_expr.clone())), - }) - .with_span(text_expr.span)); - } - }; - - let res = { - let span = format.span; - let format = format - .try_cast(ExprKind::into_ident, Some("format"), "ident")? - .to_string(); - match format.as_str() { - "csv" => from_text::parse_csv(&text) - .map_err(|r| Error::new_simple(r).with_span(span))?, - "json" => from_text::parse_json(&text) - .map_err(|r| Error::new_simple(r).with_span(span))?, - - _ => { - return Err(Error::new(Reason::Expected { - who: Some("`format`".to_string()), - expected: "csv or json".to_string(), - found: format, - }) - .with_span(span)) - } - } - }; - - let expr_id = text_expr.id.unwrap(); - let input_name = text_expr.alias.unwrap_or_else(|| "text".to_string()); - - let columns: Vec<_> = res - .columns - .iter() - .cloned() - .map(|x| TyTupleField::Single(Some(x), None)) - .collect(); - - let frame = - self.declare_table_for_literal(expr_id, Some(columns), Some(input_name)); - - let res = Expr::new(ExprKind::Array( - res.rows - .into_iter() - .map(|row| { - Expr::new(ExprKind::Tuple( - row.into_iter() - .map(|lit| Expr::new(ExprKind::Literal(lit))) - .collect(), - )) - }) - .collect(), - )); - let res = Expr { - lineage: Some(frame), - id: text_expr.id, - ..res - }; - return Ok(res); - } - - "prql_version" => { - // yes, this is not a transform, but this is the most appropriate place for it - let ver = compiler_version().to_string(); - return Ok(Expr::new(ExprKind::Literal(Literal::String(ver)))); - } - - "count" | "row_number" => { - // HACK: these functions get `this`, resolved to `{x = {_self}}`, which - // throws an error during lowering. - // But because these functions don't *really* need an arg, we can just pass - // a null instead. - return Ok(Expr { - needs_window, - ..Expr::new(ExprKind::RqOperator { - name: format!("std.{internal_name}"), - args: vec![Expr::new(Literal::Null)], - }) - }); - } - - _ => { - return Err( - Error::new_simple(format!("unknown operator {internal_name}")) - .push_hint("this is a bug in prqlc") - .with_span(func.body.span), - ) - } - }; - - let transform_call = TransformCall { - kind: Box::new(kind), - input: Box::new(input), - partition: None, - frame: WindowFrame::default(), - sort: Vec::new(), - }; - let ty = self.infer_type_of_special_func(&transform_call)?; - Ok(Expr { - ty, - ..Expr::new(ExprKind::TransformCall(transform_call)) - }) - } - - /// Wraps non-tuple Exprs into a singleton Tuple. - pub(super) fn coerce_into_tuple(&mut self, expr: Expr) -> Result { - let is_tuple_ty = expr.ty.as_ref().unwrap().kind.is_tuple() && !expr.kind.is_all(); - Ok(if is_tuple_ty { - // a helpful check for a common anti-pattern - if let Some(alias) = expr.alias { - return Err(Error::new(Reason::Unexpected { - found: format!("assign to `{alias}`"), - }) - .push_hint(format!("move assign into the tuple: `[{alias} = ...]`")) - .with_span(expr.span)); - } - - expr - } else { - let span = expr.span; - let mut expr = Expr::new(ExprKind::Tuple(vec![expr])); - expr.span = span; - - self.fold_expr(expr)? - }) - } - - /// Figure out the type of a function call, if this function is a *special function*. - /// (declared in std module & requires special handling). - pub fn infer_type_of_special_func( - &mut self, - transform_call: &TransformCall, - ) -> Result> { - // Long term plan is to make this function obsolete with generic function parameters. - // In other words, I hope to make our type system powerful enough to express return - // type of all std module functions. - - Ok(match transform_call.kind.as_ref() { - TransformKind::Select { assigns } => assigns - .ty - .clone() - .map(|x| Ty::new(TyKind::Array(Box::new(x)))), - TransformKind::Derive { assigns } => { - let input = transform_call.input.ty.clone().unwrap(); - let input = input.into_relation().unwrap(); - - let derived = assigns.ty.clone().unwrap(); - let derived = derived.kind.into_tuple().unwrap(); - - Some(Ty::new(TyKind::Array(Box::new(Ty::new(ty_tuple_kind( - [input, derived].concat(), - )))))) - } - TransformKind::Aggregate { assigns } => { - let tuple = assigns.ty.clone().unwrap(); - - Some(Ty::new(TyKind::Array(Box::new(tuple)))) - } - TransformKind::Filter { .. } - | TransformKind::Sort { .. } - | TransformKind::Take { .. } => transform_call.input.ty.clone(), - TransformKind::Join { with, .. } => { - let input = transform_call.input.ty.clone().unwrap(); - let input = input.into_relation().unwrap(); - - let with_name = with.alias.clone(); - let with = with.ty.clone().unwrap(); - let with = with.kind.into_array().unwrap(); - let with = TyTupleField::Single(with_name, Some(*with)); - - Some(Ty::new(TyKind::Array(Box::new(Ty::new(ty_tuple_kind( - [input, vec![with]].concat(), - )))))) - } - TransformKind::Group { pipeline, by } => { - let by = by.ty.clone().unwrap(); - let by = by.kind.into_tuple().unwrap(); - - let pipeline = pipeline.ty.clone().unwrap(); - let pipeline = pipeline.kind.into_function().unwrap().unwrap(); - let pipeline = pipeline.return_ty.unwrap().into_relation().unwrap(); - - Some(Ty::new(TyKind::Array(Box::new(Ty::new(ty_tuple_kind( - [by, pipeline].concat(), - )))))) - } - TransformKind::Window { pipeline, .. } | TransformKind::Loop(pipeline) => { - let pipeline = pipeline.ty.clone().unwrap(); - let pipeline = pipeline.kind.into_function().unwrap().unwrap(); - pipeline.return_ty.map(|x| *x) - } - TransformKind::Append(bottom) => { - let top = transform_call.input.ty.clone().unwrap(); - let bottom = bottom.ty.clone().unwrap(); - - Some(type_intersection(top, bottom)) - } - }) - } -} - -fn range_is_empty(range: &(Option, Option)) -> bool { - match (&range.0, &range.1) { - (Some(s), Some(e)) => s > e, - _ => false, - } -} - -fn range_from_ints(start: Option, end: Option) -> Range { - let start = start.map(|x| Box::new(Expr::new(ExprKind::Literal(Literal::Integer(x))))); - let end = end.map(|x| Box::new(Expr::new(ExprKind::Literal(Literal::Integer(x))))); - Range { start, end } -} - -fn into_literal_range(range: (Expr, Expr)) -> Result<(Option, Option)> { - fn into_int(bound: Expr) -> Result> { - match bound.kind { - ExprKind::Literal(Literal::Null) => Ok(None), - ExprKind::Literal(Literal::Integer(i)) => Ok(Some(i)), - _ => Err(Error::new_simple("expected an int literal").with_span(bound.span)), - } - } - Ok((into_int(range.0)?, into_int(range.1)?)) -} - -impl Resolver<'_> { - /// Simulate evaluation of the inner pipeline of group or window - // Creates a dummy node that acts as value that pipeline can be resolved upon. - fn fold_by_simulating_eval(&mut self, pipeline: Expr, val: &Expr) -> Result { - log::debug!("fold by simulating evaluation"); - let span = pipeline.span; - - let param_name = "_tbl"; - let param_id = self.id.gen(); - - // resolver will not resolve a function call if any arguments are missing - // but would instead return a closure to be resolved later. - // because the pipeline of group is a function that takes a table chunk - // and applies the transforms to it, it would not get resolved. - // thats why we trick the resolver with a dummy node that acts as table - // chunk and instruct resolver to apply the transform on that. - - let mut dummy = Expr::new(ExprKind::Ident(Ident::from_name(param_name))); - dummy.lineage.clone_from(&val.lineage); - dummy.ty.clone_from(&val.ty); - - let pipeline = Expr::new(ExprKind::FuncCall(FuncCall::new_simple( - pipeline, - vec![dummy], - ))); - - let env = Module::singleton(param_name, Decl::from(DeclKind::Column(param_id))); - self.root_mod.module.stack_push(NS_PARAM, env); - - let mut pipeline = self.fold_expr(pipeline)?; - - // attach the span to the TransformCall, as this is what will - // be preserved after resolving is complete - pipeline.span = pipeline.span.or(span); - - self.root_mod.module.stack_pop(NS_PARAM).unwrap(); - - // now, we need wrap the result into a closure and replace - // the dummy node with closure's parameter. - - // validate that the return type is a relation - // this can be removed after we have proper type checking for all std functions - let expected = Some(Ty::relation(vec![TyTupleField::Wildcard(None)])); - self.validate_expr_type(&mut pipeline, expected.as_ref(), &|| { - Some("pipeline".to_string()) - })?; - - // construct the function back - let func = Box::new(Func { - name_hint: None, - body: Box::new(pipeline), - return_ty: None, - - args: vec![], - params: vec![FuncParam { - name: param_id.to_string(), - ty: None, - default_value: None, - }], - named_params: vec![], - - env: Default::default(), - generic_type_params: Default::default(), - }); - Ok(*expr_of_func(func, span)) - } -} - -impl TransformCall { - pub fn infer_lineage(&self) -> Result { - use TransformKind::*; - - fn lineage_or_default(expr: &Expr) -> Result { - expr.lineage.clone().ok_or_else(|| { - Error::new_simple("expected {expr:?} to have table type").with_span(expr.span) - }) - } - - Ok(match self.kind.as_ref() { - Select { assigns } => { - let mut lineage = lineage_or_default(&self.input)?; - - lineage.clear(); - lineage.apply_assigns(assigns, false); - lineage - } - Derive { assigns } => { - let mut lineage = lineage_or_default(&self.input)?; - - lineage.apply_assigns(assigns, false); - lineage - } - Group { pipeline, by, .. } => { - let mut lineage = lineage_or_default(&self.input)?; - lineage.clear(); - lineage.apply_assigns(by, false); - - // pipeline's body is resolved, just use its type - let Func { body, .. } = pipeline.kind.as_func().unwrap().as_ref(); - - let partition_lin = lineage_or_default(body).unwrap(); - lineage.columns.extend(partition_lin.columns); - - log::debug!(".. type={lineage}"); - lineage - } - Window { pipeline, .. } => { - // pipeline's body is resolved, just use its type - let Func { body, .. } = pipeline.kind.as_func().unwrap().as_ref(); - - lineage_or_default(body).unwrap() - } - Aggregate { assigns } => { - let mut lineage = lineage_or_default(&self.input)?; - lineage.clear(); - - lineage.apply_assigns(assigns, false); - lineage - } - Join { with, .. } => { - let left = lineage_or_default(&self.input)?; - let right = lineage_or_default(with)?; - join(left, right) - } - Append(bottom) => { - let top = lineage_or_default(&self.input)?; - let bottom = lineage_or_default(bottom)?; - append(top, bottom)? - } - Loop(_) => lineage_or_default(&self.input)?, - Sort { .. } | Filter { .. } | Take { .. } => lineage_or_default(&self.input)?, - }) - } -} - -fn join(mut lhs: Lineage, rhs: Lineage) -> Lineage { - lhs.columns.extend(rhs.columns); - lhs.inputs.extend(rhs.inputs); - lhs -} - -fn append(mut top: Lineage, bottom: Lineage) -> Result { - if top.columns.len() != bottom.columns.len() { - return Err(Error::new_simple( - "cannot append two relations with non-matching number of columns.", - )) - .push_hint(format!( - "top has {} columns, but bottom has {}", - top.columns.len(), - bottom.columns.len() - )); - } - - // TODO: I'm not sure what to use as input_name and expr_id... - let mut columns = Vec::with_capacity(top.columns.len()); - for (t, b) in zip(top.columns, bottom.columns) { - columns.push(match (t, b) { - (LineageColumn::All { input_id, except }, LineageColumn::All { .. }) => { - LineageColumn::All { input_id, except } - } - ( - LineageColumn::Single { - name: name_t, - target_id, - target_name, - }, - LineageColumn::Single { name: name_b, .. }, - ) => match (name_t, name_b) { - (None, None) => { - let name = None; - LineageColumn::Single { - name, - target_id, - target_name, - } - } - (None, Some(name)) | (Some(name), _) => { - let name = Some(name); - LineageColumn::Single { - name, - target_id, - target_name, - } - } - }, - (t, b) => return Err(Error::new_simple(format!( - "cannot match columns `{t:?}` and `{b:?}`" - )) - .push_hint( - "make sure that top and bottom relations of append has the same column layout", - )), - }); - } - - top.columns = columns; - Ok(top) -} - -impl Lineage { - pub fn clear(&mut self) { - self.prev_columns.clear(); - self.prev_columns.append(&mut self.columns); - } - - pub fn apply_assigns(&mut self, assigns: &Expr, inline_refs: bool) { - match &assigns.kind { - ExprKind::Tuple(fields) => { - for expr in fields { - self.apply_assigns(expr, inline_refs); - } - - // hack for making `x | select { y = this }` work - if let Some(alias) = &assigns.alias { - if self.columns.len() == 1 { - let col = self.columns.first().unwrap(); - if let LineageColumn::All { input_id, .. } = col { - let input = self.inputs.iter_mut().find(|i| i.id == *input_id).unwrap(); - input.name.clone_from(alias); - } - } - } - } - _ => self.apply_assign(assigns, inline_refs), - } - } - - pub fn apply_assign(&mut self, expr: &Expr, inline_refs: bool) { - // special case: all except - if let ExprKind::All { within, except } = &expr.kind { - let mut within_lineage = Lineage::default(); - within_lineage.inputs.extend(self.inputs.clone()); - within_lineage.apply_assigns(within, true); - - let mut except_lineage = Lineage::default(); - except_lineage.inputs.extend(self.inputs.clone()); - except_lineage.apply_assigns(except, true); - - 'within: for col in within_lineage.columns { - match col { - LineageColumn::Single { - ref name, - ref target_id, - ref target_name, - .. - } => { - let is_excluded = except_lineage.columns.iter().any(|e| match e { - LineageColumn::Single { name: e_name, .. } => name == e_name, - - LineageColumn::All { - input_id: e_iid, - except: e_except, - } => { - target_id == e_iid - && !e_except.contains(target_name.as_ref().unwrap()) - } - }); - if !is_excluded { - self.columns.push(col); - } - } - LineageColumn::All { - input_id, - mut except, - } => { - for excluded in &except_lineage.columns { - match excluded { - LineageColumn::Single { - name: Some(name), .. - } => { - let input = self.find_input(input_id).unwrap(); - let ex_input_name = name.iter().next().unwrap(); - if ex_input_name == &input.name { - except.insert(name.name.clone()); - } - } - LineageColumn::Single { .. } => {} - LineageColumn::All { - input_id: e_iid, - except: e_e, - } => { - if *e_iid == input_id { - // The two `All`s match and will erase each other. - // The only remaining columns are those from the first wildcard - // that are not excluded, but are excluded in the second wildcard. - let input = self.find_input(input_id).unwrap(); - let input_name = input.name.clone(); - for remaining in e_e.difference(&except).sorted() { - self.columns.push(LineageColumn::Single { - name: Some(Ident { - path: vec![input_name.clone()], - name: remaining.clone(), - }), - target_id: input_id, - target_name: Some(remaining.clone()), - }) - } - continue 'within; - } - } - } - } - self.columns.push(LineageColumn::All { input_id, except }); - } - } - } - return; - } - - // special case: include a tuple - if expr.ty.as_ref().map_or(false, |x| x.kind.is_tuple()) && expr.kind.is_ident() { - // this ident is a tuple, which means it much point to an input - let input_id = expr.target_id.unwrap(); - - self.columns.push(LineageColumn::All { - input_id, - except: Default::default(), - }); - return; - } - - // special case: an ref that should be inlined because this node - // might not exist in the resulting AST - if inline_refs && expr.target_id.is_some() { - let ident = expr.kind.as_ident().unwrap().clone().pop_front().1.unwrap(); - let target_id = expr.target_id.unwrap(); - let input = &self.find_input(target_id); - - self.columns.push(if input.is_some() { - LineageColumn::Single { - target_name: Some(ident.name.clone()), - name: Some(ident), - target_id, - } - } else { - LineageColumn::Single { - target_name: None, - name: Some(ident), - target_id, - } - }); - return; - }; - - // base case: define the expr as a new lineage column - let (target_id, target_name) = (expr.id.unwrap(), None); - - let alias = expr.alias.as_ref().map(Ident::from_name); - let name = alias.or_else(|| expr.kind.as_ident()?.clone().pop_front().1); - - // remove names from columns with the same name - if name.is_some() { - for c in &mut self.columns { - if let LineageColumn::Single { name: n, .. } = c { - if n.as_ref().map(|i| &i.name) == name.as_ref().map(|i| &i.name) { - *n = None; - } - } - } - } - - self.columns.push(LineageColumn::Single { - name, - target_id, - target_name, - }); - } - - pub fn find_input_by_name(&self, input_name: &str) -> Option<&LineageInput> { - self.inputs.iter().find(|i| i.name == input_name) - } - - pub fn find_input(&self, input_id: usize) -> Option<&LineageInput> { - self.inputs.iter().find(|i| i.id == input_id) - } - - /// Renames all frame inputs to the given alias. - pub fn rename(&mut self, alias: String) { - for input in &mut self.inputs { - input.name.clone_from(&alias); - } - - for col in &mut self.columns { - match col { - LineageColumn::All { .. } => {} - LineageColumn::Single { - name: Some(name), .. - } => name.path = vec![alias.clone()], - _ => {} - } - } - } -} - -/// Expects closure's args to be resolved. -/// Note that named args are before positional args, in order of declaration. -fn unpack(func_args: Vec) -> [Expr; P] { - func_args.try_into().expect("bad special function cast") -} - -mod from_text { - use super::*; - use crate::ir::rq::RelationLiteral; - - // TODO: Can we dynamically get the types, like in pandas? We need to put - // quotes around strings and not around numbers. - // https://stackoverflow.com/questions/64369887/how-do-i-read-csv-data-without-knowing-the-structure-at-compile-time - pub fn parse_csv(text: &str) -> Result { - let text = text.trim(); - let mut rdr = csv::Reader::from_reader(text.as_bytes()); - - fn parse_header(row: &csv::StringRecord) -> Vec { - row.into_iter().map(|x| x.to_string()).collect() - } - - fn parse_row(row: csv::StringRecord) -> Vec { - row.into_iter() - .map(|x| Literal::String(x.to_string())) - .collect() - } - - Ok(RelationLiteral { - columns: parse_header(rdr.headers().map_err(|e| e.to_string())?), - rows: rdr - .records() - .map(|row_result| row_result.map(parse_row)) - .try_collect() - .map_err(|e| e.to_string())?, - }) - } - - type JsonFormat1Row = HashMap; - - #[derive(Deserialize)] - struct JsonFormat2 { - columns: Vec, - data: Vec>, - } - - fn map_json_primitive(primitive: serde_json::Value) -> Literal { - use serde_json::Value::*; - match primitive { - Null => Literal::Null, - Bool(bool) => Literal::Boolean(bool), - Number(number) if number.is_i64() => Literal::Integer(number.as_i64().unwrap()), - Number(number) if number.is_f64() => Literal::Float(number.as_f64().unwrap()), - Number(_) => Literal::Null, - String(string) => Literal::String(string), - Array(_) => Literal::Null, - Object(_) => Literal::Null, - } - } - - fn object_to_vec( - mut row_map: HashMap, - columns: &[String], - ) -> Vec { - columns - .iter() - .map(|c| { - row_map - .remove(c) - .map(map_json_primitive) - .unwrap_or(Literal::Null) - }) - .collect_vec() - } - - pub fn parse_json(text: &str) -> Result { - parse_json1(text).or_else(|err1| { - parse_json2(text) - .map_err(|err2| format!("While parsing rows: {err1}\nWhile parsing object: {err2}")) - }) - } - - fn parse_json1(text: &str) -> Result { - let data: Vec = serde_json::from_str(text).map_err(|e| e.to_string())?; - let mut columns = data - .first() - .ok_or("json: no rows")? - .keys() - .cloned() - .collect_vec(); - - // JSON object keys are not ordered, so have to apply some order to produce - // deterministic results - columns.sort(); - - let rows = data - .into_iter() - .map(|row_map| object_to_vec(row_map, &columns)) - .collect_vec(); - Ok(RelationLiteral { columns, rows }) - } - - fn parse_json2(text: &str) -> Result { - let JsonFormat2 { columns, data } = - serde_json::from_str(text).map_err(|x| x.to_string())?; - - Ok(RelationLiteral { - columns, - rows: data - .into_iter() - .map(|row| row.into_iter().map(map_json_primitive).collect_vec()) - .collect_vec(), - }) - } -} - -#[cfg(test)] -mod tests { - use insta::assert_yaml_snapshot; - - use crate::semantic::test::parse_resolve_and_lower; - - #[test] - fn test_aggregate_positional_arg() { - // distinct query #292 - - assert_yaml_snapshot!(parse_resolve_and_lower(" - from c_invoice - select invoice_no - group invoice_no ( - take 1 - ) - ").unwrap(), @r###" - --- - def: - version: ~ - other: {} - tables: - - id: 0 - name: ~ - relation: - kind: - ExternRef: - LocalTable: - - c_invoice - columns: - - Single: invoice_no - - Wildcard - relation: - kind: - Pipeline: - - From: - source: 0 - columns: - - - Single: invoice_no - - 0 - - - Wildcard - - 1 - name: c_invoice - - Select: - - 0 - - Take: - range: - start: ~ - end: - kind: - Literal: - Integer: 1 - span: ~ - partition: - - 0 - sort: [] - - Select: - - 0 - columns: - - Single: invoice_no - "###); - - // oops, two arguments #339 - let result = parse_resolve_and_lower( - " - from c_invoice - aggregate average amount - ", - ); - assert!(result.is_err()); - - // oops, two arguments - let result = parse_resolve_and_lower( - " - from c_invoice - group issued_at (aggregate average amount) - ", - ); - assert!(result.is_err()); - - // correct function call - let ctx = crate::semantic::test::parse_and_resolve( - " - from c_invoice - group issued_at ( - aggregate (average amount) - ) - ", - ) - .unwrap(); - let (res, _) = ctx.find_main_rel(&[]).unwrap().clone(); - let expr = res.clone().into_relation_var().unwrap(); - let expr = super::super::test::erase_ids(*expr); - assert_yaml_snapshot!(expr); - } - - #[test] - fn test_transform_sort() { - assert_yaml_snapshot!(parse_resolve_and_lower(" - from invoices - sort {issued_at, -amount, +num_of_articles} - sort issued_at - sort (-issued_at) - sort {issued_at} - sort {-issued_at} - ").unwrap(), @r###" - --- - def: - version: ~ - other: {} - tables: - - id: 0 - name: ~ - relation: - kind: - ExternRef: - LocalTable: - - invoices - columns: - - Single: issued_at - - Single: amount - - Single: num_of_articles - - Wildcard - relation: - kind: - Pipeline: - - From: - source: 0 - columns: - - - Single: issued_at - - 0 - - - Single: amount - - 1 - - - Single: num_of_articles - - 2 - - - Wildcard - - 3 - name: invoices - - Sort: - - direction: Asc - column: 0 - - direction: Desc - column: 1 - - direction: Asc - column: 2 - - Sort: - - direction: Asc - column: 0 - - Sort: - - direction: Desc - column: 0 - - Sort: - - direction: Asc - column: 0 - - Sort: - - direction: Desc - column: 0 - - Select: - - 0 - - 1 - - 2 - - 3 - columns: - - Single: issued_at - - Single: amount - - Single: num_of_articles - - Wildcard - "###); - } -} diff --git a/prqlc/prqlc/src/semantic/resolver/tuple.rs b/prqlc/prqlc/src/semantic/resolver/tuple.rs new file mode 100644 index 000000000000..ed0408a7c0e5 --- /dev/null +++ b/prqlc/prqlc/src/semantic/resolver/tuple.rs @@ -0,0 +1,617 @@ +use std::borrow::Cow; + +use itertools::Itertools; + +use crate::pr::{Ident, Ty, TyKind, TyTupleField, PrimitiveSet}; +use crate::codegen::write_ty; +use crate::ir::decl::DeclKind; +use crate::ir::pl::{Expr, ExprKind, IndirectionKind}; +use crate::{Error, Result, WithErrorInfo}; + +// TODO: i'm not proud of the naming scheme in this file + +pub fn lookup_position_in_tuple(base: &Ty, position: usize) -> Result> { + // get base fields + let TyKind::Tuple(fields) = &base.kind else { + return Ok(None); + }; + + let unpack = fields.last().and_then(|f| match f { + TyTupleField::Single(_, _) => None, + TyTupleField::Unpack(Some(t)) => Some(t), + TyTupleField::Unpack(None) => todo!(), + }); + let singles = if unpack.is_some() { + &fields[0..fields.len() - 1] + } else { + fields.as_slice() + }; + + Ok(if position < singles.len() { + fields.get(position).map(|f| match f { + TyTupleField::Single(_, Some(ty)) => StepOwned { + position, + target_ty: ty.clone(), + }, + TyTupleField::Single(_, None) => todo!(), + TyTupleField::Unpack(_) => unreachable!(), + }) + } else if let Some(unpack) = unpack { + let pos_here = singles.len(); + lookup_position_in_tuple(unpack, position - pos_here)?.map(|mut step| { + step.position += pos_here; + step + }) + } else { + None + }) +} + +impl super::Resolver<'_> { + /// Performs tuple indirection by name. + pub fn lookup_name_in_tuple<'a>( + &'a mut self, + ty: &'a Ty, + name: &str, + ) -> Result>> { + log::debug!("looking up `.{name}` in {}", write_ty(ty)); + + // find existing field + let found = self.find_name_in_tuple(ty, name); + match found.len() { + // no match: pass though + 0 => {} + + // single match, great! + 1 => { + let found = found.into_iter().next().unwrap(); + return Ok(Some( + found.into_iter().map(|s| s.into_owned()).collect_vec(), + )); + } + + // ambiguous + _ => return Err(ambiguous_error(found)), + } + + // field was not found, find a generic where it could be added + let generics = self.find_tuple_generic(ty, false); + match generics.len() { + // no match: pass though + 0 => {} + + // single match, great! + 1 => { + let loc = generics.into_iter().next().unwrap(); + let pos_gen = loc.position; + let ident_of_generic = loc.ident_of_generic.clone(); + + let mut steps: Vec = loc + .steps_to_base + .into_iter() + .map(|s| s.into_owned()) + .collect(); + + let indirection = IndirectionKind::Name(name.to_string()); + let (pos_within, target_ty) = + self.infer_generic_as_tuple(&ident_of_generic, indirection); + + steps.push(StepOwned { + position: pos_gen + pos_within, + target_ty, + }); + return Ok(Some(steps)); + } + + // ambiguous + _ => { + let dummy = Ty::new(PrimitiveSet::Bool); + let name = name.to_string(); + let candidates = generics + .into_iter() + .map(|mut loc| { + loc.steps_to_base.push(Step { + position: loc.position, + name: Some(&name), + target_ty: Cow::Borrowed(&dummy), + }); + loc.steps_to_base + }) + .collect(); + return Err(ambiguous_error(candidates)); + } + } + + Ok(None) + } + + fn get_tuple_or_generic_candidate<'a>(&'a self, ty: &'a Ty) -> &'a Ty { + let TyKind::Ident(ident) = &ty.kind else { + return ty; + }; + let decl = self.get_ident(ident).unwrap(); + let DeclKind::GenericParam(Some((candidate, _))) = &decl.kind else { + return ty; + }; + + candidate + } + + /// Find in fields of this tuple (including the unpack) + fn find_name_in_tuple<'a>(&'a self, ty: &'a Ty, name: &str) -> Vec> { + let ty = self.get_tuple_or_generic_candidate(ty); + + let TyKind::Tuple(fields) = &ty.kind else { + return vec![]; + }; + + if let Some(step) = self.find_name_in_tuple_direct(ty, name) { + return vec![vec![step]]; + }; + + let mut res = vec![]; + for (position, field) in fields.iter().enumerate() { + match field { + TyTupleField::Single(n, Some(ty)) => { + for mut x in self.find_name_in_tuple(ty, name) { + x.insert( + 0, + Step { + position, + name: n.as_ref(), + target_ty: Cow::Borrowed(ty), + }, + ); + res.push(x); + } + } + TyTupleField::Unpack(Some(unpack_ty)) => { + res.extend(self.find_name_in_tuple(unpack_ty, name)); + } + TyTupleField::Single(_, None) => { + todo!() + } + TyTupleField::Unpack(None) => { + todo!() + } + } + } + res + } + + /// Find in this tuple (including the unpack) + fn find_name_in_tuple_direct<'a>(&'a self, ty: &'a Ty, name: &str) -> Option> { + let ty = self.get_tuple_or_generic_candidate(ty); + + let TyKind::Tuple(fields) = &ty.kind else { + return None; + }; + + for (position, field) in fields.iter().enumerate() { + match field { + TyTupleField::Single(n, Some(ty)) => { + if n.as_ref().map_or(false, |n| n == name) { + return Some(Step { + position, + name: n.as_ref(), + target_ty: Cow::Borrowed(ty), + }); + } + } + TyTupleField::Unpack(Some(unpack_ty)) => { + if let Some(mut step) = self.find_name_in_tuple_direct(unpack_ty, name) { + step.position += position; + return Some(step); + } + } + TyTupleField::Single(_, None) => todo!(), + TyTupleField::Unpack(None) => todo!(), + } + } + None + } + + /// Utility function for wrapping an expression into additional indirections. + /// For example, when we have `x.a`, but `x = {b = {a = int}}`, lookup will return steps `[b, a]`. + /// This function converts `x` and `[b, a]` into `((x).b).a`. + pub fn apply_indirections(&mut self, mut base: Expr, steps: Vec) -> Expr { + for step in steps { + base = Expr { + id: Some(self.id.gen()), + ty: Some(step.target_ty), + ..Expr::new(ExprKind::Indirection { + base: Box::new(base), + field: IndirectionKind::Position(step.position as i64), + }) + } + } + base + } + + /// Find identifier of the generic that must receive a new field, + /// if we push a new name into this tuple. + fn find_tuple_generic<'a>( + &self, + ty: &'a Ty, + require_tuple: bool, + ) -> Vec> { + if let TyKind::Ident(ident_of_generic) = &ty.kind { + if require_tuple { + let Some(decl) = self.get_ident(ident_of_generic) else { + return vec![]; + }; + let Some((cand, _)) = decl.kind.as_generic_param().and_then(|p| p.as_ref()) else { + return vec![]; + }; + if !cand.kind.is_tuple() { + return vec![]; + } + } + + return vec![LocationOfGeneric { + ident_of_generic, + position: 0, + steps_to_base: vec![], + }]; + }; + + let TyKind::Tuple(fields) = &ty.kind else { + return vec![]; + }; + + if let Some(TyTupleField::Unpack(Some(unpack_ty))) = fields.last() { + let mut found = self.find_tuple_generic(unpack_ty, require_tuple); + if !found.is_empty() { + for x in &mut found { + if x.steps_to_base.is_empty() { + x.position += fields.len() - 1; + } else { + x.steps_to_base.first_mut().unwrap().position += fields.len() - 1; + } + } + return found; + } + } + + let mut res = vec![]; + for (position, field) in fields.iter().enumerate() { + if let TyTupleField::Single(n, Some(ty)) = field { + for mut x in self.find_tuple_generic(ty, true) { + x.steps_to_base.insert( + 0, + Step { + position, + name: n.as_ref(), + target_ty: Cow::Borrowed(ty), + }, + ); + res.push(x); + } + } + } + res + } +} + +#[derive(Debug, Clone)] +pub struct Step<'a> { + position: usize, + name: Option<&'a String>, + target_ty: Cow<'a, Ty>, +} + +impl<'a> Step<'a> { + #[allow(dead_code)] + fn into_indirection(self) -> IndirectionKind { + if let Some(name) = self.name { + IndirectionKind::Name(name.clone()) + } else { + IndirectionKind::Position(self.position as i64) + } + } + + fn as_str(&self) -> Cow { + if let Some(name) = self.name { + name.into() + } else { + self.position.to_string().into() + } + } + + fn into_owned(self) -> StepOwned { + StepOwned { + position: self.position, + target_ty: self.target_ty.into_owned(), + } + } +} + +#[derive(PartialEq)] +pub struct StepOwned { + position: usize, + target_ty: Ty, +} + +impl std::fmt::Debug for StepOwned { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StepOwned") + .field("position", &self.position) + .field("target_ty", &write_ty(&self.target_ty)) + .finish() + } +} + +struct LocationOfGeneric<'a> { + ident_of_generic: &'a Ident, + position: usize, + steps_to_base: Vec>, +} + +fn ambiguous_error(candidates: Vec>) -> Error { + let mut candidates_str = Vec::new(); + for steps in candidates { + let mut steps = steps.into_iter(); + + let first = steps.next().unwrap(); + let mut r = first.as_str().to_string(); + for step in steps { + r += "."; + r += &step.as_str(); + } + candidates_str.push(r); + } + let hint = format!("could be any of: {}", candidates_str.join(", ")); + Error::new_simple("Ambiguous name").push_hint(hint) +} + +#[cfg(test)] +mod test { + use crate::pr::Ty; + use crate::ir::decl::RootModule; + use crate::parser::parse; + use crate::semantic::resolver::tuple::StepOwned; + use crate::semantic::resolver::Resolver; + use crate::{Error, Result, SourceTree}; + + fn parse_ty(source: &str) -> Ty { + let s = SourceTree::from(format!("type X = {source}")); + let mod_def = parse(&s).unwrap(); + let stmt = mod_def.stmts.into_iter().next().unwrap(); + let ty_def = stmt.kind.into_type_def().unwrap(); + + ty_def.value.unwrap() + } + + fn tuple_lookup(tuple: &str, name: &str) -> Result> { + let mut root_module = RootModule::default(); + let mut r = Resolver::new(&mut root_module); + + r.lookup_name_in_tuple(&parse_ty(tuple), name) + .and_then(|x| match x { + Some(x) => Ok(x), + None => Err(Error::new_simple("unknown name")), + }) + } + + fn tuple_lookup_with_generic(tuple: &str, name: &str) -> Result<(Vec, Ty)> { + let mut root_module = RootModule::default(); + let mut r = Resolver::new(&mut root_module); + + // generate a new generic type (tests expect it to get name 'X1') + let ident = r.init_new_global_generic("X"); + assert_eq!(ident.to_string(), "_generic.X1"); + + // do the lookup + let res = r.lookup_name_in_tuple(&parse_ty(tuple), name)?.unwrap(); + + // get the generic candidate that was inferred + let decl = r.get_ident(&ident).unwrap(); + let generic = decl.kind.as_generic_param().unwrap(); + let generic_candidate = generic.clone().unwrap().0; + Ok((res, generic_candidate)) + } + + // ```prql + // let x = { + // a = 1, + // b = { + // a = 2, + // b = 3, + // c = 4, + // d = { + // e = 5, + // ..G101 + // } + // }, + // c = 3, + // ..G100 + // } + // + // let y5 = x.f # G102 (indirections: .3) + // let y6 = x.b.f # G103 (indirections: .1.3.1) + + #[test] + fn simple() { + assert_eq!( + tuple_lookup("{a = int}", "a").unwrap(), + vec![StepOwned { + position: 0, + target_ty: parse_ty("int") + }] + ); + + assert_eq!( + tuple_lookup("{a = int, int, b = bool}", "b").unwrap(), + vec![StepOwned { + position: 2, + target_ty: parse_ty("bool") + }] + ); + } + + #[test] + fn unpack() { + assert_eq!( + tuple_lookup("{a = int, ..{b = bool}}", "b").unwrap(), + vec![StepOwned { + position: 1, + target_ty: parse_ty("bool") + }] + ); + + assert_eq!( + tuple_lookup( + "{a = int, ..{b = bool, ..{c = int, bool, ..{d = bool}}}}", + "d" + ) + .unwrap(), + vec![StepOwned { + position: 4, + target_ty: parse_ty("bool") + }] + ); + } + + #[test] + fn nested() { + assert_eq!( + tuple_lookup("{a = int, b = {bool, bool, c = int}}", "c").unwrap(), + vec![ + StepOwned { + position: 1, + target_ty: parse_ty("{bool, bool, c = int}") + }, + StepOwned { + position: 2, + target_ty: parse_ty("int") + } + ] + ); + + assert_eq!( + tuple_lookup("{a = int, {b = int, {{c = int}, d = bool}, e = bool}}", "c").unwrap(), + vec![ + StepOwned { + position: 1, + target_ty: parse_ty("{b = int, {{c = int}, d = bool}, e = bool}") + }, + StepOwned { + position: 1, + target_ty: parse_ty("{{c = int}, d = bool}") + }, + StepOwned { + position: 0, + target_ty: parse_ty("{c = int}") + }, + StepOwned { + position: 0, + target_ty: parse_ty("int") + }, + ] + ); + + // ambiguous + tuple_lookup("{a = {c = int}, b = {c = int}}", "c").unwrap_err(); + + // ambiguous + tuple_lookup("{{c = int}, {c = int}}", "c").unwrap_err(); + + // ambiguous + tuple_lookup("{a = {c = int}, ..{b = {c = int}}}", "c").unwrap_err(); + + assert_eq!( + tuple_lookup("{a = int, b = {a = int}}", "a").unwrap(), + vec![StepOwned { + position: 0, + target_ty: parse_ty("int") + }] + ); + + assert_eq!( + tuple_lookup("{a = int, b = {a = int}}", "a").unwrap(), + vec![StepOwned { + position: 0, + target_ty: parse_ty("int") + }] + ); + } + + #[test] + fn generic() { + assert_eq!( + tuple_lookup_with_generic("{a = int, .._generic.X1}", "b").unwrap(), + ( + vec![StepOwned { + position: 1, + target_ty: parse_ty("_generic.F2") + }], + parse_ty("{b = _generic.F2}") + ) + ); + + assert_eq!( + tuple_lookup_with_generic( + "{a = int, b = {c = int, .._generic.X1}, .._generic.X1}", + "d" + ) + .unwrap(), + ( + vec![StepOwned { + position: 2, + target_ty: parse_ty("_generic.F2") + }], + parse_ty("{d = _generic.F2}") + ) + ); + + assert_eq!( + tuple_lookup_with_generic( + "{a = int, b = {c = int, .._generic.X1}, ..{c = int, .._generic.X1}}", + "d" + ) + .unwrap(), + ( + vec![StepOwned { + position: 3, + target_ty: parse_ty("_generic.F2") + }], + parse_ty("{d = _generic.F2}") + ) + ); + + assert_eq!( + tuple_lookup_with_generic("{a = int, b = {c = int, .._generic.X1}, ..{c = int}}", "d") + .unwrap(), + ( + vec![ + StepOwned { + position: 1, + target_ty: parse_ty("{c = int, .._generic.X1}") + }, + StepOwned { + position: 1, + target_ty: parse_ty("_generic.F2") + } + ], + parse_ty("{d = _generic.F2}") + ) + ); + + assert_eq!( + tuple_lookup_with_generic("{a = _generic.X1, .._generic.X1}", "b").unwrap(), + ( + vec![ + StepOwned { + position: 0, + target_ty: parse_ty("_generic.X1") + }, + StepOwned { + position: 0, + target_ty: parse_ty("_generic.F2") + } + ], + parse_ty("{b = _generic.F2}") + ) + ); + } +} diff --git a/prqlc/prqlc/src/semantic/resolver/types.rs b/prqlc/prqlc/src/semantic/resolver/types.rs index f4fc388ba8d6..955929f5ac7c 100644 --- a/prqlc/prqlc/src/semantic/resolver/types.rs +++ b/prqlc/prqlc/src/semantic/resolver/types.rs @@ -1,24 +1,108 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use itertools::Itertools; -use super::Resolver; use crate::codegen::{write_ty, write_ty_kind}; -use crate::ir::decl::DeclKind; +use crate::ir::decl::{Decl, DeclKind}; use crate::ir::pl::*; use crate::pr::{PrimitiveSet, Ty, TyFunc, TyKind, TyTupleField}; +use crate::semantic::{NS_GENERIC, NS_LOCAL}; use crate::Result; -use crate::{Error, Reason, WithErrorInfo}; +use crate::{Error, Reason, Span, WithErrorInfo}; + +use super::Resolver; impl Resolver<'_> { - pub fn infer_type(expr: &Expr) -> Result> { + /// Visit a type in the main resolver pass. It will: + /// - resolve [TyKind::Ident] to material types (expect for the ones that point to generic type arguments), + /// - inline [TyTupleField::Unpack], + /// - inline [TyKind::Exclude]. + // This function is named fold_type_actual, because fold_type must be in + // expr.rs, where we implement PlFold. + pub fn fold_type_actual(&mut self, ty: Ty) -> Result { + Ok(match ty.kind { + TyKind::Ident(ident) => { + let decl = self.get_ident(&ident).ok_or_else(|| { + Error::new_assert("cannot find type ident") + .push_hint(format!("ident={ident:?}")) + })?; + + let mut fold_again = false; + let ty = match &decl.kind { + DeclKind::Ty(ref_ty) => { + // materialize into the referred type + fold_again = true; + let inferred_name = if ident.starts_with_part(NS_GENERIC) + || ident.starts_with_part(NS_LOCAL) + { + None + } else { + Some(ident.name) + }; + Ty { + kind: ref_ty.kind.clone(), + name: ref_ty.name.clone().or(inferred_name), + span: ty.span, + } + } + + DeclKind::GenericParam(_) => { + // leave as an ident + Ty { + name: Some(ident.name.clone()), + kind: TyKind::Ident(ident), + ..ty + } + } + + DeclKind::Unresolved(_) => { + return Err(Error::new_assert(format!( + "bad resolution order: unresolved {ident} while resolving {}", + self.debug_current_decl + )) + .with_span(ty.span)) + } + _ => { + return Err(Error::new(Reason::Expected { + who: None, + expected: "a type".to_string(), + found: decl.to_string(), + }) + .with_span(ty.span)) + } + }; + + if fold_again { + self.fold_type_actual(ty)? + } else { + ty + } + } + TyKind::Tuple(fields) => Ty { + kind: TyKind::Tuple(ty_fold_and_inline_tuple_fields(self, fields)?), + ..ty + }, + TyKind::Exclude { base, except } => { + let base = self.fold_type(*base)?; + let except = self.fold_type(*except)?; + + Ty { + kind: self.ty_tuple_exclusion(base, except)?, + ..ty + } + } + _ => fold_type(self, ty)?, + }) + } + + pub fn infer_type(&mut self, expr: &Expr) -> Result> { if let Some(ty) = &expr.ty { return Ok(Some(ty.clone())); } let kind = match &expr.kind { ExprKind::Literal(ref literal) => match literal { - Literal::Null => TyKind::Singleton(Literal::Null), + Literal::Null => return Ok(None), // TODO Literal::Integer(_) => TyKind::Primitive(PrimitiveSet::Int), Literal::Float(_) => TyKind::Primitive(PrimitiveSet::Float), Literal::Boolean(_) => TyKind::Primitive(PrimitiveSet::Bool), @@ -38,59 +122,103 @@ impl Resolver<'_> { ExprKind::TransformCall(_) => return Ok(None), // TODO ExprKind::Tuple(fields) => { let mut ty_fields: Vec = Vec::with_capacity(fields.len()); - let has_other = false; for field in fields { - let ty = Resolver::infer_type(field)?; + let ty = self.infer_type(field)?; if field.flatten { - if let Some(fields) = ty.as_ref().and_then(|x| x.kind.as_tuple()) { - ty_fields.extend(fields.iter().cloned()); - continue; + let ty = ty.clone().unwrap(); + match ty.kind { + TyKind::Tuple(inner_fields) => { + ty_fields.extend(inner_fields); + } + _ => ty_fields.push(TyTupleField::Unpack(Some(ty))), } + + continue; } - // TODO: move this into de-sugar stage (expand PL) - // TODO: this will not infer nested namespaces let name = field .alias .clone() - .or_else(|| field.kind.as_ident().map(|i| i.name.clone())); + .or_else(|| self.infer_tuple_field_name(field)); ty_fields.push(TyTupleField::Single(name, ty)); } - - if has_other { - ty_fields.push(TyTupleField::Wildcard(None)); - } ty_tuple_kind(ty_fields) } ExprKind::Array(items) => { let mut variants = Vec::with_capacity(items.len()); for item in items { - let item_ty = Resolver::infer_type(item)?; + let item_ty = self.infer_type(item)?; if let Some(item_ty) = item_ty { - variants.push((None, item_ty)); + variants.push(item_ty); } } - let items_ty = Ty::new(TyKind::Union(variants)); - let items_ty = normalize_type(items_ty); + let items_ty = match variants.len() { + 0 => { + // no items, so we must infer the type + let generic_ident = self.init_new_global_generic("A"); + Ty::new(TyKind::Ident(generic_ident)) + } + 1 => { + // single item, use its type + variants.into_iter().exactly_one().unwrap() + } + 2.. => { + // ideally, we would enforce that all of items have + // the same type, but currently we don't have a good + // strategy for dealing with nullable types, which + // causes problems here. + // HACK: use only the first type + variants.into_iter().next().unwrap() + } + }; TyKind::Array(Box::new(items_ty)) } ExprKind::All { within, except } => { - let base = Box::new(Resolver::infer_type(within)?.unwrap()); - let exclude = Box::new(Resolver::infer_type(except)?.unwrap()); + let Some(within_ty) = self.infer_type(within)? else { + return Ok(None); + }; + let Some(except_ty) = self.infer_type(except)? else { + return Ok(None); + }; + self.ty_tuple_exclusion(within_ty, except_ty)? + } + + ExprKind::Case(cases) => { + let case_tys: Vec> = cases + .iter() + .map(|c| self.infer_type(&c.value)) + .try_collect()?; + + let Some(inferred_ty) = case_tys.iter().find_map(|x| x.as_ref()) else { + return Err(Error::new_simple( + "cannot infer type of any of the branches of this case statement", + ) + .with_span(expr.span)); + }; - normalize_type(Ty::new(TyKind::Difference { base, exclude })).kind + return Ok(Some(inferred_ty.clone())); } + ExprKind::Func(func) => TyKind::Function(Some(TyFunc { + params: func.params.iter().map(|p| p.ty.clone()).collect_vec(), + return_ty: func + .return_ty + .clone() + .or_else(|| func.body.ty.clone()) + .map(Box::new), + generic_type_params: func.generic_type_params.clone(), + })), + _ => return Ok(None), }; Ok(Some(Ty { kind, name: None, - span: None, + span: expr.span, })) } @@ -104,615 +232,352 @@ impl Resolver<'_> { where F: Fn() -> Option, { - if expected.is_none() { - // expected is none: there is no validation to be done + let Some(expected) = expected else { + // expected is none: there is no validation to be done and no generic to be inferred return Ok(()); }; let Some(found_ty) = &mut found.ty else { // found is none: infer from expected - - if found.lineage.is_none() && expected.unwrap().is_relation() { - // special case: infer a table type - // inferred tables are needed for s-strings that represent tables - // similarly as normal table references, we want to be able to infer columns - // of this table, which means it needs to be defined somewhere - // in the module structure. - let frame = self.declare_table_for_literal( - found - .clone() - .id - // This is quite rare but possible with something like - // `a -> b` at the moment. - .ok_or_else(|| Error::new_bug(4280))?, - None, - found.alias.clone(), - ); - - // override the empty frame with frame of the new table - found.lineage = Some(frame) - } - - // base case: infer expected type - found.ty = expected.cloned(); - + found.ty = Some(expected.clone()); return Ok(()); }; - self.validate_type(found_ty, expected, who) - .with_span(found.span) + self.validate_type(found_ty, expected, found.span, who) } /// Validates that found node has expected type. Returns assumed type of the node. pub fn validate_type( &mut self, - found: &mut Ty, - expected: Option<&Ty>, + found: &Ty, + expected: &Ty, + span: Option, who: &F, ) -> Result<(), Error> where F: Fn() -> Option, { - // infer - let Some(expected) = expected else { - // expected is none: there is no validation to be done - return Ok(()); - }; - - let expected_is_above = is_super_type_of(expected, found); - if expected_is_above { - return Ok(()); - } - - // A temporary hack for allowing calling window functions from within - // aggregate and derive. - if expected.kind.is_array() && !found.kind.is_function() { - return Ok(()); - } - - // if type is a generic, infer the constraint - if let TyKind::GenericArg(generic_id) = &expected.kind { - let domain = std::mem::take(self.generics.get_mut(generic_id).unwrap()); - - let (new_domain, _rejected): (Vec<_>, _) = domain - .into_iter() - .partition(|possible_type| is_super_type_of(possible_type, found)); - - if new_domain.is_empty() { - return Err(Error::new_simple( - "this argument does not match any of the generic types", - )); + match (&found.kind, &expected.kind) { + // base case + (TyKind::Primitive(f), TyKind::Primitive(e)) if e == f => Ok(()), + + // generics: infer + (_, TyKind::Ident(expected_fq)) => { + // if expected type is a generic, infer that it must be the found type + self.infer_generic_as_ty(expected_fq, found.clone(), found.span)?; + Ok(()) } - - // infer the new constraint - *self.generics.get_mut(generic_id).unwrap() = new_domain; - return Ok(()); - } - - Err(compose_type_error(found, expected, who)) - } - - /// Saves information that declaration identified by `fq_ident` must be of type `sub_ty`. - /// Param `sub_ty` must be a sub type of the current type of the declaration. - #[allow(dead_code)] - pub fn push_type_info(&mut self, fq_ident: &Ident, sub_ty: Ty) { - let decl = self.root_mod.module.get_mut(fq_ident).unwrap(); - - match &mut decl.kind { - DeclKind::Expr(expr) => { - restrict_type_opt(&mut expr.ty, Some(sub_ty)); - } - - DeclKind::Module(_) - | DeclKind::LayeredModules(_) - | DeclKind::Column(_) - | DeclKind::Infer(_) - | DeclKind::TableDecl(_) - | DeclKind::Ty(_) - | DeclKind::InstanceOf { .. } - | DeclKind::QueryDef(_) - | DeclKind::Import(_) => { - panic!("declaration {decl} is not able to have a type") + (TyKind::Ident(found_fq), _) => { + // if found type is a generic, infer that it must be the expected type + self.infer_generic_as_ty(found_fq, expected.clone(), span)?; + Ok(()) } - } - } - - pub fn resolve_generic_args(&mut self, mut ty: Ty) -> Result { - ty.kind = match ty.kind { - // the meaningful part - TyKind::GenericArg(id) => { - let domain = self.generics.remove(&id).unwrap(); - if domain.len() > 1 { - return Err(Error::new_simple( - "cannot determine the type of generic arg", - )); - } - // there will always be at least one, since we will never restrict to an empty domain - return Ok(domain.into_iter().next().unwrap()); + // containers: recurse + (TyKind::Array(found_items), TyKind::Array(expected_items)) => { + // co-variant contained type + self.validate_type(found_items, expected_items, span, who) } - - // recurse into container types - // this could probably be implemented with folding, but I don't want another full fold impl - TyKind::Tuple(fields) => TyKind::Tuple( - fields - .into_iter() - .map(|field| -> Result<_, Error> { - Ok(match field { - TyTupleField::Single(name, ty) => { - TyTupleField::Single(name, self.resolve_generic_args_opt(ty)?) - } - TyTupleField::Wildcard(ty) => { - TyTupleField::Wildcard(self.resolve_generic_args_opt(ty)?) - } - }) + (TyKind::Tuple(found_fields), TyKind::Tuple(expected_fields)) => { + // here we need to check that found tuple has all fields that are expected. + + // build index of found fields + let found_types: HashMap<_, _> = found_fields + .iter() + .filter_map(|e| match e { + TyTupleField::Single(Some(n), ty) => Some((n, ty)), + TyTupleField::Single(None, _) => None, + TyTupleField::Unpack(_) => None, // handled later }) - .try_collect()?, - ), - TyKind::Array(ty) => TyKind::Array(Box::new(self.resolve_generic_args(*ty)?)), - TyKind::Function(func) => TyKind::Function( - func.map(|f| -> Result<_, Error> { - Ok(TyFunc { - params: f - .params - .into_iter() - .map(|a| self.resolve_generic_args_opt(a)) - .try_collect()?, - return_ty: self - .resolve_generic_args_opt(f.return_ty.map(|x| *x))? - .map(Box::new), - name_hint: f.name_hint, - }) - }) - .transpose()?, - ), - - _ => ty.kind, - }; - Ok(ty) - } - - pub fn resolve_generic_args_opt(&mut self, ty: Option) -> Result, Error> { - ty.map(|x| self.resolve_generic_args(x)).transpose() - } -} - -pub fn ty_tuple_kind(fields: Vec) -> TyKind { - let mut res: Vec = Vec::with_capacity(fields.len()); - for field in fields { - if let TyTupleField::Single(name, _) = &field { - // remove names from previous fields with the same name - if name.is_some() { - for f in res.iter_mut() { - if f.as_single().and_then(|x| x.0.as_ref()) == name.as_ref() { - *f.as_single_mut().unwrap().0 = None; + .collect(); + + let mut expected_but_not_found = Vec::new(); + for e_field in expected_fields { + match e_field { + TyTupleField::Single(Some(e_name), e_ty) => { + // when a named field is expected + + // if it was found + if let Some(f_ty) = found_types.get(e_name) { + // check its type + if let Some((f_ty, e_ty)) = f_ty.as_ref().zip(e_ty.as_ref()) { + // co-variant contained type + self.validate_type(f_ty, e_ty, span, who)?; + } + } else { + expected_but_not_found.push(e_field); + } + } + TyTupleField::Single(None, _) => { + // TODO: positional expected fields + } + TyTupleField::Unpack(_) => {} // handled later } } - } - } - res.push(field); - } - TyKind::Tuple(res) -} - -/// Sink type difference operators down in the type expression, -/// float unions operators up, simplify type expression. -/// -/// For more info, read web/book/src/reference/spec/type-system.md -pub(crate) fn normalize_type(ty: Ty) -> Ty { - match ty.kind { - TyKind::Union(variants) => { - let variants = sink_union_into_array_and_tuple(variants); - let mut res: Vec<(_, Ty)> = Vec::with_capacity(variants.len()); + if !expected_but_not_found.is_empty() { + // not all fields were found - for (variant_name, variant_ty) in variants { - let variant_ty = normalize_type(variant_ty); - - // (A || ()) = A - // skip never - if variant_ty.is_never() && variant_name.is_none() { - continue; + // try looking into the unpack + if let Some(found_unpack) = found_fields.last().and_then(|f| f.as_unpack()) { + if let Some(f_unpack_ty) = found_unpack { + let remaining = Ty::new(TyKind::Tuple( + expected_but_not_found.into_iter().cloned().collect_vec(), + )); + self.validate_type(&remaining, f_unpack_ty, span, who)?; + } else { + // we don't know the type of unpack, so we cannot fully check if it has the fields + } + } else { + // there is no unpack, not_found fields are an error + return Err(compose_type_error(found, expected, who).with_span(span)); + } } - // (A || A || B) = A || B - // skip duplicates - let already_included = res.iter().any(|(_, r)| is_super_type_of(r, &variant_ty)); - if already_included { - continue; + // if there is an expected unpack, check it too + if let Some(Some(e_unpack)) = expected_fields.last().and_then(|f| f.as_unpack()) { + self.validate_type(found, e_unpack, span, who)?; } - res.push((variant_name, variant_ty)); + Ok(()) } + (TyKind::Function(Some(f_func)), TyKind::Function(Some(e_func))) + if f_func.params.len() == e_func.params.len() => + { + for (f_arg, e_arg) in itertools::zip_eq(&f_func.params, &e_func.params) { + if let Some((f_arg, e_arg)) = Option::zip(f_arg.as_ref(), e_arg.as_ref()) { + // contra-variant contained types + self.validate_type(e_arg, f_arg, span, who)?; + } + } - if res.len() == 1 { - res.into_iter().next().unwrap().1 - } else { - Ty { - kind: TyKind::Union(res), - ..ty + // return types + if let Some((f_ret, e_ret)) = Option::zip( + Option::as_ref(&f_func.return_ty), + Option::as_ref(&e_func.return_ty), + ) { + // co-variant contained type + self.validate_type(f_ret, e_ret, span, who)?; } + Ok(()) } + _ => Err(compose_type_error(found, expected, who).with_span(span)), } + } - TyKind::Difference { base, exclude } => { - let (base, exclude) = match (*base, *exclude) { - // (A || B) - C = (A - C) || (B - C) - ( - Ty { - kind: TyKind::Union(variants), - name, - span, - }, - c, - ) => { - let kind = TyKind::Union( - variants - .into_iter() - .map(|(name, ty)| { - ( - name, - Ty::new(TyKind::Difference { - base: Box::new(ty), - exclude: Box::new(c.clone()), - }), - ) - }) - .collect(), - ); - return normalize_type(Ty { kind, name, span }); - } - // (A - B) - C = A - (B || C) - ( - Ty { - kind: - TyKind::Difference { - base: a, - exclude: b, - }, - .. - }, - c, - ) => { - let kind = TyKind::Difference { - base: a, - exclude: Box::new(union_and_flatten(*b, c)), - }; - return normalize_type(Ty { kind, ..ty }); - } + fn infer_tuple_field_name(&self, field: &Expr) -> Option { + // at this stage, this expr should already be fully resolved + // this means that any indirections will be tuple positional + // so we check for that and pull the name from the type of the base - // A - (B - C) = - // = A & not (B & not C) - // = A & (not B || C) - // = (A & not B) || (A & C) - // = (A - B) || (A & C) - ( - a, - Ty { - kind: - TyKind::Difference { - base: b, - exclude: c, - }, - .. - }, - ) => { - let first = Ty::new(TyKind::Difference { - base: Box::new(a.clone()), - exclude: b, - }); - let second = type_intersection(a, *c); - let kind = TyKind::Union(vec![(None, first), (None, second)]); - return normalize_type(Ty { kind, ..ty }); - } + let ExprKind::Indirection { + base, + field: IndirectionKind::Position(pos), + } = &field.kind + else { + return None; + }; - // [A] - [B] = [A - B] - ( - Ty { - kind: TyKind::Array(base), - .. - }, - Ty { - kind: TyKind::Array(exclude), - .. - }, - ) => { - let item = Ty::new(TyKind::Difference { base, exclude }); - let kind = TyKind::Array(Box::new(item)); - return normalize_type(Ty { kind, ..ty }); - } - // [A] - non-array = [A] - ( - Ty { - kind: TyKind::Array(item), - .. - }, - _, - ) => { - return normalize_type(Ty { - kind: TyKind::Array(item), - ..ty - }); - } - // non-array - [B] = non-array - ( - base, - Ty { - kind: TyKind::Array(_), - .. - }, - ) => { - return normalize_type(base); - } + let ty = base.ty.as_ref()?; + self.apply_ty_tuple_indirection(ty, *pos as usize) + } - // {A, B} - {C, D} = {A - C, B - D} - ( - Ty { - kind: TyKind::Tuple(base_fields), - .. - }, - Ty { - kind: TyKind::Tuple(exclude_fields), - .. - }, - ) => { - let exclude_fields: HashMap<&String, &Option> = exclude_fields - .iter() - .flat_map(|field| match field { - TyTupleField::Single(Some(name), ty) => Some((name, ty)), - _ => None, - }) - .collect(); - - let mut res = Vec::new(); - for field in base_fields { - // TODO: this whole block should be redone - I'm not sure it fully correct. - match field { - TyTupleField::Single(Some(name), Some(ty)) => { - if let Some(right_field) = exclude_fields.get(&name) { - let right_tuple = - right_field.as_ref().map_or(false, |x| x.kind.is_tuple()); - - if right_tuple { - // recursively erase selection - let ty = Ty::new(TyKind::Difference { - base: Box::new(ty), - exclude: Box::new((*right_field).clone().unwrap()), - }); - let ty = normalize_type(ty); - res.push(TyTupleField::Single(Some(name), Some(ty))) - } else { - // erase completely - } - } else { - res.push(TyTupleField::Single(Some(name), Some(ty))) - } - } - TyTupleField::Single(Some(name), None) => { - if exclude_fields.contains_key(&name) { - // TODO: I'm not sure what should happen in this case - continue; - } else { - res.push(TyTupleField::Single(Some(name), None)) - } - } - TyTupleField::Single(None, ty) => { - res.push(TyTupleField::Single(None, ty)); - } - TyTupleField::Wildcard(_) => res.push(field), - } - } - return Ty { - kind: TyKind::Tuple(res), - ..ty - }; - } + fn apply_ty_tuple_indirection(&self, ty: &Ty, pos: usize) -> Option { + match &ty.kind { + TyKind::Tuple(fields) => { + // this tuple might contain Unpacks (which affect positions of fields after them) + // so we need to resolve this type full first. - // noop - (a, b) => (a, b), - }; + let unpack_pos = (fields.iter()) + .position(|f| f.is_unpack()) + .unwrap_or(fields.len()); + if pos < unpack_pos { + // unpacks don't interfere with preceding fields + let field = fields.get(pos)?; - let base = Box::new(normalize_type(base)); - let exclude = Box::new(normalize_type(exclude)); + field.as_single().unwrap().0.clone() + } else { + let pos_within_unpack = pos - unpack_pos; - // A - (A || B) = () - if let TyKind::Union(excluded) = &exclude.kind { - for (_, e) in excluded { - if base.as_ref() == e { - return Ty::never(); - } + let unpack_ty = fields.get(unpack_pos)?.as_unpack().unwrap(); + let unpack_ty = unpack_ty.as_ref().unwrap(); + + self.apply_ty_tuple_indirection(unpack_ty, pos_within_unpack) } } - let kind = TyKind::Difference { base, exclude }; - Ty { kind, ..ty } - } - TyKind::Array(items_ty) => Ty { - kind: TyKind::Array(Box::new(normalize_type(*items_ty))), - ..ty - }, + TyKind::Ident(fq_ident) => { + let decl = self.root_mod.module.get(fq_ident).unwrap(); + let inferred_type = decl.kind.as_generic_param()?; + let (inferred_type, _) = inferred_type.as_ref()?; - kind => Ty { kind, ..ty }, - } -} + self.apply_ty_tuple_indirection(inferred_type, pos) + } -/// Sinks union into arrays and tuples. -/// [A] || [B] -> [A || B] -/// {a = A, B} || {c = C, D} -> {a = A, c = C, B || D} -fn sink_union_into_array_and_tuple( - variants: Vec<(Option, Ty)>, -) -> Vec<(Option, Ty)> { - let mut remaining = Vec::with_capacity(variants.len()); - - let mut array_variants = Vec::new(); - let mut tuple_variants = Vec::new(); - for (variant_name, variant_ty) in variants { - // handle array variants separately - if let TyKind::Array(item) = variant_ty.kind { - array_variants.push((None, *item)); - continue; - } - // handle tuple variants separately - if let TyKind::Tuple(fields) = variant_ty.kind { - tuple_variants.push(fields); - continue; + _ => None, + } + } + + /// Instantiate generic type parameters into generic type arguments. + /// + /// When resolving a type of reference to a variable, we cannot just use the type + /// of the variable as the type of the reference. That's because the variable might contain + /// generic type arguments that need to differ between references to the same variable. + /// + /// For example: + /// ```prql + /// let plus_one = func x -> x + 1 + /// + /// let a = plus_one 1 + /// let b = plus_one 1.5 + /// ``` + /// + /// Here, the first reference to `plus_one` must resolve with T=int and the second with T=float. + /// + /// This struct makes sure that distinct instanced of T are created from generic type param T. + pub fn instantiate_type(&mut self, ty: Ty, id: usize) -> Ty { + let TyKind::Function(Some(ty_func)) = &ty.kind else { + return ty; + }; + if ty_func.generic_type_params.is_empty() { + return ty; } - remaining.push((variant_name, variant_ty)); - } + let prev_scope = Ident::from_path(vec![NS_LOCAL]); + let new_scope = Ident::from_path(vec![NS_GENERIC.to_string(), id.to_string()]); - match array_variants.len() { - 2.. => { - let item_ty = Ty::new(TyKind::Union(array_variants)); - remaining.push((None, Ty::new(TyKind::Array(Box::new(item_ty))))); - } - 1 => { - let item_ty = array_variants.into_iter().next().unwrap().1; - remaining.push((None, Ty::new(TyKind::Array(Box::new(item_ty))))); - } - _ => {} - } + let mut ident_mapping: HashMap = + HashMap::with_capacity(ty_func.generic_type_params.len()); - match tuple_variants.len() { - 2.. => { - remaining.push((None, union_of_tuples(tuple_variants))); - } - 1 => { - let fields = tuple_variants.into_iter().next().unwrap(); - remaining.push((None, Ty::new(TyKind::Tuple(fields)))); - } - _ => {} - } + for gtp in &ty_func.generic_type_params { + let new_ident = new_scope.clone() + Ident::from_name(>p.name); - remaining -} + let decl = Decl::from(DeclKind::GenericParam( + gtp.bound.as_ref().map(|t| (t.clone(), None)), + )); + self.root_mod + .module + .insert(new_ident.clone(), decl) + .unwrap(); -fn union_of_tuples(tuple_variants: Vec>) -> Ty { - let mut fields = Vec::::new(); - let mut has_wildcard = false; - - for tuple_variant in tuple_variants { - for field in tuple_variant { - match field { - TyTupleField::Single(Some(name), ty) => { - // find by name - let existing = fields.iter_mut().find_map(|f| match f { - TyTupleField::Single(n, t) if n.as_ref() == Some(&name) => Some(t), - _ => None, - }); - if let Some(existing) = existing { - // union with the existing - *existing = maybe_union(existing.take(), ty); - } else { - // push - fields.push(TyTupleField::Single(Some(name), ty)); - } - } - TyTupleField::Single(None, ty) => { - // push - fields.push(TyTupleField::Single(None, ty)); - } - TyTupleField::Wildcard(_) => has_wildcard = true, - } + ident_mapping.insert( + prev_scope.clone() + Ident::from_name(>p.name), + Ty::new(TyKind::Ident(new_ident)), + ); } - } - if has_wildcard { - fields.push(TyTupleField::Wildcard(None)); - } - Ty::new(TyKind::Tuple(fields)) -} -fn restrict_type_opt(ty: &mut Option, sub_ty: Option) { - let Some(sub_ty) = sub_ty else { - return; - }; - if let Some(ty) = ty { - restrict_type(ty, sub_ty) - } else { - *ty = Some(sub_ty); + TypeReplacer::on_ty(ty, ident_mapping) } -} -fn restrict_type(ty: &mut Ty, sub_ty: Ty) { - match (&mut ty.kind, sub_ty.kind) { - (TyKind::Any, sub) => ty.kind = sub, - - (TyKind::Union(variants), sub_kind) => { - let sub_ty = Ty { - kind: sub_kind, - ..sub_ty - }; - let drained = variants - .drain(..) - .filter(|(_, variant)| is_super_type_of(variant, &sub_ty)) - .map(|(name, mut ty)| { - restrict_type(&mut ty, sub_ty.clone()); - (name, ty) - }) - .collect_vec(); - variants.extend(drained); - } + pub fn ty_tuple_exclusion(&self, base: Ty, except: Ty) -> Result { + let mask = self.ty_tuple_exclusion_mask(&base, &except)?; - (kind, TyKind::Union(sub_variants)) => { - todo!("restrict {kind:?} to union of {sub_variants:?}") - } + if let Some(mask) = mask { + let new_fields = itertools::zip_eq(base.kind.as_tuple().unwrap(), mask) + .filter(|(_, p)| *p) + .map(|(x, _)| x.clone()) + .collect(); - (TyKind::Primitive(_), _) => {} + Ok(TyKind::Tuple(new_fields)) + } else { + Ok(TyKind::Exclude { + base: Box::new(base), + except: Box::new(except), + }) + } + } + + /// Computes the "field mask", which is a vector of booleans indicating if a field of + /// base tuple type should appear in the resulting type. + /// + /// Returns `None` if: + /// - base or exclude is a generic type argument, or + /// - either of the types contains Unpack. + pub fn ty_tuple_exclusion_mask(&self, base: &Ty, except: &Ty) -> Result>> { + let within_fields = match &base.kind { + TyKind::Tuple(f) => f, + + // this is a generic, exclusion cannot be inlined + TyKind::Ident(_) => return Ok(None), + + _ => { + return Err( + Error::new_simple("fields can only be excluded from a tuple") + .push_hint(format!("got {}", write_ty_kind(&base.kind))) + .with_span(base.span), + ) + } + }; + if within_fields.last().map_or(false, |f| f.is_unpack()) { + return Ok(None); + } - (TyKind::Singleton(_), _) => {} + let except_fields = match &except.kind { + TyKind::Tuple(f) => f, - (TyKind::Tuple(tuple), TyKind::Tuple(sub_tuple)) => { - for sub_field in sub_tuple { - match sub_field { - TyTupleField::Single(sub_name, sub_ty) => { - if let Some(sub_name) = sub_name { - let existing = tuple - .iter_mut() - .filter_map(|x| x.as_single_mut()) - .find(|f| f.0.as_ref() == Some(&sub_name)); + // this is a generic, exclusion cannot be inlined + TyKind::Ident(_) => return Ok(None), - if let Some((_, existing)) = existing { - restrict_type_opt(existing, sub_ty) - } else { - tuple.push(TyTupleField::Single(Some(sub_name), sub_ty)); - } - } else { - // TODO: insert unnamed fields? - } - } - TyTupleField::Wildcard(_) => todo!("remove TupleField::Wildcard"), - } + _ => { + return Err(Error::new_simple("expected excluded fields to be a tuple") + .push_hint(format!("got {}", write_ty_kind(&except.kind))) + .with_span(except.span)); } + }; + if except_fields.last().map_or(false, |f| f.is_unpack()) { + return Ok(None); } - (TyKind::Array(ty), TyKind::Array(sub_ty)) => restrict_type(ty, *sub_ty), + let except_fields: HashSet<&String> = except_fields + .iter() + .map(|field| match field { + TyTupleField::Single(Some(name), _) => Ok(name), + TyTupleField::Single(None, _) => { + Err(Error::new_simple("excluded fields must be named")) + } + _ => unreachable!(), + }) + .collect::>() + .with_span(except.span)?; - (TyKind::Function(ty), TyKind::Function(sub_ty)) => { - if sub_ty.is_none() { - return; - } - if ty.is_none() { - *ty = sub_ty; - return; - } - if let (Some(func), Some(sub_func)) = (ty, sub_ty) { - todo!("restrict function {func:?} to function {sub_func:?}") - } + let mut mask = Vec::new(); + for field in within_fields { + mask.push(match &field { + TyTupleField::Single(Some(name), _) => !except_fields.contains(&name), + TyTupleField::Single(None, _) => true, + + TyTupleField::Unpack(_) => unreachable!(), + }); } + Ok(Some(mask)) + } +} - _ => { - panic!("trying to restrict a type with a non sub type") +pub fn ty_tuple_kind(fields: Vec) -> TyKind { + let mut res: Vec = Vec::with_capacity(fields.len()); + for field in fields { + if let TyTupleField::Single(name, _) = &field { + // remove names from previous fields with the same name + if name.is_some() { + for f in res.iter_mut() { + if f.as_single().and_then(|x| x.0.as_ref()) == name.as_ref() { + *f.as_single_mut().unwrap().0 = None; + } + } + } } + res.push(field); } + TyKind::Tuple(res) } -fn compose_type_error(found_ty: &mut Ty, expected: &Ty, who: &F) -> Error +fn compose_type_error(found_ty: &Ty, expected: &Ty, who: &F) -> Error where F: Fn() -> Option, { fn display_ty(ty: &Ty) -> String { if ty.name.is_none() { if let TyKind::Tuple(fields) = &ty.kind { - if fields.len() == 1 && fields[0].is_wildcard() { + if fields.len() == 1 && fields[0].is_unpack() { return "a tuple".to_string(); } } @@ -733,15 +598,7 @@ where }); if found_ty.kind.is_function() && !expected.kind.is_function() { - let found = found_ty.kind.as_function().unwrap(); - let func_name = if let Some(func) = found { - func.name_hint.as_ref() - } else { - None - }; - let to_what = func_name - .map(|n| format!("to function {n}")) - .unwrap_or_else(|| "in this function call?".to_string()); + let to_what = "in this function call?"; e = e.push_hint(format!("Have you forgotten an argument {to_what}?")); } @@ -757,276 +614,203 @@ where e } -/// Analogous to [crate::ir::pl::Lineage::rename()] -pub fn rename_relation(ty_kind: &mut TyKind, alias: String) { - if let TyKind::Array(items_ty) = ty_kind { - rename_tuples(&mut items_ty.kind, alias); - } -} - -fn rename_tuples(ty_kind: &mut TyKind, alias: String) { - flatten_tuples(ty_kind); - - if let TyKind::Tuple(fields) = ty_kind { - let inner_fields = std::mem::take(fields); - - let ty = Ty::new(TyKind::Tuple(inner_fields)); - fields.push(TyTupleField::Single(Some(alias), Some(ty))); - } -} - -fn flatten_tuples(ty_kind: &mut TyKind) { - if let TyKind::Tuple(fields) = ty_kind { - let mut new_fields = Vec::new(); - - for field in fields.drain(..) { - let TyTupleField::Single(name, Some(ty)) = field else { - new_fields.push(field); - continue; - }; - - // recurse - // let ty = ty.flatten_tuples(); - - let TyKind::Tuple(inner_fields) = ty.kind else { +pub fn ty_fold_and_inline_tuple_fields( + fold: &mut F, + fields: Vec, +) -> Result> { + let mut new_fields = Vec::new(); + for field in fields { + match field { + TyTupleField::Single(name, Some(ty)) => { + // standard folding + let ty = fold.fold_type(ty)?; new_fields.push(TyTupleField::Single(name, Some(ty))); - continue; - }; - new_fields.extend(inner_fields); + } + TyTupleField::Unpack(Some(ty)) => { + let ty = fold.fold_type(ty)?; + + // inline unpack if it contains a tuple + if let TyKind::Tuple(inner_fields) = ty.kind { + new_fields.extend(inner_fields); + } else { + new_fields.push(TyTupleField::Unpack(Some(ty))); + } + } + _ => { + // standard folding + new_fields.push(field); + } } - - fields.extend(new_fields); } + Ok(new_fields) } -pub fn is_super_type_of(superset: &Ty, subset: &Ty) -> bool { - if superset.is_relation() && subset.is_relation() { - return true; - } - is_super_type_of_kind(&superset.kind, &subset.kind) +/// Replaces references to generic type parameters with (partially) resolved argument types +/// and makes makes the type "human friendly". +pub struct TypePreviewer<'r> { + resolver: &'r super::Resolver<'r>, } -pub fn is_sub_type_of_array(ty: &Ty) -> bool { - let array = TyKind::Array(Box::new(Ty::new(TyKind::Any))); - is_super_type_of_kind(&array, &ty.kind) +impl<'r> TypePreviewer<'r> { + pub fn run(resolver: &'r super::Resolver<'r>, ty: Ty) -> Ty { + TypePreviewer { resolver }.fold_type(ty).unwrap() + } } -fn is_super_type_of_kind(superset: &TyKind, subset: &TyKind) -> bool { - match (superset, subset) { - (TyKind::Any, _) => true, - (_, TyKind::Any) => false, - - (TyKind::Primitive(l0), TyKind::Primitive(r0)) => l0 == r0, +impl PlFold for TypePreviewer<'_> { + fn fold_type(&mut self, mut ty: Ty) -> Result { + ty.kind = match ty.kind { + TyKind::Ident(fq_ident) => { + let root_mod = &self.resolver.root_mod.module; + let decl = root_mod.get(&fq_ident).unwrap(); - (one, TyKind::Union(many)) => many - .iter() - .all(|(_, each)| is_super_type_of_kind(one, &each.kind)), + let candidate = decl.kind.as_generic_param().unwrap(); - (TyKind::Union(many), one) => many - .iter() - .any(|(_, any)| is_super_type_of_kind(&any.kind, one)), + if let Some((candidate, _)) = candidate { + let mut previewed = self.fold_type(candidate.clone()).unwrap(); + if let TyKind::Tuple(fields) = &mut previewed.kind { + fields.push(TyTupleField::Unpack(None)); + } - (TyKind::Function(None), TyKind::Function(_)) => true, - (TyKind::Function(Some(_)), TyKind::Function(None)) => true, - (TyKind::Function(Some(sup)), TyKind::Function(Some(sub))) => { - if is_not_super_type_of(sup.return_ty.as_deref(), sub.return_ty.as_deref()) { - return false; - } - if sup.params.len() != sub.params.len() { - return false; - } - for (sup_arg, sub_arg) in sup.params.iter().zip(&sub.params) { - if is_not_super_type_of(sup_arg.as_ref(), sub_arg.as_ref()) { - return false; + previewed.kind + } else { + TyKind::Ident(Ident::from_name("?")) } } - - true - } - - (TyKind::Array(sup), TyKind::Array(sub)) => is_super_type_of(sup, sub), - - (TyKind::Tuple(sup_tuple), TyKind::Tuple(sub_tuple)) => { - let sup_has_wildcard = sup_tuple - .iter() - .any(|f| matches!(f, TyTupleField::Wildcard(_))); - let sub_has_wildcard = sub_tuple - .iter() - .any(|f| matches!(f, TyTupleField::Wildcard(_))); - - let mut sup_fields = sup_tuple.iter().filter(|f| f.is_single()); - let mut sub_fields = sub_tuple.iter().filter(|f| f.is_single()); - - loop { - let sup = sup_fields.next(); - let sub = sub_fields.next(); - - match (sup, sub) { - (Some(TyTupleField::Single(_, sup)), Some(TyTupleField::Single(_, sub))) => { - if is_not_super_type_of(sup.as_ref(), sub.as_ref()) { - return false; - } - } - (_, Some(_)) => { - if !sup_has_wildcard { - return false; - } - } - (Some(_), None) => { - if !sub_has_wildcard { - return false; - } + TyKind::Tuple(fields) => { + let mut fields = ty_fold_and_inline_tuple_fields(self, fields)?; + + // clear types of fields that are just Ident("?") + for field in &mut fields { + let ty = match field { + TyTupleField::Single(_, ty) => ty, + TyTupleField::Unpack(ty) => ty, + }; + let is_unknown = ty + .as_ref() + .and_then(|t| t.kind.as_ident()) + .map_or(false, |i| i.name == "?"); + if is_unknown { + *ty = None } - (None, None) => break, } + TyKind::Tuple(fields) } - true - } - - (l, r) => l == r, + _ => return fold_type(self, ty), + }; + Ok(ty) } } -fn is_not_super_type_of(sup: Option<&Ty>, sub: Option<&Ty>) -> bool { - if let Some(sub_ret) = sub { - if let Some(sup_ret) = sup { - if !is_super_type_of(sup_ret, sub_ret) { - return true; - } - } - } - false +pub struct TypeReplacer { + mapping: HashMap, } -fn maybe_type_intersection(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(type_intersection(a, b)), - (x, None) | (None, x) => x, +impl TypeReplacer { + pub fn on_ty(ty: Ty, mapping: HashMap) -> Ty { + TypeReplacer { mapping }.fold_type(ty).unwrap() } -} -pub fn type_intersection(a: Ty, b: Ty) -> Ty { - match (a.kind, b.kind) { - (TyKind::Any, b_kind) => Ty { kind: b_kind, ..b }, - (a_kind, TyKind::Any) => Ty { kind: a_kind, ..a }, + pub fn on_func(func: Func, mapping: HashMap) -> Func { + TypeReplacer { mapping }.fold_func(func).unwrap() + } +} - // union - (TyKind::Union(a_variants), b_kind) => { - let b = Ty { kind: b_kind, ..b }; - type_intersection_with_union(a_variants, b) - } - (a_kind, TyKind::Union(b_variants)) => { - let a = Ty { kind: a_kind, ..a }; - type_intersection_with_union(b_variants, a) - } +impl PlFold for TypeReplacer { + fn fold_type(&mut self, mut ty: Ty) -> Result { + ty.kind = match ty.kind { + TyKind::Ident(ident) => { + if let Some(new_ty) = self.mapping.get(&ident) { + return Ok(new_ty.clone()); + } else { + TyKind::Ident(ident) + } + } + _ => return fold_type(self, ty), + }; + Ok(ty) + } +} - // difference - (TyKind::Difference { base, exclude }, b_kind) => { - let b = Ty { kind: b_kind, ..b }; - let base = Box::new(type_intersection(*base, b)); - Ty::new(TyKind::Difference { base, exclude }) - } - (a_kind, TyKind::Difference { base, exclude }) => { - let a = Ty { kind: a_kind, ..a }; - let base = Box::new(type_intersection(a, *base)); - Ty::new(TyKind::Difference { base, exclude }) - } +#[cfg(test)] +mod test { + use super::*; + use crate::ir::decl::RootModule; - (a_kind, b_kind) if a_kind == b_kind => Ty { kind: a_kind, ..a }, + #[track_caller] + fn validate_type(found: &str, expected: &str) -> crate::Result<()> { + let mut root_mod = RootModule::default(); + let mut r = Resolver::new(&mut root_mod); - // tuple - (TyKind::Tuple(a_fields), TyKind::Tuple(b_fields)) => { - type_intersection_of_tuples(a_fields, b_fields) - } + let found = parse_ty(found); + let expected = parse_ty(expected); - // array - (TyKind::Array(a), TyKind::Array(b)) => { - Ty::new(TyKind::Array(Box::new(type_intersection(*a, *b)))) - } + r.validate_type(&found, &expected, None, &|| None) + } - _ => Ty::never(), + #[track_caller] + fn parse_ty(source: &str) -> Ty { + let source = format!("type x = {source}"); + let stmts = crate::parser::parse_source(&source, 0).unwrap(); + let stmt = stmts.into_iter().next().unwrap(); + stmt.kind.into_type_def().unwrap().value.unwrap() } -} -fn type_intersection_with_union(variants: Vec<(Option, Ty)>, b: Ty) -> Ty { - let variants = variants - .into_iter() - .map(|(name, variant)| { - let inter = type_intersection(variant, b.clone()); - (name, inter) - }) - .collect_vec(); + #[test] + fn validate_type_00() { + validate_type("{a = int, b = bool}", "{a = int}").unwrap(); + } - Ty::new(TyKind::Union(variants)) -} + #[test] + fn validate_type_01() { + // should fail because field b is expected, but not found + validate_type("{a = int}", "{a = int, b = int}").unwrap_err(); + } -fn type_intersection_of_tuples(a: Vec, b: Vec) -> Ty { - let a_has_other = a.iter().any(|f| f.is_wildcard()); - let b_has_other = b.iter().any(|f| f.is_wildcard()); - - let mut a_fields = a.into_iter().filter_map(|f| f.into_single().ok()); - let mut b_fields = b.into_iter().filter_map(|f| f.into_single().ok()); - - let mut fields = Vec::new(); - let mut has_other = false; - loop { - match (a_fields.next(), b_fields.next()) { - (None, None) => break, - (None, Some(b_field)) => { - if !a_has_other { - return Ty::never(); - } - has_other = true; - fields.push(TyTupleField::Single(b_field.0, b_field.1)); - } - (Some(a_field), None) => { - if !b_has_other { - return Ty::never(); - } - has_other = true; - fields.push(TyTupleField::Single(a_field.0, a_field.1)); - } - (Some((a_name, a_ty)), Some((b_name, b_ty))) => { - let name = match (a_name, b_name) { - (Some(a), Some(b)) if a == b => Some(a), - (None, None) | (Some(_), Some(_)) => None, - (None, Some(n)) | (Some(n), None) => Some(n), - }; - let ty = maybe_type_intersection(a_ty, b_ty); + #[test] + fn validate_type_02() { + validate_type( + "{a = int, b = {b1 = int, b2 = bool}}", + "{a = int, b = {b1 = int}}", + ) + .unwrap(); + } - fields.push(TyTupleField::Single(name, ty)); - } - } + #[test] + fn validate_type_03() { + // should fail because field b.b2 is expected, but not found + validate_type( + "{a = int, b = {b1 = int}}", + "{a = int, b = {b1 = int, b2 = bool}}", + ) + .unwrap_err(); } - if has_other { - fields.push(TyTupleField::Wildcard(None)); + + #[test] + fn validate_type_04() { + // should fail because found b is bool instead of int + validate_type("{a = int, b = bool}", "{a = int, b = int}").unwrap_err(); } - Ty::new(TyKind::Tuple(fields)) -} + #[test] + fn validate_type_05() { + validate_type("{a = int, ..{b = bool}}", "{a = int, b = bool}").unwrap(); + } -/// Converts: -/// - A, B into A | B and -/// - A, B | C into A | B | C and -/// - A | B, C into A | B | C. -fn union_and_flatten(a: Ty, b: Ty) -> Ty { - let mut variants = Vec::with_capacity(2); - if let TyKind::Union(v) = a.kind { - variants.extend(v) - } else { - variants.push((None, a)); + #[test] + fn validate_type_06() { + // should fail because found b is bool instead of int + validate_type("{a = int, ..{b = bool}}", "{a = int, b = int}").unwrap_err(); } - if let TyKind::Union(v) = b.kind { - variants.extend(v) - } else { - variants.push((None, b)); + + #[test] + fn validate_type_07() { + validate_type("{a = int, b = bool}", "{a = int, ..{b = bool}}").unwrap(); } - Ty::new(TyKind::Union(variants)) -} -fn maybe_union(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(Ty::new(TyKind::Union(vec![(None, a), (None, b)]))), - (None, x) | (x, None) => x, + #[test] + fn validate_type_08() { + // should fail because found b is bool instead of int + validate_type("{a = int, b = bool}", "{a = int, ..{b = int}}").unwrap_err(); } } diff --git a/prqlc/prqlc/src/semantic/std.prql b/prqlc/prqlc/src/semantic/std.prql index 2c60440de567..34c1525869b7 100644 --- a/prqlc/prqlc/src/semantic/std.prql +++ b/prqlc/prqlc/src/semantic/std.prql @@ -15,12 +15,12 @@ # Operators -let mul = left right -> internal std.mul -let div_i = left right -> internal std.div_i -let div_f = left right -> internal std.div_f -let mod = left right -> internal std.mod -let add = left right -> internal std.add -let sub = left right -> internal std.sub +let mul = left right -> internal std.mul +let div_i = left right -> internal std.div_i +let div_f = left right -> internal std.div_f +let mod = left right -> internal std.mod +let add = left right -> internal std.add +let sub = left right -> internal std.sub let eq = left right -> internal std.eq let ne = left right -> internal std.ne let gt = left right -> internal std.gt @@ -32,7 +32,7 @@ let or = left right -> internal std.or let coalesce = left right -> internal std.coalesce let regex_search = text pattern -> internal std.regex_search -let neg = expr -> internal std.neg +let neg = expr -> internal std.neg let not = expr -> internal std.not # Types @@ -46,162 +46,164 @@ type date = date type time = time type timestamp = timestamp type `func` = func -type anytype = anytype ## Generic array -# TODO: an array of anything, not just nulls -type array = [anytype] - -## Scalar -type scalar = int || float || bool || text || date || time || timestamp || null -type tuple = {..anytype} ## Range -type range = {start = scalar, end = scalar} - -## Relation (an array of tuples) -type relation = [tuple] +# type range = {start = T, end = T} ## Transform -type transform = func relation -> relation +# type transform = func [I] -> [O] # Functions ## Relational transforms -let from = func - `default_db.source` - -> source - -let select = func - columns - tbl - -> internal select - -let filter = func +let from = func + tbl + -> tbl + +@(coerce_tuple 0) +@(implicit_closure 0 this:1) +let select = func + assigns + tbl <[I]> + -> <[A]> internal select + +@(implicit_closure 0 this:1) +let filter = func condition - tbl - -> internal filter - -let derive = func - columns - tbl - -> internal derive - -let aggregate = func - columns - tbl - -> internal aggregate - -let sort = func - by - tbl - -> internal sort - -let take = func - expr - tbl - -> internal take - -let join = func - `default_db.with` + tbl <[T]> + -> <[T]> internal filter + +@(coerce_tuple 0) +@(implicit_closure 0 this:1) +let derive = func + assigns + tbl <[I]> + -> <[{I, ..A}]> internal derive + +@(coerce_tuple 0) +@(implicit_closure 0 this:1) +let aggregate = func + assigns + tbl <[I]> + -> <[A]> internal aggregate + +@(coerce_tuple 0) +@(implicit_closure 0 this:1) +let sort = func + by + tbl <[I]> + -> <[I]> internal sort + +let take = func + expr + tbl <[I]> + -> <[I]> internal take + +@(implicit_closure 1 this:2 that:0) +let join = func + with <[B]> condition `noresolve.side`:inner - tbl - -> internal join + tbl <[A]> + -> <[{A, B}]> internal join -let group = func - by - pipeline - tbl - -> internal group +@(coerce_tuple 0) +@(implicit_closure 0 this:2) +let group = func + by + pipeline [PO]> + tbl <[I]> + -> <[{B, ..PO}]> internal group let window = func rows:0..-1 range:0..-1 expanding :false rolling :0 - pipeline - tbl - -> internal window - -let append = `default_db.bottom` top -> internal append -let intersect = `default_db.bottom` top -> ( - t = top - join (b = bottom) (tuple_every (tuple_map _eq (tuple_zip t.* b.*))) - select t.* -) -let remove = `default_db.bottom` top -> ( - t = top - join side:left (b = bottom) (tuple_every (tuple_map _eq (tuple_zip t.* b.*))) - filter (tuple_every (tuple_map _is_null b.*)) - select t.* -) + pipeline + tbl + -> internal window + +let append = bottom top -> internal append +# let intersect = bottom top -> ( +# t = top +# join (b = bottom) (std.tuple_every (std.tuple_map std._eq (std.tuple_zip t.* b.*))) +# select t.* +# ) +# let remove = bottom top -> ( +# t = top +# join side:left (b = bottom) (std.tuple_every (std.tuple_map std._eq (std.tuple_zip t.* b.*))) +# filter (std.tuple_every (std.tuple_map std._is_null b.*)) +# select t.* +# ) let loop = func - pipeline - top - -> internal loop + pipeline + top + -> internal loop ## Aggregate functions # These return either a scalar when used within `aggregate`, or a column when used anywhere else. -let min = column -> internal std.min +let min = column -> internal std.min -let max = column -> internal std.max +let max = column -> internal std.max -let sum = column -> internal std.sum +let sum = column -> internal std.sum -let average = column -> internal std.average +let average = column -> internal std.average -let stddev = column -> internal std.stddev +let stddev = column -> internal std.stddev -let all = column -> internal std.all +let all = column -> internal std.all -let any = column -> internal std.any +let any = column -> internal std.any -let concat_array = column -> internal std.concat_array +let concat_array = column -> internal std.concat_array # Counts number of items in the column. # Note that the count will include null values. -let count = column -> internal count +let count = colum -> internal count # Deprecated in favour of filterning input to the [std.count] function (not yet implemented). @{deprecated} -let count_distinct = column -> internal std.count_distinct +let count_distinct = column -> internal std.count_distinct ## Window functions -let lag = offset column -> internal std.lag -let lead = offset column -> internal std.lead -let first = column -> internal std.first -let last = column -> internal std.last -let rank = column -> internal std.rank -let rank_dense = column -> internal std.rank_dense -let row_number = column -> internal row_number +let lag = offset column -> internal std.lag +let lead = offset column -> internal std.lead +let first = column -> internal std.first +let last = column -> internal std.last +let rank = column -> internal std.rank +let rank_dense = column -> internal std.rank_dense +let row_number = column -> internal row_number # Mathematical functions module math { - let abs = column -> internal std.math.abs + let abs = column -> internal std.math.abs let floor = column -> internal std.math.floor let ceil = column -> internal std.math.ceil - let pi = -> internal std.math.pi - let exp = column -> internal std.math.exp - let ln = column -> internal std.math.ln - let log10 = column -> internal std.math.log10 - let log = base column -> internal std.math.log - let sqrt = column -> internal std.math.sqrt - let degrees = column -> internal std.math.degrees - let radians = column -> internal std.math.radians - let cos = column -> internal std.math.cos - let acos = column -> internal std.math.acos - let sin = column -> internal std.math.sin - let asin = column -> internal std.math.asin - let tan = column -> internal std.math.tan - let atan = column -> internal std.math.atan - let pow = exponent column -> internal std.math.pow - let round = n_digits column -> internal std.math.round + let pi = internal std.math.pi + let exp = column -> internal std.math.exp + let ln = column -> internal std.math.ln + let log10 = column -> internal std.math.log10 + let log = base column -> internal std.math.log + let sqrt = column -> internal std.math.sqrt + let degrees = column -> internal std.math.degrees + let radians = column -> internal std.math.radians + let cos = column -> internal std.math.cos + let acos = column -> internal std.math.acos + let sin = column -> internal std.math.sin + let asin = column -> internal std.math.asin + let tan = column -> internal std.math.tan + let atan = column -> internal std.math.atan + let pow = exponent column -> internal std.math.pow + let round = n_digits column -> internal std.math.round } ## Misc functions -let as = `noresolve.type` column -> internal std.as +let as = `noresolve.type` column -> internal std.as let in = pattern value -> internal in ## Tuple functions @@ -209,10 +211,10 @@ let tuple_every = func list -> internal tuple_every let tuple_map = func fn list -> internal tuple_map let tuple_zip = func a b -> internal tuple_zip let _eq = func a -> internal _eq -let _is_null = func a -> _param.a == null +let _is_null = func a -> a == null ## Misc -let from_text = input `noresolve.format`:csv -> internal from_text +let from_text = func input `noresolve.format`:csv -> <[R]> internal from_text ## Text functions module text { @@ -235,14 +237,13 @@ module date { } ## File-reading functions, primarily for DuckDB -let read_parquet = source -> internal std.read_parquet -let read_csv = source -> internal std.read_csv - +let read_parquet = func source -> <[R]> internal std.read_parquet +# let read_csv = func source -> <[R]> internal std.read_csv ## PRQL compiler functions module `prql` { - let version = -> internal prql_version + let version = internal prql_version } # Deprecated, will be removed in 0.12.0 -let prql_version = -> internal prql_version +let prql_version = internal prql_version diff --git a/prqlc/prqlc/src/sql/gen_projection.rs b/prqlc/prqlc/src/sql/gen_projection.rs index fecfa41852e4..d67f08ab40fc 100644 --- a/prqlc/prqlc/src/sql/gen_projection.rs +++ b/prqlc/prqlc/src/sql/gen_projection.rs @@ -64,7 +64,7 @@ pub(super) fn translate_wildcards(ctx: &AnchorContext, cols: Vec) -> (Vec 3 "#; diff --git a/prqlc/prqlc/src/sql/mod.rs b/prqlc/prqlc/src/sql/mod.rs index ac0bcd630d51..90246bcb4d26 100644 --- a/prqlc/prqlc/src/sql/mod.rs +++ b/prqlc/prqlc/src/sql/mod.rs @@ -138,7 +138,7 @@ mod test { #[test] fn test_end_with_new_line() { - let sql = compile("from a", &Options::default().no_signature()).unwrap(); + let sql = compile("from db.a", &Options::default().no_signature()).unwrap(); assert_eq!(sql, "SELECT\n *\nFROM\n a\n") } } diff --git a/prqlc/prqlc/src/sql/operators.rs b/prqlc/prqlc/src/sql/operators.rs index aba525ed2138..96f8ff74a211 100644 --- a/prqlc/prqlc/src/sql/operators.rs +++ b/prqlc/prqlc/src/sql/operators.rs @@ -20,7 +20,7 @@ fn std() -> &'static decl::Module { let std_lib = crate::SourceTree::new( [( - PathBuf::from("std.prql"), + PathBuf::from("std.sql.prql"), include_str!("./std.sql.prql").to_string(), )], None, @@ -45,20 +45,14 @@ pub(super) fn translate_operator( args: Vec, ctx: &mut Context, ) -> Result { - let (func_def, binding_strength, window_frame, coalesce) = + let (operator_impl, binding_strength, window_frame, coalesce) = find_operator_impl(&name, ctx.dialect_enum).unwrap(); let parent_binding_strength = binding_strength.unwrap_or(100); - let params = func_def - .named_params - .iter() - .chain(func_def.params.iter()) - .map(|x| x.name.split('.').last().unwrap_or(x.name.as_str())); - - let args: HashMap<&str, _> = zip(params, args).collect(); + let args: HashMap<&str, _> = zip(operator_impl.params, args).collect(); // body can only be an s-string - let body = match &func_def.body.kind { + let body = match &operator_impl.body.kind { pl::ExprKind::Literal(pl::Literal::Null) => { return Err(Error::new_simple(format!( "operator {} is not supported for dialect {}", @@ -66,7 +60,11 @@ pub(super) fn translate_operator( ))) } pl::ExprKind::SString(items) => items, - _ => panic!("Bad RQ operator implementation. Expected s-string or null"), + _ => { + return Err(Error::new_assert( + "bad RQ operator implementation for SQL: expected function or a plain s-string", + )) + } }; let mut text = String::new(); @@ -120,10 +118,15 @@ pub(super) fn translate_operator( }) } +struct OperatorImpl<'a> { + body: &'a pl::Expr, + params: Vec<&'a str>, +} + fn find_operator_impl( operator_name: &str, dialect: Dialect, -) -> Option<(&pl::Func, Option, bool, Option)> { +) -> Option<(OperatorImpl<'_>, Option, bool, Option)> { let operator_name = operator_name.strip_prefix("std.").unwrap(); let operator_ident = pl::Ident::from_path( operator_name @@ -134,23 +137,40 @@ fn find_operator_impl( let dialect_module = std().get(&pl::Ident::from_name(dialect.to_string())); - let mut func_def = None; + let mut impl_decl = None; if let Some(dialect_module) = dialect_module { let module = dialect_module.kind.as_module().unwrap(); - func_def = module.get(&operator_ident); + impl_decl = module.get(&operator_ident); } - if func_def.is_none() { - func_def = std().get(&operator_ident); + if impl_decl.is_none() { + impl_decl = std().get(&operator_ident); } - let decl = func_def?; - - let func_def = decl.kind.as_expr().unwrap(); - let func_def = func_def.kind.as_func().unwrap(); + let impl_decl = impl_decl?; + + let impl_expr = impl_decl.kind.as_expr().unwrap(); + let operator_impl = match &impl_expr.kind { + pl::ExprKind::Func(func) => { + let params: Vec<_> = func + .named_params + .iter() + .chain(func.params.iter()) + .map(|x| x.name.split('.').last().unwrap_or(x.name.as_str())) + .collect(); + OperatorImpl { + body: func.body.as_ref(), + params, + } + } + _ => OperatorImpl { + body: impl_expr.as_ref(), + params: Vec::new(), + }, + }; - let annotation = decl.annotations.iter().exactly_one().ok(); + let annotation = impl_decl.annotations.iter().exactly_one().ok(); let mut annotation = annotation .and_then(|x| into_tuple_items(*x.expr.clone()).ok()) .unwrap_or_default(); @@ -166,7 +186,7 @@ fn find_operator_impl( let coalesce = pluck_annotation(&mut annotation, "coalesce").and_then(|val| val.into_string().ok()); - Some((func_def.as_ref(), binding_strength, window_frame, coalesce)) + Some((operator_impl, binding_strength, window_frame, coalesce)) } fn pluck_annotation( diff --git a/prqlc/prqlc/src/sql/pq/mod.rs b/prqlc/prqlc/src/sql/pq/mod.rs index 9d29e5dc68e8..d60a3d6a4fc7 100644 --- a/prqlc/prqlc/src/sql/pq/mod.rs +++ b/prqlc/prqlc/src/sql/pq/mod.rs @@ -41,7 +41,7 @@ mod test { fn test_ctes_of_pipeline() { // One aggregate, take at the end let prql: &str = r#" - from employees + from db.employees filter country == "USA" aggregate {sal = average salary} sort sal @@ -52,7 +52,7 @@ mod test { // One aggregate, but take at the top let prql: &str = r#" - from employees + from db.employees take 20 filter country == "USA" aggregate {sal = average salary} @@ -63,7 +63,7 @@ mod test { // A take, then two aggregates let prql: &str = r#" - from employees + from db.employees take 20 filter country == "USA" aggregate {sal = average salary} @@ -75,7 +75,7 @@ mod test { // A take, then a select let prql: &str = r###" - from employees + from db.employees take 20 select first_name "###; diff --git a/prqlc/prqlc/src/sql/std.sql.prql b/prqlc/prqlc/src/sql/std.sql.prql index e2bfada1bfca..556ed69ed2cc 100644 --- a/prqlc/prqlc/src/sql/std.sql.prql +++ b/prqlc/prqlc/src/sql/std.sql.prql @@ -54,11 +54,11 @@ let first = column -> s"FIRST_VALUE({column:0})" let last = column -> s"LAST_VALUE({column:0})" -let rank = -> s"RANK()" +let rank = column -> s"RANK()" -let rank_dense = -> s"DENSE_RANK()" +let rank_dense = column -> s"DENSE_RANK()" -let row_number = -> s"ROW_NUMBER()" +let row_number = column -> s"ROW_NUMBER()" # Mathematical functions module math { @@ -74,7 +74,7 @@ module math { let abs = column -> s"ABS({column:0})" let floor = column -> s"FLOOR({column:0})" let ceil = column -> s"CEIL({column:0})" - let pi = -> s"PI()" + let pi = s"PI()" let exp = column -> s"EXP({column:0})" let ln = column -> s"LN({column:0})" let log10 = column -> s"LOG10({column:0})" @@ -174,7 +174,7 @@ let or = l r -> null let coalesce = l r -> s"COALESCE({l:0}, {r:0})" -let regex_search = text pattern -> s"REGEXP({text:0}, {pattern:0})" +let regex_search = haystack pattern -> s"REGEXP({haystack:0}, {pattern:0})" @{binding_strength=13} let neg = l -> s"-{l}" @@ -198,7 +198,7 @@ module bigquery { let radians = column -> s"({column:0} * PI() / 180)" } - let regex_search = text pattern -> s"REGEXP_CONTAINS({text:0}, {pattern:0})" + let regex_search = haystack pattern -> s"REGEXP_CONTAINS({haystack:0}, {pattern:0})" } module clickhouse { @@ -215,7 +215,7 @@ module clickhouse { let to_text = format column -> s"formatDateTimeInJodaSyntax({column:0}, {format:0})" } - let regex_search = text pattern -> s"match({text:0}, {pattern:0})" + let regex_search = haystack pattern -> s"match({haystack:0}, {pattern:0})" let read_csv = source -> s"file({source:0}, 'CSV')" @@ -241,7 +241,7 @@ module duckdb { let to_text = format column -> s"strftime({column:0}, {format:0})" } - let regex_search = text pattern -> s"REGEXP_MATCHES({text:0}, {pattern:0})" + let regex_search = haystack pattern -> s"REGEXP_MATCHES({haystack:0}, {pattern:0})" let read_csv = source -> s"read_csv_auto({source:0})" } @@ -290,7 +290,7 @@ module mysql { } # 'c' for case-sensitive - let regex_search = text pattern -> s"REGEXP_LIKE({text:0}, {pattern:0}, 'c')" + let regex_search = haystack pattern -> s"REGEXP_LIKE({haystack:0}, {pattern:0}, 'c')" } module postgres { @@ -320,7 +320,7 @@ module postgres { } @{binding_strength=9} - let regex_search = text pattern -> s"{text} ~ {pattern}" + let regex_search = haystack pattern -> s"{haystack} ~ {pattern}" } module glaredb { @@ -338,7 +338,7 @@ module glaredb { } @{binding_strength=9} - let regex_search = text pattern -> s"{text} ~ {pattern}" + let regex_search = haystack pattern -> s"{haystack} ~ {pattern}" let read_csv = source -> s"csv_scan({source:0})" @@ -371,7 +371,7 @@ module sqlite { } @{binding_strength=9} - let regex_search = text pattern -> s"{text} REGEXP {pattern}" + let regex_search = haystack pattern -> s"{haystack} REGEXP {pattern}" } module snowflake { diff --git a/prqlc/prqlc/src/utils/id_gen.rs b/prqlc/prqlc/src/utils/id_gen.rs index b12ac953f3f5..54fe71db2daf 100644 --- a/prqlc/prqlc/src/utils/id_gen.rs +++ b/prqlc/prqlc/src/utils/id_gen.rs @@ -14,7 +14,7 @@ impl> IdGenerator { Self::default() } - fn skip(&mut self, id: usize) { + pub fn skip(&mut self, id: usize) { self.next_id = self.next_id.max(id + 1); } diff --git a/prqlc/prqlc/tests/integration/resolving.rs b/prqlc/prqlc/tests/integration/resolving.rs index 170b613a96a7..d443dbf7a998 100644 --- a/prqlc/prqlc/tests/integration/resolving.rs +++ b/prqlc/prqlc/tests/integration/resolving.rs @@ -12,118 +12,316 @@ fn resolve(prql_source: &str) -> Result { // resolved PL, restricted back into AST let mut root_module = prqlc::semantic::ast_expand::restrict_module(root_module.module); - drop_module_defs(&mut root_module.stmts, &["std", "default_db"]); + drop_irrelevant_stuff( + &mut root_module.stmts, + &["std", "_local", "_infer", "_generic"], + ); prqlc::pl_to_prql(&root_module) } -fn drop_module_defs(stmts: &mut Vec, to_drop: &[&str]) { - stmts.retain(|x| { - x.kind - .as_module_def() - .map_or(true, |m| !to_drop.contains(&m.name.as_str())) +fn drop_irrelevant_stuff(stmts: &mut Vec, to_drop: &[&str]) { + stmts.retain_mut(|x| { + match &mut x.kind { + pr::StmtKind::ModuleDef(m) => { + if to_drop.contains(&m.name.as_str()) { + return false; + } + + drop_irrelevant_stuff(&mut m.stmts, to_drop); + } + pr::StmtKind::VarDef(v) => { + if to_drop.contains(&v.name.as_str()) { + return false; + } + } + _ => (), + } + true }); } #[test] fn resolve_basic_01() { assert_snapshot!(resolve(r#" - from x + module db { + let x <[{a = int, b = text, c = float}]> + } + + from db.x select {a, b} "#).unwrap(), @r###" - let main <[{a = ?, b = ?}]> = `(Select ...)` + module db { + let x <[{a = int, b = text, c = float}]> = $x + } + + let main <[{a = int, b = text}]> = std.select {this.0, this.1} ( + std.from db.x + ) "###) } #[test] -fn resolve_function_01() { +fn resolve_ty_tuple_unpack() { assert_snapshot!(resolve(r#" - let my_func = func param_1 -> ( - param_1 + 1 - ) + type Employee = {first_name = text, age = int} + + let employees <[{ id = int, ..module.Employee }]> "#).unwrap(), @r###" - let my_func = func param_1 -> ( - std.add param_1 1 - ) + type Employee = {first_name = text, age = int} + + module db { + } + + let employees <[{id = int, first_name = text, age = int}]> = $employees "###) } #[test] -fn resolve_types_01() { +fn resolve_ty_exclude() { assert_snapshot!(resolve(r#" - type A = int || int + type X = {a = int, b = text} + type Y = {b = text} + type Z = module.X - module.Y "#).unwrap(), @r###" - type A = int - "###) + type X = {a = int, b = text} + + type Y = {b = text} + + type Z = {a = int} + + module db { + } + "###); + + assert_snapshot!(resolve(r#" + type X = {a = int, b = text} + type Y = text + type Z = module.X - module.Y + "#).unwrap_err(), @r###" + Error: + ╭─[:4:25] + │ + 4 │ type Z = module.X - module.Y + │ ────┬─── + │ ╰───── expected excluded fields to be a tuple + │ + │ Help: got text + ───╯ + "###); + + assert_snapshot!(resolve(r#" + type X = text + type Y = {a = int} + type Z = module.X - module.Y + "#).unwrap_err(), @r###" + Error: + ╭─[:4:14] + │ + 4 │ type Z = module.X - module.Y + │ ────┬─── + │ ╰───── fields can only be excluded from a tuple + │ + │ Help: got text + ───╯ + "###); } #[test] -fn resolve_types_02() { - assert_snapshot!(resolve(r#" - type A = int || {} - "#).unwrap(), @r###" - type A = int || {} - "###) +#[ignore] +fn resolve_generics_01() { + assert_snapshot!(resolve( + r#" + let add_one = func a -> a + 1 + + let my_int = module.add_one 1 + let my_float = module.add_one 1.0 + "#, + ) + .unwrap(), @r###" + let add_one = func a -> ( + std.add a 1 + ) + + module db { + } + + let my_float = `(std.add ...)` + + let my_int = `(std.add ...)` + "###); } #[test] -fn resolve_types_03() { - assert_snapshot!(resolve(r#" - type A = {a = int, bool} || {b = text, float} - "#).unwrap(), @r###" - type A = {a = int, bool, b = text, float} - "###) +#[ignore] +fn resolve_generics_02() { + assert_snapshot!(resolve( + r#" + let neg = func num -> -num + let map = func + mapper M> + elements <[E]> + -> <[M]> s"an array of mapped elements" + + let ints = [1, 2, 3] + let negated = (module.map module.neg module.ints) + "#, + ) + .unwrap(), @r###" + let add_one = func a -> ( + std.add a 1 + ) + + module db { + } + + let my_float = `(std.add ...)` + + let my_int = `(std.add ...)` + "###); } #[test] -fn resolve_types_04() { +fn table_inference_01() { assert_snapshot!(resolve( r#" - type Status = enum { - Paid = {}, - Unpaid = float, - Canceled = {reason = text, cancelled_at = timestamp}, + from db.employees + "#, + ) + .unwrap(), @r###" + module db { + let employees <[{.._generic.T0}]> = $ } + + let main <[{..T0}]> = std.from db.employees + "###); +} + +#[test] +fn table_inference_02() { + assert_snapshot!(resolve( + r#" + from db.employees + select {id, age} "#, ) .unwrap(), @r###" - type Status = ( - Unpaid = float || - {reason = text, cancelled_at = timestamp} || + module db { + let employees <[{.._generic.T0}]> = $ + } + + let main <[{id = F378, age = F381}]> = std.select {this.0, this.1} ( + std.from db.employees ) "###); } #[test] -fn resolve_types_05() { - // TODO: this is very strange, it should only be allowed in std +fn table_inference_03() { assert_snapshot!(resolve( r#" - type A + from db.employees + select {e = this} + select {e.name} "#, ) .unwrap(), @r###" - type A = null + module db { + let employees <[{.._generic.T0}]> = $ + } + + let main <[{name = F385}]> = std.select {this.0.0} ( + std.select {e = this} (std.from db.employees) + ) "###); } #[test] -fn resolve_generics_01() { +fn table_inference_04() { assert_snapshot!(resolve( r#" - let add_one = func a -> a + 1 - - let my_int = add_one 1 - let my_float = add_one 1.0 + let len_of_ints = func arr<[int]> -> 0 + + module.len_of_ints [] "#, ) .unwrap(), @r###" - let add_one = func a -> ( - std.add a 1 + module db { + } + + let len_of_ints int> = func arr <[int]> -> 0 + + let main = len_of_ints [] + "###); +} + +#[test] +#[ignore] +fn high_order_func_01() { + assert_snapshot!(resolve( + r#" + let neg = func num -> -num + let map = func + mapper M> + elements <[E]> + -> <[M]> s"an array of mapped elements" + + let ints = [1, 2, 3] + let negated = (module.map module.neg module.ints) + "#, ) + .unwrap(), @r###" + module db { + } - let my_float = `(std.add ...)` + let ints <[int]> = [1, 2, 3] - let my_int = `(std.add ...)` + let map = func mapper M> elements <[E]> -> <[M]> s"an array of mapped elements" + + let neg = func num -> std.neg num + + let negated <[int]> = s"an array of mapped elements" + "###); +} + +#[test] +#[ignore] +fn high_order_func_02() { + assert_snapshot!(resolve( + r#" + let convert_to_one = func x -> 1 + + let both = func + mapper O> + input <{I, I}> + -> <{O, O}> { + (mapper input.0), + (mapper input.1), + } + + let both_ones = (module.both module.convert_to_one) + + let main = (module.both_ones {'hello', 'world'}) + "#, + ) + .unwrap(), @r###" + let both = func mapper O> input <{I, I}> -> <{O, O}> { + mapper input.0, + mapper input.1, + } + + let both_ones {O, O}> = ( + func mapper O> input <{I, I}> -> <{O, O}> { + mapper input.0, + mapper input.1, + } + ) convert_to_one + + let convert_to_one = func x -> 1 + + module db { + } + + let main <{int, int}> = {1, 1} "###); } diff --git a/prqlc/prqlc/tests/integration/sql.rs b/prqlc/prqlc/tests/integration/sql.rs index 26f7f2f478d3..a1e13badce77 100644 --- a/prqlc/prqlc/tests/integration/sql.rs +++ b/prqlc/prqlc/tests/integration/sql.rs @@ -25,7 +25,7 @@ fn compile_with_sql_dialect(prql: &str, dialect: sql::Dialect) -> Result 0, ltz = !(a > 0), @@ -383,7 +388,7 @@ fn test_precedence_04() { fn test_precedence_05() { assert_snapshot!(compile( r###" - from numbers + from db.numbers derive x = (y - z) select { c - (a + b), @@ -417,11 +422,9 @@ fn test_precedence_05() { } #[test] -#[ignore] -// FIXME: right associativity of `pow` is not implemented yet fn test_pow_is_right_associative() { assert_snapshot!(compile(r#" - from numbers + from db.numbers select { c ** a ** b } @@ -435,10 +438,11 @@ fn test_pow_is_right_associative() { } #[test] +#[ignore] fn test_append() { assert_snapshot!(compile(r###" - from employees - append managers + from db.employees + append db.managers "###).unwrap(), @r###" SELECT * @@ -453,11 +457,11 @@ fn test_append() { "###); assert_snapshot!(compile(r###" - from employees + from db.employees select {name, cost = salary} take 3 append ( - from employees + from db.employees select {name, cost = salary + bonuses} take 10 ) @@ -493,10 +497,10 @@ fn test_append() { assert_snapshot!(compile(r###" let distinct = rel -> (_param.rel | group this (take 1)) - let union = func `default_db.bottom` top -> (top | append bottom | distinct) + let union = func bottom top -> (top | append bottom | module.distinct) - from employees - union (from managers) + from db.employees + module.union db.managers "###).unwrap(), @r###" SELECT * @@ -512,11 +516,11 @@ fn test_append() { assert_snapshot!(compile(r###" let distinct = rel -> (_param.rel | group this (take 1)) - let union = func `default_db.bottom` top -> (top | append bottom | distinct) + let union = func bottom top -> (top | append bottom | module.distinct) - from employees - append managers - union all_employees_of_some_other_company + from db.employees + append db.managers + module.union db.all_employees_of_some_other_company "###).unwrap(), @r###" SELECT * @@ -538,10 +542,11 @@ fn test_append() { } #[test] +#[ignore] fn test_remove_01() { assert_snapshot!(compile(r#" - from albums - remove artists + from db.albums + remove db.artists "#).unwrap(), @r###" SELECT @@ -559,12 +564,13 @@ fn test_remove_01() { } #[test] +#[ignore] fn test_remove_02() { assert_snapshot!(compile(r#" - from album + from db.album select artist_id remove ( - from artist | select artist_id + from db.artist | select artist_id ) "#).unwrap(), @r###" @@ -589,12 +595,13 @@ fn test_remove_02() { } #[test] +#[ignore] fn test_remove_03() { assert_snapshot!(compile(r#" - from album + from db.album select {artist_id, title} remove ( - from artist | select artist_id + from db.artist | select artist_id ) "#).unwrap(), @r###" @@ -617,12 +624,13 @@ fn test_remove_03() { } #[test] +#[ignore] fn test_remove_04() { assert_snapshot!(compile(r#" prql target:sql.sqlite - from album - remove artist + from db.album + remove db.artist "#).unwrap_err(), @r###" Error: The dialect SQLiteDialect does not support EXCEPT ALL @@ -632,16 +640,18 @@ fn test_remove_04() { } #[test] +#[ignore] fn test_remove_05() { assert_snapshot!(compile(r#" prql target:sql.sqlite - let distinct = rel -> (from t = _param.rel | group {t.*} (take 1)) - let except = `default_db.bottom` top -> (top | distinct | remove bottom) + let distinct = rel -> (_param.rel | group this (take 1)) + let except = func bottom top -> (top | module.distinct | remove bottom) - from album + from db.album + from db.album select {artist_id, title} - except (from artist | select {artist_id, name}) + module.except (from db.artist | select {artist_id, name}) "#).unwrap(), @r###" WITH table_0 AS ( @@ -666,21 +676,22 @@ fn test_remove_05() { } #[test] +#[ignore] fn test_remove_06() { assert_snapshot!(compile(r#" prql target:sql.sqlite - let distinct = rel -> (from t = _param.rel | group {t.*} (take 1)) - let except = func `default_db.bottom` top -> (top | distinct | remove bottom) + let distinct = rel -> (_param.rel | group this (take 1)) + let except = func bottom top -> (top | module.distinct | remove bottom) - from album - except artist + from db.album + module.except db.artist "#).unwrap(), @r###" SELECT * FROM - album AS t + album EXCEPT SELECT * @@ -691,10 +702,11 @@ fn test_remove_06() { } #[test] +#[ignore] fn test_intersect_01() { assert_snapshot!(compile(r#" - from album - intersect artist + from db.album + intersect db.artist "#).unwrap(), @r###" SELECT @@ -712,12 +724,13 @@ fn test_intersect_01() { } #[test] +#[ignore] fn test_intersect_02() { assert_snapshot!(compile(r#" - from album + from db.album select artist_id intersect ( - from artist | select artist_id + from db.artist | select artist_id ) "#).unwrap(), @r###" @@ -742,17 +755,18 @@ fn test_intersect_02() { } #[test] +#[ignore] fn test_intersect_03() { assert_snapshot!(compile(r#" let distinct = rel -> (_param.rel | group this (take 1)) - from album + from db.album select artist_id - distinct + module.distinct intersect ( - from artist | select artist_id + from db.artist | select artist_id ) - distinct + module.distinct "#).unwrap(), @r###" WITH table_0 AS ( @@ -782,16 +796,17 @@ fn test_intersect_03() { } #[test] +#[ignore] fn test_intersect_04() { assert_snapshot!(compile(r#" let distinct = rel -> (_param.rel | group this (take 1)) - from album + from db.album select artist_id intersect ( - from artist | select artist_id + from db.artist | select artist_id ) - distinct + module.distinct "#).unwrap(), @r###" WITH table_0 AS ( @@ -821,15 +836,16 @@ fn test_intersect_04() { } #[test] +#[ignore] fn test_intersect_05() { assert_snapshot!(compile(r#" let distinct = rel -> (_param.rel | group this (take 1)) - from album + from db.album select artist_id - distinct + module.distinct intersect ( - from artist | select artist_id + from db.artist | select artist_id ) "#).unwrap(), @r###" @@ -854,12 +870,13 @@ fn test_intersect_05() { } #[test] +#[ignore] fn test_intersect_06() { assert_snapshot!(compile(r#" prql target:sql.sqlite - from album - intersect artist + from db.album + intersect db.artist "#).unwrap_err(), @r###" Error: The dialect SQLiteDialect does not support INTERSECT ALL @@ -869,10 +886,11 @@ fn test_intersect_06() { } #[test] +#[ignore] fn test_intersect_07() { assert_snapshot!(compile(r#" - from ds2 = foo.t1 - join side:inner ds1 = bar.t2 (ds2.idx==ds1.idx) + from ds2 = db.foo.t1 + join side:inner ds1 = db.bar.t2 (ds2.idx==ds1.idx) aggregate { count this } "#).unwrap(), @r###" @@ -886,10 +904,11 @@ fn test_intersect_07() { } #[test] +#[ignore] fn test_rn_ids_are_unique() { // this is wrong, output will have duplicate y_id and x_id assert_snapshot!((compile(r###" - from y_orig + from db.y_orig group {y_id} ( take 2 # take 1 uses `distinct` instead of partitioning, which might be a separate bug ) @@ -928,10 +947,10 @@ fn test_quoting_01() { assert_snapshot!((compile(r###" prql target:sql.postgres let UPPER = ( - default_db.lower + from db.lower ) - from UPPER - join `some_schema.tablename` (==id) + module.UPPER + join db.`some_schema.tablename` (==id) derive `from` = 5 "###).unwrap()), @r###" WITH "UPPER" AS ( @@ -954,7 +973,7 @@ fn test_quoting_01() { fn test_quoting_02() { // GH-1493 let query = r###" - from `dir/*.parquet` + from db.`dir/*.parquet` "###; assert_snapshot!((compile(query).unwrap()), @r###" SELECT @@ -965,29 +984,30 @@ fn test_quoting_02() { } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_quoting_03() { // GH-#852 assert_snapshot!((compile(r###" prql target:sql.bigquery - from `schema.table` - join `schema.table2` (==id) - join c = `schema.t-able` (`schema.table`.id == c.id) + from db.`schema.table` + join db.`schema.table2` (==id) + join c = db.`schema.t-able` (`db.schema.table`.id == c.id) "###).unwrap()), @r###" SELECT - `schema.table`.*, - `schema.table2`.*, + `db.schema.table`.*, + `db.schema.table2`.*, c.* FROM - `schema.table` - JOIN `schema.table2` ON `schema.table`.id = `schema.table2`.id - JOIN `schema.t-able` AS c ON `schema.table`.id = c.id + `db.schema.table` + JOIN `db.schema.table2` ON `db.schema.table`.id = `db.schema.table2`.id + JOIN `db.schema.t-able` AS c ON `db.schema.table`.id = c.id "###); } #[test] fn test_quoting_04() { assert_snapshot!((compile(r###" - from table + from db.table select `first name` "###).unwrap()), @r###" SELECT @@ -1000,19 +1020,20 @@ fn test_quoting_04() { #[test] fn test_quoting_05() { assert_snapshot!((compile(r###" - from as = Assessment + from db.Assessment + select {as = this} "###).unwrap()), @r###" SELECT * FROM - "Assessment" AS "as" + "Assessment" "###); } #[test] fn test_sorts_01() { assert_snapshot!((compile(r###" - from invoices + from db.invoices sort {issued_at, -amount, +num_of_articles} "### ).unwrap()), @r###" @@ -1027,25 +1048,18 @@ fn test_sorts_01() { "###); assert_snapshot!((compile(r#" - from x + from db.x derive somefield = "something" sort {somefield} select {renamed = somefield} "# ).unwrap()), @r###" - WITH table_0 AS ( - SELECT - 'something' AS renamed, - 'something' AS _expr_0 - FROM - x - ) SELECT - renamed + 'something' AS renamed FROM - table_0 + x ORDER BY - _expr_0 + renamed "###); } @@ -1055,11 +1069,11 @@ fn test_sorts_02() { assert_snapshot!((compile(r###" let x = ( - from table + from db.table sort index select {fieldA} ) - from x + module.x "### ).unwrap()), @r###" WITH table_0 AS ( @@ -1086,11 +1100,12 @@ fn test_sorts_02() { } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_sorts_03() { // TODO: this is invalid SQL: a._expr_0 does not exist assert_snapshot!((compile(r#" - from a - join b side:left (==col) + from db.a + join db.b side:left (==col) sort a.col select !{a.col} take 5 @@ -1121,7 +1136,7 @@ fn test_sorts_03() { #[test] fn test_numbers() { let query = r###" - from numbers + from db.numbers select { v = 5.000_000_1, w = 5_000, @@ -1146,7 +1161,7 @@ fn test_numbers() { #[test] fn test_ranges() { assert_snapshot!((compile(r###" - from employees + from db.employees derive { close = (distance | in ..50), middle = (distance | in 50..100), @@ -1168,7 +1183,7 @@ fn test_ranges() { #[test] fn test_in_values_01() { assert_snapshot!((compile(r#" - from employees + from db.employees filter (title | in ["Sales Manager", "Sales Support Agent"]) filter (employee_id | in [1, 2, 5]) filter (f"{emp_group}.{role}" | in ["sales_ne.mgr", "sales_mw.mgr"]) @@ -1192,7 +1207,7 @@ fn test_in_values_02() { assert_snapshot!((compile(r#" let allowed_titles = ["Sales Manager", "Sales Support Agent"] - from employees + from db.employees derive {allowed_ids = [1, 2, 5]} filter (title | in allowed_titles) filter (title | in allowed_ids) @@ -1210,7 +1225,7 @@ fn test_in_values_02() { #[ignore] // unimplemented, column ref type resolution required fn test_in_values_03() { assert_snapshot!((compile(r#" - from employees + from db.employees derive allowed_titles = case [ is_guest => ["Sales Manager"], true => ["Sales Manager", "Sales Support Agent"], @@ -1229,7 +1244,7 @@ fn test_in_values_03() { #[test] fn test_not_in_values() { assert_snapshot!((compile(r#" - from employees + from db.employees filter !(title | in ["Sales Manager", "Sales Support Agent"]) "#).unwrap()), @r#" SELECT @@ -1244,7 +1259,7 @@ fn test_not_in_values() { #[test] fn test_in_no_values() { assert_snapshot!((compile(r#" - from employees + from db.employees filter (title | in []) "#).unwrap()), @r#" SELECT @@ -1259,7 +1274,7 @@ fn test_in_no_values() { #[test] fn test_in_values_err_01() { assert_snapshot!((compile(r###" - from employees + from db.employees derive { ng = ([1, 2] | in [3, 4]) } "###).unwrap_err()), @r###" Error: @@ -1275,7 +1290,7 @@ fn test_in_values_err_01() { #[test] fn test_interval() { let query = r###" - from projects + from db.projects derive first_check_in = start + 10days "###; @@ -1290,7 +1305,7 @@ fn test_interval() { let query = r###" prql target:sql.postgres - from projects + from db.projects derive first_check_in = start + 10days "###; assert_snapshot!((compile(query).unwrap()), @r###" @@ -1304,7 +1319,7 @@ fn test_interval() { let query = r###" prql target:sql.glaredb - from projects + from db.projects derive first_check_in = start + 10days "###; assert_snapshot!((compile(query).unwrap()), @r###" @@ -1319,7 +1334,7 @@ fn test_interval() { #[test] fn test_dates() { assert_snapshot!((compile(r###" - from to_do_empty_table + from db.to_do_empty_table derive { date = @2011-02-01, timestamp = @2011-02-01T10:00, @@ -1338,9 +1353,10 @@ fn test_dates() { } #[test] +#[ignore] fn test_window_functions_00() { assert_snapshot!((compile(r###" - from employees + from db.employees group last_name ( derive {count first_name} ) @@ -1354,10 +1370,12 @@ fn test_window_functions_00() { } #[test] +#[ignore] fn test_window_functions_02() { let query = r#" - from co=cust_order - join ol=order_line (==order_id) + from db.cust_order + select {co = this} + join (db.order_line | select {ol = this}) (==order_id) derive { order_month = s"TO_CHAR({co.order_date}, '%Y-%m')", order_day = s"TO_CHAR({co.order_date}, '%Y-%m-%d')", @@ -1382,17 +1400,23 @@ fn test_window_functions_02() { assert_snapshot!((compile(query).unwrap()), @r###" WITH table_0 AS ( SELECT - TO_CHAR(co.order_date, '%Y-%m') AS order_month, - TO_CHAR(co.order_date, '%Y-%m-%d') AS order_day, - COUNT(DISTINCT co.order_id) AS num_orders, + * + FROM + order_line + ), + table_1 AS ( + SELECT + TO_CHAR(cust_order.order_date, '%Y-%m') AS order_month, + TO_CHAR(cust_order.order_date, '%Y-%m-%d') AS order_day, + COUNT(DISTINCT cust_order.order_id) AS num_orders, COUNT(*) AS num_books, - COALESCE(SUM(ol.price), 0) AS total_price + COALESCE(SUM(table_0.price), 0) AS total_price FROM - cust_order AS co - JOIN order_line AS ol ON co.order_id = ol.order_id + cust_order + JOIN table_0 ON cust_order.order_id = table_0.order_id GROUP BY - TO_CHAR(co.order_date, '%Y-%m'), - TO_CHAR(co.order_date, '%Y-%m-%d') + TO_CHAR(cust_order.order_date, '%Y-%m'), + TO_CHAR(cust_order.order_date, '%Y-%m-%d') ) SELECT order_month, @@ -1410,18 +1434,19 @@ fn test_window_functions_02() { order_day ) AS num_books_last_week FROM - table_0 + table_1 ORDER BY order_day "###); } #[test] +#[ignore] fn test_window_functions_03() { // lag must be recognized as window function, even outside of group context // rank must not have two OVER clauses let query = r###" - from daily_orders + from db.daily_orders derive {last_week = lag 7 num_orders} derive {first_count = first num_orders} derive {last_count = last num_orders} @@ -1443,10 +1468,11 @@ fn test_window_functions_03() { } #[test] +#[ignore] fn test_window_functions_04() { // sort does not affects into groups, group undoes sorting let query = r###" - from daily_orders + from db.daily_orders sort day group month (derive {total_month = rank day}) derive {last_week = lag 7 num_orders} @@ -1463,10 +1489,11 @@ fn test_window_functions_04() { } #[test] +#[ignore] fn test_window_functions_05() { // sort does not leak out of groups let query = r###" - from daily_orders + from db.daily_orders sort day group month (sort num_orders | window expanding:true (derive {rank day})) derive {num_orders_last_week = lag 7 num_orders} @@ -1486,10 +1513,11 @@ fn test_window_functions_05() { } #[test] +#[ignore] fn test_window_functions_06() { // detect sum as a window function, even without group or window assert_snapshot!((compile(r###" - from foo + from db.foo derive {a = sum b} group c ( derive {d = sum b} @@ -1505,9 +1533,10 @@ fn test_window_functions_06() { } #[test] +#[ignore] fn test_window_functions_07() { assert_snapshot!((compile(r###" - from foo + from db.foo window expanding:true ( derive {running_total = sum b} ) @@ -1521,9 +1550,10 @@ fn test_window_functions_07() { } #[test] +#[ignore] fn test_window_functions_08() { assert_snapshot!((compile(r###" - from foo + from db.foo window rolling:3 ( derive {last_three = sum b} ) @@ -1537,9 +1567,10 @@ fn test_window_functions_08() { } #[test] +#[ignore] fn test_window_functions_09() { assert_snapshot!((compile(r###" - from foo + from db.foo window rows:0..4 ( derive {next_four_rows = sum b} ) @@ -1556,9 +1587,10 @@ fn test_window_functions_09() { } #[test] +#[ignore] fn test_window_functions_10() { assert_snapshot!((compile(r###" - from foo + from db.foo sort day window range:-4..4 ( derive {next_four_days = sum b} @@ -1578,9 +1610,10 @@ fn test_window_functions_10() { } #[test] +#[ignore] fn test_window_functions_11() { assert_snapshot!((compile(r###" - from employees + from db.employees sort age derive {num = row_number this} "###).unwrap()), @r###" @@ -1598,11 +1631,12 @@ fn test_window_functions_11() { } #[test] +#[ignore] fn test_window_functions_12() { // window params need to be simple expressions assert_snapshot!((compile(r###" - from x + from db.x derive {b = lag 1 a} window ( sort b @@ -1629,7 +1663,7 @@ fn test_window_functions_12() { "###); assert_snapshot!((compile(r###" - from x + from db.x derive {b = lag 1 a} group b ( derive {c = lag 1 a} @@ -1651,11 +1685,12 @@ fn test_window_functions_12() { } #[test] +#[ignore] fn test_window_functions_13() { // window params need to be simple expressions assert_snapshot!((compile(r###" - from tracks + from db.tracks group {album_id} ( window (derive {grp = milliseconds - (row_number this)}) ) @@ -1680,9 +1715,10 @@ fn test_window_functions_13() { } #[test] +#[ignore] fn test_window_single_item_range() { assert_snapshot!(compile(r###" - from login_event + from db.login_event window rows:1..1 ( sort time_upload derive { @@ -1704,9 +1740,10 @@ fn test_window_single_item_range() { } #[test] +#[ignore] // change behavior: forbid references to fields of current tuple fn test_name_resolving() { let query = r###" - from numbers + from db.numbers derive x = 5 select {y = 6, z = x + y + a} "###; @@ -1720,9 +1757,10 @@ fn test_name_resolving() { } #[test] +#[ignore] // change behavior: forbid references to fields of current tuple fn test_strings() { let query = r#" - from empty_table_to_do + from db.empty_table_to_do select { x = "two households'", y = 'two households"', @@ -1757,14 +1795,14 @@ fn test_strings() { fn test_filter() { // https://github.com/PRQL/prql/issues/469 let query = r###" - from employees + from db.employees filter {age > 25, age < 40} "###; assert!(compile(query).is_err()); assert_snapshot!((compile(r###" - from employees + from db.employees filter age > 25 && age < 40 "###).unwrap()), @r###" SELECT @@ -1777,7 +1815,7 @@ fn test_filter() { "###); assert_snapshot!((compile(r###" - from employees + from db.employees filter age > 25 filter age < 40 "###).unwrap()), @r###" @@ -1794,7 +1832,7 @@ fn test_filter() { #[test] fn test_nulls_01() { assert_snapshot!((compile(r###" - from employees + from db.employees select amount = null "###).unwrap()), @r###" SELECT @@ -1808,7 +1846,7 @@ fn test_nulls_01() { fn test_nulls_02() { // coalesce assert_snapshot!((compile(r###" - from employees + from db.employees derive amount = amount + 2 ?? 3 * 5 "###).unwrap()), @r###" SELECT @@ -1823,7 +1861,7 @@ fn test_nulls_02() { fn test_nulls_03() { // IS NULL assert_snapshot!((compile(r###" - from employees + from db.employees filter first_name == null && null == last_name "###).unwrap()), @r###" SELECT @@ -1840,7 +1878,7 @@ fn test_nulls_03() { fn test_nulls_04() { // IS NOT NULL assert_snapshot!((compile(r###" - from employees + from db.employees filter first_name != null && null != last_name "###).unwrap()), @r###" SELECT @@ -1856,7 +1894,7 @@ fn test_nulls_04() { #[test] fn test_take_01() { assert_snapshot!((compile(r###" - from employees + from db.employees take ..10 "###).unwrap()), @r###" SELECT @@ -1871,7 +1909,7 @@ fn test_take_01() { #[test] fn test_take_02() { assert_snapshot!((compile(r###" - from employees + from db.employees take 5..10 "###).unwrap()), @r###" SELECT @@ -1886,7 +1924,7 @@ fn test_take_02() { #[test] fn test_take_03() { assert_snapshot!((compile(r###" - from employees + from db.employees take 5.. "###).unwrap()), @r###" SELECT @@ -1899,7 +1937,7 @@ fn test_take_03() { #[test] fn test_take_04() { assert_snapshot!((compile(r###" - from employees + from db.employees take 5..5 "###).unwrap()), @r###" SELECT @@ -1915,7 +1953,7 @@ fn test_take_04() { fn test_take_05() { // should be one SELECT assert_snapshot!((compile(r###" - from employees + from db.employees take 11..20 take 1..5 "###).unwrap()), @r###" @@ -1932,7 +1970,7 @@ fn test_take_05() { fn test_take_06() { // should be two SELECTs assert_snapshot!((compile(r###" - from employees + from db.employees take 11..20 sort name take 1..5 @@ -1959,7 +1997,7 @@ fn test_take_06() { #[test] fn test_take_07() { assert_snapshot!((compile(r###" - from employees + from db.employees take 0..1 "###).unwrap_err()), @r###" Error: @@ -1967,7 +2005,7 @@ fn test_take_07() { │ 3 │ take 0..1 │ ────┬──── - │ ╰────── take expected a positive int range, but found 0..1 + │ ╰────── take expected a positive int range ───╯ "###); } @@ -1975,7 +2013,7 @@ fn test_take_07() { #[test] fn test_take_08() { assert_snapshot!((compile(r###" - from employees + from db.employees take (-1..) "###).unwrap_err()), @r###" Error: @@ -1983,7 +2021,7 @@ fn test_take_08() { │ 3 │ take (-1..) │ ─────┬───── - │ ╰─────── take expected a positive int range, but found -1.. + │ ╰─────── take expected a positive int range ───╯ "###); } @@ -1991,7 +2029,7 @@ fn test_take_08() { #[test] fn test_take_09() { assert_snapshot!((compile(r###" - from employees + from db.employees select a take 5..5.6 "###).unwrap_err()), @r###" @@ -2000,7 +2038,7 @@ fn test_take_09() { │ 4 │ take 5..5.6 │ ─────┬───── - │ ╰─────── take expected a positive int range, but found 5..? + │ ╰─────── take expected a positive int range ───╯ "###); } @@ -2008,15 +2046,15 @@ fn test_take_09() { #[test] fn test_take_10() { assert_snapshot!((compile(r###" - from employees + from db.employees take (-1) "###).unwrap_err()), @r###" Error: - ╭─[:3:5] + ╭─[:3:11] │ 3 │ take (-1) - │ ────┬──── - │ ╰────── take expected a positive int range, but found ..-1 + │ ─┬ + │ ╰── `take` expected int or range, but found `(std.neg ...)` ───╯ "###); } @@ -2026,7 +2064,7 @@ fn test_take_mssql() { assert_snapshot!((compile(r#" prql target:sql.mssql - from tracks + from db.tracks take 3..5 "#).unwrap()), @r###" SELECT @@ -2045,7 +2083,7 @@ fn test_take_mssql() { assert_snapshot!((compile(r#" prql target:sql.mssql - from tracks + from db.tracks take ..5 "#).unwrap()), @r###" SELECT @@ -2064,7 +2102,7 @@ fn test_take_mssql() { assert_snapshot!((compile(r#" prql target:sql.mssql - from tracks + from db.tracks take 3.. "#).unwrap()), @r###" SELECT @@ -2075,10 +2113,11 @@ fn test_take_mssql() { } #[test] +#[ignore] fn test_distinct_01() { // window functions cannot materialize into where statement: CTE is needed assert_snapshot!((compile(r###" - from employees + from db.employees derive {rn = row_number id} filter rn > 2 "###).unwrap()), @r###" @@ -2099,10 +2138,11 @@ fn test_distinct_01() { } #[test] +#[ignore] fn test_distinct_02() { // basic distinct assert_snapshot!((compile(r###" - from employees + from db.employees select first_name group first_name (take 1) "###).unwrap()), @r###" @@ -2114,10 +2154,11 @@ fn test_distinct_02() { } #[test] +#[ignore] fn test_distinct_03() { // distinct on two columns assert_snapshot!((compile(r###" - from employees + from db.employees select {first_name, last_name} group {first_name, last_name} (take 1) "###).unwrap()), @r###" @@ -2129,11 +2170,12 @@ fn test_distinct_03() { "###); } #[test] +#[ignore] fn test_distinct_04() { // We want distinct only over first_name and last_name, so we can't use a // `DISTINCT *` here. assert_snapshot!((compile(r###" - from employees + from db.employees group {first_name, last_name} (take 1) "###).unwrap()), @r###" WITH table_0 AS ( @@ -2152,19 +2194,21 @@ fn test_distinct_04() { "###); } #[test] +#[ignore] fn test_distinct_05() { // Check that a different order doesn't stop distinct from being used. assert!(compile( - "from employees | select {first_name, last_name} | group {last_name, first_name} (take 1)" + "from db.employees | select {first_name, last_name} | group {last_name, first_name} (take 1)" ) .unwrap() .contains("DISTINCT")); } #[test] +#[ignore] fn test_distinct_06() { // head assert_snapshot!((compile(r###" - from employees + from db.employees group department (take 3) "###).unwrap()), @r###" WITH table_0 AS ( @@ -2183,9 +2227,10 @@ fn test_distinct_06() { "###); } #[test] +#[ignore] fn test_distinct_07() { assert_snapshot!((compile(r###" - from employees + from db.employees group department (sort salary | take 2..3) "###).unwrap()), @r###" WITH table_0 AS ( @@ -2208,9 +2253,10 @@ fn test_distinct_07() { "###); } #[test] +#[ignore] fn test_distinct_08() { assert_snapshot!((compile(r###" - from employees + from db.employees group department (sort salary | take 4..4) "###).unwrap()), @r###" WITH table_0 AS ( @@ -2234,9 +2280,10 @@ fn test_distinct_08() { } #[test] +#[ignore] fn test_distinct_09() { assert_snapshot!(compile(" - from invoices + from db.invoices select {billing_country, billing_city} group {billing_city} ( take 1 @@ -2264,11 +2311,12 @@ fn test_distinct_09() { } #[test] +#[ignore] fn test_distinct_on_01() { assert_snapshot!((compile(r###" prql target:sql.postgres - from employees + from db.employees group department ( sort age take 1 @@ -2285,11 +2333,12 @@ fn test_distinct_on_01() { } #[test] +#[ignore] fn test_distinct_on_02() { assert_snapshot!((compile(r###" prql target:sql.duckdb - from x + from db.x select {class, begins} group {begins} (take 1) "###).unwrap()), @r###" @@ -2302,11 +2351,12 @@ fn test_distinct_on_02() { } #[test] +#[ignore] fn test_distinct_on_03() { assert_snapshot!((compile(r###" prql target:sql.duckdb - from tab1 + from db.tab1 group col1 ( take 1 ) @@ -2327,12 +2377,13 @@ fn test_distinct_on_03() { } #[test] +#[ignore] fn test_distinct_on_04() { assert_snapshot!((compile(r###" prql target:sql.duckdb - from a - join b (b.a_id == a.id) + from db.a + join db.b (b.a_id == a.id) group {a.id} ( sort b.x take 1 @@ -2352,11 +2403,12 @@ fn test_distinct_on_04() { } #[test] +#[ignore] fn test_group_take_n_01() { assert_snapshot!((compile(r###" prql target:sql.postgres - from employees + from db.employees group department ( sort age take 2 @@ -2383,11 +2435,12 @@ fn test_group_take_n_01() { } #[test] +#[ignore] fn test_group_take_n_02() { assert_snapshot!((compile(r###" prql target:sql.postgres - from employees + from db.employees group department ( sort age take 2.. @@ -2416,8 +2469,8 @@ fn test_group_take_n_02() { #[test] fn test_join() { assert_snapshot!((compile(r###" - from x - join y (==id) + from db.x + join db.y (==id) "###).unwrap()), @r###" SELECT x.*, @@ -2427,16 +2480,17 @@ fn test_join() { JOIN y ON x.id = y.id "###); - compile("from x | join y {==x.id}").unwrap_err(); + compile("from x | join db.y {==x.id}").unwrap_err(); } #[test] +#[ignore] // TODO: join side fn test_join_side_literal() { assert_snapshot!((compile(r###" let my_side = "right" - from x - join y (==id) side:my_side + from db.x + join db.y (==id) side:my_side "###).unwrap()), @r###" SELECT x.*, @@ -2448,32 +2502,34 @@ fn test_join_side_literal() { } #[test] +#[ignore] // TODO: join side fn test_join_side_literal_err() { assert_snapshot!((compile(r###" let my_side = 42 - from x - join y (==id) side:my_side + from db.x + join db.y (==id) side:my_side "###).unwrap_err()), @r###" Error: - ╭─[:5:24] + ╭─[:5:27] │ - 5 │ join y (==id) side:my_side - │ ───┬─── - │ ╰───── `side` expected inner, left, right or full, but found 42 + 5 │ join db.y (==id) side:my_side + │ ───┬─── + │ ╰───── `side` expected inner, left, right or full, but found 42 ───╯ "###); } #[test] +#[ignore] fn test_join_side_literal_via_func() { assert_snapshot!((compile(r###" let my_join = func m c s :"right" tbl -> ( join side:_param.s m (c == that.k) tbl ) - from x - my_join default_db.y this.id s:"left" + from db.x + my_join db.y this.id s:"left" "###).unwrap()), @r###" SELECT x.*, @@ -2485,14 +2541,15 @@ fn test_join_side_literal_via_func() { } #[test] +#[ignore] fn test_join_side_literal_via_func_err() { assert_snapshot!((compile(r###" let my_join = func m c s :"right" tbl -> ( - join side:_param.s m (c == that.k) tbl + join side:s m (c == that.k) tbl ) - from x - my_join default_db.y this.id s:"four" + from db.x + my_join db.y this.id s:"four" "###).unwrap_err()), @r###" Error: ╭─[:3:25] @@ -2505,18 +2562,20 @@ fn test_join_side_literal_via_func_err() { } #[test] +#[ignore] fn test_from_json() { // Test that the SQL generated from the JSON of the PRQL is the same as the raw PRQL let original_prql = r#" - from e=employees - join salaries (==emp_no) + from db.employees + select {e = this} + join db.salaries (==emp_no) group {e.emp_no, e.gender} ( aggregate { emp_salary = average salaries.salary } ) - join de=dept_emp (==emp_no) - join dm=dept_manager ( + join (db.dept_emp | select {de = this}) (==emp_no) + join (db.dept_manager | select {dm = this}) ( (dm.dept_no == de.dept_no) && s"(de.from_date, de.to_date) OVERLAPS (dm.from_date, dm.to_date)" ) group {dm.emp_no, gender} ( @@ -2526,7 +2585,7 @@ fn test_from_json() { } ) derive mng_no = emp_no - join managers=employees (==emp_no) + join (db.employees | select {managers = this}) (==emp_no) derive mng_name = s"managers.first_name || ' ' || managers.last_name" select {mng_name, managers.gender, salary_avg, salary_sd} "#; @@ -2551,7 +2610,7 @@ fn test_from_json() { #[test] fn test_f_string() { let query = r#" - from employees + from db.employees derive age = year_born - s'now()' select { f"Hello my name is {prefix}{first_name} {last_name}", @@ -2595,20 +2654,17 @@ fn test_f_string() { } #[test] -fn test_sql_of_ast_1() { - let query = r#" - from employees +#[ignore] +fn test_sql_of_ast_01() { + assert_snapshot!(compile(r#" + from db.employees filter country == "USA" group {title, country} ( aggregate {average salary} ) sort title take 20 - "#; - - let sql = compile(query).unwrap(); - assert_snapshot!(sql, - @r###" + "#).unwrap(), @r###" SELECT title, country, @@ -2631,7 +2687,7 @@ fn test_sql_of_ast_1() { #[test] fn test_sql_of_ast_02() { assert_snapshot!(compile(r#" - from employees + from db.employees aggregate sum_salary = s"sum({salary})" filter sum_salary > 100 "#).unwrap(), @r###" @@ -2645,6 +2701,7 @@ fn test_sql_of_ast_02() { } #[test] +#[ignore] fn test_bare_s_string() { let query = r#" let grouping = s""" @@ -2654,7 +2711,7 @@ fn test_bare_s_string() { GROUPING SETS ((b, c, d), (d), (b, d)) """ - from grouping + from module.grouping "#; let sql = compile(query).unwrap(); @@ -2677,13 +2734,13 @@ fn test_bare_s_string() { } #[test] +#[ignore] fn test_bare_s_string_01() { // Test that case insensitive SELECT is accepted. We allow it as it is valid SQL. assert_snapshot!(compile(r#" let a = s"select insensitive from rude" - from a - "#).unwrap(), - @r###" + from module.a + "#).unwrap(), @r###" WITH table_0 AS ( SELECT insensitive @@ -2699,11 +2756,13 @@ fn test_bare_s_string_01() { } #[test] +#[ignore] fn test_bare_s_string_02() { // Check a mixture of cases for good measure. assert_snapshot!(compile(r#" + assert_snapshot!(compile(r#" let a = s"sElEcT insensitive from rude" - from a + module.a "#).unwrap(), @r###" WITH table_0 AS ( @@ -2721,6 +2780,7 @@ fn test_bare_s_string_02() { } #[test] +#[ignore] fn test_bare_s_string_03() { // Check SELECT\n. assert_snapshot!(compile(r#" @@ -2746,6 +2806,7 @@ fn test_bare_s_string_03() { } #[test] +#[ignore] fn test_bare_s_string_04() { assert_snapshot!(compile(r#" s"SELECTfoo" @@ -2756,11 +2817,12 @@ fn test_bare_s_string_04() { } #[test] -// Confirm that a regular expr_call in a table definition works as expected. fn test_table_definition_with_expr_call() { + // Confirm that a regular expr_call in a table definition works as expected. let query = r###" - let e = take 4 (from employees) - from e + let e = take 4 (from db.employees) + + module.e "###; let sql = compile(query).unwrap(); @@ -2783,9 +2845,10 @@ fn test_table_definition_with_expr_call() { } #[test] +#[ignore] fn test_prql_to_sql_1() { assert_snapshot!(compile(r#" - from employees + from db.employees aggregate { count salary, sum salary, @@ -2800,7 +2863,7 @@ fn test_prql_to_sql_1() { ); assert_snapshot!(compile(r#" prql target:sql.postgres - from developers + from db.developers group team ( aggregate { skill_width = count_distinct specialty, @@ -2822,7 +2885,7 @@ fn test_prql_to_sql_1() { #[ignore] fn test_prql_to_sql_2() { let query = r#" -from employees +from db.employees filter country == "USA" # Each line transforms the previous result. derive { # This adds columns / variables. gross_salary = salary + payroll_tax, @@ -2886,24 +2949,25 @@ take 20 } #[test] +#[ignore] fn test_prql_to_sql_table() { // table let query = r#" let newest_employees = ( - from employees + from db.employees sort tenure take 50 ) let average_salaries = ( - from salaries + from db.salaries group country ( aggregate { average_country_salary = average salary } ) ) - from newest_employees - join average_salaries (==country) + module.newest_employees + join module.average_salaries (==country) select {name, salary, average_country_salary} "#; let sql = compile(query).unwrap(); @@ -2941,10 +3005,11 @@ fn test_prql_to_sql_table() { } #[test] +#[ignore] fn test_nonatomic() { // A take, then two aggregates let query = r#" - from employees + from db.employees take 20 filter country == "USA" group {title, country} ( @@ -2998,7 +3063,7 @@ fn test_nonatomic() { // A aggregate, then sort and filter let query = r###" - from employees + from db.employees group {title, country} ( aggregate { sum_gross_cost = average salary @@ -3027,16 +3092,17 @@ fn test_nonatomic() { #[test] /// Confirm a nonatomic table works. +#[ignore] fn test_nonatomic_table() { // A take, then two aggregates let query = r#" let a = ( - from employees + from db.employees take 50 group country (aggregate {s"count(*)"}) ) - from a - join b (==country) + module.a + join db.b (==country) select {name, salary, average_country_salary} "#; @@ -3068,48 +3134,61 @@ fn test_nonatomic_table() { } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_table_names_between_splits_01() { assert_snapshot!(compile(r###" - from employees - join d = department (==dept_no) + from db.employees + join (d = db.department) (==dept_no) take 10 derive emp_no = employees.emp_no - join s = salaries (==emp_no) + join s = db.salaries (==emp_no) select {employees.emp_no, d.name, s.salary} "###).unwrap(), @r###" WITH table_0 AS ( + SELECT + * + FROM + department + ), + table_2 AS ( SELECT employees.emp_no, - d.name + table_0.name FROM employees - JOIN department AS d ON employees.dept_no = d.dept_no + JOIN table_0 ON employees.dept_no = table_0.dept_no LIMIT 10 + ), table_1 AS ( + SELECT + * + FROM + salaries ) SELECT - table_0.emp_no, - table_0.name, - s.salary + table_2.emp_no, + table_2.name, + table_1.salary FROM - table_0 - JOIN salaries AS s ON table_0.emp_no = s.emp_no + table_2 + JOIN table_1 ON table_2.emp_no = table_1.emp_no "###); } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_table_names_between_splits_02() { assert_snapshot!(compile(r###" - from e = employees + from e = db.employees take 10 - join salaries (==emp_no) + join db.salaries (==emp_no) select {e.*, salaries.salary} "###).unwrap(), @r###" WITH table_0 AS ( SELECT * FROM - employees AS e + employees LIMIT 10 ) @@ -3123,10 +3202,12 @@ fn test_table_names_between_splits_02() { } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_table_alias_01() { assert_snapshot!((compile(r###" - from e = employees - join salaries side:left (salaries.emp_no == e.emp_no) + from db.employees + select {e = this} + join db.salaries side:left (salaries.emp_no == e.emp_no) group {e.emp_no} ( aggregate { emp_salary = average salaries.salary @@ -3135,27 +3216,29 @@ fn test_table_alias_01() { select {emp_no, emp_salary} "###).unwrap()), @r###" SELECT - e.emp_no, + employees.emp_no, AVG(salaries.salary) AS emp_salary FROM - employees AS e - LEFT JOIN salaries ON salaries.emp_no = e.emp_no + employees + LEFT JOIN salaries ON salaries.emp_no = employees.emp_no GROUP BY - e.emp_no + employees.emp_no "###); } #[test] +#[ignore] // change behavior: `x.a` will infer name `a` only fn test_table_alias_02() { assert_snapshot!((compile(r#" - from e = employees + from db.employees + select {e = this} select e.first_name filter e.first_name == "Fred" "#).unwrap()), @r###" SELECT first_name FROM - employees AS e + employees WHERE first_name = 'Fred' "###); @@ -3166,7 +3249,7 @@ fn test_targets() { // Generic let query = r###" prql target:sql.generic - from Employees + from db.Employees select {FirstName, `last name`} take 3 "###; @@ -3184,7 +3267,7 @@ fn test_targets() { // SQL server let query = r###" prql target:sql.mssql - from Employees + from db.Employees select {FirstName, `last name`} take 3 "###; @@ -3207,7 +3290,7 @@ fn test_targets() { // MySQL let query = r###" prql target:sql.mysql - from Employees + from db.Employees select {FirstName, `last name`} take 3 "###; @@ -3228,7 +3311,7 @@ fn test_target_clickhouse() { let query = r###" prql target:sql.clickhouse - from github_json + from db.github_json derive {event_type_dotted = `event.type`} "###; @@ -3245,7 +3328,7 @@ fn test_target_clickhouse() { fn test_ident_escaping() { // Generic let query = r#" - from `anim"ls` + from db.`anim"ls` derive {`čebela` = BeeName, medved = `bear's_name`} "#; @@ -3262,7 +3345,7 @@ fn test_ident_escaping() { let query = r#" prql target:sql.mysql - from `anim"ls` + from db.`anim"ls` derive {`čebela` = BeeName, medved = `bear's_name`} "#; @@ -3279,7 +3362,7 @@ fn test_ident_escaping() { #[test] fn test_literal() { let query = r###" - from employees + from db.employees derive {always_true = true} "###; @@ -3300,17 +3383,17 @@ fn test_same_column_names() { // #820 let query = r###" let x = ( -from x_table +from db.x_table select only_in_x = foo ) let y = ( -from y_table +from db.y_table select foo ) -from x -join y (foo == only_in_x) +module.x +join module.y (foo == only_in_x) "###; assert_snapshot!(compile(query).unwrap(), @@ -3338,11 +3421,12 @@ join y (foo == only_in_x) } #[test] +#[ignore] fn test_double_aggregate() { // #941 compile( r###" - from numbers + from db.numbers group {type} ( aggregate { total_amt = sum amount, @@ -3356,7 +3440,7 @@ fn test_double_aggregate() { .unwrap_err(); assert_snapshot!(compile(r###" - from numbers + from db.numbers group {`type`} ( aggregate { total_amt = sum amount, @@ -3378,10 +3462,11 @@ fn test_double_aggregate() { } #[test] +#[ignore] fn test_window_function_coalesce() { // #3587 assert_snapshot!(compile(r###" - from x + from db.x select {a, b=a} window ( select { @@ -3401,9 +3486,10 @@ fn test_window_function_coalesce() { } #[test] +#[ignore] fn test_casting() { assert_snapshot!(compile(r###" - from x + from db.x select {a} derive { b = (a | as int) + 10, @@ -3431,14 +3517,14 @@ fn test_toposort() { assert_snapshot!(compile(r###" let b = ( - from somesource + from db.somesource ) let a = ( - from b + project.b ) - from b + project.b "###).unwrap(), @r###" WITH b AS ( @@ -3456,13 +3542,14 @@ fn test_toposort() { } #[test] +#[ignore] fn test_inline_tables() { assert_snapshot!(compile(r###" ( - from employees + from db.employees select {emp_id, name, surname, `type`, amount} ) - join s = (from salaries | select {emp_id, salary}) (==emp_id) + join (db.salaries | select {emp_id, salary}) (==emp_id) "###).unwrap(), @r###" WITH table_0 AS ( @@ -3488,11 +3575,12 @@ fn test_inline_tables() { } #[test] +#[ignore] fn test_filter_and_select_unchanged_alias() { // #1185 assert_snapshot!(compile(r###" - from account + from db.account filter account.name != null select {name = account.name} "###).unwrap(), @@ -3508,10 +3596,11 @@ fn test_filter_and_select_unchanged_alias() { } #[test] +#[ignore] fn test_filter_and_select_changed_alias() { // #1185 assert_snapshot!(compile(r###" - from account + from db.account filter account.name != null select {renamed_name = account.name} "###).unwrap(), @@ -3527,7 +3616,7 @@ fn test_filter_and_select_changed_alias() { // #1207 assert_snapshot!(compile(r#" - from x + from db.x filter name != "Bob" select name = name ?? "Default" "#).unwrap(), @@ -3546,7 +3635,7 @@ fn test_filter_and_select_changed_alias() { fn test_unused_alias() { // #1308 assert_snapshot!(compile(r###" - from account + from db.account select n = {account.name} "###).unwrap_err(), @r###" Error: @@ -3556,7 +3645,7 @@ fn test_unused_alias() { │ ───────┬────── │ ╰──────── unexpected assign to `n` │ - │ Help: move assign into the tuple: `[n = ...]` + │ Help: move assign into the tuple: `{n = ...}` ───╯ "###) } @@ -3564,7 +3653,7 @@ fn test_unused_alias() { #[test] fn test_table_s_string_01() { assert_snapshot!(compile(r#" - let main = s"SELECT DISTINCT ON first_name, age FROM employees ORDER BY age ASC" + let main <[{first_name = text, age = int}]> = s"SELECT DISTINCT ON first_name, age FROM employees ORDER BY age ASC" "#).unwrap(), @r###" WITH table_0 AS ( @@ -3577,19 +3666,21 @@ fn test_table_s_string_01() { age ASC ) SELECT - * + first_name, + age FROM table_0 "### ); } #[test] +#[ignore] fn test_table_s_string_02() { assert_snapshot!(compile(r#" s""" SELECT DISTINCT ON first_name, id, age FROM employees ORDER BY age ASC """ - join s = s"SELECT * FROM salaries" (==id) + join s"SELECT * FROM salaries" (==id) "#).unwrap(), @r###" WITH table_0 AS ( @@ -3618,6 +3709,7 @@ fn test_table_s_string_02() { ); } #[test] +#[ignore] fn test_table_s_string_03() { assert_snapshot!(compile(r#" s"""SELECT * FROM employees""" @@ -3640,6 +3732,7 @@ fn test_table_s_string_03() { ); } #[test] +#[ignore] fn test_table_s_string_04() { assert_snapshot!(compile(r#" s"""SELECT * FROM employees""" @@ -3663,12 +3756,13 @@ fn test_table_s_string_04() { ); } #[test] +#[ignore] fn test_table_s_string_05() { assert_snapshot!(compile(r#" let weeks_between = start end -> s"SELECT generate_series({start}, {end}, '1 week') as date" let current_week = -> s"date(date_trunc('week', current_date))" - weeks_between @2022-06-03 (current_week + 4) + module.weeks_between @2022-06-03 (module.current_week + 4) "#).unwrap(), @r###" WITH table_0 AS ( @@ -3687,9 +3781,10 @@ fn test_table_s_string_05() { ); } #[test] +#[ignore] fn test_table_s_string_06() { assert_snapshot!(compile(r#" - s"SELECT * FROM {default_db.x}" + s"SELECT * FROM {db.x}" "#).unwrap(), @r###" WITH table_0 AS ( @@ -3707,14 +3802,15 @@ fn test_table_s_string_06() { } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_direct_table_references() { assert_snapshot!(compile( r#" - from x + from db.x select s"{x}.field" "#, ) - .unwrap_err(), @r###" + .unwrap(), @r###" Error: ╭─[:3:15] │ @@ -3728,7 +3824,7 @@ fn test_direct_table_references() { assert_snapshot!(compile( r###" - from x + from db.x select x "###, ) @@ -3741,10 +3837,11 @@ fn test_direct_table_references() { } #[test] +#[ignore] // change behavior: forbid references to fields of current tuple fn test_name_shadowing() { assert_snapshot!(compile( r###" - from x + from db.x select {a, a, a = a + 1} "###).unwrap(), @r###" @@ -3759,7 +3856,7 @@ fn test_name_shadowing() { assert_snapshot!(compile( r###" - from x + from db.x select a derive a derive a = a + 1 @@ -3778,12 +3875,14 @@ fn test_name_shadowing() { } #[test] +#[ignore] fn test_group_all() { assert_snapshot!(compile( r###" prql target:sql.sqlite - from a=albums + from db.albums + select {a = this} group a.* (aggregate {count this}) "###).unwrap_err(), @r###" Error: Target dialect does not support * in this position. @@ -3791,7 +3890,8 @@ fn test_group_all() { assert_snapshot!(compile( r###" - from e=albums + from db.albums + select {e = this} group !{genre_id} (aggregate {count this}) "###).unwrap_err(), @r###" Error: Excluding columns not supported as this position @@ -3803,7 +3903,7 @@ fn test_output_column_deduplication() { // #1249 assert_snapshot!(compile( r#" - from report + from db.report derive r = s"RANK() OVER ()" filter r == 1 "#).unwrap(), @@ -3829,7 +3929,7 @@ fn test_output_column_deduplication() { fn test_case_01() { assert_snapshot!(compile( r###" - from employees + from db.employees derive display_name = case [ nickname != null => nickname, true => f'{first_name} {last_name}' @@ -3852,7 +3952,7 @@ fn test_case_01() { fn test_case_02() { assert_snapshot!(compile( r###" - from employees + from db.employees derive display_name = case [ nickname != null => nickname, first_name != null => f'{first_name} {last_name}' @@ -3873,10 +3973,11 @@ fn test_case_02() { } #[test] +#[ignore] fn test_case_03() { assert_snapshot!(compile( r###" - from tracks + from db.tracks select category = case [ length > avg_length => 'long' ] @@ -3908,23 +4009,24 @@ fn test_case_03() { #[test] fn test_sql_options() { let options = Options::default(); - let sql = prqlc::compile("from x", &options).unwrap(); + let sql = prqlc::compile("from db.x", &options).unwrap(); assert!(sql.contains('\n')); assert!(sql.contains("-- Generated by")); let options = Options::default().no_signature().no_format(); - let sql = prqlc::compile("from x", &options).unwrap(); + let sql = prqlc::compile("from db.x", &options).unwrap(); assert!(!sql.contains('\n')); assert!(!sql.contains("-- Generated by")); } #[test] +#[ignore] fn test_static_analysis() { assert_snapshot!(compile( r###" - from x + from db.x select { a = (- (-3)), b = !(!(!(!(!(true))))), @@ -3956,19 +4058,20 @@ fn test_static_analysis() { } #[test] +#[ignore] fn test_closures_and_pipelines() { assert_snapshot!(compile( - r#" + r#" let addthree = a b c -> s"{a} || {b} || {c}" let arg = myarg myfunc -> ( myfunc myarg ) - from y + from db.y select x = ( - addthree "apples" - arg "bananas" - arg "citrus" + module.addthree "apples" + module.arg "bananas" + module.arg "citrus" ) - "#).unwrap(), + "#).unwrap(), @r###" SELECT 'apples' || 'bananas' || 'citrus' AS x @@ -3981,7 +4084,7 @@ fn test_closures_and_pipelines() { #[test] fn test_basic_agg() { assert_snapshot!(compile(r#" - from employees + from db.employees aggregate { count salary, count this, @@ -4000,7 +4103,7 @@ fn test_basic_agg() { #[test] fn test_exclude_columns_01() { assert_snapshot!(compile(r#" - from tracks + from db.tracks select {track_id, title, composer, bytes} select !{title, composer} "#).unwrap(), @@ -4015,9 +4118,10 @@ fn test_exclude_columns_01() { } #[test] +#[ignore] fn test_exclude_columns_02() { assert_snapshot!(compile(r#" - from tracks + from db.tracks select {track_id, title, composer, bytes} group !{title, composer} (aggregate {count this}) "#).unwrap(), @@ -4036,9 +4140,10 @@ fn test_exclude_columns_02() { } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_exclude_columns_03() { assert_snapshot!(compile(r#" - from artists + from db.artists derive nick = name select !{artists.*} "#).unwrap(), @@ -4055,7 +4160,7 @@ fn test_exclude_columns_03() { fn test_exclude_columns_04() { assert_snapshot!(compile(r#" prql target:sql.bigquery - from tracks + from db.tracks select !{milliseconds,bytes} "#).unwrap(), @r###" @@ -4073,7 +4178,7 @@ fn test_exclude_columns_04() { fn test_exclude_columns_05() { assert_snapshot!(compile(r#" prql target:sql.snowflake - from tracks + from db.tracks select !{milliseconds,bytes} "#).unwrap(), @r###" @@ -4089,7 +4194,7 @@ fn test_exclude_columns_05() { fn test_exclude_columns_06() { assert_snapshot!(compile(r#" prql target:sql.duckdb - from tracks + from db.tracks select !{milliseconds,bytes} "#).unwrap(), @r###" @@ -4102,10 +4207,11 @@ fn test_exclude_columns_06() { } #[test] +#[ignore] fn test_exclude_columns_07() { assert_snapshot!(compile(r#" prql target:sql.duckdb - from s"SELECT * FROM foo" + s"SELECT * FROM foo" select !{bar} "#).unwrap(), @r###" @@ -4124,6 +4230,7 @@ fn test_exclude_columns_07() { } #[test] +#[ignore] fn test_custom_transforms() { assert_snapshot!(compile(r#" let my_transform = ( @@ -4131,8 +4238,8 @@ fn test_custom_transforms() { sort name ) - from tab - my_transform + from db.tab + module.my_transform take 3 "#).unwrap(), @r###" @@ -4150,11 +4257,11 @@ fn test_custom_transforms() { } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_name_inference() { assert_snapshot!(compile(r#" - from albums + from db.albums select {artist_id + album_id} - # nothing inferred infer "#).unwrap(), @r###" SELECT @@ -4166,7 +4273,7 @@ fn test_name_inference() { let sql1 = compile( r#" - from albums + from db.albums select {artist_id} select {albums.artist_id} "#, @@ -4174,7 +4281,7 @@ fn test_name_inference() { .unwrap(); let sql2 = compile( r#" - from albums + from db.albums select {albums.artist_id} select {albums.artist_id} "#, @@ -4194,9 +4301,10 @@ fn test_name_inference() { } #[test] +#[ignore] fn test_from_text_01() { assert_snapshot!(compile(r#" - from_text format:csv """ + std.from_text format:csv """ a,b,c 1,2,3 4,5,6 @@ -4226,9 +4334,10 @@ a,b,c } #[test] +#[ignore] fn test_from_text_02() { assert_snapshot!(compile(r#" - from_text format:json ''' + std.from_text format:json ''' [{"a": 1, "b": "x", "c": false }, {"a": 4, "b": "y", "c": null }] ''' select {b, c} @@ -4256,6 +4365,7 @@ fn test_from_text_02() { } #[test] +#[ignore] fn test_from_text_03() { assert_snapshot!(compile(r#" std.from_text format:json '''{ @@ -4290,6 +4400,7 @@ fn test_from_text_03() { } #[test] +#[ignore] fn test_from_text_04() { assert_snapshot!(compile(r#" std.from_text 'a,b' @@ -4312,6 +4423,7 @@ fn test_from_text_04() { } #[test] +#[ignore] fn test_from_text_05() { assert_snapshot!(compile(r#" std.from_text format:json '''{"columns": ["a", "b", "c"], "data": []}''' @@ -4336,6 +4448,7 @@ fn test_from_text_05() { } #[test] +#[ignore] fn test_from_text_06() { assert_snapshot!(compile(r#" std.from_text '' @@ -4356,6 +4469,7 @@ fn test_from_text_06() { } #[test] +#[ignore] fn test_from_text_07() { assert_snapshot!(compile(r#" std.from_text format:json '''{"columns": [], "data": [[], []]}''' @@ -4388,7 +4502,7 @@ fn test_header() { assert_snapshot!(compile(format!(r#" {header} - from a + from db.a take 5 "#).as_str()).unwrap(),@r###" SELECT @@ -4408,21 +4522,21 @@ fn test_header() { fn test_header_target_error() { assert_snapshot!(compile(r#" prql target:foo - from a + from db.a "#).unwrap_err(),@r###" Error: target `"foo"` not found "###); assert_snapshot!(compile(r#" prql target:sql.foo - from a + from db.a "#).unwrap_err(),@r###" Error: target `"sql.foo"` not found "###); assert_snapshot!(compile(r#" prql target:foo.bar - from a + from db.a "#).unwrap_err(),@r###"Error: target `"foo.bar"` not found"###); // TODO: Can we use the span of: @@ -4430,7 +4544,7 @@ fn test_header_target_error() { // - At least not the first empty line? assert_snapshot!(compile(r#" prql dialect:foo.bar - from a + from db.a "#).unwrap_err(),@r###" Error: ╭─[:1:1] @@ -4460,7 +4574,7 @@ fn shortest_prql_version() { "###); assert_snapshot!(compile(r#" - from x + from db.x derive y = std.prql.version "#).unwrap(),@r###" SELECT @@ -4473,6 +4587,7 @@ fn shortest_prql_version() { } #[test] +#[ignore] fn test_loop() { assert_snapshot!(compile(r#" [{n = 1}] @@ -4519,12 +4634,13 @@ fn test_loop() { } #[test] +#[ignore] fn test_loop_2() { assert_snapshot!(compile(r#" - read_csv 'employees.csv' + std.read_csv 'employees.csv' filter last_name=="Mitchell" loop ( - join manager=employees (manager.employee_id==this.reports_to) + join (db.employees | select {manager = this}) (manager.employee_id==this.reports_to) select manager.* ) "#).unwrap(), @@ -4535,7 +4651,7 @@ fn test_loop_2() { FROM read_csv('employees.csv') ), - table_1 AS ( + table_2 AS ( SELECT * FROM @@ -4545,23 +4661,29 @@ fn test_loop_2() { UNION ALL SELECT - manager.* + table_4.* FROM - table_1 - JOIN employees AS manager ON manager.employee_id = table_1.reports_to + table_2 + JOIN ( + SELECT + * + FROM + employees + ) AS table_4 ON table_4.employee_id = table_2.reports_to ) SELECT * FROM - table_1 AS table_2 + table_2 AS table_3 "### ); } #[test] +#[ignore] // change behavior: `x.a` will infer name `a` only fn test_params() { assert_snapshot!(compile(r#" - from invoices + from db.invoices select {i = this} filter $1 <= i.date || i.date <= $2 select { @@ -4589,21 +4711,17 @@ fn test_params() { // for #1969 #[test] fn test_datetime() { - let query = &r#" - from test_table - select {date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800} - "#; - - assert_snapshot!( - compile(query).unwrap(), - @r###"SELECT - DATE '2022-12-31' AS date, - TIME '08:30' AS time, - TIMESTAMP '2020-01-01T13:19:55-0800' AS timestamp -FROM - test_table -"### - ) + assert_snapshot!(compile(r#" + from db.test_table + select {date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800} + "#).unwrap(), @r###" + SELECT + DATE '2022-12-31' AS date, + TIME '08:30' AS time, + TIMESTAMP '2020-01-01T13:19:55-0800' AS timestamp + FROM + test_table + "###); } #[test] @@ -4613,7 +4731,7 @@ fn test_datetime_sqlite() { assert_snapshot!(compile(r#" prql target:sql.sqlite - from x + from db.x select { date = @2022-12-31, time = @08:30, @@ -4642,7 +4760,7 @@ fn test_datetime_sqlite() { #[test] fn test_datetime_parsing() { assert_snapshot!(compile(r#" - from test_tables + from db.test_tables select {date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800} "#).unwrap(), @r###" @@ -4659,8 +4777,8 @@ fn test_datetime_parsing() { #[test] fn test_lower() { assert_snapshot!(compile(r#" - from test_tables - derive {lower_name = (name | text.lower)} + from db.test_tables + derive {lower_name = (name | std.text.lower)} "#).unwrap(), @r###" SELECT @@ -4675,8 +4793,8 @@ fn test_lower() { #[test] fn test_upper() { assert_snapshot!(compile(r#" - from test_tables - derive {upper_name = text.upper name} + from db.test_tables + derive {upper_name = std.text.upper name} select {upper_name} "#).unwrap(), @r###" @@ -4691,7 +4809,7 @@ fn test_upper() { #[test] fn test_1535() { assert_snapshot!(compile(r#" - from x.y.z + from db.x.y.z "#).unwrap(), @r###" SELECT @@ -4737,7 +4855,7 @@ fn test_read_parquet_duckdb() { fn test_excess_columns() { // https://github.com/PRQL/prql/issues/2079 assert_snapshot!(compile(r#" - from tracks + from db.tracks derive d = track_id sort d select {title} @@ -4746,7 +4864,7 @@ fn test_excess_columns() { WITH table_0 AS ( SELECT title, - track_id AS _expr_0 + track_id FROM tracks ) @@ -4755,7 +4873,7 @@ fn test_excess_columns() { FROM table_0 ORDER BY - _expr_0 + track_id "### ); } @@ -4763,7 +4881,7 @@ fn test_excess_columns() { #[test] fn test_regex_search() { assert_snapshot!(compile(r#" - from tracks + from db.tracks derive is_bob_marley = artist_name ~= "Bob\\sMarley" "#).unwrap(), @r###" @@ -4779,7 +4897,7 @@ fn test_regex_search() { #[test] fn test_intervals() { assert_snapshot!(compile(r#" - from foo + from db.foo select dt = 1years + 1months + 1weeks + 1days + 1hours + 1minutes + 1seconds + 1milliseconds + 1microseconds "#).unwrap(), @r###" @@ -4794,10 +4912,10 @@ fn test_intervals() { #[test] fn test_into() { assert_snapshot!(compile(r#" - from data + from db.data into table_a - from table_a + module.table_a select {x, y} "#).unwrap(), @r###" @@ -4822,7 +4940,7 @@ fn test_array_01() { r#" let a = [1, 2, false] - from x + from db.x "#, ) .unwrap(); @@ -4833,7 +4951,7 @@ fn test_array_01() { {a = 4, b = true}, ] - let main = (my_relation | filter b) + let main = (module.my_relation | filter b) "#).unwrap(), @r###" WITH table_0 AS ( @@ -4867,7 +4985,7 @@ fn test_array_01() { #[test] fn test_array_02() { assert_snapshot!(compile(r#" - from [ + [ {x = null}, {x = '1'}, ] @@ -4889,10 +5007,11 @@ fn test_array_02() { } #[test] +#[ignore] // change behavior: table names don't make it into tuples fn test_double_stars() { assert_snapshot!(compile(r#" - from tb1 - join tb2 (==c2) + from db.tb1 + join db.tb2 (==c2) take 5 filter (tb2.c3 < 100) "#).unwrap(), @@ -4919,8 +5038,8 @@ fn test_double_stars() { assert_snapshot!(compile(r#" prql target:sql.duckdb - from tb1 - join tb2 (==c2) + from db.tb1 + join db.tb2 (==c2) take 5 filter (tb2.c3 < 100) "#).unwrap(), @@ -4946,10 +5065,11 @@ fn test_double_stars() { } #[test] +#[ignore] fn test_lineage() { // #2627 assert_snapshot!(compile(r#" - from_text """ + std.from_text """ a 1 2 @@ -4980,7 +5100,7 @@ fn test_lineage() { // #2392 assert_snapshot!(compile(r#" - from_text format:json """{ + std.from_text format:json """{ "columns": ["a"], "data": [[1]] }""" @@ -5001,6 +5121,7 @@ fn test_lineage() { } #[test] +#[ignore] fn test_type_as_column_name() { // #2503 assert_snapshot!(compile(r#" @@ -5009,8 +5130,9 @@ fn test_type_as_column_name() { select t.date ) - from foo - f"#) + from db.foo + module.f + "#) .unwrap(), @r###" SELECT date @@ -5023,7 +5145,7 @@ fn test_type_as_column_name() { fn test_error_code() { let err = compile( r###" - let a = (from x) + let a = (from db.x) "###, ) .unwrap_err(); @@ -5031,11 +5153,12 @@ fn test_error_code() { } #[test] +#[ignore] fn large_query() { // This was causing a stack overflow on Windows, ref https://github.com/PRQL/prql/issues/2857 compile( r###" -from employees +from db.employees filter gross_cost > 0 group {title} ( aggregate { @@ -5063,7 +5186,7 @@ take 20 fn test_returning_constants_only() { assert_snapshot!(compile( r###" - from tb1 + from db.tb1 sort {a} select {c = b} select {d = 10} @@ -5087,7 +5210,7 @@ fn test_returning_constants_only() { assert_snapshot!(compile( r###" - from tb1 + from db.tb1 take 10 filter true take 20 @@ -5123,14 +5246,15 @@ fn test_returning_constants_only() { } #[test] +#[ignore] fn test_conflicting_names_at_split() { // issue #2697 assert_snapshot!(compile( r#" - from s = workflow_steps - join wp=workflow_phases (s.phase_id == wp.id) + from db.workflow_steps | select {s = this} + join (db.workflow_phases | select {wp = this}) (s.phase_id == wp.id) filter wp.name == "CREATE_OUTLET" - join w=workflow (wp.workflow_id == w.id) + join (db.workflow | select {w = this}) (wp.workflow_id == w.id) select { step_id = s.id, phase_id = wp.id, @@ -5140,21 +5264,33 @@ fn test_conflicting_names_at_split() { .unwrap(), @r###" WITH table_0 AS ( SELECT - wp.id, - s.id AS _expr_0, - wp.workflow_id + * + FROM + workflow_phases + ), + table_2 AS ( + SELECT + table_0.id, + workflow_steps.id AS _expr_0, + table_0.workflow_id FROM - workflow_steps AS s - JOIN workflow_phases AS wp ON s.phase_id = wp.id + workflow_steps + JOIN table_0 ON workflow_steps.phase_id = table_0.id WHERE - wp.name = 'CREATE_OUTLET' + table_0.name = 'CREATE_OUTLET' + ), + table_1 AS ( + SELECT + * + FROM + workflow ) SELECT - table_0._expr_0 AS step_id, - table_0.id AS phase_id + table_2._expr_0 AS step_id, + table_2.id AS phase_id FROM - table_0 - JOIN workflow AS w ON table_0.workflow_id = w.id + table_2 + JOIN table_1 ON table_2.workflow_id = table_1.id "###); } @@ -5163,7 +5299,7 @@ fn test_relation_literal_quoting() { // issue #3484 assert_snapshot!(compile( r###" - from [ + [ {`small number`=1e-10, `large number`=1e10}, ] select {`small number`, `large number`} @@ -5187,9 +5323,10 @@ fn test_relation_literal_quoting() { fn test_relation_var_name_clashes_01() { assert_snapshot!(compile( r###" - let table_0 = (from a) + let table_0 = (from db.a) + let table_0 = (from db.a) - from table_0 + project.table_0 take 10 filter x > 0 "###, @@ -5223,8 +5360,8 @@ fn test_relation_var_name_clashes_02() { // issue #3713 assert_snapshot!(compile( r###" - from t - join t (==x) + from db.t + join db.t (==x) "###, ) .unwrap(), @r###" @@ -5238,14 +5375,10 @@ fn test_relation_var_name_clashes_02() { } #[test] -#[ignore] fn test_select_this() { - // Currently broken for a few reasons: - // - type of `this` is not resolved as tuple, but an union? - // - lineage is not computed correctly assert_snapshot!(compile( r###" - from x + from db.x select {a, b} select this "###, @@ -5260,10 +5393,11 @@ fn test_select_this() { } #[test] +#[ignore] fn test_group_exclude() { assert_snapshot!(compile( r###" - from x + from db.x select {a, b} group {a} (derive c = a + 1) "###, @@ -5300,7 +5434,7 @@ fn test_group_exclude() { fn test_table_declarations() { assert_snapshot!(compile( r###" - module default_db { + module db { module my_schema { let my_table <[{ id = int, a = text }]> } @@ -5308,7 +5442,7 @@ fn test_table_declarations() { let another_table <[{ id = int, b = text }]> } - from my_schema.my_table | join another_table (==id) | take 10 + from module.db.my_schema.my_table | join module.db.another_table (==id) | take 10 "###, ) .unwrap(), @r###" @@ -5331,7 +5465,7 @@ fn test_param_declarations() { r###" let a - from x | filter b == a + from db.x | filter b == module.a "###, ) .unwrap(), @r###" @@ -5348,7 +5482,7 @@ fn test_param_declarations() { fn test_relation_aliasing() { assert_snapshot!(compile( r###" - from x | select {y = this} | select {y.hello} + from db.x | select {y = this} | select {y.hello} "###, ) .unwrap(), @r###" @@ -5367,9 +5501,9 @@ fn test_import() { let world = 1 } - import a = hello.world + import a = module.hello.world - from x | select a + from db.x | select module.a "###, ) .unwrap(), @r###" @@ -5379,3 +5513,326 @@ fn test_import() { x "###); } + +#[test] +fn test_ordering_declarations() { + // declarations must be resolved in the correct order: + // - hello.world + // - foo.bar.baz + // - main + + assert_snapshot!(compile( + r###" + let main = (from db.h | filter j == (module.foo.bar.baz + 1)) + + module foo { + module bar { + let baz = project.hello.world + } + } + + module hello { + let world = 1 + } + "###).unwrap(), @r###" + SELECT + * + FROM + h + WHERE + j = 1 + 1 + "###); +} + +#[test] +fn test_local_ref() { + assert_snapshot!(compile( + r###" + let hello = 10 + + from db.x + select {this.hello} + select {hello} + "###, + ) + .unwrap(), @r###" + SELECT + hello + FROM + x + "###); +} + +#[test] +fn query_01() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ first_name = text, age = int}]> + } + + db.employees + select {e = {this.first_name}} + select {this.e.first_name} + + "###).unwrap(), @r###" + SELECT + first_name + FROM + employees + "### + ); +} + +#[test] +fn query_02() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ first_name = text, age = int}]> + } + + db.employees + select {e = {x = this.first_name}} + select {this.e.x} + + "###).unwrap(), @r###" + SELECT + first_name AS x + FROM + employees + "### + ); +} + +#[test] +fn query_03() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ first_name = text, age = int}]> + } + + db.employees + select {e = this} + select {this.e.first_name} + + "###).unwrap(), @r###" + SELECT + first_name + FROM + employees + "### + ); +} + +#[test] +fn query_04() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ first_name = text, age = int}]> + } + + e = db.employees + select {this.e.first_name} + + "###).unwrap(), @r###" + SELECT + first_name + FROM + employees + "### + ); +} + +#[test] +fn query_05() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ first_name = text, age = int}]> + } + + e = db.employees + select {this.e.first_name} + filter (std.text.starts_with 'Jo' this.first_name) + + "###).unwrap(), @r###" + SELECT + first_name + FROM + employees + WHERE + first_name LIKE CONCAT('Jo', '%') + "### + ); +} + +#[test] +fn query_06() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ first_name = text, age = int}]> + } + + db.employees + take 5..10 + + "###).unwrap(), @r###" + SELECT + first_name, + age + FROM + employees + LIMIT + 6 OFFSET 4 + "### + ); +} + +#[test] +fn query_07() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ first_name = text, age = int}]> + } + + db.employees + select {e = {this}} + + "###).unwrap(), @r###" + SELECT + first_name AS "e.first_name", + age AS "e.age" + FROM + employees + "### + ); +} + +#[test] +fn query_08() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ id = int, first_name = text, age = int}]> + + let projects <[{ title = text, owner = int}]> + } + + from db.employees + select {e = this} + join db.projects (this.e.id == that.owner) + "###).unwrap(), @r###" + SELECT + employees.id AS "e.id", + employees.first_name AS "e.first_name", + employees.age AS "e.age", + projects.title, + projects.owner + FROM + employees + JOIN projects ON employees.id = projects.owner + "### + ); +} + +#[test] +fn query_09() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ id = int, first_name = text, age = int}]> + } + + from db.employees + select {a = !{this.first_name}} + "###).unwrap(), @r###" + SELECT + id AS "a.id", + age AS "a.age" + FROM + employees + "### + ); +} + +#[test] +fn query_10() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ id = int, first_name = text, age = int}]> + } + + from db.employees + select !{this.first_name} + "###).unwrap(), @r###" + SELECT + id, + age + FROM + employees + "### + ); +} + +#[test] +fn query_11() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ id = int, first_name = text, age = int}]> + } + + from db.employees + select !{first_name} + "###).unwrap(), @r###" + SELECT + id, + age + FROM + employees + "### + ); +} + +#[test] +fn query_12() { + assert_snapshot!(compile( + r###" + module db { + let employees <[{ id = int, first_name = text, age = int}]> + } + + from db.employees + select {e = {x = id, y = age}} + select {..e} + "###).unwrap(), @r###" + SELECT + id AS x, + age AS y + FROM + employees + "### + ); +} + +#[test] +fn query_13() { + assert_snapshot!(compile( + r###" + from db.employees + select {x = id, y = age} + "###).unwrap(), @r###" + SELECT + id AS x, + age AS y + FROM + employees + "### + ); +}