From db244272d0d1a059f41983bd6a4ffc4352449c32 Mon Sep 17 00:00:00 2001
From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com>
Date: Sun, 12 May 2024 15:15:08 -0700
Subject: [PATCH] feat: Add `prqlc lex` command to the CLI (#4467)

---
 CHANGELOG.md                                  |  1 +
 prqlc/prqlc-parser/src/lib.rs                 |  7 +-
 prqlc/prqlc/src/cli/mod.rs                    | 69 +++++++++++++++++--
 prqlc/prqlc/src/error_message.rs              |  6 ++
 prqlc/prqlc/src/lib.rs                        | 11 +++
 prqlc/prqlc/tests/integration/cli.rs          | 47 +++++++++++++
 .../integration__cli__shell_completion-2.snap | 34 +++++----
 .../integration__cli__shell_completion-3.snap | 17 ++++-
 .../integration__cli__shell_completion-4.snap | 32 ++++++++-
 .../integration__cli__shell_completion.snap   | 51 ++++++++++++--
 10 files changed, 239 insertions(+), 36 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0de89d986399..c19639d05c8d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@
 - Initial implementation of an experimental documentation generator that
   generates Markdown documentation from `.prql` files. (@vanillajonathan,
   #4152).
+- Add `prqlc lex` command to the CLI (@max-sixty)
 
 **Fixes**:
 
diff --git a/prqlc/prqlc-parser/src/lib.rs b/prqlc/prqlc-parser/src/lib.rs
index 025c9a75401b..6a150c9f12cf 100644
--- a/prqlc/prqlc-parser/src/lib.rs
+++ b/prqlc/prqlc-parser/src/lib.rs
@@ -10,13 +10,12 @@ mod types;
 use chumsky::error::SimpleReason;
 use chumsky::{prelude::*, Stream};
 
-use prqlc_ast::error::Reason;
-use prqlc_ast::error::{Error, WithErrorInfo};
+use prqlc_ast::error::{Error, Reason, WithErrorInfo};
 use prqlc_ast::stmt::*;
 use prqlc_ast::Span;
 
-use lexer::TokenKind;
-use lexer::{Token, TokenVec};
+use lexer::Token;
+pub use lexer::{TokenKind, TokenVec};
 use span::ParserSpan;
 
 /// Build PRQL AST from a PRQL query string.
diff --git a/prqlc/prqlc/src/cli/mod.rs b/prqlc/prqlc/src/cli/mod.rs
index f1ac926d100b..fccc0dd0e7aa 100644
--- a/prqlc/prqlc/src/cli/mod.rs
+++ b/prqlc/prqlc/src/cli/mod.rs
@@ -22,10 +22,10 @@ use std::path::Path;
 use std::process::exit;
 use std::str::FromStr;
 
-use prqlc::ast;
 use prqlc::semantic;
 use prqlc::semantic::reporting::{collect_frames, label_references};
 use prqlc::semantic::NS_DEFAULT_DB;
+use prqlc::{ast, prql_to_tokens};
 use prqlc::{ir::pl::Lineage, ir::Span};
 use prqlc::{pl_to_prql, pl_to_rq_tree, prql_to_pl, prql_to_pl_tree, rq_to_sql, SourceTree};
 use prqlc::{Options, Target};
@@ -79,6 +79,14 @@ enum Command {
         format: Format,
     },
 
+    /// Lex into Tokens
+    Lex {
+        #[command(flatten)]
+        io_args: IoArgs,
+        #[arg(value_enum, long, default_value = "yaml")]
+        format: Format,
+    },
+
     /// Parse & generate PRQL code back
     #[command(name = "fmt")]
     Format {
@@ -288,6 +296,17 @@ impl Command {
                     Format::Yaml => serde_yaml::to_string(&ast)?.into_bytes(),
                 }
             }
+            Command::Lex { format, .. } => {
+                let s = sources.sources.values().exactly_one().or_else(|_| {
+                    // TODO: allow multiple sources
+                    bail!("Currently `lex` only works with a single source, but found multiple sources")
+                })?;
+                let tokens = prql_to_tokens(s)?;
+                match format {
+                    Format::Json => serde_json::to_string_pretty(&tokens)?.into_bytes(),
+                    Format::Yaml => serde_yaml::to_string(&tokens)?.into_bytes(),
+                }
+            }
             Command::Collect(_) => {
                 let mut root_module_def = prql_to_pl_tree(sources)?;
 
@@ -429,7 +448,7 @@ impl Command {
                 }
             }
 
-            _ => unreachable!(),
+            _ => unreachable!("Other commands shouldn't reach `execute`"),
         })
     }
 
@@ -438,11 +457,10 @@ impl Command {
         // `input`, rather than matching on them and grabbing `input` from
         // `self`? But possibly if everything moves to `io_args`, then this is
         // quite reasonable?
-        use Command::{
-            Collect, Debug, Experimental, Parse, Resolve, SQLAnchor, SQLCompile, SQLPreprocess,
-        };
+        use Command::*;
         let io_args = match self {
             Parse { io_args, .. }
+            | Lex { io_args, .. }
             | Collect(io_args)
             | Resolve { io_args, .. }
             | SQLCompile { io_args, .. }
@@ -481,10 +499,11 @@ impl Command {
 
     fn write_output(&mut self, data: &[u8]) -> std::io::Result<()> {
         use Command::{
-            Collect, Debug, Experimental, Parse, Resolve, SQLAnchor, SQLCompile, SQLPreprocess,
+            Collect, Debug, Experimental, Lex, Parse, Resolve, SQLAnchor, SQLCompile, SQLPreprocess,
         };
         let mut output = match self {
             Parse { io_args, .. }
+            | Lex { io_args, .. }
             | Collect(io_args)
             | Resolve { io_args, .. }
             | SQLCompile { io_args, .. }
@@ -815,4 +834,42 @@ sort full
               column: 2
         "###);
     }
+
+    #[test]
+    fn lex() {
+        let output = Command::execute(
+            &Command::Lex {
+                io_args: IoArgs::default(),
+                format: Format::Yaml,
+            },
+            &mut "from x | select y".into(),
+            "",
+        )
+        .unwrap();
+
+        // TODO: terser output; maybe serialize span as `0..4`? Remove the
+        // `!Ident` complication?
+        assert_snapshot!(String::from_utf8(output).unwrap().trim(), @r###"
+        - kind: !Ident from
+          span:
+            start: 0
+            end: 4
+        - kind: !Ident x
+          span:
+            start: 5
+            end: 6
+        - kind: !Control '|'
+          span:
+            start: 7
+            end: 8
+        - kind: !Ident select
+          span:
+            start: 9
+            end: 15
+        - kind: !Ident y
+          span:
+            start: 16
+            end: 17
+        "###);
+    }
 }
diff --git a/prqlc/prqlc/src/error_message.rs b/prqlc/prqlc/src/error_message.rs
index 5c50054ca725..89e7c6ae9559 100644
--- a/prqlc/prqlc/src/error_message.rs
+++ b/prqlc/prqlc/src/error_message.rs
@@ -87,6 +87,12 @@ impl From<Error> for ErrorMessage {
     }
 }
 
+impl From<Vec<ErrorMessage>> for ErrorMessages {
+    fn from(errors: Vec<ErrorMessage>) -> Self {
+        ErrorMessages { inner: errors }
+    }
+}
+
 #[derive(Debug, Clone, Serialize)]
 pub struct ErrorMessages {
     pub inner: Vec<ErrorMessage>,
diff --git a/prqlc/prqlc/src/lib.rs b/prqlc/prqlc/src/lib.rs
index cb680056d9e4..6f38a6d1f317 100644
--- a/prqlc/prqlc/src/lib.rs
+++ b/prqlc/prqlc/src/lib.rs
@@ -113,6 +113,7 @@ pub static COMPILER_VERSION: Lazy<Version> =
     Lazy::new(|| Version::parse(env!("CARGO_PKG_VERSION")).expect("Invalid prqlc version number"));
 
 use once_cell::sync::Lazy;
+use prqlc_parser::TokenVec;
 use semver::Version;
 use serde::{Deserialize, Serialize};
 use std::{collections::HashMap, path::PathBuf, str::FromStr};
@@ -306,6 +307,16 @@ pub enum DisplayOptions {
 #[cfg(doctest)]
 pub struct ReadmeDoctests;
 
+/// Lex PRQL source into tokens.
+pub fn prql_to_tokens(prql: &str) -> Result<TokenVec, ErrorMessages> {
+    prqlc_parser::lex_source(prql).map_err(|e| {
+        e.into_iter()
+            .map(|e| e.into())
+            .collect::<Vec<ErrorMessage>>()
+            .into()
+    })
+}
+
 /// Parse PRQL into a PL AST
 // TODO: rename this to `prql_to_pl_simple`
 pub fn prql_to_pl(prql: &str) -> Result<ast::ModuleDef, ErrorMessages> {
diff --git a/prqlc/prqlc/tests/integration/cli.rs b/prqlc/prqlc/tests/integration/cli.rs
index 87a70b4348ed..c9f7aa07972b 100644
--- a/prqlc/prqlc/tests/integration/cli.rs
+++ b/prqlc/prqlc/tests/integration/cli.rs
@@ -17,6 +17,7 @@ fn help() {
 
     Commands:
       parse             Parse into PL AST
+      lex               Lex into Tokens
       fmt               Parse & generate PRQL code back
       collect           Parse the whole project and collect it into a single PRQL source file
       debug             Commands for meant for debugging, prone to change
@@ -471,3 +472,49 @@ fn compile_no_prql_files() {
 
     "###);
 }
+
+#[test]
+fn lex() {
+    assert_cmd_snapshot!(prqlc_command().args(["lex"]).pass_stdin("from tracks"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    - kind: !Ident from
+      span:
+        start: 0
+        end: 4
+    - kind: !Ident tracks
+      span:
+        start: 5
+        end: 11
+
+    ----- stderr -----
+    "###);
+
+    assert_cmd_snapshot!(prqlc_command().args(["lex", "--format=json"]).pass_stdin("from tracks"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    [
+      {
+        "kind": {
+          "Ident": "from"
+        },
+        "span": {
+          "start": 0,
+          "end": 4
+        }
+      },
+      {
+        "kind": {
+          "Ident": "tracks"
+        },
+        "span": {
+          "start": 5,
+          "end": 11
+        }
+      }
+    ]
+    ----- stderr -----
+    "###);
+}
diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-2.snap b/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-2.snap
index 7d477a77b1ec..708f67dd20cf 100644
--- a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-2.snap
+++ b/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-2.snap
@@ -7,10 +7,10 @@ info:
     - shell-completion
     - fish
   env:
+    CLICOLOR_FORCE: ""
     NO_COLOR: "1"
     RUST_BACKTRACE: ""
     RUST_LOG: ""
-    CLICOLOR_FORCE: ""
 ---
 success: true
 exit_code: 0
@@ -19,6 +19,7 @@ complete -c prqlc -n "__fish_use_subcommand" -l color -d 'Controls when to use c
 complete -c prqlc -n "__fish_use_subcommand" -s h -l help -d 'Print help'
 complete -c prqlc -n "__fish_use_subcommand" -s V -l version -d 'Print version'
 complete -c prqlc -n "__fish_use_subcommand" -f -a "parse" -d 'Parse into PL AST'
+complete -c prqlc -n "__fish_use_subcommand" -f -a "lex" -d 'Lex into Tokens'
 complete -c prqlc -n "__fish_use_subcommand" -f -a "fmt" -d 'Parse & generate PRQL code back'
 complete -c prqlc -n "__fish_use_subcommand" -f -a "collect" -d 'Parse the whole project and collect it into a single PRQL source file'
 complete -c prqlc -n "__fish_use_subcommand" -f -a "debug" -d 'Commands for meant for debugging, prone to change'
@@ -34,6 +35,9 @@ complete -c prqlc -n "__fish_use_subcommand" -f -a "help" -d 'Print this message
 complete -c prqlc -n "__fish_seen_subcommand_from parse" -l format -r -f -a "{json	'',yaml	''}"
 complete -c prqlc -n "__fish_seen_subcommand_from parse" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
 complete -c prqlc -n "__fish_seen_subcommand_from parse" -s h -l help -d 'Print help'
+complete -c prqlc -n "__fish_seen_subcommand_from lex" -l format -r -f -a "{json	'',yaml	''}"
+complete -c prqlc -n "__fish_seen_subcommand_from lex" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
+complete -c prqlc -n "__fish_seen_subcommand_from lex" -s h -l help -d 'Print help'
 complete -c prqlc -n "__fish_seen_subcommand_from fmt" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
 complete -c prqlc -n "__fish_seen_subcommand_from fmt" -s h -l help -d 'Print help'
 complete -c prqlc -n "__fish_seen_subcommand_from collect" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
@@ -91,19 +95,20 @@ complete -c prqlc -n "__fish_seen_subcommand_from list-targets" -l color -d 'Con
 complete -c prqlc -n "__fish_seen_subcommand_from list-targets" -s h -l help -d 'Print help'
 complete -c prqlc -n "__fish_seen_subcommand_from shell-completion" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
 complete -c prqlc -n "__fish_seen_subcommand_from shell-completion" -s h -l help -d 'Print help'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "parse" -d 'Parse into PL AST'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "fmt" -d 'Parse & generate PRQL code back'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "collect" -d 'Parse the whole project and collect it into a single PRQL source file'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "debug" -d 'Commands for meant for debugging, prone to change'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "experimental" -d 'Experimental commands are prone to change'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "resolve" -d 'Parse, resolve & lower into RQ'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "sql:preprocess" -d 'Parse, resolve, lower into RQ & preprocess SRQ'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "sql:anchor" -d 'Parse, resolve, lower into RQ & preprocess & anchor SRQ'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "compile" -d 'Parse, resolve, lower into RQ & compile to SQL'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "watch" -d 'Watch a directory and compile .prql files to .sql files'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "list-targets" -d 'Show available compile target names'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "shell-completion" -d 'Print a shell completion for supported shells'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "parse" -d 'Parse into PL AST'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "lex" -d 'Lex into Tokens'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "fmt" -d 'Parse & generate PRQL code back'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "collect" -d 'Parse the whole project and collect it into a single PRQL source file'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "debug" -d 'Commands for meant for debugging, prone to change'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "experimental" -d 'Experimental commands are prone to change'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "resolve" -d 'Parse, resolve & lower into RQ'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "sql:preprocess" -d 'Parse, resolve, lower into RQ & preprocess SRQ'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "sql:anchor" -d 'Parse, resolve, lower into RQ & preprocess & anchor SRQ'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "compile" -d 'Parse, resolve, lower into RQ & compile to SQL'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "watch" -d 'Watch a directory and compile .prql files to .sql files'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "list-targets" -d 'Show available compile target names'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "shell-completion" -d 'Print a shell completion for supported shells'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from parse; and not __fish_seen_subcommand_from lex; and not __fish_seen_subcommand_from fmt; and not __fish_seen_subcommand_from collect; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from sql:preprocess; and not __fish_seen_subcommand_from sql:anchor; and not __fish_seen_subcommand_from compile; and not __fish_seen_subcommand_from watch; and not __fish_seen_subcommand_from list-targets; and not __fish_seen_subcommand_from shell-completion; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from expand-pl; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from eval; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from ast" -f -a "expand-pl" -d 'Parse & and expand into PL, but don\'t resolve'
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from expand-pl; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from eval; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from ast" -f -a "resolve" -d 'Parse & resolve, but don\'t lower into RQ'
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from expand-pl; and not __fish_seen_subcommand_from resolve; and not __fish_seen_subcommand_from eval; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from ast" -f -a "eval" -d 'Parse & evaluate expression down to a value'
@@ -112,4 +117,3 @@ complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcomma
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc" -f -a "doc" -d 'Generate Markdown documentation'
 
 ----- stderr -----
-
diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-3.snap b/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-3.snap
index ef8c87779589..3ff91c69d646 100644
--- a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-3.snap
+++ b/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-3.snap
@@ -8,9 +8,9 @@ info:
     - powershell
   env:
     CLICOLOR_FORCE: ""
-    RUST_LOG: ""
-    RUST_BACKTRACE: ""
     NO_COLOR: "1"
+    RUST_BACKTRACE: ""
+    RUST_LOG: ""
 ---
 success: true
 exit_code: 0
@@ -44,6 +44,7 @@ Register-ArgumentCompleter -Native -CommandName 'prqlc' -ScriptBlock {
             [CompletionResult]::new('-V', 'V ', [CompletionResultType]::ParameterName, 'Print version')
             [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Print version')
             [CompletionResult]::new('parse', 'parse', [CompletionResultType]::ParameterValue, 'Parse into PL AST')
+            [CompletionResult]::new('lex', 'lex', [CompletionResultType]::ParameterValue, 'Lex into Tokens')
             [CompletionResult]::new('fmt', 'fmt', [CompletionResultType]::ParameterValue, 'Parse & generate PRQL code back')
             [CompletionResult]::new('collect', 'collect', [CompletionResultType]::ParameterValue, 'Parse the whole project and collect it into a single PRQL source file')
             [CompletionResult]::new('debug', 'debug', [CompletionResultType]::ParameterValue, 'Commands for meant for debugging, prone to change')
@@ -65,6 +66,13 @@ Register-ArgumentCompleter -Native -CommandName 'prqlc' -ScriptBlock {
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
             break
         }
+        'prqlc;lex' {
+            [CompletionResult]::new('--format', 'format', [CompletionResultType]::ParameterName, 'format')
+            [CompletionResult]::new('--color', 'color', [CompletionResultType]::ParameterName, 'Controls when to use color')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
+            break
+        }
         'prqlc;fmt' {
             [CompletionResult]::new('--color', 'color', [CompletionResultType]::ParameterName, 'Controls when to use color')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
@@ -223,6 +231,7 @@ Register-ArgumentCompleter -Native -CommandName 'prqlc' -ScriptBlock {
         }
         'prqlc;help' {
             [CompletionResult]::new('parse', 'parse', [CompletionResultType]::ParameterValue, 'Parse into PL AST')
+            [CompletionResult]::new('lex', 'lex', [CompletionResultType]::ParameterValue, 'Lex into Tokens')
             [CompletionResult]::new('fmt', 'fmt', [CompletionResultType]::ParameterValue, 'Parse & generate PRQL code back')
             [CompletionResult]::new('collect', 'collect', [CompletionResultType]::ParameterValue, 'Parse the whole project and collect it into a single PRQL source file')
             [CompletionResult]::new('debug', 'debug', [CompletionResultType]::ParameterValue, 'Commands for meant for debugging, prone to change')
@@ -240,6 +249,9 @@ Register-ArgumentCompleter -Native -CommandName 'prqlc' -ScriptBlock {
         'prqlc;help;parse' {
             break
         }
+        'prqlc;help;lex' {
+            break
+        }
         'prqlc;help;fmt' {
             break
         }
@@ -307,4 +319,3 @@ Register-ArgumentCompleter -Native -CommandName 'prqlc' -ScriptBlock {
 }
 
 ----- stderr -----
-
diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-4.snap b/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-4.snap
index 2de4acf50ab4..2be136899be2 100644
--- a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-4.snap
+++ b/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-4.snap
@@ -7,10 +7,10 @@ info:
     - shell-completion
     - zsh
   env:
-    RUST_BACKTRACE: ""
-    RUST_LOG: ""
     CLICOLOR_FORCE: ""
     NO_COLOR: "1"
+    RUST_BACKTRACE: ""
+    RUST_LOG: ""
 ---
 success: true
 exit_code: 0
@@ -57,6 +57,17 @@ _arguments "${_arguments_options[@]}" \
 '::main_path -- Identifier of the main pipeline:' \
 && ret=0
 ;;
+(lex)
+_arguments "${_arguments_options[@]}" \
+'--format=[]:FORMAT:(json yaml)' \
+'--color=[Controls when to use color]:WHEN:(auto always never)' \
+'-h[Print help]' \
+'--help[Print help]' \
+'::input:_files' \
+'::output:_files' \
+'::main_path -- Identifier of the main pipeline:' \
+&& ret=0
+;;
 (fmt)
 _arguments "${_arguments_options[@]}" \
 '--color=[Controls when to use color]:WHEN:(auto always never)' \
@@ -321,6 +332,10 @@ _arguments "${_arguments_options[@]}" \
 _arguments "${_arguments_options[@]}" \
 && ret=0
 ;;
+(lex)
+_arguments "${_arguments_options[@]}" \
+&& ret=0
+;;
 (fmt)
 _arguments "${_arguments_options[@]}" \
 && ret=0
@@ -430,6 +445,7 @@ esac
 _prqlc_commands() {
     local commands; commands=(
 'parse:Parse into PL AST' \
+'lex:Lex into Tokens' \
 'fmt:Parse & generate PRQL code back' \
 'collect:Parse the whole project and collect it into a single PRQL source file' \
 'debug:Commands for meant for debugging, prone to change' \
@@ -622,6 +638,7 @@ _prqlc__experimental__help__help_commands() {
 _prqlc__help_commands() {
     local commands; commands=(
 'parse:Parse into PL AST' \
+'lex:Lex into Tokens' \
 'fmt:Parse & generate PRQL code back' \
 'collect:Parse the whole project and collect it into a single PRQL source file' \
 'debug:Commands for meant for debugging, prone to change' \
@@ -642,6 +659,16 @@ _prqlc__help__help_commands() {
     local commands; commands=()
     _describe -t commands 'prqlc help help commands' commands "$@"
 }
+(( $+functions[_prqlc__help__lex_commands] )) ||
+_prqlc__help__lex_commands() {
+    local commands; commands=()
+    _describe -t commands 'prqlc help lex commands' commands "$@"
+}
+(( $+functions[_prqlc__lex_commands] )) ||
+_prqlc__lex_commands() {
+    local commands; commands=()
+    _describe -t commands 'prqlc lex commands' commands "$@"
+}
 (( $+functions[_prqlc__help__list-targets_commands] )) ||
 _prqlc__help__list-targets_commands() {
     local commands; commands=()
@@ -735,4 +762,3 @@ else
 fi
 
 ----- stderr -----
-
diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion.snap b/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion.snap
index ce4340c9b006..fba7d359e4a5 100644
--- a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion.snap
+++ b/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion.snap
@@ -7,10 +7,10 @@ info:
     - shell-completion
     - bash
   env:
-    RUST_LOG: ""
     CLICOLOR_FORCE: ""
-    RUST_BACKTRACE: ""
     NO_COLOR: "1"
+    RUST_BACKTRACE: ""
+    RUST_LOG: ""
 ---
 success: true
 exit_code: 0
@@ -47,6 +47,9 @@ _prqlc() {
             prqlc,help)
                 cmd="prqlc__help"
                 ;;
+            prqlc,lex)
+                cmd="prqlc__lex"
+                ;;
             prqlc,list-targets)
                 cmd="prqlc__list__targets"
                 ;;
@@ -134,6 +137,9 @@ _prqlc() {
             prqlc__help,help)
                 cmd="prqlc__help__help"
                 ;;
+            prqlc__help,lex)
+                cmd="prqlc__help__lex"
+                ;;
             prqlc__help,list-targets)
                 cmd="prqlc__help__list__targets"
                 ;;
@@ -180,7 +186,7 @@ _prqlc() {
 
     case "${cmd}" in
         prqlc)
-            opts="-h -V --color --help --version parse fmt collect debug experimental resolve sql:preprocess sql:anchor compile watch list-targets shell-completion help"
+            opts="-h -V --color --help --version parse lex fmt collect debug experimental resolve sql:preprocess sql:anchor compile watch list-targets shell-completion help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -544,7 +550,7 @@ _prqlc() {
             return 0
             ;;
         prqlc__help)
-            opts="parse fmt collect debug experimental resolve sql:preprocess sql:anchor compile watch list-targets shell-completion help"
+            opts="parse lex fmt collect debug experimental resolve sql:preprocess sql:anchor compile watch list-targets shell-completion help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -725,6 +731,20 @@ _prqlc() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
+        prqlc__help__lex)
+            opts=""
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
         prqlc__help__list__targets)
             opts=""
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
@@ -823,6 +843,28 @@ _prqlc() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
+        prqlc__lex)
+            opts="-h --format --color --help [INPUT] [OUTPUT] [MAIN_PATH]"
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                --format)
+                    COMPREPLY=($(compgen -W "json yaml" -- "${cur}"))
+                    return 0
+                    ;;
+                --color)
+                    COMPREPLY=($(compgen -W "auto always never" -- "${cur}"))
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
         prqlc__list__targets)
             opts="-h --color --help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
@@ -971,4 +1013,3 @@ else
 fi
 
 ----- stderr -----
-