Skip to content

Commit b65c7d0

Browse files
bvanjoibvanjoi
committed
parse tpl
1 parent 6753d99 commit b65c7d0

File tree

8 files changed

+360
-94
lines changed

8 files changed

+360
-94
lines changed

‎crates/swc_ecma_lexer/src/input.rs

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ pub trait Tokens: Clone + Iterator<Item = TokenAndSpan> {
5454
/// If the program was parsed as a script, this contains the module
5555
/// errors should the program be identified as a module in the future.
5656
fn take_script_module_errors(&mut self) -> Vec<Error>;
57+
58+
fn rescan_template_token(
59+
&mut self,
60+
start: BytePos,
61+
start_with_back_tick: bool,
62+
) -> Option<TokenAndSpan>;
5763
}
5864

5965
#[derive(Clone)]
@@ -161,6 +167,14 @@ impl Tokens for TokensInput {
161167
.map(|t| t.span.hi)
162168
.unwrap_or(self.start_pos)
163169
}
170+
171+
fn rescan_template_token(
172+
&mut self,
173+
_start: BytePos,
174+
_start_with_back_tick: bool,
175+
) -> Option<TokenAndSpan> {
176+
unreachable!()
177+
}
164178
}
165179

166180
/// Note: Lexer need access to parser's context to lex correctly.
@@ -285,6 +299,14 @@ impl<I: Tokens> Tokens for Capturing<I> {
285299
fn end_pos(&self) -> BytePos {
286300
self.inner.end_pos()
287301
}
302+
303+
fn rescan_template_token(
304+
&mut self,
305+
_start: BytePos,
306+
_start_with_back_tick: bool,
307+
) -> Option<TokenAndSpan> {
308+
unreachable!()
309+
}
288310
}
289311

290312
/// This struct is responsible for managing current token and peeked token.
@@ -404,6 +426,19 @@ impl<I: Tokens> Buffer<I> {
404426
}
405427
}
406428

429+
pub fn rescan_template_token(
430+
&mut self,
431+
start: BytePos,
432+
start_with_back_tick: bool,
433+
) -> Option<&Token> {
434+
debug_assert!(self.cur.is_some());
435+
self.cur = self.iter.rescan_template_token(start, start_with_back_tick);
436+
match &self.cur {
437+
Some(v) => Some(&v.token),
438+
None => None,
439+
}
440+
}
441+
407442
#[inline]
408443
pub fn cut_lshift(&mut self) {
409444
debug_assert!(

‎crates/swc_ecma_lexer/src/lexer/state.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,14 @@ impl Tokens for Lexer<'_> {
200200
fn end_pos(&self) -> BytePos {
201201
self.input.end_pos()
202202
}
203+
204+
fn rescan_template_token(
205+
&mut self,
206+
_start: BytePos,
207+
_start_with_back_tick: bool,
208+
) -> Option<TokenAndSpan> {
209+
unreachable!()
210+
}
203211
}
204212

205213
impl Lexer<'_> {

‎crates/swc_ecma_lexer/src/token.rs

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,10 @@ pub enum TokenKind {
187187
Comma,
188188
BackQuote,
189189
Template,
190+
NoSubstitutionTemplateLiteral,
191+
TemplateHead,
192+
TemplateMiddle,
193+
TemplateTail,
190194
Colon,
191195
BinOp(BinOpToken),
192196
AssignOp(AssignOp),
@@ -257,6 +261,22 @@ pub enum Token {
257261
raw: Atom,
258262
cooked: LexResult<Atom>,
259263
},
264+
NoSubstitutionTemplateLiteral {
265+
raw: Atom,
266+
cooked: LexResult<Atom>,
267+
},
268+
TemplateHead {
269+
raw: Atom,
270+
cooked: LexResult<Atom>,
271+
},
272+
TemplateMiddle {
273+
raw: Atom,
274+
cooked: LexResult<Atom>,
275+
},
276+
TemplateTail {
277+
raw: Atom,
278+
cooked: LexResult<Atom>,
279+
},
260280
/// ':'
261281
Colon,
262282
BinOp(BinOpToken),
@@ -329,6 +349,10 @@ impl Token {
329349
Self::Comma => TokenKind::Comma,
330350
Self::BackQuote => TokenKind::BackQuote,
331351
Self::Template { .. } => TokenKind::Template,
352+
Self::NoSubstitutionTemplateLiteral { .. } => TokenKind::NoSubstitutionTemplateLiteral,
353+
Self::TemplateHead { .. } => TokenKind::TemplateHead,
354+
Self::TemplateMiddle { .. } => TokenKind::TemplateMiddle,
355+
Self::TemplateTail { .. } => TokenKind::TemplateTail,
332356
Self::Colon => TokenKind::Colon,
333357
Self::BinOp(op) => TokenKind::BinOp(*op),
334358
Self::AssignOp(op) => TokenKind::AssignOp(*op),
@@ -969,6 +993,10 @@ impl Debug for Token {
969993
Comma => write!(f, ",")?,
970994
BackQuote => write!(f, "`")?,
971995
Template { raw, .. } => write!(f, "template token ({})", raw)?,
996+
NoSubstitutionTemplateLiteral { raw, .. } => write!(f, "`{}`", raw)?,
997+
TemplateHead { raw, .. } => write!(f, "`{}${{", raw)?,
998+
TemplateMiddle { raw, .. } => write!(f, "{}${{", raw)?,
999+
TemplateTail { raw, .. } => write!(f, "}}{}`", raw)?,
9721000
Colon => write!(f, ":")?,
9731001
BinOp(op) => write!(f, "{}", BinaryOp::from(*op).as_str())?,
9741002
AssignOp(op) => write!(f, "{}", op.as_str())?,

‎crates/swc_ecma_parser/src/lexer/mod.rs

Lines changed: 34 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -657,14 +657,23 @@ impl<'a> Lexer<'a> {
657657

658658
fn read_token_back_quote(&mut self) -> LexResult<Option<Token>> {
659659
let start = self.cur_pos();
660-
self.rescan_token_back_quote(start).map(|t| Some(t))
660+
self.scan_template_token(start, true).map(Some)
661661
}
662662

663-
fn rescan_token_back_quote(&mut self, start: BytePos) -> LexResult<Token> {
664-
let mut cooked = Ok(String::new());
665-
let mut cooked_slice_start = start;
666-
let raw_slice_start = start;
667-
663+
fn scan_template_token(
664+
&mut self,
665+
start: BytePos,
666+
started_with_backtick: bool,
667+
) -> LexResult<Token> {
668+
let mut cooked = Ok(String::with_capacity(8));
669+
self.bump();
670+
let mut cooked_slice_start = self.cur_pos();
671+
let raw_slice_start = cooked_slice_start;
672+
let raw_atom = |this: &mut Self| {
673+
let last_pos = this.cur_pos();
674+
let s = unsafe { this.input.slice(raw_slice_start, last_pos) };
675+
this.atoms.atom(s)
676+
};
668677
macro_rules! consume_cooked {
669678
() => {{
670679
if let Ok(cooked) = &mut cooked {
@@ -679,48 +688,27 @@ impl<'a> Lexer<'a> {
679688
}
680689

681690
while let Some(c) = self.cur() {
682-
if c == '`' || (c == '$' && self.peek() == Some('{')) {
683-
if start == self.cur_pos() && self.state.last_was_tpl_element() {
684-
if c == '$' {
685-
self.bump();
686-
self.bump();
687-
return Ok(tok!("${"));
688-
} else {
689-
self.bump();
690-
return Ok(tok!('`'));
691-
}
692-
}
693-
694-
// If we don't have any escape
695-
let cooked = if cooked_slice_start == raw_slice_start {
696-
let last_pos = self.cur_pos();
697-
let s = unsafe {
698-
// Safety: Both of start and last_pos are valid position because we got them
699-
// from `self.input`
700-
self.input.slice(cooked_slice_start, last_pos)
701-
};
702-
703-
Ok(self.atoms.atom(s))
691+
if c == '`' {
692+
consume_cooked!();
693+
let cooked = cooked.map(|cooked| self.atoms.atom(cooked));
694+
let raw = raw_atom(self);
695+
self.bump();
696+
return Ok(if started_with_backtick {
697+
Token::NoSubstitutionTemplateLiteral { cooked, raw }
704698
} else {
705-
consume_cooked!();
706-
707-
cooked.map(|s| self.atoms.atom(s))
708-
};
709-
710-
// TODO: Handle error
711-
let end = self.input.cur_pos();
712-
let raw = unsafe {
713-
// Safety: Both of start and last_pos are valid position because we got them
714-
// from `self.input`
715-
self.input.slice(raw_slice_start, end)
716-
};
717-
return Ok(Token::Template {
718-
cooked,
719-
raw: self.atoms.atom(raw),
699+
Token::TemplateTail { cooked, raw }
720700
});
721-
}
722-
723-
if c == '\\' {
701+
} else if c == '$' && self.input.peek() == Some('{') {
702+
consume_cooked!();
703+
let cooked = cooked.map(|cooked| self.atoms.atom(cooked));
704+
let raw = raw_atom(self);
705+
self.input.bump_bytes(2);
706+
return Ok(if started_with_backtick {
707+
Token::TemplateHead { cooked, raw }
708+
} else {
709+
Token::TemplateMiddle { cooked, raw }
710+
});
711+
} else if c == '\\' {
724712
consume_cooked!();
725713

726714
match self.read_escaped_char(true) {

‎crates/swc_ecma_parser/src/lexer/state.rs

Lines changed: 64 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,46 @@ impl Tokens for Lexer<'_> {
119119
fn end_pos(&self) -> BytePos {
120120
self.input.end_pos()
121121
}
122+
123+
fn rescan_template_token(
124+
&mut self,
125+
start: BytePos,
126+
start_with_back_tick: bool,
127+
) -> Option<TokenAndSpan> {
128+
unsafe { self.input.reset_to(start) };
129+
130+
let res = self
131+
.scan_template_token(start, start_with_back_tick)
132+
.map(Some);
133+
let token = match res.map_err(Token::Error).map_err(Some) {
134+
Ok(t) => t,
135+
Err(e) => e,
136+
};
137+
let span = self.span(start);
138+
if let Some(ref token) = token {
139+
if let Some(comments) = self.comments_buffer.as_mut() {
140+
for comment in comments.take_pending_leading() {
141+
comments.push(BufferedComment {
142+
kind: BufferedCommentKind::Leading,
143+
pos: start,
144+
comment,
145+
});
146+
}
147+
}
148+
149+
self.state.update(token.kind());
150+
self.state.prev_hi = self.last_pos();
151+
self.state.had_line_break_before_last = self.had_line_break_before_last();
152+
}
153+
token.map(|token| {
154+
// Attach span to token.
155+
TokenAndSpan {
156+
token,
157+
had_line_break: self.had_line_break_before_last(),
158+
span,
159+
}
160+
})
161+
}
122162
}
123163

124164
impl Lexer<'_> {
@@ -354,10 +394,6 @@ impl State {
354394
}
355395
}
356396

357-
pub fn last_was_tpl_element(&self) -> bool {
358-
matches!(self.token_type, Some(TokenType::Template))
359-
}
360-
361397
fn update(&mut self, next: TokenKind) {
362398
if cfg!(feature = "debug") {
363399
trace!(
@@ -411,6 +447,14 @@ impl State {
411447
return false;
412448
}
413449

450+
// ${} in template
451+
if out == TokenContext::TplQuasi {
452+
match context.current() {
453+
Some(TokenContext::Tpl { .. }) => return false,
454+
_ => return true,
455+
}
456+
}
457+
414458
// expression cannot follow expression
415459
!out.is_expr()
416460
}
@@ -482,6 +526,8 @@ impl State {
482526
let cur = context.current();
483527
if syntax.jsx() && cur == Some(TokenContext::JSXOpeningTag) {
484528
context.push(TokenContext::BraceExpr)
529+
} else if syntax.jsx() && cur == Some(TokenContext::JSXExpr) {
530+
context.push(TokenContext::TplQuasi);
485531
} else {
486532
let next_ctxt =
487533
if context.is_brace_block(prev, had_line_break, is_expr_allowed) {
@@ -503,7 +549,10 @@ impl State {
503549
false
504550
}
505551

506-
TokenKind::DollarLBrace => true,
552+
TokenKind::DollarLBrace => {
553+
context.push(TokenContext::TplQuasi);
554+
true
555+
}
507556

508557
TokenKind::LParen => {
509558
// if, for, with, while is statement
@@ -524,7 +573,16 @@ impl State {
524573
// remains unchanged.
525574
TokenKind::PlusPlus | TokenKind::MinusMinus => is_expr_allowed,
526575

527-
TokenKind::BackQuote => false,
576+
TokenKind::BackQuote => {
577+
// If we are in template, ` terminates template.
578+
if let Some(TokenContext::Tpl { .. }) = context.current() {
579+
context.pop();
580+
} else {
581+
// self.tpl_start = start;
582+
context.push(TokenContext::Tpl);
583+
}
584+
false
585+
}
528586

529587
// tt.jsxTagStart.updateContext
530588
TokenKind::JSXTagStart => {

0 commit comments

Comments
 (0)