fix: using tokens from parser to handle asi

crates/rspack_javascript_compiler/Cargo.toml

@@ -32,9 +32,11 @@ swc_core = { workspace = true, features = [
   "swc_ecma_ast",
   "ecma_transforms_react",
   "ecma_transforms_module",
+  "ecma_parser_unstable",
   "swc_ecma_codegen",
   "swc_ecma_visit",
 ] }
+swc_ecma_lexer = { workspace = true }
 swc_ecma_minifier = { workspace = true, features = ["concurrent"] }
 swc_error_reporters = { workspace = true }
 swc_node_comments = { workspace = true }

crates/rspack_javascript_compiler/src/compiler/parse.rs

@@ -6,7 +6,7 @@ use swc_core::{
   common::{FileName, SourceFile, comments::Comments, input::SourceFileInput},
   ecma::{
     ast::{EsVersion, Program as SwcProgram},
-    parser::{self, Parser, Syntax, lexer::Lexer},
+    parser::{self, Parser, Syntax, lexer::Lexer, unstable::TokenAndSpan},
   },
 };
 use swc_node_comments::SwcComments;
@@ -52,8 +52,8 @@ impl JavaScriptCompiler {
       comments.as_ref().map(|c| c as &dyn Comments),
     );
 
-    parse_with_lexer(lexer, is_module)
-      .map(|program| {
+    parse_with_lexer(lexer, is_module, false)
+      .map(|(program, _)| {
         Ast::new(program, self.cm.clone(), comments)
           .with_context(ast::Context::new(self.cm, Some(self.globals)))
       })
@@ -74,11 +74,15 @@ impl JavaScriptCompiler {
     lexer: Lexer,
     is_module: IsModule,
     comments: Option<SwcComments>,
-  ) -> Result<Ast, BatchErrors> {
-    parse_with_lexer(lexer, is_module)
-      .map(|program| {
-        Ast::new(program, self.cm.clone(), comments)
-          .with_context(ast::Context::new(self.cm.clone(), Some(self.globals)))
+    with_tokens: bool,
+  ) -> Result<(Ast, Option<Vec<TokenAndSpan>>), BatchErrors> {
+    parse_with_lexer(lexer, is_module, with_tokens)
+      .map(|(program, tokens)| {
+        (
+          Ast::new(program, self.cm.clone(), comments)
+            .with_context(ast::Context::new(self.cm.clone(), Some(self.globals))),
+          tokens,
+        )
       })
       .map_err(|errs| {
         BatchErrors(
@@ -101,38 +105,60 @@ impl JavaScriptCompiler {
     comments: Option<&dyn Comments>,
   ) -> Result<SwcProgram, BatchErrors> {
     let lexer = Lexer::new(syntax, target, SourceFileInput::from(&*fm), comments);
-    parse_with_lexer(lexer, is_module).map_err(|errs| {
-      BatchErrors(
-        errs
-          .dedup_ecma_errors()
-          .into_iter()
-          .map(|err| ecma_parse_error_deduped_to_rspack_error(err, &fm))
-          .collect::<Vec<_>>(),
-      )
-    })
+    parse_with_lexer(lexer, is_module, false)
+      .map(|(program, _)| program)
+      .map_err(|errs| {
+        BatchErrors(
+          errs
+            .dedup_ecma_errors()
+            .into_iter()
+            .map(|err| ecma_parse_error_deduped_to_rspack_error(err, &fm))
+            .collect::<Vec<_>>(),
+        )
+      })
   }
 }
-
+use swc_core::ecma::parser::unstable::Capturing;
 fn parse_with_lexer(
   lexer: Lexer,
   is_module: IsModule,
-) -> Result<SwcProgram, Vec<parser::error::Error>> {
+  with_tokens: bool,
+) -> Result<(SwcProgram, Option<Vec<TokenAndSpan>>), Vec<parser::error::Error>> {
+  use swc_ecma_lexer::common::parser::{Parser as _, buffer::Buffer};
   let inner = || {
-    let mut parser = Parser::new_from(lexer);
-    let program_result = match is_module {
-      IsModule::Bool(true) => parser.parse_module().map(SwcProgram::Module),
-      IsModule::Bool(false) => parser.parse_script().map(SwcProgram::Script),
-      IsModule::Unknown => parser.parse_program(),
-      IsModule::CommonJS => parser.parse_commonjs().map(SwcProgram::Script),
+    // don't call capturing when with_tokens is false to avoid performance cost
+    let (tokens, program_result, mut errors) = if with_tokens {
+      let lexer = Capturing::new(lexer);
+      let mut parser = Parser::new_from(lexer);
+      let program_result = match is_module {
+        IsModule::Bool(true) => parser.parse_module().map(SwcProgram::Module),
+        IsModule::Bool(false) => parser.parse_script().map(SwcProgram::Script),
+        IsModule::Unknown => parser.parse_program(),
+        IsModule::CommonJS => parser.parse_commonjs().map(SwcProgram::Script),
+      };
+      (
+        Some(parser.input_mut().iter_mut().take()),
+        program_result,
+        parser.take_errors(),
+      )
+    } else {
+      let mut parser = Parser::new_from(lexer);
+      let program_result = match is_module {
+        IsModule::Bool(true) => parser.parse_module().map(SwcProgram::Module),
+        IsModule::Bool(false) => parser.parse_script().map(SwcProgram::Script),
+        IsModule::Unknown => parser.parse_program(),
+        IsModule::CommonJS => parser.parse_commonjs().map(SwcProgram::Script),
+      };
+      (None, program_result, parser.take_errors())
     };
-    let mut errors = parser.take_errors();
+
     // Using combinator will let rustc unhappy.
     match program_result {
       Ok(program) => {
         if !errors.is_empty() {
           return Err(errors);
         }
-        Ok(program)
+        Ok((program, tokens))
       }
       Err(err) => {
         errors.push(err);

crates/rspack_plugin_javascript/Cargo.toml

@@ -50,7 +50,6 @@ swc_core = { workspace = true, features = [
   "base",
   "ecma_quote",
 ] }
-swc_ecma_lexer = { workspace = true }
 swc_node_comments = { workspace = true }
 tokio = { workspace = true, features = ["sync"] }
 tracing = { workspace = true }

crates/rspack_plugin_javascript/src/parser_and_generator/mod.rs

@@ -1,6 +1,5 @@
 use std::{borrow::Cow, sync::Arc};
 
-use itertools::Itertools;
 use rspack_cacheable::{cacheable, cacheable_dyn, with::Skip};
 use rspack_core::{
   AsyncDependenciesBlockIdentifier, BuildMetaExportsType, COLLECTED_TYPESCRIPT_INFO_PARSE_META_KEY,
@@ -196,27 +195,14 @@ impl ParserAndGenerator for JavaScriptParserAndGenerator {
       Some(&comments),
     );
 
-    let lexer = swc_ecma_lexer::Lexer::new(
-      Syntax::Es(EsSyntax {
-        allow_return_outside_function: matches!(
-          module_type,
-          ModuleType::JsDynamic | ModuleType::JsAuto
-        ),
-        import_attributes: true,
-        ..Default::default()
-      }),
-      target,
-      SourceFileInput::from(&*fm),
-      Some(&comments),
-    );
-
     let javascript_compiler = JavaScriptCompiler::new();
 
-    let mut ast = match javascript_compiler.parse_with_lexer(
+    let (mut ast, tokens) = match javascript_compiler.parse_with_lexer(
       &fm,
       parser_lexer,
       module_type_to_is_module(module_type),
       Some(comments.clone()),
+      true,
     ) {
       Ok(ast) => ast,
       Err(e) => {
@@ -235,7 +221,8 @@ impl ParserAndGenerator for JavaScriptParserAndGenerator {
       ));
       program.visit_with(&mut semicolon::InsertedSemicolons {
         semicolons: &mut semicolons,
-        tokens: &lexer.collect_vec(),
+        // safety: it's safe to assert tokens is some since we pass with_tokens = true
+        tokens: &tokens.expect("should get tokens from parser"),
       });
     });
 

crates/rspack_plugin_javascript/src/visitors/semicolon.rs

@@ -3,10 +3,10 @@ use swc_core::{
   common::{BytePos, Span},
   ecma::{
     ast::ClassMember,
+    parser::unstable::{Token, TokenAndSpan},
     visit::{Visit, VisitWith},
   },
 };
-use swc_ecma_lexer::token::{Token, TokenAndSpan};
 
 /// Auto inserted semicolon
 /// See: https://262.ecma-international.org/7.0/#sec-rules-of-automatic-semicolon-insertion