Merge remote-tracking branch 'origin/bugs'

maciejhirsz · maciejhirsz · commit cdd9918f4ad6 · 2016-11-09T23:21:14.000+01:00
diff --git a/core/src/codegen.rs b/core/src/codegen.rs
@@ -333,6 +333,7 @@ impl Code for Parameter {
 impl Code for Expression {
     fn to_code(&self, gen: &mut Generator) {
         match *self {
+            Expression::Void => {},
 
             Expression::This => gen.write_bytes(b"this"),
 
@@ -381,6 +382,12 @@ impl Code for Expression {
             Expression::Array(ref items) => {
                 gen.write_byte(b'[');
                 gen.write_list(items);
+
+                // Add dangling comma if the array ends with a spare element
+                if let Some(&Expression::Void) = items.iter().rev().next() {
+                    gen.write_byte(b',');
+                }
+
                 gen.write_byte(b']');
             },
 
diff --git a/core/src/grammar.rs b/core/src/grammar.rs
@@ -21,6 +21,7 @@ pub struct Parameter {
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Expression {
+    Void,
     This,
     Identifier(OwnedSlice),
     Literal(Value),
diff --git a/core/src/parser.rs b/core/src/parser.rs
@@ -132,6 +132,10 @@ impl<'a> Parser<'a> {
         loop {
             match next!(self) {
                 BracketClose => break,
+                Comma        => {
+                    list.push(Expression::Void);
+                    continue;
+                }
                 token        => {
                     let expression = try!(self.expression_from_token(token, 0));
                     list.push(expression);
@@ -566,7 +570,7 @@ impl<'a> Parser<'a> {
                 },
 
                 ParenOpen => {
-                    if lbp > 0 {
+                    if lbp > 18 {
                         break;
                     }
 
@@ -579,7 +583,7 @@ impl<'a> Parser<'a> {
                 },
 
                 BracketOpen => {
-                    if lbp > 0 {
+                    if lbp > 19 {
                         break;
                     }
 
@@ -616,6 +620,14 @@ impl<'a> Parser<'a> {
 
     /// Helper for the `for` loops that doesn't consume semicolons
     fn variable_declaration(&mut self, kind: VariableDeclarationKind) -> Result<Statement> {
+        Ok(Statement::VariableDeclaration {
+            kind: kind,
+            declarators: try!(self.variable_declarators()),
+        })
+    }
+
+    #[inline]
+    fn variable_declarators(&mut self) -> Result<Vec<VariableDeclarator>> {
         let mut declarators = Vec::new();
 
         loop {
@@ -636,10 +648,7 @@ impl<'a> Parser<'a> {
             break;
         }
 
-        Ok(Statement::VariableDeclaration {
-            kind: kind,
-            declarators: declarators,
-        })
+        Ok(declarators)
     }
 
     #[inline]
@@ -799,9 +808,40 @@ impl<'a> Parser<'a> {
         let init = match next!(self) {
             Semicolon         => None,
 
-            Declaration(kind) => Some(Box::new(try!(self.variable_declaration(kind)))),
+            Declaration(kind) => {
+                let mut declarators = try!(self.variable_declarators());
+
+                if declarators.len() == 1 {
+                    let value = declarators[0].value.take();
 
-            token             => {
+                    match value {
+                        Some(Expression::Binary {
+                            operator: In,
+                            left,
+                            right,
+                            ..
+                        }) => {
+                            declarators[0].value = Some(*left);
+
+                            let left = Statement::VariableDeclaration {
+                                kind: kind,
+                                declarators: declarators,
+                            };
+
+                            return self.for_in_statement_from_parts(left, *right);
+                        },
+
+                        _ => declarators[0].value = value
+                    }
+                }
+
+                Some(Box::new(Statement::VariableDeclaration {
+                    kind: kind,
+                    declarators: declarators,
+                }))
+            }
+
+            token => {
                 let expression = try!(self.sequence_or_expression_from_token(token));
 
                 if let Expression::Binary {
@@ -810,7 +850,7 @@ impl<'a> Parser<'a> {
                     right,
                     ..
                 } = expression {
-                    return self.for_in_statement_from_expressions(*left, *right);
+                    return self.for_in_statement_from_parts(*left, *right);
                 }
 
                 Some(Box::new(expression.into()))
@@ -858,8 +898,8 @@ impl<'a> Parser<'a> {
         })
     }
 
-    fn for_in_statement_from_expressions(&mut self, left: Expression, right: Expression)
-    -> Result<Statement> {
+    fn for_in_statement_from_parts<S>(&mut self, left: S, right: Expression) -> Result<Statement>
+    where S: Into<Statement> {
         let left = Box::new(left.into());
 
         expect!(self, ParenClose);
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
@@ -43,7 +43,7 @@ static BYTE_HANDLERS: [fn(&mut Tokenizer, u8) -> Result<Token>; 256] = [
     ___, EXL, QOT, ___, IDT, PRC, AMP, QOT, PNO, PNC, ATR, PLS, COM, MIN, PRD, SLH, // 2
     ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3
     ___, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4
-    IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, ___, BTC, CRT, IDT, // 5
+    IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, IDT, BTC, CRT, IDT, // 5
     TPL, IDT, L_B, L_C, L_D, L_E, L_F, IDT, IDT, L_I, IDT, IDT, L_L, IDT, L_N, IDT, // 6
     L_P, IDT, L_R, L_S, L_T, L_U, L_V, L_W, IDT, L_Y, IDT, BEO, PIP, BEC, TLD, ___, // 7
     UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // 8
@@ -829,6 +829,7 @@ mod ident_lookup {
     pub const DO: bool = true; // dollar sign $
     pub const US: bool = true; // underscore
     pub const UN: bool = true; // unicode
+    pub const BS: bool = true; // backslash
     pub const __: bool = false;
 
     pub static TABLE: [bool; 256] = [
@@ -838,7 +839,7 @@ mod ident_lookup {
       __, __, __, __, DO, __, __, __, __, __, __, __, __, __, __, __, // 2
       NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, __, __, __, __, __, __, // 3
       __, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, // 4
-      AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, __, __, __, __, US, // 5
+      AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, __, BS, __, __, US, // 5
       __, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, // 6
       AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, AL, __, __, __, __, __, // 7
       UN, UN, UN, UN, UN, UN, UN, UN, UN, UN, UN, UN, UN, UN, UN, UN, // 8
@@ -1048,6 +1049,8 @@ impl<'a> Tokenizer<'a> {
 
     #[inline]
     fn consume_label_characters(&mut self) -> &str {
+        // TODO: Reject invalid unicode and escaped unicode character
+
         let start = self.index;
 
         self.bump();
diff --git a/core/src/transformer.rs b/core/src/transformer.rs
@@ -29,7 +29,7 @@ impl<T> Take for Vec<T> {
 impl Take for Expression {
     #[inline]
     fn take(&mut self) -> Self {
-        mem::replace(self, Expression::This)
+        mem::replace(self, Expression::Void)
     }
 }
 
@@ -490,6 +490,7 @@ impl Transformable for Expression {
                 return;
             },
 
+            Expression::Void           |
             Expression::This           |
             Expression::Literal(..)    |
             Expression::RegEx {..}     |
diff --git a/core/tests/codegen.rs b/core/tests/codegen.rs
@@ -192,3 +192,14 @@ fn class_method_default_parameters() {
 fn try_catch() {
     assert_compile!("try{foo();}catch(err){console.error(err);}", "try{foo();}catch(err){console.error(err);}");
 }
+
+#[test]
+fn ternary_expression() {
+    assert_compile!("const a=1?console.log('foo'):null;", "var a=1?console.log('foo'):null;")
+}
+
+#[test]
+fn sparse_array_expression() {
+    assert_compile!("[,,1];", "[,,1];");
+    assert_compile!("[,,];", "[,,];");
+}
diff --git a/core/tests/parser.rs b/core/tests/parser.rs
@@ -688,6 +688,25 @@ fn for_in_declare_statement() {
     });
 }
 
+#[test]
+fn for_in_declare_statement_initial_value() {
+    assert_statement!("for (let item = 0 in object) {}", Statement::ForIn {
+        left: Box::new(Statement::VariableDeclaration {
+            kind: VariableDeclarationKind::Let,
+            declarators: vec![
+                VariableDeclarator {
+                    name: "item".into(),
+                    value: Some(num!("0")),
+                }
+            ],
+        }),
+        right: ident!("object"),
+        body: Box::new(Statement::Block {
+            body: Vec::new(),
+        }),
+    });
+}
+
 #[test]
 fn for_in_declare_no_block_statement() {
     assert_statement!("for (let item in object);", Statement::ForIn {
@@ -748,6 +767,25 @@ fn for_of_declare_statement() {
     });
 }
 
+#[test]
+fn for_of_declare_statement_initial_value() {
+    assert_statement!("for (let item = 0 of array) {}", Statement::ForOf {
+        left: Box::new(Statement::VariableDeclaration {
+            kind: VariableDeclarationKind::Let,
+            declarators: vec![
+                VariableDeclarator {
+                    name: "item".into(),
+                    value: Some(num!("0")),
+                }
+            ],
+        }),
+        right: ident!("array"),
+        body: Box::new(Statement::Block {
+            body: Vec::new(),
+        }),
+    });
+}
+
 #[test]
 fn for_of_declare_no_block_statement() {
     assert_statement!("for (let item of array);", Statement::ForOf {
diff --git a/core/tests/tokenizer.rs b/core/tests/tokenizer.rs
@@ -30,6 +30,18 @@ macro_rules! assert_token {
     }
 }
 
+#[test]
+fn test_tokenizer_identifier() {
+    assert_token!("foo", Identifier("foo".into()), "read a plain identifier");
+    assert_token!("_foo", Identifier("_foo".into()), "read an identifier starting with _");
+    assert_token!("$foo", Identifier("$foo".into()), "read an identifier starting with $");
+    assert_token!("_0", Identifier("_0".into()), "read an identifier with _ and digit");
+    assert_token!("$0", Identifier("$0".into()), "read an identifier with $ and digit");
+    assert_token!("π", Identifier("π".into()), "read a unicode identifier");
+    assert_token!("\\u0050", Identifier("\\u0050".into()), "read an escaped unicode identifier");
+    assert_token!("_\\u0050", Identifier("_\\u0050".into()), "read an identifier with escaped unicode character");
+}
+
 #[test]
 fn test_tokenizer_chars() {
     assert_token!(";", Semicolon, "read a Semicolon");

Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,7 @@ impl<T> Take for Vec<T> {`
`29`	`29`	`impl Take for Expression {`
`30`	`30`	`#[inline]`
`31`	`31`	`fn take(&mut self) -> Self {`
`32`		`- mem::replace(self, Expression::This)`
	`32`	`+ mem::replace(self, Expression::Void)`
`33`	`33`	`}`
`34`	`34`	`}`
`35`	`35`
`@@ -490,6 +490,7 @@ impl Transformable for Expression {`
`490`	`490`	`return;`
`491`	`491`	`},`
`492`	`492`
	`493`	`+ Expression::Void \|`
`493`	`494`	`Expression::This \|`
`494`	`495`	`Expression::Literal(..) \|`
`495`	`496`	`Expression::RegEx {..} \|`