Merge pull request #533 from meh/fix-utf8

Fix one_of, none_of, and char when meeting UTF-8 chars

Author: Geal

src/character.rs

@@ -13,12 +13,12 @@ macro_rules! one_of (
       use $crate::InputIter;
 
       match ($i).iter_elements().next().map(|c| {
-        c.find_token($inp)
+        (c, c.find_token($inp))
       }) {
-        None        => $crate::IResult::Incomplete::<_, _>($crate::Needed::Size(1)),
-        Some(false) => $crate::IResult::Error(error_position!($crate::ErrorKind::OneOf, $i)),
+        None             => $crate::IResult::Incomplete::<_, _>($crate::Needed::Size(1)),
+        Some((_, false)) => $crate::IResult::Error(error_position!($crate::ErrorKind::OneOf, $i)),
         //the unwrap should be safe here
-        Some(true)  => $crate::IResult::Done($i.slice(1..), $i.iter_elements().next().unwrap().as_char())
+        Some((c, true))  => $crate::IResult::Done($i.slice(c.len()..), $i.iter_elements().next().unwrap().as_char())
       }
     }
   );
@@ -35,12 +35,12 @@ macro_rules! none_of (
       use $crate::InputIter;
 
       match ($i).iter_elements().next().map(|c| {
-        !c.find_token($inp)
+        (c, !c.find_token($inp))
       }) {
-        None        => $crate::IResult::Incomplete::<_, _>($crate::Needed::Size(1)),
-        Some(false) => $crate::IResult::Error(error_position!($crate::ErrorKind::NoneOf, $i)),
+        None             => $crate::IResult::Incomplete::<_, _>($crate::Needed::Size(1)),
+        Some((_, false)) => $crate::IResult::Error(error_position!($crate::ErrorKind::NoneOf, $i)),
         //the unwrap should be safe here
-        Some(true)  => $crate::IResult::Done($i.slice(1..), $i.iter_elements().next().unwrap().as_char())
+        Some((c, true))  => $crate::IResult::Done($i.slice(c.len()..), $i.iter_elements().next().unwrap().as_char())
       }
     }
   );
@@ -56,12 +56,12 @@ macro_rules! char (
       use $crate::InputIter;
 
       match ($i).iter_elements().next().map(|c| {
-        c.as_char() == $c
+        (c, c.as_char() == $c)
       }) {
-        None        => $crate::IResult::Incomplete::<_, _>($crate::Needed::Size(1)),
-        Some(false) => $crate::IResult::Error(error_position!($crate::ErrorKind::Char, $i)),
+        None             => $crate::IResult::Incomplete::<_, _>($crate::Needed::Size(1)),
+        Some((_, false)) => $crate::IResult::Error(error_position!($crate::ErrorKind::Char, $i)),
         //the unwrap should be safe here
-        Some(true)  => $crate::IResult::Done($i.slice(1..), $i.iter_elements().next().unwrap().as_char())
+        Some((c, true))  => $crate::IResult::Done($i.slice(c.len()..), $i.iter_elements().next().unwrap().as_char())
       }
     }
   );
@@ -93,6 +93,12 @@ mod tests {
 
     let b = &b"cde"[..];
     assert_eq!(f(b), Error(error_position!(ErrorKind::OneOf, b)));
+
+    named!(utf8(&str) -> char,
+      one_of!("+\u{FF0B}"));
+
+    assert!(utf8("+").is_done());
+    assert!(utf8("\u{FF0B}").is_done());
   }
 
   #[test]

src/traits.rs

@@ -70,6 +70,9 @@ pub trait AsChar {
     /// tests that self is an octal digit
     #[inline]
     fn is_oct_digit(self) -> bool;
+    /// gets the len in bytes for self
+    #[inline]
+    fn len(self) -> usize;
 }
 
 impl AsChar for u8 {
@@ -95,6 +98,10 @@ impl AsChar for u8 {
     fn is_oct_digit(self) -> bool {
       self >= 0x30 && self <= 0x37
     }
+    #[inline]
+    fn len(self) -> usize {
+      1
+    }
 }
 impl<'a> AsChar for &'a u8 {
     #[inline]
@@ -119,6 +126,10 @@ impl<'a> AsChar for &'a u8 {
     fn is_oct_digit(self)   -> bool {
       *self >= 0x30 && *self <= 0x37
     }
+    #[inline]
+    fn len(self) -> usize {
+      1
+    }
 }
 
 impl AsChar for char {
@@ -134,6 +145,8 @@ impl AsChar for char {
     fn is_hex_digit(self) -> bool { self.is_digit(16) }
     #[inline]
     fn is_oct_digit(self) -> bool { self.is_digit(8) }
+    #[inline]
+    fn len(self) -> usize { self.len_utf8() }
 }
 
 impl<'a> AsChar for &'a char {
@@ -149,6 +162,8 @@ impl<'a> AsChar for &'a char {
     fn is_hex_digit(self) -> bool { self.is_digit(16) }
     #[inline]
     fn is_oct_digit(self) -> bool { self.is_digit(8) }
+    #[inline]
+    fn len(self) -> usize { self.len_utf8() }
 }
 
 /// abstracts common iteration operations on the input type

tests/issues.rs

@@ -155,4 +155,3 @@ fn issue_302(input: &[u8]) -> IResult<&[u8], Option<Vec<u64>> > {
         ( entries )
     )
 }
-