Messy, but effective subset-alternation for Utf8/ASCII text

Author: archaephyrryx

src/bin/doodle/format/text.rs

@@ -132,15 +132,17 @@ pub fn main(module: &mut FormatModule, base: &BaseModule) -> FormatRef {
         ),
     );
 
-    let ascii_str = module.define_format("text.string.ascii", repeat1(base.ascii_char_strict()));
-    let utf8_str = module.define_format("text.string.utf8", repeat(utf8_char.call()));
+    let ascii_char = module.define_format(
+        "text.char.ascii",
+        Format::Map(
+            Box::new(base.ascii_char_strict()),
+            Expr::Lambda("byte".into(), Box::new(Expr::AsChar(Box::new(var("byte"))))),
+        ),
+    );
 
     module.define_format(
         "text.string",
-        Format::NondetUnion(vec![
-            ("ascii".into(), ascii_str.call()),
-            ("utf8".into(), utf8_str.call()),
-        ]),
+        Format::RepeatFallback(Box::new(ascii_char.call()), Box::new(utf8_char.call())),
     )
 }
 

src/lib.rs

@@ -133,6 +133,7 @@ pub enum Value {
     Variant(Cow<'static, str>, Box<Value>),
     Seq(Vec<Value>),
     Mapped(Box<Value>, Box<Value>),
+    Fallback(bool, Box<Value>),
     Format(Box<Format>),
 }
 
@@ -466,6 +467,8 @@ pub enum Format {
     RepeatUntilLast(Expr, Box<Format>),
     /// Repeat a format until a condition is satisfied by the sequence
     RepeatUntilSeq(Expr, Box<Format>),
+    /// Repeat an eager narrow format, but continue with a broader item if necessary upon recoverable failure
+    RepeatFallback(Box<Format>, Box<Format>),
     /// Parse a format without advancing the stream position afterwards
     Peek(Box<Format>),
     /// Attempt to parse a format and fail if it succeeds
@@ -660,6 +663,11 @@ impl FormatModule {
                 let t = self.infer_format_type(scope, a)?;
                 Ok(ValueType::Seq(Box::new(t)))
             }
+            Format::RepeatFallback(narrow, wide) => {
+                let mut t = self.infer_format_type(scope, narrow)?;
+                t = t.unify(&self.infer_format_type(scope, wide)?)?;
+                Ok(ValueType::Seq(Box::new(t)))
+            }
             Format::Peek(a) => self.infer_format_type(scope, a),
             Format::PeekNot(_a) => Ok(ValueType::Tuple(vec![])),
             Format::Slice(_expr, a) => self.infer_format_type(scope, a),
@@ -771,6 +779,7 @@ enum Decoder {
     Record(Vec<(Cow<'static, str>, Decoder)>),
     While(MatchTree, Box<Decoder>),
     Until(MatchTree, Box<Decoder>),
+    RepeatFallback(MatchTree, Box<Decoder>, Box<Decoder>),
     RepeatCount(Expr, Box<Decoder>),
     RepeatUntilLast(Expr, Box<Decoder>),
     RepeatUntilSeq(Expr, Box<Decoder>),
@@ -1310,6 +1319,7 @@ impl Format {
             Format::RepeatCount(expr, f) => f.match_bounds(module) * expr.bounds(),
             Format::RepeatUntilLast(_, f) => f.match_bounds(module) * Bounds::new(1, None),
             Format::RepeatUntilSeq(_, _f) => Bounds::new(0, None),
+            Format::RepeatFallback(_narrow, _wide) => Bounds::new(0, None),
             Format::Peek(_) => Bounds::exact(0),
             Format::PeekNot(_) => Bounds::exact(0),
             Format::Slice(expr, _) => expr.bounds(),
@@ -1354,6 +1364,7 @@ impl Format {
             Format::Record(fields) => fields.iter().any(|(_, f)| f.depends_on_next(module)),
             Format::Repeat(_) => true,
             Format::Repeat1(_) => true,
+            Format::RepeatFallback(_narrow, _wide) => true,
             Format::RepeatCount(_, _f) => false,
             Format::RepeatUntilLast(_, _f) => false,
             Format::RepeatUntilSeq(_, _f) => false,
@@ -1409,6 +1420,14 @@ impl Format {
         }
     }
 
+    pub fn is_char_format(&self, module: &FormatModule) -> bool {
+        match self {
+            // NOTE - currently only true for named formats matching `/.*char.*/`
+            Format::ItemVar(level, _args) => module.get_name(*level).contains("char"),
+            _ => false,
+        }
+    }
+
     /// Returns `true` if values associated to this format should be handled as multi-character ASCII strings
     pub fn is_ascii_string_format(&self, module: &FormatModule) -> bool {
         match self {
@@ -1666,6 +1685,19 @@ impl<'a> MatchTreeStep<'a> {
             Format::RepeatUntilSeq(_expr, _a) => {
                 Self::accept() // FIXME
             }
+            Format::RepeatFallback(narrow, wide) => {
+                let tree = Self::add_next(module, next.clone());
+                tree.union(Self::add(
+                    module,
+                    narrow,
+                    Rc::new(Next::Repeat(narrow, next.clone())),
+                ))
+                .union(Self::add(
+                    module,
+                    wide,
+                    Rc::new(Next::Repeat(wide, next.clone())),
+                ))
+            }
             Format::Peek(a) => {
                 let tree = Self::add_next(module, next.clone());
                 let peek = Self::add(module, a, Rc::new(Next::Empty));
@@ -2334,6 +2366,33 @@ impl Decoder {
                     Err(format!("cannot build match tree for {:?}", format))
                 }
             }
+            Format::RepeatFallback(narrow, wide) => {
+                if narrow.is_nullable(compiler.module) || wide.is_nullable(compiler.module) {
+                    return Err(format!(
+                        "Cannot repeat nullable format: Repeat({narrow:?} ⊂ {wide:?})"
+                    ));
+                }
+
+                let dnarrow = Box::new(Decoder::compile_next(
+                    compiler,
+                    narrow,
+                    Rc::new(Next::Repeat(narrow, next.clone())),
+                )?);
+
+                let dwide = Box::new(Decoder::compile_next(
+                    compiler,
+                    wide,
+                    Rc::new(Next::Repeat(wide, next.clone())),
+                )?);
+
+                if let Some(tree) =
+                    MatchTree::build(compiler.module, std::slice::from_ref(format), next)
+                {
+                    Ok(Decoder::RepeatFallback(tree, dnarrow, dwide))
+                } else {
+                    Err(format!("canot build match treee for {:?}", format))
+                }
+            }
             Format::Repeat1(a) => {
                 if a.is_nullable(compiler.module) {
                     return Err(format!("cannot repeat nullable format: {a:?}"));
@@ -2545,6 +2604,44 @@ impl Decoder {
                 }
                 Ok((Value::Seq(v), input))
             }
+            Decoder::RepeatFallback(_tree, subset, superset) => {
+                let mut accum = Vec::new();
+                let mut incr_input = input;
+
+                loop {
+                    match subset.parse(program, scope, incr_input) {
+                        Ok((b, next_input)) => {
+                            accum.push(b);
+                            incr_input = next_input;
+                        }
+                        Err(_err) => break,
+                    }
+                }
+
+                // track the last offset we reached to see if we get any further
+                let checkpoint = incr_input.offset;
+
+                loop {
+                    match superset.parse(program, scope, incr_input) {
+                        Ok((b, next_input)) => {
+                            accum.push(b);
+                            incr_input = next_input;
+                        }
+                        Err(_err) => break,
+                    }
+                }
+
+                if let Err(e) = (Decoder::EndOfInput).parse(program, scope, incr_input) {
+                    return Err(e);
+                }
+
+                let fellthrough = incr_input.offset > checkpoint;
+
+                Ok((
+                    Value::Fallback(fellthrough, Box::new(Value::Seq(accum))),
+                    incr_input,
+                )) // index 1
+            }
             Decoder::RepeatCount(expr, a) => {
                 let mut input = input;
                 let count = expr.eval_value(scope).unwrap_usize();

src/output/flat.rs

@@ -155,6 +155,10 @@ fn check_covered(
         | Format::RepeatUntilSeq(_, format) => {
             check_covered(module, path, format)?;
         }
+        Format::RepeatFallback(narrow, wide) => {
+            check_covered(module, path, narrow)?;
+            check_covered(module, path, wide)?;
+        }
         Format::Peek(_) => {}    // FIXME
         Format::PeekNot(_) => {} // FIXME
         Format::Slice(_, format) => {
@@ -260,6 +264,15 @@ impl<'module, W: io::Write> Context<'module, W> {
                 }
                 _ => panic!("expected sequence"),
             },
+            Format::RepeatFallback(_narrow, _wide) => match value {
+                Value::Seq(values) => {
+                    for _v in values {
+                        (); // FIXME
+                    }
+                    Ok(())
+                }
+                _ => panic!("expected sequence"),
+            },
             Format::Peek(format) => self.write_flat(scope, value, format),
             Format::PeekNot(format) => self.write_flat(scope, value, format),
             Format::Slice(_, format) => self.write_flat(scope, value, format),

src/output/tree.rs

@@ -120,6 +120,13 @@ impl<'module> MonoidalPrinter<'module> {
             Value::Char(_) => true,
             Value::Bool(_) => true,
             Value::U8(_) | Value::U16(_) | Value::U32(_) => true,
+            Value::Fallback(is_fallback, v) => match format {
+                Some(Format::RepeatFallback(a, b)) => {
+                    let format = if *is_fallback { b } else { a };
+                    self.is_atomic_value(v, Some(format))
+                }
+                _ => self.is_atomic_value(v, None),
+            },
             Value::Tuple(values) => values.is_empty(),
             Value::Record(fields) => fields.is_empty(),
             Value::Seq(values) => values.is_empty(),
@@ -250,6 +257,30 @@ impl<'module> MonoidalPrinter<'module> {
                 }
                 _ => panic!("expected sequence, found {value:?}"),
             },
+            Format::RepeatFallback(narrow, wide) => match value {
+                Value::Fallback(is_wide, v) => match v.deref() {
+                    Value::Seq(values) => {
+                        let format = if *is_wide { wide } else { narrow };
+                        if self.flags.tables_for_record_sequences
+                            && self.is_record_with_atomic_fields(format).is_some()
+                        {
+                            self.compile_seq_records(values, format)
+                        } else if self.flags.pretty_ascii_strings
+                            && format.is_ascii_char_format(self.module)
+                        {
+                            self.compile_ascii_seq(values)
+                        } else if self.flags.pretty_utf8_strings
+                            && format.is_char_format(self.module)
+                        {
+                            self.compile_char_seq(values)
+                        } else {
+                            self.compile_seq(scope, values, Some(format))
+                        }
+                    }
+                    _ => panic!("expected sequence, found {v:?}"),
+                },
+                _ => panic!("expected Fallback, found {value:?}"),
+            },
             Format::Peek(format) => self.compile_decoded_value(scope, value, format),
             Format::PeekNot(_format) => self.compile_value(scope, value),
             Format::Slice(_, format) => self.compile_decoded_value(scope, value, format),
@@ -315,6 +346,7 @@ impl<'module> MonoidalPrinter<'module> {
             Value::Seq(vals) => self.compile_seq(scope, vals, None),
             Value::Record(fields) => self.compile_record(scope, fields, None),
             Value::Variant(label, value) => self.compile_variant(scope, label, value, None),
+            Value::Fallback(_, value) => self.compile_value(scope, value),
             Value::Mapped(orig, value) => {
                 if self.flags.collapse_mapped_values {
                     self.compile_value(scope, value)
@@ -1010,6 +1042,15 @@ impl<'module> MonoidalPrinter<'module> {
                 prec,
                 Precedence::FORMAT_COMPOUND,
             ),
+            Format::RepeatFallback(narrow, wide) => {
+                let wide_frag = self.compile_format(wide, Precedence::FORMAT_ATOM);
+
+                cond_paren(
+                    self.compile_nested_format("repeat_fallback", Some(&[wide_frag]), narrow, prec),
+                    prec,
+                    Precedence::FORMAT_COMPOUND,
+                )
+            }
             Format::Repeat(format) => cond_paren(
                 self.compile_nested_format("repeat", None, format, prec),
                 prec,

tests/expected/decode/test.txt.stdout

@@ -1,3 +1,3 @@
 ├── data <- _ |...| _ :=
-│   └── text <- text.string := { ascii := "GIF89a is a popular format\n" }
+│   └── text <- text.string := "GIF89a is a popular format\n"
 └── end <- end-of-input

tests/expected/decode/test.utf8.stdout

@@ -1,4 +1,3 @@
 ├── data <- _ |...| _ :=
-│   └── text <- text.string :=
-│       └── utf8 <- text.string.utf8 := "この🦀は擂り身ではなく、本物のカニです。\n"
+│   └── text <- text.string := "この🦀は擂り身ではなく、本物のカニです。\n"
 └── end <- end-of-input