Fix parsing RS-separated streams

Fix the parsing loop to use the proper interface, thus fixing parsing
RS-separated streams (RFC 7464). Without this, the program would abort
on assertion failure, like this:

    python3: src/jv_parse.c:684: jv_parser_set_buf: Assertion
    `(p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length) &&
    "previous buffer not exhausted"' failed.

Author: spbnick

jq.pyx

@@ -43,6 +43,7 @@ cdef extern from "jv.h":
     jv_parser* jv_parser_new(int)
     void jv_parser_free(jv_parser*)
     void jv_parser_set_buf(jv_parser*, const char*, int, int)
+    int jv_parser_remaining(jv_parser*)
     jv jv_parser_next(jv_parser*)
 
 
@@ -171,11 +172,11 @@ cdef class _JSONParser(object):
         """
         cdef jv value
         while True:
-            # If we have no bytes to parse
-            if self._bytes is None:
-                # Ready some more
+            # If the parser has no buffer set/left
+            if not jv_parser_remaining(self._parser):
+                # Supply it with some bytes
                 self._ready_next_bytes()
-            # Parse whatever we've readied, if any
+            # Get next value from the parser
             value = jv_parser_next(self._parser)
             if jv_is_valid(value):
                 if self._packed:
@@ -189,12 +190,10 @@ cdef class _JSONParser(object):
                 message = jv_string_value(error_message).decode("utf8")
                 jv_free(error_message)
                 raise JSONParseError(message)
-            else:
-                jv_free(value)
-                # If we didn't ready any bytes
-                if self._bytes is None:
-                    raise StopIteration
-                self._bytes = None
+            jv_free(value)
+            # If we supplied no bytes last time
+            if self._bytes is None:
+                raise StopIteration
 
     cdef bint _ready_next_bytes(self) except 1:
         cdef char* cbytes

tests/jq_tests.py

@@ -177,6 +177,58 @@ def unicode_strings_can_be_used_as_input():
     )
 
 
+@istest
+def record_separator_character_accepted_in_input():
+    assert_equal(
+        [],
+        list(jq.compile(".").input(text='\x1e'))
+    )
+    assert_equal(
+        [],
+        list(jq.compile(".").input(text='\x1e\x1e'))
+    )
+    assert_equal(
+        [{}],
+        list(jq.compile(".").input(text='\x1e{}'))
+    )
+    assert_equal(
+        [{}],
+        list(jq.compile(".").input(text='\x1e\x1e{}'))
+    )
+    assert_equal(
+        [{}],
+        list(jq.compile(".").input(text='{}\x1e'))
+    )
+    assert_equal(
+        [{}],
+        list(jq.compile(".").input(text='{}\x1e\x1e'))
+    )
+    assert_equal(
+        [{}],
+        list(jq.compile(".").input(text='\x1e{}\x1e'))
+    )
+    assert_equal(
+        [{},[]],
+        list(jq.compile(".").input(text='{}\x1e[]'))
+    )
+    assert_equal(
+        [{},[]],
+        list(jq.compile(".").input(text='{}\x1e\x1e[]'))
+    )
+    assert_equal(
+        [{},[]],
+        list(jq.compile(".").input(text='\x1e{}\x1e[]'))
+    )
+    assert_equal(
+        [{},[]],
+        list(jq.compile(".").input(text='{}\x1e[]\x1e'))
+    )
+    assert_equal(
+        [{},[]],
+        list(jq.compile(".").input(text='\x1e{}\x1e[]\x1e'))
+    )
+
+
 @istest
 def unicode_strings_can_be_used_as_programs():
     assert_equal(
@@ -204,6 +256,21 @@ def parse_json_all_inputs_accepted():
     assert_equal(True, next(jq.parse_json(text=b"true")))
     assert_equal(True, next(jq.parse_json(text_iter=iter([b"true"]))))
 
+@istest
+def parse_json_record_separator_character_accepted():
+    assert_equal([], list(jq.parse_json(text='\x1e')))
+    assert_equal([], list(jq.parse_json(text='\x1e\x1e')))
+    assert_equal([{}], list(jq.parse_json(text='\x1e{}')))
+    assert_equal([{}], list(jq.parse_json(text='\x1e\x1e{}')))
+    assert_equal([{}], list(jq.parse_json(text='{}\x1e')))
+    assert_equal([{}], list(jq.parse_json(text='{}\x1e\x1e')))
+    assert_equal([{}], list(jq.parse_json(text='\x1e{}\x1e')))
+    assert_equal([{},[]], list(jq.parse_json(text='{}\x1e[]')))
+    assert_equal([{},[]], list(jq.parse_json(text='{}\x1e\x1e[]')))
+    assert_equal([{},[]], list(jq.parse_json(text='\x1e{}\x1e[]')))
+    assert_equal([{},[]], list(jq.parse_json(text='{}\x1e[]\x1e')))
+    assert_equal([{},[]], list(jq.parse_json(text='\x1e{}\x1e[]\x1e')))
+
 @istest
 def parse_json_file_works():
     fp = io.StringIO('{"abc": "def"}')