---
hide:
  - toc
search:
  exclude: true
---
```````pony linenums="1"
class ref JSONTokenParser
  """
  Streaming, incremental JSON token parser.

  Feed bytes as they arrive with `feed()`; the parser walks the JSON
  structure to any depth and pushes tokens to a `JSONTokenNotify`
  callback as they complete. It builds no tree — its working memory
  is the container-depth stack, the single string or number it is
  part-way through, and the fed bytes it has not yet consumed. A
  value split across a chunk boundary is held and stitched onto the
  next `feed()`, so you never manage leftover bytes.

  ```pony
  let parser = JSONTokenParser(
    object is JSONTokenNotify
      fun ref apply(p: JSONTokenParser, token: JSONToken) =>
        match token
        | let k: JSONTokenKey    => // k.value
        | let s: JSONTokenString => // s.value
        | let n: JSONTokenNumber => // n.value
        | JSONTokenObjectStart   => // ...
        end
    end)
  parser.feed(chunk)?
  ```

  It parses a *stream* of top-level values: after one value's
  tokens end it continues to the next, so newline-delimited JSON or
  a socket delivering messages back to back works with no per-value
  setup. Feeding a whole document at once is just the case where
  every byte is already in hand.

  A number is the one value with no self-delimiter — the parser
  cannot know a number is finished until a following non-number byte
  arrives, so a number at the very end of the fed bytes is not
  emitted until more bytes come or you call `finish()`. `finish()`
  says "no more bytes are coming" and completes it.

  Malformed input raises from `feed()`/`finish()`; read
  `describe_error()`, `token_end()`, and `line()` for the location.
  After a raise the parser latches: every later `feed()`/`finish()`
  raises again. `abort()` (called from the notifier) stops the
  parse the same way. For untrusted input, `JSONParseLimits`
  caps nesting depth and the length of a single string or number.

  To turn a token stream back into a `JSONValue`, use `JSONReassembler`. For a
  whole in-memory document, `JSONParser.parse()` is simpler.
  """

  let _notify: JSONTokenNotify
  let _limits: JSONParseLimits
  embed _reader: _ChunkReader
  embed _frames: Array[_StreamFrame]
  var _scan: (_LeafScan | None) = None
  var _offset: USize = 0
  var _line: USize = 1
  var _token_start: USize = 0
  var _state: _ParserState = _Open
  var _error_message: String = ""
  var _running: Bool = false

  new ref create(
    notify: JSONTokenNotify,
    limits: JSONParseLimits = JSONParseLimits)
  =>
    _notify = notify
    _limits = limits
    _reader = _ChunkReader
    _frames = Array[_StreamFrame]

  fun ref feed(data: ByteSeq) ? =>
    """
    Append a chunk of bytes and emit every token now derivable from the bytes in
    hand. Holds any partial value to be finished by a later `feed()`. Raises on
    malformed input or `abort()`; after that the parser is done and every later
    `feed()`/`finish()` raises. Do not call `feed()` or `finish()`
    from within the notifier — the parse is running and a re-entrant
    call raises.
    """
    if (_state isnt _Open) or _running then error end
    // Drop empty chunks: a producer that loops feeding zero-byte reads would
    // otherwise pile up empty buffer nodes that never drain.
    if data.size() > 0 then _reader.append(data) end
    _running = true
    try _run()? else _running = false; error end
    _running = false

  fun ref finish() ? =>
    """
    Signal that no more bytes will be fed. Completes a pending number (the one
    value that otherwise waits for a following byte) and emits its
    token. Raises if that number's text is not valid, or if
    aborted/already done. A value left structurally incomplete (an
    open container, an unterminated string) is simply
    not completed — the consumer sees the truncation via `incomplete()`. After
    `finish()` the parser is done; feeding more raises.
    """
    if (_state isnt _Open) or _running then error end
    _running = true
    try
      match _scan
      | let ns: _NumberScan =>
        _token_start = ns.start
        _scan = None
        match \exhaustive\ _NumberParse(ns.buf)
        | let v: (I64 | F64) => _emit(JSONTokenNumber(v))?
        | _NumberMalformed => _fail_msg("invalid number")?
        | _NumberOutOfRange => _fail_msg("Number out of range")?
        end
      end
      _running = false
    else
      _running = false
      error
    end
    _state = _Finished

  fun ref abort() =>
    """
    Signal the parser to stop. The current `feed()`/`finish()` then
    raises, and the parser is done. Call this from the notifier when
    you have seen enough.
    """
    _state = _Aborted

  fun token_start(): USize =>
    """
    Byte offset where the current token starts, absolute across all feeds. Valid
    only during the notify callback.
    """
    _token_start

  fun token_end(): USize =>
    """
    Byte offset just past the current token, absolute across all
    feeds. Valid only during the notify callback.
    """
    _offset

  fun line(): USize =>
    """
    Current line number (1-based).
    """
    _line

  fun incomplete(): Bool =>
    """
    True while a value is part-way through — either mid-scan on a
    leaf or with a container still open. After feeding a complete
    document and calling `finish()` this is false; if it is still
    true, the input ended in the middle of a value. This is the
    byte-level truncation check; `JSONReassembler.mid_value()`
    reports only whether the reassembler is holding a partial
    value's tokens.
    """
    (_scan isnt None) or (_frames.size() > 0)

  fun describe_error(): String =>
    """
    Human-readable description of the most recent error.
    """
    match _state
    | _Aborted => "Parse aborted by the notifier"
    | _Finished => "Input already finished"
    else
      if _error_message != "" then
        _error_message
      else
        "Invalid JSON at byte offset " + _offset.string() +
          ", line " + _line.string()
      end
    end

  fun parse_error(): JSONParseError =>
    """
    The most recent error as a `JSONParseError` (message, byte offset, line).
    Meaningful after a raise from `feed()`/`finish()`.
    """
    JSONParseError(describe_error(), _offset, _line)

  // --- driving loop -------------------------------------------------------
  fun ref _run() ? =>
    var progress = true
    while progress do
      match \exhaustive\ _scan
      | let ss: _StringScan =>
        match \exhaustive\ _resume_string(ss)
        | _ScanNeedMore => progress = false
        | let e: JSONParseError => _fail(e)?
        | _ScanComplete =>
          let value: String =
            try ss.result as String val
            else _Unreachable(); "" end
          _token_start = ss.start
          _scan = None
          if ss.is_key then
            _emit(JSONTokenKey(value))?
            _set_object_pos(_ObjColon)
          else
            _emit(JSONTokenString(value))?
          end
        end
      | let ns: _NumberScan =>
        match \exhaustive\ _resume_number(ns)
        | _ScanNeedMore => progress = false
        | let e: JSONParseError => _fail(e)?
        | _ScanComplete =>
          _token_start = ns.start
          _scan = None
          match \exhaustive\ _NumberParse(ns.buf)
          | let v: (I64 | F64) => _emit(JSONTokenNumber(v))?
          | _NumberMalformed => _fail_msg("invalid number")?
          | _NumberOutOfRange => _fail_msg("Number out of range")?
          end
        end
      | let ks: _KeywordScan =>
        match \exhaustive\ _resume_keyword(ks)
        | _ScanNeedMore => progress = false
        | let e: JSONParseError => _fail(e)?
        | _ScanComplete =>
          _token_start = ks.start
          let t = ks.token
          _scan = None
          _emit(t)?
        end
      | None =>
        match \exhaustive\ _dispatch()?
        | _StepGo => None
        | _ScanNeedMore => progress = false
        end
      end
    end

  // --- dispatch between tokens --------------------------------------------
  fun ref _dispatch(): (_StepGo | _ScanNeedMore) ? =>
    _skip_whitespace()
    let b = try _reader.peek()? else return _ScanNeedMore end
    if _frames.size() == 0 then
      _begin_value(b)?
    else
      match \exhaustive\ _top_frame()
      | let f: _ArrayFrame => _dispatch_array(f, b)?
      | let f: _ObjectFrame => _dispatch_object(f, b)?
      end
    end

  fun ref _top_frame(): _StreamFrame =>
    try _frames(_frames.size() - 1)?
    else _Unreachable(); _ArrayFrame(_ArrValueOrEnd)
    end

  fun ref _dispatch_array(f: _ArrayFrame, b: U8): _StepGo ? =>
    match \exhaustive\ f.pos
    | _ArrValueOrEnd =>
      if b == ']' then _close_array()?
      else f.pos = _ArrCommaOrEnd; _begin_value(b)? end
    | _ArrValue =>
      if b == ']' then _fail_msg("trailing comma in array")?
      else f.pos = _ArrCommaOrEnd; _begin_value(b)? end
    | _ArrCommaOrEnd =>
      if b == ',' then _consume(); f.pos = _ArrValue; _StepGo
      elseif b == ']' then _close_array()?
      else _fail_msg("expected ',' or ']' in array")? end
    end

  fun ref _dispatch_object(f: _ObjectFrame, b: U8): _StepGo ? =>
    match \exhaustive\ f.pos
    | _ObjKeyOrEnd =>
      if b == '}' then _close_object()?
      elseif b == '"' then _begin_key()
      else _fail_msg("expected a key or '}' in object")? end
    | _ObjKey =>
      if b == '"' then _begin_key()
      else _fail_msg("expected a key in object")? end
    | _ObjColon =>
      if b == ':' then _consume(); f.pos = _ObjValue; _StepGo
      else _fail_msg("expected ':' in object")? end
    | _ObjValue =>
      f.pos = _ObjCommaOrEnd; _begin_value(b)?
    | _ObjCommaOrEnd =>
      if b == ',' then _consume(); f.pos = _ObjKey; _StepGo
      elseif b == '}' then _close_object()?
      else _fail_msg("expected ',' or '}' in object")? end
    end

  fun ref _begin_value(b: U8): _StepGo ? =>
    match b
    | '{' => _open_object()?
    | '[' => _open_array()?
    | '"' =>
      let start = _offset
      _consume() // opening '"'
      _reader.scan_reset() // scan cursor starts at the string content
      _scan = _StringScan(false, start)
      _StepGo
    | 't' => _scan = _KeywordScan(_offset, "true", JSONTokenTrue); _StepGo
    | 'f' => _scan = _KeywordScan(_offset, "false", JSONTokenFalse); _StepGo
    | 'n' => _scan = _KeywordScan(_offset, "null", JSONTokenNull); _StepGo
    else
      if (b == '-') or ((b >= '0') and (b <= '9')) then
        _scan = _NumberScan(_offset)
        _StepGo
      else
        _fail_msg("expected a value")?
      end
    end

  fun ref _begin_key(): _StepGo =>
    let start = _offset
    _consume() // opening '"'
    _reader.scan_reset() // scan cursor starts at the string content
    _scan = _StringScan(true, start)
    _StepGo

  fun ref _open_object(): _StepGo ? =>
    if _frames.size() >= _limits.max_depth then
      _fail_msg("maximum nesting depth exceeded")?
    end
    _token_start = _offset
    _consume() // '{'
    _emit(JSONTokenObjectStart)?
    _frames.push(_ObjectFrame(_ObjKeyOrEnd))
    _StepGo

  fun ref _open_array(): _StepGo ? =>
    if _frames.size() >= _limits.max_depth then
      _fail_msg("maximum nesting depth exceeded")?
    end
    _token_start = _offset
    _consume() // '['
    _emit(JSONTokenArrayStart)?
    _frames.push(_ArrayFrame(_ArrValueOrEnd))
    _StepGo

  fun ref _close_object(): _StepGo ? =>
    _token_start = _offset
    _consume() // '}'
    _emit(JSONTokenObjectEnd)?
    try _frames.pop()? else _Unreachable() end
    _StepGo

  fun ref _close_array(): _StepGo ? =>
    _token_start = _offset
    _consume() // ']'
    _emit(JSONTokenArrayEnd)?
    try _frames.pop()? else _Unreachable() end
    _StepGo

  fun ref _set_object_pos(pos: _ObjPos) =>
    match _top_frame()
    | let f: _ObjectFrame => f.pos = pos
    else _Unreachable()
    end

  // --- leaf scanners (suspendable) ----------------------------------------
  fun ref _resume_string(s: _StringScan): _ScanOutcome =>
    // Scan forward for the closing quote WITHOUT consuming,
    // tracking whether any escape appears. A string with no escapes
    // that lies within one chunk can then be handed back as a
    // zero-copy view; anything else is decoded. The scan resumes
    // across feeds from where it stopped, so it visits each byte
    // once.
    while not _reader.scan_at_end() do
      if _reader.scan_len() > _limits.max_string_len then
        return _err("string exceeds maximum length")
      end
      let c = try _reader.scan_byte()? else return _ScanNeedMore end
      if s.escaped then
        s.escaped = false
        _reader.scan_advance()
      elseif c == '"' then
        return _finish_string(s)
      elseif c == '\\' then
        s.had_escape = true
        s.escaped = true
        _reader.scan_advance()
      elseif c < 0x20 then
        return _err("control character in string")
      else
        _reader.scan_advance()
      end
    end
    _ScanNeedMore

  fun ref _finish_string(s: _StringScan): _ScanOutcome =>
    // The scan cursor is at the closing quote; the content is [consume, scan).
    if (not s.had_escape) and _reader.scan_single_chunk() then
      match _reader.view()
      | let v: String val =>
        let n = _reader.scan_len()
        _reader.skip(n + 1) // content + closing quote
        // The content holds no raw newline (control bytes are
        // rejected above), so only the byte offset moves; the line
        // number is unchanged.
        _offset = _offset + n + 1
        s.result = v
        return _ScanComplete
      end
    end
    // Copy path: escapes, or the content spans chunks — decode it. The whole
    // string is present now, so the decode runs in one pass and cannot suspend,
    // and it re-checks no limit: the extent scan already bounded the source
    // length, and every JSON escape decodes to no more bytes than it spans
    // (`\n` 2->1, `\uXXXX` 6->at most 3), so the decoded buffer is bounded too.
    // A new escape whose decoded form could exceed its source would break that.
    _decode_string(s)

  fun ref _decode_string(s: _StringScan): _ScanOutcome =>
    while _reader.size() > 0 do
      let c = _take()
      match \exhaustive\ s.phase
      | _StrNormal =>
        // A pending high surrogate must be followed immediately by `\u`; this
        // guard (and its twin in _StrEscape) is what lets _apply_unicode assume
        // a low surrogate follows. Don't drop it.
        if (s.pending_high isnt None) and (c != '\\') then
          return _err("unpaired high surrogate")
        end
        if c == '"' then
          s.result = s.buf = recover iso String end
          return _ScanComplete
        elseif c == '\\' then
          s.phase = _StrEscape
        elseif c < 0x20 then
          return _err("control character in string")
        else
          s.buf.push(c)
        end
      | _StrEscape =>
        if (s.pending_high isnt None) and (c != 'u') then
          return _err("unpaired high surrogate")
        end
        match c
        | '"' => s.buf.push('"'); s.phase = _StrNormal
        | '\\' => s.buf.push('\\'); s.phase = _StrNormal
        | '/' => s.buf.push('/'); s.phase = _StrNormal
        | 'b' => s.buf.push(0x08); s.phase = _StrNormal
        | 'f' => s.buf.push(0x0C); s.phase = _StrNormal
        | 'n' => s.buf.push('\n'); s.phase = _StrNormal
        | 'r' => s.buf.push('\r'); s.phase = _StrNormal
        | 't' => s.buf.push('\t'); s.phase = _StrNormal
        | 'u' => s.phase = _StrUnicode; s.hex_value = 0; s.hex_count = 0
        else return _err("invalid escape sequence")
        end
      | _StrUnicode =>
        match \exhaustive\ _hex_digit(c)
        | let hd: U32 =>
          s.hex_value = (s.hex_value << 4) or hd
          s.hex_count = s.hex_count + 1
          if s.hex_count == 4 then
            match _apply_unicode(s)
            | let e: JSONParseError => return e
            end
          end
        | None => return _err("invalid unicode escape")
        end
      end
    end
    // The extent scan already found the closing quote, so we never
    // run dry here.
    _Unreachable()
    _ScanNeedMore

  fun ref _apply_unicode(s: _StringScan): (None | JSONParseError) =>
    let value = s.hex_value
    match \exhaustive\ s.pending_high
    | let high: U32 =>
      if (value >= 0xDC00) and (value < 0xE000) then
        let combined: U32 =
          0x10000 + (((high and 0x3FF) << 10) or (value and 0x3FF))
        s.buf.append(recover val String.from_utf32(combined) end)
        s.pending_high = None
        s.phase = _StrNormal
        None
      else
        _err("expected a low surrogate")
      end
    | None =>
      if (value >= 0xD800) and (value < 0xDC00) then
        // High surrogate: a low-surrogate `\u` escape must follow immediately.
        // The `pending_high isnt None` guards in _resume_string reject any byte
        // other than `\` then `u` while a high surrogate is pending.
        s.pending_high = value
        s.phase = _StrNormal
        None
      elseif (value >= 0xDC00) and (value < 0xE000) then
        _err("unexpected low surrogate")
      else
        s.buf.append(recover val String.from_utf32(value) end)
        s.phase = _StrNormal
        None
      end
    end

  fun ref _resume_number(s: _NumberScan): _ScanOutcome =>
    while _reader.size() > 0 do
      // The loop guard guarantees a byte, so the peek cannot fail here.
      let c = try _reader.peek()? else _Unreachable(); U8(0) end
      if not _is_number_byte(c) then
        // A following non-number byte terminates the number; leave it in the
        // reader for dispatch. A number is never completed on exhaustion.
        return _ScanComplete
      end
      // `>=` keeps the bound tight: the terminator is peeked, not
      // consumed here, so reject the byte that would exceed the
      // limit before pushing it.
      if s.buf.size() >= _limits.max_number_len then
        return _err("number exceeds maximum length")
      end
      _consume()
      s.buf.push(c)
    end
    _ScanNeedMore

  fun ref _resume_keyword(s: _KeywordScan): _ScanOutcome =>
    while s.matched < s.word.size() do
      if _reader.size() == 0 then return _ScanNeedMore end
      let c = _take()
      let expected = try s.word(s.matched)? else _Unreachable(); U8(0) end
      if c != expected then return _err("invalid literal") end
      s.matched = s.matched + 1
    end
    _ScanComplete

  // --- byte primitives ----------------------------------------------------
  fun ref _skip_whitespace() =>
    while _reader.size() > 0 do
      match try _reader.peek()? else return end
      | ' ' | '\t' | '\r' | '\n' => _consume()
      else return
      end
    end

  fun ref _consume() =>
    try
      let c = _reader.peek()?
      _reader.skip1()
      _offset = _offset + 1
      if c == '\n' then _line = _line + 1 end
    else
      _Unreachable()
    end

  fun ref _take(): U8 =>
    try
      let c = _reader.peek()?
      _reader.skip1()
      _offset = _offset + 1
      if c == '\n' then _line = _line + 1 end
      c
    else
      _Unreachable()
      U8(0)
    end

  fun _hex_digit(c: U8): (U32 | None) =>
    if (c >= '0') and (c <= '9') then (c - '0').u32()
    elseif (c >= 'a') and (c <= 'f') then ((c - 'a') + 10).u32()
    elseif (c >= 'A') and (c <= 'F') then ((c - 'A') + 10).u32()
    else None
    end

  fun _is_number_byte(c: U8): Bool =>
    ((c >= '0') and (c <= '9')) or (c == '-') or (c == '+') or (c == '.') or
      (c == 'e') or (c == 'E')

  // --- token emission and failure -----------------------------------------
  fun ref _emit(token: JSONToken) ? =>
    _notify(this, token)
    if _state is _Aborted then error end

  fun _err(message: String): JSONParseError =>
    JSONParseError(message, _offset, _line)

  fun ref _set_error(message: String) =>
    _state = _Failed
    _error_message = message

  fun ref _fail(e: JSONParseError) ? =>
    _set_error(e.message)
    error

  fun ref _fail_msg(message: String): _StepGo ? =>
    // Always raises; the `_StepGo` return type is only so a
    // dispatch branch that fails can stand where a `_StepGo` value
    // is expected.
    _set_error(message)
    error

```````