Document and test parser type internals

src_v2/parser/parser_api.nim

@@ -189,7 +189,7 @@ proc anyUntil*(stopFn: parserFnT): parserFnT {.inline.} =
   manyUntil(anyCh, stopFn)
 
 proc choice*(parserFns: seq[parserFnT]): parserFnT {.inline} =
-  ## Creates parser function that checks any of the `parserFns`.
+  ## creates parser function that checks any of the `parserFns`.
   ## Needs one match for a `ParserResult.ok`.
   return proc(parser: Parser): ParserResult {.closure.} =
     var errors: seq[ParserResult] = newSeq[ParserResult]()

src_v2/parser/parser_types.nim

@@ -1,5 +1,4 @@
 import std/[
-  options,
   strutils,
   strformat,
   collections/sequtils,
@@ -47,8 +46,12 @@ type
 
 # -- Initalizers
 
-func initParserToken*(x: char): ParserToken = ParserToken(kind: parserTokenChar, charValue: x)
-func initParserToken*(x: string): ParserToken = ParserToken(kind: parserTokenString, stringValue: x)
+func initParserToken*(x: char): ParserToken =
+  ## Initialize `ParserToken` as `parserTokenChar` with value `x`.
+  ParserToken(kind: parserTokenChar, charValue: x)
+func initParserToken*(x: string): ParserToken =
+  ## Initialize `ParserToken` as `parserTokenString` with value `x`.
+  ParserToken(kind: parserTokenString, stringValue: x)
 
 func initParser*(
   stream: string,
@@ -56,6 +59,7 @@ func initParser*(
   position = -1,
   lastPosition = 0,
 ): Parser =
+  ## Initialize `Parser` with `stream`.
   Parser(
     state: ParserState(
       stream: stream,
@@ -71,6 +75,7 @@ func initParserResult*(
   position = -1,
   lastPosition = 0,
 ): ParserResult =
+  ## Initialize `ParserResult.ok` with `stream`.
   ParserResult.ok(initParser(
     stream,
     tokens,
@@ -78,10 +83,19 @@ func initParserResult*(
     lastPosition,
   ))
 
+# -- Equalizers
+
+func `==`*(a: ParserToken, b: ParserToken): bool =
+  ## Compare two `ParserToken` objects.
+  case ((a.kind, b.kind)):
+    of (parserTokenChar, parserTokenChar): a.charValue == b.charValue
+    of (parserTokenString, parserTokenString): a.stringValue == b.stringValue
+    else: false
+
 # -- Getters
 
 func toString*(x: ParserToken): string =
-  ## Get the Token value `x` as a string.
+  ## Get the `ParserToken` value `x` as a string.
   case x.kind:
     of parserTokenChar:
       $x.charValue
@@ -89,11 +103,13 @@ func toString*(x: ParserToken): string =
       x.stringValue
 
 func toString*(tokens: seq[ParserToken]): string =
+  ## Get the seq `ParserToken` value `x` as a string.
   tokens.foldl(a & b.toString(), "")
 
 # -- Modifiers
 
 func flattenTokens*(parser: Parser): Parser =
+  ## Flatten tokens in `parser` as a string `ParserToken`.
   let token = initParserToken(parser.tokens.foldl(a & b.toString(), ""))
   Parser(
     state: parser.state,
@@ -101,6 +117,7 @@ func flattenTokens*(parser: Parser): Parser =
   )
 
 func emptyTokens*(parser: Parser): Parser =
+  ## Empty `parser` tokens.
   Parser(
     state: parser.state,
     tokens: newSeq[ParserToken](),
@@ -108,14 +125,15 @@ func emptyTokens*(parser: Parser): Parser =
 
 func foldTokens*[T](
   parserResult: ParserResult,
-  onError: ParserError -> T,
-  onSuccess: seq[ParserToken] -> T,
+  onErrorFn: ParserError -> T,
+  onSuccessFn: seq[ParserToken] -> T,
 ): T =
+  ## Fold over `tokens` inside `ParserResult` with `onSuccessFn` or `onErrorFn`.
   if parserResult.isOk():
-    onSuccess(parserResult.unsafeGet().tokens)
+    onSuccessFn(parserResult.unsafeGet().tokens)
   else:
     let err = parserResult.error()
-    onError(err)
+    onErrorFn(err)
 
 func tokensToString*(parserResult: ParserResult, fallback = ""): string =
   parserResult.foldTokens(
@@ -264,30 +282,48 @@ func `$`*(x: ParserError): string =
     else: "ParseError"
 
 when isMainModule:
-  let test1 = """ABC
+  block testEqualizers:
+    assert initParserToken("a") == initParserToken("a")
+    assert initParserToken('a') == initParserToken('a')
+    assert initParserToken("a") != initParserToken('a')
 
-D
+  block testGetters:
+    # toString
+    assert initParserToken("a").toString() == "a"
+    assert initParserToken('a').toString() == "a"
 
-EFG"""
-  # echo test1.highlightStreamPosition2(test1.find("B"))
-  # echo "=============="
-  # echo test1.highlightStreamPosition2(test1.find("C"))
-  # echo "=============="
-  # echo test1.highlightStreamPosition2(test1.find("D"))
-  # echo "=============="
-  # echo test1.highlightStreamPosition2(test1.find("\n"))
-  # echo "=============="
-  # echo test1.highlightStreamPosition2(test1.find("\n", test1.find("\n") + 1))
+  block testModifiers:
+    let testTokensSeq = @[initParserToken("a"), initParserToken('b'), initParserToken("c")]
+    let testExpectedStr = "abc"
+    let testTokensSeqParser = initParser(
+      stream = testExpectedStr,
+      tokens = testTokensSeq,
+      position = testTokensSeq.len - 1,
+    )
+    let testTokensSeqParserResult = ParserResult.ok(testTokensSeqParser)
+    let testTokensSeqParserResultErr = ParserResult.err(ParserError(
+      kind: charMismatchErr,
+      unexpected: "a",
+      expected: "b",
+      index: 0,
+      parser: testTokensSeqParser,
+    ))
 
+    # flattenTokens
+    assert testTokensSeqParser.flattenTokens().tokens[0] == initParserToken("abc")
 
-  # echo "\n1\n".rfind('\n', 0, 2)
+    # emptyTokens
+    assert testTokensSeqParser.emptyTokens().tokens.len == 0
 
-  # block highlightStreamPosition:
-  #   let s = "abc"
-  #   # Out of bounds
-  #   assert s.highlightStreamPosition(-1) == s
-  #   assert s.highlightStreamPosition(s.len) == s
-  #   # Regular highlighting
-  #   assert s.highlightStreamPosition(0) == LEFT_HIGHLIGHT_CHAR & "a" & RIGHT_HIGHLIGHT_CHAR & "bc"
-  #   assert s.highlightStreamPosition(1) == "a" & LEFT_HIGHLIGHT_CHAR & "b" & RIGHT_HIGHLIGHT_CHAR & "c"
-  #   assert s.highlightStreamPosition(2) == "ab" & LEFT_HIGHLIGHT_CHAR & "c" & RIGHT_HIGHLIGHT_CHAR
+    # foldTokens
+    assert testTokensSeqParserResult.foldTokens(
+      err => "Failure",
+      xs => xs.toString(),
+    ) == testExpectedStr
+    assert testTokensSeqParserResultErr.foldTokens(
+      err => "Failure",
+      xs => xs.toString(),
+    ) == "Failure"
+
+    # tokensToString
+    assert testTokensSeqParserResult.tokensToString() == testExpectedStr