From 4679cda4412483d70b1d094988ef3794751cf092 Mon Sep 17 00:00:00 2001 From: Florian Schroedl Date: Fri, 26 Aug 2022 20:05:54 +0200 Subject: [PATCH] Document and test parser type internals --- src_v2/parser/parser_api.nim | 2 +- src_v2/parser/parser_types.nim | 96 +++++++++++++++++++++++----------- 2 files changed, 67 insertions(+), 31 deletions(-) diff --git a/src_v2/parser/parser_api.nim b/src_v2/parser/parser_api.nim index baea12c..704b61f 100644 --- a/src_v2/parser/parser_api.nim +++ b/src_v2/parser/parser_api.nim @@ -189,7 +189,7 @@ proc anyUntil*(stopFn: parserFnT): parserFnT {.inline.} = manyUntil(anyCh, stopFn) proc choice*(parserFns: seq[parserFnT]): parserFnT {.inline} = - ## Creates parser function that checks any of the `parserFns`. + ## creates parser function that checks any of the `parserFns`. ## Needs one match for a `ParserResult.ok`. return proc(parser: Parser): ParserResult {.closure.} = var errors: seq[ParserResult] = newSeq[ParserResult]() diff --git a/src_v2/parser/parser_types.nim b/src_v2/parser/parser_types.nim index e3bf7eb..f51ac25 100644 --- a/src_v2/parser/parser_types.nim +++ b/src_v2/parser/parser_types.nim @@ -1,5 +1,4 @@ import std/[ - options, strutils, strformat, collections/sequtils, @@ -47,8 +46,12 @@ type # -- Initalizers -func initParserToken*(x: char): ParserToken = ParserToken(kind: parserTokenChar, charValue: x) -func initParserToken*(x: string): ParserToken = ParserToken(kind: parserTokenString, stringValue: x) +func initParserToken*(x: char): ParserToken = + ## Initialize `ParserToken` as `parserTokenChar` with value `x`. + ParserToken(kind: parserTokenChar, charValue: x) +func initParserToken*(x: string): ParserToken = + ## Initialize `ParserToken` as `parserTokenString` with value `x`. + ParserToken(kind: parserTokenString, stringValue: x) func initParser*( stream: string, @@ -56,6 +59,7 @@ func initParser*( position = -1, lastPosition = 0, ): Parser = + ## Initialize `Parser` with `stream`. Parser( state: ParserState( stream: stream, @@ -71,6 +75,7 @@ func initParserResult*( position = -1, lastPosition = 0, ): ParserResult = + ## Initialize `ParserResult.ok` with `stream`. ParserResult.ok(initParser( stream, tokens, @@ -78,10 +83,19 @@ func initParserResult*( lastPosition, )) +# -- Equalizers + +func `==`*(a: ParserToken, b: ParserToken): bool = + ## Compare two `ParserToken` objects. + case ((a.kind, b.kind)): + of (parserTokenChar, parserTokenChar): a.charValue == b.charValue + of (parserTokenString, parserTokenString): a.stringValue == b.stringValue + else: false + # -- Getters func toString*(x: ParserToken): string = - ## Get the Token value `x` as a string. + ## Get the `ParserToken` value `x` as a string. case x.kind: of parserTokenChar: $x.charValue @@ -89,11 +103,13 @@ func toString*(x: ParserToken): string = x.stringValue func toString*(tokens: seq[ParserToken]): string = + ## Get the seq `ParserToken` value `x` as a string. tokens.foldl(a & b.toString(), "") # -- Modifiers func flattenTokens*(parser: Parser): Parser = + ## Flatten tokens in `parser` as a string `ParserToken`. let token = initParserToken(parser.tokens.foldl(a & b.toString(), "")) Parser( state: parser.state, @@ -101,6 +117,7 @@ func flattenTokens*(parser: Parser): Parser = ) func emptyTokens*(parser: Parser): Parser = + ## Empty `parser` tokens. Parser( state: parser.state, tokens: newSeq[ParserToken](), @@ -108,14 +125,15 @@ func emptyTokens*(parser: Parser): Parser = func foldTokens*[T]( parserResult: ParserResult, - onError: ParserError -> T, - onSuccess: seq[ParserToken] -> T, + onErrorFn: ParserError -> T, + onSuccessFn: seq[ParserToken] -> T, ): T = + ## Fold over `tokens` inside `ParserResult` with `onSuccessFn` or `onErrorFn`. if parserResult.isOk(): - onSuccess(parserResult.unsafeGet().tokens) + onSuccessFn(parserResult.unsafeGet().tokens) else: let err = parserResult.error() - onError(err) + onErrorFn(err) func tokensToString*(parserResult: ParserResult, fallback = ""): string = parserResult.foldTokens( @@ -264,30 +282,48 @@ func `$`*(x: ParserError): string = else: "ParseError" when isMainModule: - let test1 = """ABC + block testEqualizers: + assert initParserToken("a") == initParserToken("a") + assert initParserToken('a') == initParserToken('a') + assert initParserToken("a") != initParserToken('a') -D + block testGetters: + # toString + assert initParserToken("a").toString() == "a" + assert initParserToken('a').toString() == "a" -EFG""" - # echo test1.highlightStreamPosition2(test1.find("B")) - # echo "==============" - # echo test1.highlightStreamPosition2(test1.find("C")) - # echo "==============" - # echo test1.highlightStreamPosition2(test1.find("D")) - # echo "==============" - # echo test1.highlightStreamPosition2(test1.find("\n")) - # echo "==============" - # echo test1.highlightStreamPosition2(test1.find("\n", test1.find("\n") + 1)) + block testModifiers: + let testTokensSeq = @[initParserToken("a"), initParserToken('b'), initParserToken("c")] + let testExpectedStr = "abc" + let testTokensSeqParser = initParser( + stream = testExpectedStr, + tokens = testTokensSeq, + position = testTokensSeq.len - 1, + ) + let testTokensSeqParserResult = ParserResult.ok(testTokensSeqParser) + let testTokensSeqParserResultErr = ParserResult.err(ParserError( + kind: charMismatchErr, + unexpected: "a", + expected: "b", + index: 0, + parser: testTokensSeqParser, + )) + # flattenTokens + assert testTokensSeqParser.flattenTokens().tokens[0] == initParserToken("abc") - # echo "\n1\n".rfind('\n', 0, 2) + # emptyTokens + assert testTokensSeqParser.emptyTokens().tokens.len == 0 - # block highlightStreamPosition: - # let s = "abc" - # # Out of bounds - # assert s.highlightStreamPosition(-1) == s - # assert s.highlightStreamPosition(s.len) == s - # # Regular highlighting - # assert s.highlightStreamPosition(0) == LEFT_HIGHLIGHT_CHAR & "a" & RIGHT_HIGHLIGHT_CHAR & "bc" - # assert s.highlightStreamPosition(1) == "a" & LEFT_HIGHLIGHT_CHAR & "b" & RIGHT_HIGHLIGHT_CHAR & "c" - # assert s.highlightStreamPosition(2) == "ab" & LEFT_HIGHLIGHT_CHAR & "c" & RIGHT_HIGHLIGHT_CHAR + # foldTokens + assert testTokensSeqParserResult.foldTokens( + err => "Failure", + xs => xs.toString(), + ) == testExpectedStr + assert testTokensSeqParserResultErr.foldTokens( + err => "Failure", + xs => xs.toString(), + ) == "Failure" + + # tokensToString + assert testTokensSeqParserResult.tokensToString() == testExpectedStr