From 305bd1e503d6d5a5ec7be9f052902da5ea647231 Mon Sep 17 00:00:00 2001 From: Florian Schroedl Date: Thu, 20 Jan 2022 17:00:00 +0100 Subject: [PATCH] Restructure --- src/parser/parser_internals.nim | 194 +++++++++----------------------- src/parser/parser_types.nim | 95 ++++++++++++++++ 2 files changed, 150 insertions(+), 139 deletions(-) diff --git a/src/parser/parser_internals.nim b/src/parser/parser_internals.nim index 954d21b..dba9dcb 100644 --- a/src/parser/parser_internals.nim +++ b/src/parser/parser_internals.nim @@ -12,6 +12,8 @@ import ../utils/str {.experimental: "caseStmtMacros".} +# -- Parsing Functions + func ch*(expectedChars: set[char]): (Parser -> ParserResult) {.inline.} = return func(parser: Parser): ParserResult = let state = parser.state @@ -84,46 +86,7 @@ func str*(s: string): (Parser -> ParserResult) {.inline.} = return p -proc endOfStream*(parser: Parser): ParserResult = - let index = parser.state.position + 1 - if index == parser.state.stream.len: - ok(parser) - else: - err(ParserError( - kind: endOfStringErr, - expected: &"EndOfString", - index: index, - parser: parser, - )) - -func ignore*(parserFn: Parser -> ParserResult): (Parser -> ParserResult) {.inline.} = - ## Parse characters but throw success tokens away - return proc(parser: Parser): ParserResult = - return parserFn(parser) - .map((x: Parser) => Parser( - state: x.state, - tokens: parser.tokens, - )) - - -func flattenParserTokens*(parser: Parser): ParserResult = - return ParserResult.ok( - Parser( - state: parser.state, - tokens: @[ - ParserToken( - kind: parserTokenString, - stringValue: parser.tokens.foldl(a & b.tokenStringValue(), "") - ) - ] - ) - ) - -func emptyTokens*(parser: Parser): Parser = - Parser( - state: parser.state, - tokens: newSeq[ParserToken](), - ) +# -- Parsing Helper Functions func optional*(parserFn: Parser -> ParserResult): (Parser -> ParserResult) {.inline.} = ## Parse characters and ignore failure @@ -135,6 +98,15 @@ func optional*(parserFn: Parser -> ParserResult): (Parser -> ParserResult) {.inl else: parser.ok() +func ignore*(parserFn: Parser -> ParserResult): (Parser -> ParserResult) {.inline.} = + ## Parse characters but throw success tokens away + return proc(parser: Parser): ParserResult = + return parserFn(parser) + .map((x: Parser) => Parser( + state: x.state, + tokens: parser.tokens, + )) + func manyUntil*(acceptFn: Parser -> ParserResult, stopFn: Parser -> ParserResult): (Parser -> ParserResult) {.inline.} = ## Parse characters but throw success tokens away return proc(parser: Parser): ParserResult = @@ -174,19 +146,28 @@ func choice*(parsers: seq[Parser -> ParserResult]): (Parser -> ParserResult) {.i proc(x: ParserResult): ParserResult = x, ) + proc `+`*(parserFnA: Parser -> ParserResult, parserFnB: Parser -> ParserResult): (Parser -> ParserResult) {.inline.} = ## Parse characters and ignore failure return proc(parser: Parser): ParserResult = parserFnA(parser).flatMap(parserFnB) -proc setErrorExpectedField(err: ParserError, expected: string): ParserError = - ParserError( - kind: err.kind, - unexpected: err.unexpected, - expected: expected, - index: err.index, - parser: err.parser, - ) +proc parseSeq*(parser: ParserResult, xs: seq[Parser -> ParserResult]): ParserResult = + xs.foldl(a.flatMap(b), parser) + +# -- Parsing Aliases + +proc endOfStream*(parser: Parser): ParserResult = + let index = parser.state.position + 1 + if index == parser.state.stream.len: + ok(parser) + else: + err(ParserError( + kind: endOfStringErr, + expected: &"EndOfString", + index: index, + parser: parser, + )) let newlineParser = choice(@[ ch(NewLines), @@ -196,99 +177,34 @@ proc newline*(parser: Parser): ParserResult = newlineParser(parser) .mapErr((x: ParserError) => x.setErrorExpectedField("Newline")) -proc parseSeq*(parser: ParserResult, xs: seq[Parser -> ParserResult]): ParserResult = - xs.foldl(a.flatMap(b), parser) -proc foldTokens*[T]( - parserResult: ParserResult, - onError: ParserError -> T, - onSuccess: seq[ParserToken] -> T, -): T = - if parserResult.isOk(): - onSuccess(parserResult.unsafeGet().tokens) - else: - let err = parserResult.error() - onError(err) +# when isMainModule: +# proc getTokens(x: ParserResult): seq[string] = +# x.foldTokens( +# proc(err: ParserError): seq[string] = +# echo err +# @[], +# proc(xs: seq[ParserToken]): seq[string] = xs.map((x: ParserToken) => x.tokenStringValue()), +# ) -proc merge[T](t: Builder[T], parser: Parser, tree: seq[T]): Builder[T] = - Builder[T](( - parser, - tree - )) +# proc testParser(x: string, ps: seq[Parser -> ParserResult]): seq[string] = +# initParserResult(x).parseSeq(ps).getTokens() -proc mapTree[T](builder: BuilderResult[T], fn: seq[T] -> seq[T]): BuilderResult[T] = - builder.map(proc(b: Builder[T]): Builder[T] = Builder(( - parser: b[0], - tree: fn(b[1]), - ))) +# let optionalPrefixParser = @[ +# optional(ch('_')), +# str("ABC") +# ] +# assert: "_ABC".testParser(optionalPrefixParser) == @["_", "A", "B", "C"] +# assert: "ABC".testParser(optionalPrefixParser) == @["A", "B", "C"] -proc applyParsers*[T]( - builder: Builder[T], - parsers: seq[Parser -> ParserResult], - tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T], -): BuilderResult[T] = - # proc nested(b: Builder[T]): BuilderResult[T] = - let newParser = ParserResult.ok(Parser( - state: builder[0].state, - tokens: @[] - )) - .parseSeq(parsers) +# let andParser = @[ +# (ch('A') + ch('B')), +# ch('C'), +# ] +# assert: "ABC".testParser(andParser) == @["A", "B", "C"] - newParser - .foldTokens( - (err: ParserError) => BuilderResult[T].err((builder, "foo")), - (newTokens: seq[ParserToken]) => BuilderResult[T].ok( - builder.merge(newParser.unsafeGet(), tokenFoldFn(newTokens, builder[1])) - ), - ) - -proc applyParsersSeq*[T]( - builder: BuilderResult[T], - xs: seq[tuple[ - parsers: seq[Parser -> ParserResult], - tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T], - ]]): BuilderResult[T] = - xs.foldl(a.flatMap((x: Builder[T]) => x.applyParsers(b[0], b[1])), builder) - -proc foldBuilder*[T, T2]( - builderResult: BuilderResult[T], - onError: string -> T2, - onSuccess: seq[T] -> T2, -): T = - if builderResult.isOk(): - onSuccess(builderResult.unsafeGet().tree) - else: - let err = builderResult.error() - onError(err[1]) - - -when isMainModule: - proc getTokens(x: ParserResult): seq[string] = - x.foldTokens( - proc(err: ParserError): seq[string] = - echo err - @[], - proc(xs: seq[ParserToken]): seq[string] = xs.map((x: ParserToken) => x.tokenStringValue()), - ) - - proc testParser(x: string, ps: seq[Parser -> ParserResult]): seq[string] = - initParserResult(x).parseSeq(ps).getTokens() - - let optionalPrefixParser = @[ - optional(ch('_')), - str("ABC") - ] - assert: "_ABC".testParser(optionalPrefixParser) == @["_", "A", "B", "C"] - assert: "ABC".testParser(optionalPrefixParser) == @["A", "B", "C"] - - let andParser = @[ - (ch('A') + ch('B')), - ch('C'), - ] - assert: "ABC".testParser(andParser) == @["A", "B", "C"] - - let newlineParserTest = @[ - str("ABC"), - newline - ] - assert "ABC\n".testParser(newlineParserTest) == @["A", "B", "C", "\n"] +# let newlineParserTest = @[ +# str("ABC"), +# newline +# ] +# assert "ABC\n".testParser(newlineParserTest) == @["A", "B", "C", "\n"] diff --git a/src/parser/parser_types.nim b/src/parser/parser_types.nim index 9aa4561..7fc158f 100644 --- a/src/parser/parser_types.nim +++ b/src/parser/parser_types.nim @@ -1,5 +1,7 @@ import std/strutils import std/strformat +import std/collections/sequtils +import std/sugar import results import fusion/matching import ../utils/str @@ -62,6 +64,12 @@ proc initParser*(str: string): Parser = proc initParserResult*(str: string): ParserResult = ParserResult.ok(initParser(str)) +proc initBuilder*[T](t: Builder[T], parser: Parser, tree: seq[T]): Builder[T] = + Builder[T](( + parser, + tree + )) + # -- Getters proc tokenStringValue*(x: ParserToken): string = @@ -72,6 +80,93 @@ proc tokenStringValue*(x: ParserToken): string = of parserTokenString: x.stringValue +# -- Modifiers + +func flattenParserTokens*(parser: Parser): ParserResult = + return ParserResult.ok( + Parser( + state: parser.state, + tokens: @[ + ParserToken( + kind: parserTokenString, + stringValue: parser.tokens.foldl(a & b.tokenStringValue(), "") + ) + ] + ) + ) + +func emptyTokens*(parser: Parser): Parser = + Parser( + state: parser.state, + tokens: newSeq[ParserToken](), + ) + +proc foldTokens*[T]( + parserResult: ParserResult, + onError: ParserError -> T, + onSuccess: seq[ParserToken] -> T, +): T = + if parserResult.isOk(): + onSuccess(parserResult.unsafeGet().tokens) + else: + let err = parserResult.error() + onError(err) + +proc setErrorExpectedField*(err: ParserError, expected: string): ParserError = + ParserError( + kind: err.kind, + unexpected: err.unexpected, + expected: expected, + index: err.index, + parser: err.parser, + ) + +proc mapTree*[T](builder: BuilderResult[T], fn: seq[T] -> seq[T]): BuilderResult[T] = + builder.map(proc(b: Builder[T]): Builder[T] = Builder(( + parser: b[0], + tree: fn(b[1]), + ))) + +proc applyParsers*[T]( + builder: Builder[T], + parsers: seq[Parser -> ParserResult], + tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T], +): BuilderResult[T] = + # proc nested(b: Builder[T]): BuilderResult[T] = + let newParser = ParserResult.ok(Parser( + state: builder[0].state, + tokens: @[] + )) + .parseSeq(parsers) + + newParser + .foldTokens( + (err: ParserError) => BuilderResult[T].err((builder, "foo")), + (newTokens: seq[ParserToken]) => BuilderResult[T].ok( + builder.initBuilder(newParser.unsafeGet(), tokenFoldFn(newTokens, builder[1])) + ), + ) + +proc applyParsersSeq*[T]( + builder: BuilderResult[T], + xs: seq[tuple[ + parsers: seq[Parser -> ParserResult], + tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T], + ]]): BuilderResult[T] = + xs.foldl(a.flatMap((x: Builder[T]) => x.applyParsers(b[0], b[1])), builder) + +proc foldBuilder*[T, T2]( + builderResult: BuilderResult[T], + onError: string -> T2, + onSuccess: seq[T] -> T2, +): T = + if builderResult.isOk(): + onSuccess(builderResult.unsafeGet().tree) + else: + let err = builderResult.error() + onError(err[1]) + + # -- Stringifiers proc `$`*(x: ParserToken): string =