floscr/org-parser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement Multi type tokens

Browse Source
This commit is contained in:
Florian Schroedl
2022-01-20 17:00:00 +01:00
parent b5744dc40a
commit 62a56662f9
2 changed files with 93 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
src/org_parser.nim
View File

@@ -5,32 +5,68 @@ import std/strutils
import parser/parser
import results
type StringBuilderT = string
type StringBuilder = Builder[StringBuilderT]
type StringBuilderResult = BuilderResult[StringBuilderT]
proc stringConcat(typeInfo: StringBuilderT):
(seq[ParserToken], seq[StringBuilderT]) -> seq[StringBuilderT] =
return proc(xs: seq[ParserToken], ys: seq[StringBuilderT]): seq[StringBuilderT] =
return ys & xs.foldl(a & b.tokenStringValue(), typeInfo)
when isMainModule:
let parseHeadingStars = @[
manyUntil(ch('*'), ch(' ')),
ignore(ch(' '))
]
let parseHeadingText = @[
anyUntil(endOfStream),
proc createTodoKeywordParser(xs: seq[string]): (Parser -> ParserResult) =
choice(xs.map((x: string) => str(x) + ignore(ch(' '))))
let todoKeywords = @["TODO"]
let doneKeywords = @["DONE"]
let parseTodoKeyword = todoKeywords.createTodoKeywordParser()
let parseDoneKeyword = doneKeywords.createTodoKeywordParser()
let propertiesKeyValueParser = anyUntil(ch(Newlines)) + ignore(ch(NewLines))
let propertiesEndParser = str(":PROPERTIES_END:") + ch(Newlines)
let parseProperties = @[
ignore(str(":PROPERTIES:") + ch(Newlines)),
manyUntil(
propertiesKeyValueParser,
propertiesEndParser,
),
ignore(propertiesEndParser),
]
type StringBuilderT = string
type StringBuilder = Builder[StringBuilderT]
type StringBuilderResult = BuilderResult[StringBuilderT]
proc stringConcat(typeInfo: StringBuilderT):
(seq[Token], seq[StringBuilderT]) -> seq[StringBuilderT] =
return proc(xs: seq[Token], ys: seq[StringBuilderT]): seq[StringBuilderT] =
return ys & xs.foldl(a & b.value, typeInfo)
let parseHeadingText = @[
anyUntil(newline),
ignore(newline),
]
let sampleBuilder = StringBuilderResult
.ok(StringBuilder((
parser: initParser("**** Some stars"),
parser: initParser("""**** TODO Some stars
:PROPERTIES:
:PROP_NAME: Value
:PROP_NAME: Value
:PROP_NAME: Value
:PROP_NAME: Value
:PROPERTIES_END:
"""),
tree: newSeq[StringBuilderT](),
)))
.applyParsersSeq(@[
(parseHeadingStars, stringConcat("Stars: ")),
(parseHeadingText, stringConcat("Text: "))
(@[optional(parseTodoKeyword)], stringConcat("TODO: ")),
(@[optional(parseDoneKeyword)], stringConcat("DONE: ")),
(parseHeadingText, stringConcat("Text: ")),
(parseProperties, stringConcat("Properties: ")),
# (@[optional(choice(parseProperties))], stringConcat("Properties: ")),
])
.foldBuilder(
err => &"Error Parsing: {err}",

67
src/parser/parser.nim
View File

@@ -15,12 +15,26 @@ type
stream: string
position, lastPosition: int
Token* = ref object
value*: char
parserTokenCharValueT* = char
parserTokenStringValueT* = string
parserTokenKeyValuePairValueT* = tuple[k: string, v: string]
ParserTokenKind* = enum
parserTokenChar
parserTokenString
# parserTokenKeyValuePair
ParserToken* = ref object
case kind*: ParserTokenKind
of parserTokenChar:
charValue*: parserTokenCharValueT
of parserTokenString:
stringValue*: parserTokenStringValueT
# of parserTokenKeyValuePair:
# keyValuePairValue*: parserTokenKeyValuePairValueT
Parser* = ref object
state: ParserState
tokens: seq[Token]
tokens: seq[ParserToken]
ParseErrorKind = enum
choiceMismatchErr
@@ -45,9 +59,16 @@ proc indentKey(x: string, count: int): string =
y.delete(0..count - 1)
y
proc `$`*(x: Token): string =
&"""Token(
value: {x.value},
proc tokenStringValue*(x: ParserToken): string =
case x.kind:
of parserTokenChar:
$x.charValue
of parserTokenString:
x.stringValue
proc `$`*(x: ParserToken): string =
&"""ParserToken(
value: {tokenStringValue(x)},
)"""
proc `$`*(x: ParserState): string =
@@ -108,6 +129,10 @@ proc `$`*(x: ParserError): string =
else: "ParseError"
func initParserToken(x: char): ParserToken = ParserToken(kind: parserTokenChar, charValue: x)
func initParserToken(x: string): ParserToken = ParserToken(kind: parserTokenString, stringValue: x)
# func initParserToken(x: parserTokenKeyValuePairValueT): ParserToken = ParserToken(kind: parserTokenKeyValuePair, keyValuePairValue: x)
proc initParser*(str: string): Parser =
Parser(
state: ParserState(
@@ -115,7 +140,7 @@ proc initParser*(str: string): Parser =
position: -1,
lastPosition: 0,
),
tokens: newSeq[Token](),
tokens: newSeq[ParserToken](),
)
proc initParserResult*(str: string): ParserResult =
@@ -142,7 +167,7 @@ func ch*(expectedChars: set[char]): (Parser -> ParserResult) {.inline.} =
position: newIndex,
lastPosition: parser.state.position,
),
tokens: parser.tokens & Token(value: foundChar)
tokens: parser.tokens & initParserToken(foundChar)
).ok()
else:
return err(ParserError(
@@ -174,7 +199,7 @@ func ch*(expectedChar: char): (Parser -> ParserResult) {.inline.} =
position: newIndex,
lastPosition: parser.state.position,
),
tokens: parser.tokens & Token(value: foundChar)
tokens: parser.tokens & initParserToken(foundChar)
).ok()
else:
return err(ParserError(
@@ -279,7 +304,7 @@ proc parseSeq*(parser: ParserResult, xs: seq[Parser -> ParserResult]): ParserRes
proc foldTokens*[T](
parserResult: ParserResult,
onError: ParserError -> T,
onSuccess: seq[Token] -> T,
onSuccess: seq[ParserToken] -> T,
): T =
if parserResult.isOk():
onSuccess(parserResult.unsafeGet().tokens)
@@ -302,7 +327,7 @@ proc mapTree[T](builder: BuilderResult[T], fn: seq[T] -> seq[T]): BuilderResult[
proc applyParsers*[T](
builder: Builder[T],
parsers: seq[Parser -> ParserResult],
tokenFoldFn: (seq[Token], seq[T]) -> seq[T],
tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T],
): BuilderResult[T] =
# proc nested(b: Builder[T]): BuilderResult[T] =
let newParser = ParserResult.ok(Parser(
@@ -314,7 +339,7 @@ proc applyParsers*[T](
newParser
.foldTokens(
(err: ParserError) => BuilderResult[T].err((builder, "foo")),
(newTokens: seq[Token]) => BuilderResult[T].ok(
(newTokens: seq[ParserToken]) => BuilderResult[T].ok(
builder.merge(newParser.unsafeGet(), tokenFoldFn(newTokens, builder[1]))
),
)
@@ -323,7 +348,7 @@ proc applyParsersSeq*[T](
builder: BuilderResult[T],
xs: seq[tuple[
parsers: seq[Parser -> ParserResult],
tokenFoldFn: (seq[Token], seq[T]) -> seq[T],
tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T],
]]): BuilderResult[T] =
xs.foldl(a.flatMap((x: Builder[T]) => x.applyParsers(b[0], b[1])), builder)
@@ -340,32 +365,32 @@ proc foldBuilder*[T, T2](
when isMainModule:
proc getTokens(x: ParserResult): seq[char] =
proc getTokens(x: ParserResult): seq[string] =
x.foldTokens(
proc(err: ParserError): seq[char] =
proc(err: ParserError): seq[string] =
echo err
@[],
proc(xs: seq[Token]): seq[char] = xs.map((x: Token) => x.value),
proc(xs: seq[ParserToken]): seq[string] = xs.map((x: ParserToken) => x.tokenStringValue()),
)
proc testParser(x: string, ps: seq[Parser -> ParserResult]): seq[char] =
proc testParser(x: string, ps: seq[Parser -> ParserResult]): seq[string] =
initParserResult(x).parseSeq(ps).getTokens()
let optionalPrefixParser = @[
optional(ch('_')),
str("ABC")
]
assert: "_ABC".testParser(optionalPrefixParser) == @['_', 'A', 'B', 'C']
assert: "ABC".testParser(optionalPrefixParser) == @['A', 'B', 'C']
assert: "_ABC".testParser(optionalPrefixParser) == @["_", "A", "B", "C"]
assert: "ABC".testParser(optionalPrefixParser) == @["A", "B", "C"]
let andParser = @[
(ch('A') + ch('B')),
ch('C'),
]
assert: "ABC".testParser(andParser) == @['A', 'B', 'C']
assert: "ABC".testParser(andParser) == @["A", "B", "C"]
let newlineParserTest = @[
str("ABC"),
newline
]
assert "ABC\n".testParser(newlineParserTest) == @['A', 'B', 'C', '\n']
assert "ABC\n".testParser(newlineParserTest) == @["A", "B", "C", "\n"]