Implement Multi type tokens

src/org_parser.nim

@@ -5,32 +5,68 @@ import std/strutils
 import parser/parser
 import results
 
+type StringBuilderT = string
+type StringBuilder = Builder[StringBuilderT]
+type StringBuilderResult = BuilderResult[StringBuilderT]
+proc stringConcat(typeInfo: StringBuilderT):
+                  (seq[ParserToken], seq[StringBuilderT]) -> seq[StringBuilderT] =
+
+  return proc(xs: seq[ParserToken], ys: seq[StringBuilderT]): seq[StringBuilderT] =
+    return ys & xs.foldl(a & b.tokenStringValue(), typeInfo)
+
 when isMainModule:
   let parseHeadingStars = @[
     manyUntil(ch('*'), ch(' ')),
     ignore(ch(' '))
   ]
 
-  let parseHeadingText = @[
+  proc createTodoKeywordParser(xs: seq[string]): (Parser -> ParserResult) =
-    anyUntil(endOfStream),
+    choice(xs.map((x: string) => str(x) + ignore(ch(' '))))
+
+  let todoKeywords = @["TODO"]
+  let doneKeywords = @["DONE"]
+
+  let parseTodoKeyword = todoKeywords.createTodoKeywordParser()
+  let parseDoneKeyword = doneKeywords.createTodoKeywordParser()
+
+  let propertiesKeyValueParser = anyUntil(ch(Newlines)) + ignore(ch(NewLines))
+  let propertiesEndParser = str(":PROPERTIES_END:") + ch(Newlines)
+
+  let parseProperties = @[
+    ignore(str(":PROPERTIES:") + ch(Newlines)),
+    manyUntil(
+      propertiesKeyValueParser,
+      propertiesEndParser,
+    ),
+    ignore(propertiesEndParser),
   ]
 
-  type StringBuilderT = string
+  let parseHeadingText = @[
-  type StringBuilder = Builder[StringBuilderT]
+    anyUntil(newline),
-  type StringBuilderResult = BuilderResult[StringBuilderT]
+    ignore(newline),
-  proc stringConcat(typeInfo: StringBuilderT):
+  ]
-                   (seq[Token], seq[StringBuilderT]) -> seq[StringBuilderT] =
-    return proc(xs: seq[Token], ys: seq[StringBuilderT]): seq[StringBuilderT] =
-      return ys & xs.foldl(a & b.value, typeInfo)
 
   let sampleBuilder = StringBuilderResult
   .ok(StringBuilder((
-    parser: initParser("**** Some stars"),
+    parser: initParser("""**** TODO Some stars
+:PROPERTIES:
+:PROP_NAME: Value
+:PROP_NAME: Value
+:PROP_NAME: Value
+:PROP_NAME: Value
+:PROPERTIES_END:
+"""),
     tree: newSeq[StringBuilderT](),
   )))
   .applyParsersSeq(@[
     (parseHeadingStars, stringConcat("Stars: ")),
-    (parseHeadingText, stringConcat("Text: "))
+
+    (@[optional(parseTodoKeyword)], stringConcat("TODO: ")),
+    (@[optional(parseDoneKeyword)], stringConcat("DONE: ")),
+
+    (parseHeadingText, stringConcat("Text: ")),
+    (parseProperties, stringConcat("Properties: ")),
+    # (@[optional(choice(parseProperties))], stringConcat("Properties: ")),
   ])
   .foldBuilder(
     err => &"Error Parsing: {err}",

src/parser/parser.nim

@@ -15,12 +15,26 @@ type
     stream: string
     position, lastPosition: int
 
-  Token* = ref object
+  parserTokenCharValueT* = char
-    value*: char
+  parserTokenStringValueT* = string
+  parserTokenKeyValuePairValueT* = tuple[k: string, v: string]
+
+  ParserTokenKind* = enum
+    parserTokenChar
+    parserTokenString
+    # parserTokenKeyValuePair
+  ParserToken* = ref object
+    case kind*: ParserTokenKind
+      of parserTokenChar:
+        charValue*: parserTokenCharValueT
+      of parserTokenString:
+        stringValue*: parserTokenStringValueT
+      # of parserTokenKeyValuePair:
+      #   keyValuePairValue*: parserTokenKeyValuePairValueT
 
   Parser* = ref object
     state: ParserState
-    tokens: seq[Token]
+    tokens: seq[ParserToken]
 
   ParseErrorKind = enum
     choiceMismatchErr
@@ -45,9 +59,16 @@ proc indentKey(x: string, count: int): string =
   y.delete(0..count - 1)
   y
 
-proc `$`*(x: Token): string =
+proc tokenStringValue*(x: ParserToken): string =
-  &"""Token(
+  case x.kind:
-  value: {x.value},
+    of parserTokenChar:
+      $x.charValue
+    of parserTokenString:
+      x.stringValue
+
+proc `$`*(x: ParserToken): string =
+  &"""ParserToken(
+  value: {tokenStringValue(x)},
 )"""
 
 proc `$`*(x: ParserState): string =
@@ -108,6 +129,10 @@ proc `$`*(x: ParserError): string =
 
     else: "ParseError"
 
+func initParserToken(x: char): ParserToken = ParserToken(kind: parserTokenChar, charValue: x)
+func initParserToken(x: string): ParserToken = ParserToken(kind: parserTokenString, stringValue: x)
+# func initParserToken(x: parserTokenKeyValuePairValueT): ParserToken = ParserToken(kind: parserTokenKeyValuePair, keyValuePairValue: x)
+
 proc initParser*(str: string): Parser =
   Parser(
     state: ParserState(
@@ -115,7 +140,7 @@ proc initParser*(str: string): Parser =
       position: -1,
       lastPosition: 0,
     ),
-    tokens: newSeq[Token](),
+    tokens: newSeq[ParserToken](),
   )
 
 proc initParserResult*(str: string): ParserResult =
@@ -142,7 +167,7 @@ func ch*(expectedChars: set[char]): (Parser -> ParserResult) {.inline.} =
             position: newIndex,
             lastPosition: parser.state.position,
           ),
-          tokens: parser.tokens & Token(value: foundChar)
+          tokens: parser.tokens & initParserToken(foundChar)
         ).ok()
       else:
         return err(ParserError(
@@ -174,7 +199,7 @@ func ch*(expectedChar: char): (Parser -> ParserResult) {.inline.} =
             position: newIndex,
             lastPosition: parser.state.position,
           ),
-          tokens: parser.tokens & Token(value: foundChar)
+          tokens: parser.tokens & initParserToken(foundChar)
         ).ok()
       else:
         return err(ParserError(
@@ -279,7 +304,7 @@ proc parseSeq*(parser: ParserResult, xs: seq[Parser -> ParserResult]): ParserRes
 proc foldTokens*[T](
   parserResult: ParserResult,
   onError: ParserError -> T,
-  onSuccess: seq[Token] -> T,
+  onSuccess: seq[ParserToken] -> T,
 ): T =
   if parserResult.isOk():
     onSuccess(parserResult.unsafeGet().tokens)
@@ -302,7 +327,7 @@ proc mapTree[T](builder: BuilderResult[T], fn: seq[T] -> seq[T]): BuilderResult[
 proc applyParsers*[T](
   builder: Builder[T],
   parsers: seq[Parser -> ParserResult],
-  tokenFoldFn: (seq[Token], seq[T]) -> seq[T],
+  tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T],
 ): BuilderResult[T] =
   # proc nested(b: Builder[T]): BuilderResult[T] =
   let newParser = ParserResult.ok(Parser(
@@ -314,7 +339,7 @@ proc applyParsers*[T](
   newParser
   .foldTokens(
     (err: ParserError) => BuilderResult[T].err((builder, "foo")),
-    (newTokens: seq[Token]) => BuilderResult[T].ok(
+    (newTokens: seq[ParserToken]) => BuilderResult[T].ok(
       builder.merge(newParser.unsafeGet(), tokenFoldFn(newTokens, builder[1]))
     ),
   )
@@ -323,7 +348,7 @@ proc applyParsersSeq*[T](
   builder: BuilderResult[T],
   xs: seq[tuple[
     parsers: seq[Parser -> ParserResult],
-    tokenFoldFn: (seq[Token], seq[T]) -> seq[T],
+    tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T],
   ]]): BuilderResult[T] =
     xs.foldl(a.flatMap((x: Builder[T]) => x.applyParsers(b[0], b[1])), builder)
 
@@ -340,32 +365,32 @@ proc foldBuilder*[T, T2](
 
 
 when isMainModule:
-  proc getTokens(x: ParserResult): seq[char] =
+  proc getTokens(x: ParserResult): seq[string] =
     x.foldTokens(
-      proc(err: ParserError): seq[char] =
+      proc(err: ParserError): seq[string] =
         echo err
         @[],
-      proc(xs: seq[Token]): seq[char] = xs.map((x: Token) => x.value),
+      proc(xs: seq[ParserToken]): seq[string] = xs.map((x: ParserToken) => x.tokenStringValue()),
     )
 
-  proc testParser(x: string, ps: seq[Parser -> ParserResult]): seq[char] =
+  proc testParser(x: string, ps: seq[Parser -> ParserResult]): seq[string] =
     initParserResult(x).parseSeq(ps).getTokens()
 
   let optionalPrefixParser = @[
     optional(ch('_')),
     str("ABC")
   ]
-  assert: "_ABC".testParser(optionalPrefixParser) == @['_', 'A', 'B', 'C']
+  assert: "_ABC".testParser(optionalPrefixParser) == @["_", "A", "B", "C"]
-  assert: "ABC".testParser(optionalPrefixParser) == @['A', 'B', 'C']
+  assert: "ABC".testParser(optionalPrefixParser) == @["A", "B", "C"]
 
   let andParser = @[
     (ch('A') + ch('B')),
     ch('C'),
   ]
-  assert: "ABC".testParser(andParser) == @['A', 'B', 'C']
+  assert: "ABC".testParser(andParser) == @["A", "B", "C"]
 
   let newlineParserTest = @[
     str("ABC"),
     newline
   ]
-  assert "ABC\n".testParser(newlineParserTest) == @['A', 'B', 'C', '\n']
+  assert "ABC\n".testParser(newlineParserTest) == @["A", "B", "C", "\n"]