Convert tokens before

src/org/org_text_delimiter.nim

@@ -42,6 +42,12 @@ type OrgBuilderT* = OrgElement
 type OrgBuilder* = Builder[OrgBuilderT]
 type OrgBuilderResult* = BuilderResult[OrgBuilderT]
 
+proc makeRawToken*(content: string): OrgBuilderT =
+  OrgBuilderT(
+    kind: orgRawText,
+    content: content,
+  )
+
 proc makeBoldToken*(content: string): OrgBuilderT =
   OrgBuilderT(
     kind: orgBoldText,
@@ -58,6 +64,7 @@ proc parseText[T](
     parserFn: Parser -> ParserResult,
     tokenFoldFn: (seq[ParserToken], seq[T]) -> seq[T],
   ]],
+  otherWiseFn: (seq[ParserToken]) -> seq[T],
   stopFn = newline,
 ): BuilderResult[T] =
   let (parser, tree) = builder
@@ -79,11 +86,20 @@ proc parseText[T](
       if parseResult.isOk():
         let okParser = parseResult.unsafeGet()
 
+        let textTokens = parserAcc
+        .foldTokens(
+          onError = _ => newSeq[T](),
+          onSuccess = otherWiseFn,
+        )
+
         found = true
         parserAcc = parseResult
         builderAcc = builder.initBuilder(
           okParser,
-          tokenFoldFn(okParser.tokens, builderAcc[1]),
+          tokenFoldFn(
+            okParser.tokens,
+            builderAcc[1] & textTokens,
+          ),
         )
         break
 
@@ -92,16 +108,23 @@ proc parseText[T](
 
   BuilderResult[T].ok(builderAcc)
 
+proc makeRawTokenOrEmpty(xs: seq[ParserToken]): seq[OrgBuilderT] =
+  let str = xs.foldl(a & b.tokenStringValue(), "")
+  if str.len == 0: @[]
+  else: @[makeRawToken(str)]
+
 when isMainModule:
+
   let foo = OrgBuilderResult.ok(OrgBuilder((
-      parser: initParser("Just *some* text"),
+      parser: initParser("Just *some text* here"),
       tree: newSeq[OrgBuilderT](),
   )))
   .flatMap((builder: OrgBuilder) => parseText(
     builder = builder,
     builderFns = @[
       (boldParser, makeOrgToken(makeBoldToken)),
-    ]
+    ],
+    otherWiseFn = makeRawTokenOrEmpty,
   ))
   # .foldBuilder(
   #   err => &"Error Parsing: {err}",