From 084594a7d3b79a96d30457e2a400ebabdd85a364 Mon Sep 17 00:00:00 2001 From: Florian Schroedl Date: Sun, 28 Aug 2022 14:07:23 +0200 Subject: [PATCH] Restore InlineBuilder --- src_v2/org/org_builder.nim | 14 ++--- src_v2/org/org_text_delimiter.nim | 91 +++++++++++++++---------------- 2 files changed, 51 insertions(+), 54 deletions(-) diff --git a/src_v2/org/org_builder.nim b/src_v2/org/org_builder.nim index e4b4275..5fffd39 100644 --- a/src_v2/org/org_builder.nim +++ b/src_v2/org/org_builder.nim @@ -22,7 +22,7 @@ func initOrgInlineBuilder*(content: string): OrgInlineBuilderResult = tree: newSeq[OrgInlineBuilderT](), ))) -func convertTokens*(kind: orgInlineBlockKind): seq[ParserToken] -> seq[OrgInlineBuilderT] = +func tokenizeInlineTokens*(kind: orgInlineBlockKind): seq[ParserToken] -> seq[OrgInlineBuilderT] = return func(tokens: seq[ParserToken]): seq[OrgInlineBuilderT] = return @[ OrgInlineBuilderT( @@ -31,12 +31,12 @@ func convertTokens*(kind: orgInlineBlockKind): seq[ParserToken] -> seq[OrgInline ) ] -# func rawTextTokenizer*(kind: orgInlineBlockKind): string -> OrgInlineBuilderT = -# return func(content: string): OrgInlineBuilderT = -# return OrgInlineBuilderT( -# kind: kind, -# content: content, -# ) +func tokenizeRawText*(kind: orgInlineBlockKind): string -> OrgInlineBuilderT = + return func(content: string): OrgInlineBuilderT = + return OrgInlineBuilderT( + kind: kind, + content: content, + ) # ## Blocks # type OrgBuilderT* = OrgBlock diff --git a/src_v2/org/org_text_delimiter.nim b/src_v2/org/org_text_delimiter.nim index 0f01fb1..bb35ec9 100644 --- a/src_v2/org/org_text_delimiter.nim +++ b/src_v2/org/org_text_delimiter.nim @@ -1,7 +1,5 @@ import std/[ collections/sequtils, - strformat, - strutils, sugar, ] import fp/[ @@ -11,61 +9,54 @@ import ./org_types import ./org_builder import ../parser/parser # import ./org_text_link -# import ../utils/fp -# import ../parser/parser_internals -# import ../parser/parser_types -# import ../parser/builder_api # -- Parsers -let boldParser* = anyBetweenPair(ch('*')) -let italicParser* = anyBetweenPair(ch('/')) -let underlineParser* = anyBetweenPair(ch('_')) -let verbatimParser* = anyBetweenPair(ch('=')) -let codeParser* = anyBetweenPair(ch('~')) -let strikeThroughParser* = anyBetweenPair(ch('+')) +let boldParser* = anyBetweenPair(ch('*')) +let italicParser* = anyBetweenPair(ch('/')) +let underlineParser* = anyBetweenPair(ch('_')) +let verbatimParser* = anyBetweenPair(ch('=')) +let codeParser* = anyBetweenPair(ch('~')) +let strikeThroughParser* = anyBetweenPair(ch('+')) -# let rawTokenizer* = rawTextTokenizer(orgRawText) -# let boldTokenizer* = textTokenizer(orgBoldText) -# let italicTokenizer* = textTokenizer(orgItalicText) -# let underlineTokenizer* = textTokenizer(orgUnderlineText) -# let verbatimTokenizer* = textTokenizer(orgVerbatimText) -# let codeTokenizer* = textTokenizer(orgCodeText) -# let strikeThroughTokenizer* = textTokenizer(orgStrikeThroughText) +# -- Tokenizers -# let tok = linkTokenizerSeq +let rawTextTokenizer* = tokenizeRawText(orgRawText) -# let orgStyledTextBuilders = @[ -# (boldParser, boldTokenizer), -# (italicParser, italicTokenizer), -# (underlineParser, underlineTokenizer), -# (verbatimParser, verbatimTokenizer), -# (codeParser, codeTokenizer), -# (strikeThroughParser, strikeThroughTokenizer), -# (linkParser, linkTokenizerSeq), -# ] +let boldTokenizer* = tokenizeInlineTokens(orgBoldText) +let italicTokenizer* = tokenizeInlineTokens(orgItalicText) +let underlineTokenizer* = tokenizeInlineTokens(orgUnderlineText) +let verbatimTokenizer* = tokenizeInlineTokens(orgVerbatimText) +let codeTokenizer* = tokenizeInlineTokens(orgCodeText) +let strikeThroughTokenizer* = tokenizeInlineTokens(orgStrikeThroughText) -# proc makeRawTokenOrEmpty(tokens: seq[ParserToken]): seq[OrgInlineBuilderT] = -# ## Merge all parser `tokens` into a string to tokenize for the builder. -# ## Unless the string is empty, in this case return an empty list. -# let str = tokens.foldl(a & b.tokenStringValue(), "") -# if str.len == 0: @[] -# else: @[rawTokenizer(str)] +let styledTextTokenizers = @[ + (boldParser, boldTokenizer), + (italicParser, italicTokenizer), + (underlineParser, underlineTokenizer), + (verbatimParser, verbatimTokenizer), + (codeParser, codeTokenizer), + (strikeThroughParser, strikeThroughTokenizer), + # (linkParser, linkTokenizerSeq), +] -# proc tryBuildInline*(content: string): OrgInlineBuilderResult = -# initOrgInlineBuilder(content) -# .flatMap((builder: OrgInlineBuilder) => tryParseBuild( -# builder = builder, -# builderFns = orgStyledTextBuilders, -# defaultBuilderFn = makeRawTokenOrEmpty, -# )) +proc tryTokenizeRawText(tokens: seq[ParserToken]): seq[OrgInlineBuilderT] = + ## Merge all parser `tokens` into a string to tokenize for the builder. + ## Unless the string is empty, in this case return an empty list. + let str = tokens.foldl(a & b.toString(), "") + if str.len == 0: @[] + else: @[rawTextTokenizer(str)] -# when isMainModule: -# let test = tryBuildInline( -# "Regular *bold* [[placeholder.com]] /italic/ _underline_ =verbatim= ~code~ +strikethrough+ [[https://placeholder.com][title]]" -# ) +proc tryTokenizeInline*(content: string): OrgInlineBuilderResult = + initOrgInlineBuilder(content) + .flatMap((builder: OrgInlineBuilder) => tryTokenize( + builder = builder, + builderFns = styledTextTokenizers, + defaultTokenizerFn = tryTokenizeRawText, + )) -# echo test + +# -- Tests when isMainModule: block testParsers: @@ -78,3 +69,9 @@ when isMainModule: assert testParser("=verbatim=", verbatimParser) == "verbatim" assert testParser("~code~", codeParser) == "code" assert testParser("+strikeThrough+", strikeThroughParser) == "strikeThrough" + + block testTokenizers: + let test = tryTokenizeInline( + "Regular *bold* [[placeholder.com]] /italic/ _underline_ =verbatim= ~code~ +strikethrough+ [[https://placeholder.com][title]]" + ) + echo test.unsafeGet()