moustachu_tokenizer.nim

1 
2 
3 
4 
5 
6 
7 
8 
9 
10 
11 
12 
13 
14 
15 
16 
17 
18 
19 
20 
21 
22 
23 
24 
25 
26 
27 
28 
29 
30 
31 
32 
33 
34 
35 
36 
37 
38 
39 
40 
41 
42 
43 
44 
45 
46 
47 
48 
49 
50 
51 
52 
53 
54 
55 
56 
57 
58 
59 
60 
61 
62 
63 
64 
65 
66 
67 
68 
69 
70 
71 
72 
73 
74 
75 
76 
77 
78 
79 
80 
81 
82 
83 
84 
85 
86 
87 
88 
89 
90 
91 
92 
93 
94 
95 
96 
97 
98 
99 
100 
101 
102 
103 
104 
105 
106 
107 
108 
109 
110 
111 
112 
113 
114 
115 
116 
117 
118 
119 
120 
121 
122 
123 
124 
125 
126 
127 
128 
import strutils

type
  TokenType* {.pure.} = enum
    rawtext,
    escapedvariable,
    unescapedvariable,
    section,
    invertedsection,
    comment,
    ender,
    indenter

  Token* = tuple[tokentype: TokenType, value: string]


proc left_side_empty(tmplate: string, pivotindex: int): tuple[empty: bool, newlineindex: int] =
  var ls_i = 0
  var ls_empty = false
  var i = pivotindex - 1
  while i > -1 and tmplate[i] in {' ', '\t'}: dec(i)
  if (i == -1) or (tmplate[i] == '\l'):
    ls_i = i
    ls_empty = true
  return (empty: ls_empty, newlineindex: ls_i)


iterator tokenize*(tmplate: string): Token =
  let opening = "{{"
  var pos = 0

  while pos < tmplate.len:
    let originalpos = pos
    var closing = "}}"

    # find start of tag
    var opening_index = tmplate.find(opening, start=pos)
    if opening_index == -1:
      yield (tokentype: TokenType.rawtext, value: tmplate[pos..high(tmplate)])
      break

    #Check if the left side is empty
    var left_side = left_side_empty(tmplate, opening_index)
    var ls_empty = left_side.empty
    var ls_i = left_side.newlineindex

    #Deal with text before tag
    var beforetoken = (tokentype: TokenType.rawtext, value: "")
    if opening_index > pos:
      #safe bet for now
      beforetoken.value = tmplate[pos..opening_index-1]

    pos = opening_index + opening.len

    if not (pos < tmplate.len):
      yield (tokentype: TokenType.rawtext, value: tmplate[opening_index..high(tmplate)])
      break

    #Determine TokenType
    var tt = TokenType.escapedvariable

    case tmplate[pos]
    of '!':
      tt = TokenType.comment
      pos += 1
    of '&':
      tt = TokenType.unescapedvariable
      pos += 1
    of '{':
      tt = TokenType.unescapedvariable
      pos += 1
      closing &= "}"
    of '#':
      tt = TokenType.section
      pos += 1
    of '^':
      tt = TokenType.invertedsection
      pos += 1
    of '/':
      tt = TokenType.ender
      pos += 1
    else:
      tt = TokenType.escapedvariable

    #find end of tag
    var closingindex = tmplate.find(closing, start=pos)
    if closingindex == -1:
      if beforetoken.value != "": yield beforetoken
      yield (tokentype: TokenType.rawtext, value: tmplate[opening_index..pos-1])
      continue

    #Check if the right side is empty
    var rs_i = 0
    var rs_empty = false
    var i = 0
    if ls_empty:
      i = closingindex + closing.len
      while i < tmplate.len and tmplate[i] in {' ', '\t'}: inc(i)
      if i == tmplate.len:
        rs_i = i - 1
        rs_empty = true
      elif tmplate[i] == '\c' and (i+1 < tmplate.len) and (tmplate[i+1] == '\l'):
        rs_i = i + 1
        rs_empty = true
      elif tmplate[i] == '\l':
        rs_i = i
        rs_empty = true
      else:
        discard

    if tt in [TokenType.comment, TokenType.section,
              TokenType.invertedsection, TokenType.ender]:
      # Standalone tokens
      if rs_empty:
        if beforetoken.value != "":
          beforetoken.value = tmplate[originalpos..ls_i]
          yield beforetoken

        yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip)
        pos = rs_i + 1 # remove new line of this line
      else:
        if beforetoken.value != "": yield beforetoken
        yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip)
        pos = closingindex + closing.len
    else:
      if beforetoken.value != "": yield beforetoken
      yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip)
      pos = closingindex + closing.len