tcenal.dsl.parsers source code

1 module tcenal.dsl.parsers;
2 
3 import compile_time_unittest : enableCompileTimeUnittest;
4 
5 import tcenal.parser_combinator.token : Token;
6 import tcenal.parser_combinator.parsing_result : ParsingResult;
7 import tcenal.parser_combinator.parse_tree_node : ParseTreeNode;
8 import tcenal.parser_combinator.memo : Memo;
9 import tcenal.parser_combinator.combinators : parseToken, parseTokenWithType, applyRule, sequence, select, choice, option, zeroOrMore, oneOrMore, zeroOrMoreWithSeparator, oneOrMoreWithSeparator, not;
10 import tcenal.parser_combinator.util : parseWithoutMemo;
11 import tcenal.dsl.lexer : lex;
12 
13 import std.ascii : isWhite, isAlpha, isAlphaNum;
14 
15 mixin enableCompileTimeUnittest;
16 
17 
18 ParsingResult rules(Token[] input, size_t position, ref Memo memo)
19 {
20     return applyRule!(oneOrMore!rule)(input, position, memo);
21 }
22 
23 
24 ParsingResult rule(Token[] input, size_t position, ref Memo memo)
25 {
26     return applyRule!(
27         sequence!(
28             select!(0,
29                 parseTokenWithType!"identifier",
30                 parseToken!"<-",
31             ),
32             choiceExpr,
33         )
34     )(input, position, memo);
35 }
36 unittest
37 {
38     Memo memo;
39     with (rule(lex(q{foo <- foo "foo" / "foo"}), 0, memo))
40     {
41         assert (success);
42         assert (nextPosition == 6);
43         assert (node.ruleName == "rule");
44         assert (node.children[0].ruleName == "#sequence");
45         assert (node.children[0].children[0].token.value == "foo");
46         assert (node.children[0].children[0].token.type == "identifier");
47         assert (node.children[0].children[1].ruleName == "choiceExpr");
48         assert (node.children[0].children[1].children[0].ruleName == "#repeat");
49         assert (node.children[0].children[1].children[0].children[0].ruleName == "sequenceExpr");
50         assert (node.children[0].children[1].children[0].children[0].children[0].ruleName == "#repeat");
51         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].ruleName == "prefixExpr");
52         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].ruleName == "postfixExpr");
53         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].ruleName == "primaryExpr");
54         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].children[0].ruleName == "ruleName");
55         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].token.value == "foo");
56         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].token.type == "identifier");
57         assert (node.children[0].children[1].children[0].children[0].children[0].children[1].ruleName == "prefixExpr");
58         assert (node.children[0].children[1].children[0].children[0].children[0].children[1].children[0].ruleName == "postfixExpr");
59         assert (node.children[0].children[1].children[0].children[0].children[0].children[1].children[0].children[0].ruleName == "primaryExpr");
60         assert (node.children[0].children[1].children[0].children[0].children[0].children[1].children[0].children[0].children[0].ruleName == "tokenValue");
61         assert (node.children[0].children[1].children[0].children[0].children[0].children[1].children[0].children[0].children[0].children[0].token.value == "foo");
62         assert (node.children[0].children[1].children[0].children[0].children[0].children[1].children[0].children[0].children[0].children[0].token.type == "stringLiteral");
63         assert (node.children[0].children[1].children[0].children[1].ruleName == "sequenceExpr");
64         assert (node.children[0].children[1].children[0].children[1].children[0].ruleName == "#repeat");
65         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].ruleName == "prefixExpr");
66         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].children[0].ruleName == "postfixExpr");
67         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].children[0].children[0].ruleName == "primaryExpr");
68         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].children[0].children[0].children[0].ruleName == "tokenValue");
69         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].token.value == "foo");
70         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].token.type == "stringLiteral");
71     }
72 }
73 
74 
75 ParsingResult choiceExpr(Token[] input, size_t position, ref Memo memo)
76 {
77     return applyRule!(
78         oneOrMoreWithSeparator!(
79             sequenceExpr,
80             parseToken!"/"
81         )
82     )(input, position, memo);
83 }
84 unittest
85 {
86     Memo memo;
87     with (choiceExpr(lex(q{foo !bar / buz+ / "foobar"}), 0, memo))
88     {
89         assert (success);
90         assert (nextPosition == 8);
91         assert (node.ruleName == "choiceExpr");
92         assert (node.children[0].ruleName == "#repeat");
93         assert (node.children[0].children[0].ruleName == "sequenceExpr");
94         assert (node.children[0].children[0].children[0].ruleName == "#repeat");
95         assert (node.children[0].children[0].children[0].children[0].ruleName == "prefixExpr");
96         assert (node.children[0].children[0].children[0].children[0].children[0].ruleName == "postfixExpr");
97         assert (node.children[0].children[0].children[0].children[0].children[0].children[0].ruleName == "primaryExpr");
98         assert (node.children[0].children[0].children[0].children[0].children[0].children[0].children[0].ruleName == "ruleName");
99         assert (node.children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].token.value == "foo");
100         assert (node.children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].token.type == "identifier");
101         assert (node.children[0].children[0].children[0].children[1].ruleName == "prefixExpr");
102         assert (node.children[0].children[0].children[0].children[1].children[0].ruleName == "#sequence");
103         assert (node.children[0].children[0].children[0].children[1].children[0].children[0].token.value == "!");
104         assert (node.children[0].children[0].children[0].children[1].children[0].children[1].ruleName == "postfixExpr");
105         assert (node.children[0].children[0].children[0].children[1].children[0].children[1].children[0].ruleName == "primaryExpr");
106         assert (node.children[0].children[0].children[0].children[1].children[0].children[1].children[0].children[0].ruleName == "ruleName");
107         assert (node.children[0].children[0].children[0].children[1].children[0].children[1].children[0].children[0].children[0].token.value == "bar");
108         assert (node.children[0].children[0].children[0].children[1].children[0].children[1].children[0].children[0].children[0].token.type == "identifier");
109         assert (node.children[0].children[1].ruleName == "sequenceExpr");
110         assert (node.children[0].children[1].children[0].ruleName == "#repeat");
111         assert (node.children[0].children[1].children[0].children[0].ruleName == "prefixExpr");
112         assert (node.children[0].children[1].children[0].children[0].children[0].ruleName == "postfixExpr");
113         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].ruleName == "#sequence");
114         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].ruleName == "primaryExpr");
115         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].ruleName == "ruleName");
116         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].children[0].token.value == "buz");
117         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[0].children[0].children[0].token.type == "identifier");
118         assert (node.children[0].children[1].children[0].children[0].children[0].children[0].children[1].token.value == "+");
119         assert (node.children[0].children[2].ruleName == "sequenceExpr");
120         assert (node.children[0].children[2].children[0].ruleName == "#repeat");
121         assert (node.children[0].children[2].children[0].children[0].ruleName == "prefixExpr");
122         assert (node.children[0].children[2].children[0].children[0].children[0].ruleName == "postfixExpr");
123         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].ruleName == "primaryExpr");
124         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].ruleName == "tokenValue");
125         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].children[0].token.value == "foobar");
126         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].children[0].token.type == "stringLiteral");
127     }
128 }
129 
130 
131 ParsingResult sequenceExpr(Token[] input, size_t position, ref Memo memo)
132 {
133     return applyRule!(oneOrMore!prefixExpr)(input, position, memo);
134 }
135 unittest
136 {
137     Memo memo;
138     with (sequenceExpr(lex(q{foo !bar buz+ "foobar"}), 0, memo))
139     {
140         assert (success);
141         assert (nextPosition == 6);
142         assert (node.ruleName == "sequenceExpr");
143         assert (node.children[0].ruleName == "#repeat");
144         assert (node.children[0].children[0].ruleName == "prefixExpr");
145         assert (node.children[0].children[0].children[0].ruleName == "postfixExpr");
146         assert (node.children[0].children[0].children[0].children[0].ruleName == "primaryExpr");
147         assert (node.children[0].children[0].children[0].children[0].children[0].ruleName == "ruleName");
148         assert (node.children[0].children[0].children[0].children[0].children[0].children[0].token.value == "foo");
149         assert (node.children[0].children[0].children[0].children[0].children[0].children[0].token.type == "identifier");
150         assert (node.children[0].children[1].ruleName == "prefixExpr");
151         assert (node.children[0].children[1].children[0].ruleName == "#sequence");
152         assert (node.children[0].children[1].children[0].children[0].token.value == "!");
153         assert (node.children[0].children[1].children[0].children[1].ruleName == "postfixExpr");
154         assert (node.children[0].children[1].children[0].children[1].children[0].ruleName == "primaryExpr");
155         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].ruleName == "ruleName");
156         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].children[0].token.value == "bar");
157         assert (node.children[0].children[1].children[0].children[1].children[0].children[0].children[0].token.type == "identifier");
158         assert (node.children[0].children[2].ruleName == "prefixExpr");
159         assert (node.children[0].children[2].children[0].ruleName == "postfixExpr");
160         assert (node.children[0].children[2].children[0].children[0].ruleName == "#sequence");
161         assert (node.children[0].children[2].children[0].children[0].children[0].ruleName == "primaryExpr");
162         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].ruleName == "ruleName");
163         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].token.value == "buz");
164         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].token.type == "identifier");
165         assert (node.children[0].children[2].children[0].children[0].children[1].token.value == "+");
166         assert (node.children[0].children[3].ruleName == "prefixExpr");
167         assert (node.children[0].children[3].children[0].ruleName == "postfixExpr");
168         assert (node.children[0].children[3].children[0].children[0].ruleName == "primaryExpr");
169         assert (node.children[0].children[3].children[0].children[0].children[0].ruleName == "tokenValue");
170         assert (node.children[0].children[3].children[0].children[0].children[0].children[0].token.value == "foobar");
171         assert (node.children[0].children[3].children[0].children[0].children[0].children[0].token.type == "stringLiteral");
172     }
173 }
174 
175 
176 ParsingResult prefixExpr(Token[] input, size_t position, ref Memo memo)
177 {
178     return applyRule!(
179         choice!(
180             sequence!(
181                 choice!(
182                     parseToken!"&",
183                     parseToken!"!"
184                 ),
185                 postfixExpr
186             ),
187             postfixExpr
188         )
189     )(input, position, memo);
190 }
191 unittest
192 {
193     Memo memo;
194     with (prefixExpr(lex(q{&foo}), 0, memo))
195     {
196         assert (success);
197         assert (nextPosition == 2);
198         assert (node.ruleName == "prefixExpr");
199         assert (node.children[0].ruleName == "#sequence");
200         assert (node.children[0].children[0].token.value == "&");
201         assert (node.children[0].children[1].ruleName == "postfixExpr");
202         assert (node.children[0].children[1].children[0].ruleName == "primaryExpr");
203         assert (node.children[0].children[1].children[0].children[0].ruleName == "ruleName");
204         assert (node.children[0].children[1].children[0].children[0].children[0].token.value == "foo");
205         assert (node.children[0].children[1].children[0].children[0].children[0].token.type == "identifier");
206     }
207 }
208 
209 
210 ParsingResult postfixExpr(Token[] input, size_t position, ref Memo memo)
211 {
212     return applyRule!(
213         choice!(
214             sequence!(
215                 primaryExpr,
216                 choice!(
217                     parseToken!"*",
218                     parseToken!"+",
219                 ),
220                 option!(
221                     select!(1,
222                         parseToken!"<",
223                         choiceExpr,
224                         parseToken!">",
225                     )
226                 )
227             ),
228             sequence!(
229                 primaryExpr,
230                 parseToken!"?",
231             ),
232             primaryExpr
233         )
234     )(input, position, memo);
235 }
236 unittest
237 {
238     with (parseWithoutMemo!postfixExpr(lex(q{foo*})))
239     {
240         assert (success);
241         assert (nextPosition == 2);
242         assert (node.ruleName == "postfixExpr");
243         assert (node.children[0].ruleName == "#sequence");
244         assert (node.children[0].children[0].ruleName == "primaryExpr");
245         assert (node.children[0].children[0].children[0].ruleName == "ruleName");
246         assert (node.children[0].children[0].children[0].children[0].token.value == "foo");
247         assert (node.children[0].children[0].children[0].children[0].token.type == "identifier");
248         assert (node.children[0].children[1].token.value == "*");
249         assert (node.children[0].children[2].ruleName == "#option");
250         assert (node.children[0].children[2].children.length == 0);
251     }
252     with (parseWithoutMemo!postfixExpr(lex(q{foo*<foo>})))
253     {
254         assert (success);
255         assert (nextPosition == 5);
256         assert (node.ruleName == "postfixExpr");
257         assert (node.children[0].ruleName == "#sequence");
258         assert (node.children[0].children[0].ruleName == "primaryExpr");
259         assert (node.children[0].children[0].children[0].ruleName == "ruleName");
260         assert (node.children[0].children[0].children[0].children[0].token.value == "foo");
261         assert (node.children[0].children[0].children[0].children[0].token.type == "identifier");
262         assert (node.children[0].children[1].token.value == "*");
263         assert (node.children[0].children[2].ruleName == "#option");
264         assert (node.children[0].children[2].children[0].ruleName == "choiceExpr");
265         assert (node.children[0].children[2].children[0].children[0].ruleName == "#repeat");
266         assert (node.children[0].children[2].children[0].children[0].children[0].ruleName == "sequenceExpr");
267         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].ruleName == "#repeat");
268         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].ruleName == "prefixExpr");
269         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].children[0].ruleName == "postfixExpr");
270         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].children[0].children[0].ruleName == "primaryExpr");
271         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].ruleName == "ruleName");
272         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].token.value == "foo");
273         assert (node.children[0].children[2].children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].children[0].token.type == "identifier");
274     }
275 }
276 
277 
278 ParsingResult primaryExpr(Token[] input, size_t position, ref Memo memo)
279 {
280     return applyRule!(
281         choice!(
282             select!(1,
283                 parseToken!"(",
284                 choiceExpr,
285                 parseToken!")"
286             ),
287             tokenValue,
288             tokenType,
289             select!(0,
290                 ruleName,
291                 not!(parseToken!"<-")
292             )
293         )
294     )(input, position, memo);
295 }
296 unittest
297 {
298     Memo memo;
299     with (primaryExpr(lex(q{foo}), 0, memo))
300     {
301         assert (success);
302         assert (nextPosition == 1);
303         assert (node.ruleName == "primaryExpr");
304         assert (node.children[0].ruleName == "ruleName");
305         assert (node.children[0].children[0].token.value == "foo");
306         assert (node.children[0].children[0].token.type == "identifier");
307     }
308 
309     memo = memo.init;
310     assert (!primaryExpr(lex(q{foo <-}), 0, memo).success);
311 }
312 
313 
314 ParsingResult ruleName(Token[] input, size_t position, ref Memo memo)
315 {
316     return applyRule!(parseTokenWithType!"identifier")(input, position, memo);
317 }
318 
319 
320 ParsingResult tokenValue(Token[] input, size_t position, ref Memo memo)
321 {
322     return applyRule!(parseTokenWithType!"stringLiteral")(input, position, memo);
323 }
324 
325 
326 ParsingResult tokenType(Token[] input, size_t position, ref Memo memo)
327 {
328     return applyRule!(select!(1, parseToken!"@", parseTokenWithType!"identifier"))(input, position, memo);
329 }