1 ! Copyright (C) 2007 Chris Double.
2 ! See http://factorcode.org/license.txt for BSD license.
4 USING: kernel tools.test peg peg.ebnf words math math.parser
5 sequences accessors peg.parsers parser namespaces arrays
6 strings eval unicode.data multiline ;
9 { T{ ebnf-non-terminal f "abc" } } [
10 "abc" 'non-terminal' parse
13 { T{ ebnf-terminal f "55" } } [
14 "'55'" 'terminal' parse
21 V{ T{ ebnf-terminal f "1" } T{ ebnf-terminal f "2" } }
25 "digit = '1' | '2'" 'rule' parse
32 V{ T{ ebnf-terminal f "1" } T{ ebnf-terminal f "2" } }
36 "digit = '1' '2'" 'rule' parse
43 V{ T{ ebnf-non-terminal f "one" } T{ ebnf-non-terminal f "two" } }
45 T{ ebnf-non-terminal f "three" }
49 "one two | three" 'choice' parse
55 T{ ebnf-non-terminal f "one" }
58 V{ T{ ebnf-non-terminal f "two" } T{ ebnf-non-terminal f "three" } }
64 "one {two | three}" 'choice' parse
70 T{ ebnf-non-terminal f "one" }
75 V{ T{ ebnf-non-terminal f "two" } T{ ebnf-non-terminal f "three" } }
77 T{ ebnf-non-terminal f "four" }
84 "one ((two | three) four)*" 'choice' parse
90 T{ ebnf-non-terminal f "one" }
91 T{ ebnf-optional f T{ ebnf-non-terminal f "two" } }
92 T{ ebnf-non-terminal f "three" }
96 "one ( two )? three" 'choice' parse
100 "\"foo\"" 'identifier' parse
104 "'foo'" 'identifier' parse
108 "foo" 'non-terminal' parse symbol>>
112 "foo]" 'non-terminal' parse symbol>>
116 "ab" [EBNF foo='a' 'b' EBNF]
120 "ab" [EBNF foo=('a')[[ drop 1 ]] 'b' EBNF]
124 "ab" [EBNF foo=('a') [[ drop 1 ]] ('b') [[ drop 2 ]] EBNF]
128 "A" [EBNF foo=[A-Z] EBNF]
132 "Z" [EBNF foo=[A-Z] EBNF]
136 "0" [EBNF foo=[A-Z] EBNF]
140 "0" [EBNF foo=[^A-Z] EBNF]
144 "A" [EBNF foo=[^A-Z] EBNF]
148 "Z" [EBNF foo=[^A-Z] EBNF]
151 { V{ "1" "+" "foo" } } [
152 "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF]
156 "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] EBNF]
160 "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF]
164 "1-1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF]
168 "4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] EBNF]
172 "4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] EBNF]
176 { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF]
180 { "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF]
184 { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF]
188 "ab" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
191 { V{ "a" " " "b" } } [
192 "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
195 { V{ "a" "\t" "b" } } [
196 "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
199 { V{ "a" "\n" "b" } } [
200 "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
204 "ab" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
207 { V{ "a" " " "b" } } [
208 "a b" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
212 { V{ "a" "\t" "b" } } [
213 "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
216 { V{ "a" "\n" "b" } } [
217 "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
221 "ab" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
225 "a\tb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
229 "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
233 "axb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
236 { V{ V{ 49 } "+" V{ 49 } } } [
237 #! Test direct left recursion.
238 #! Using packrat, so first part of expr fails, causing 2nd choice to be used
239 "1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF]
242 { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [
243 #! Test direct left recursion.
244 #! Using packrat, so first part of expr fails, causing 2nd choice to be used
245 "1+1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF]
248 { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [
249 #! Test indirect left recursion.
250 #! Using packrat, so first part of expr fails, causing 2nd choice to be used
251 "1+1+1" [EBNF num=([0-9])+ x=expr expr=x "+" num | num EBNF]
255 "abcd='9' | ('8'):x => [[ x ]]" 'ebnf' (parse) remaining>> empty?
259 Primary = PrimaryNoNewArray
260 PrimaryNoNewArray = ClassInstanceCreationExpression
265 ClassInstanceCreationExpression = "new" ClassOrInterfaceType "(" ")"
266 | Primary "." "new" Identifier "(" ")"
267 MethodInvocation = Primary "." MethodName "(" ")"
269 FieldAccess = Primary "." Identifier
270 | "super" "." Identifier
271 ArrayAccess = Primary "[" Expression "]"
272 | ExpressionName "[" Expression "]"
273 ClassOrInterfaceType = ClassName | InterfaceTypeName
274 ClassName = "C" | "D"
275 InterfaceTypeName = "I" | "J"
276 Identifier = "x" | "y" | ClassOrInterfaceType
277 MethodName = "m" | "n"
278 ExpressionName = Identifier
279 Expression = "i" | "j"
287 { V{ "this" "." "x" } } [
291 { V{ V{ "this" "." "x" } "." "y" } } [
295 { V{ V{ "this" "." "x" } "." "m" "(" ")" } } [
299 { V{ V{ V{ "x" "[" "i" "]" } "[" "j" "]" } "." "y" } } [
303 { V{ V{ "a" "b" } "c" } } [
304 "abc" [EBNF a="a" "b" foo=(a "c") EBNF]
307 { V{ V{ "a" "b" } "c" } } [
308 "abc" [EBNF a="a" "b" foo={a "c"} EBNF]
311 { V{ V{ "a" "b" } "c" } } [
312 "abc" [EBNF a="a" "b" foo=a "c" EBNF]
316 "a bc" [EBNF a="a" "b" foo=(a "c") EBNF]
320 "a bc" [EBNF a="a" "b" foo=a "c" EBNF]
324 "a bc" [EBNF a="a" "b" foo={a "c"} EBNF]
328 "ab c" [EBNF a="a" "b" foo=a "c" EBNF]
331 { V{ V{ "a" "b" } "c" } } [
332 "ab c" [EBNF a="a" "b" foo={a "c"} EBNF]
336 "ab c" [EBNF a="a" "b" foo=(a "c") EBNF]
340 "a b c" [EBNF a="a" "b" foo=a "c" EBNF]
344 "a b c" [EBNF a="a" "b" foo=(a "c") EBNF]
348 "a b c" [EBNF a="a" "b" foo={a "c"} EBNF]
351 { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
352 "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF]
356 "ab cab c" [EBNF a="a" "b" foo=(a "c")* EBNF]
359 { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
360 "ab c ab c" [EBNF a="a" "b" foo={a "c"}* EBNF]
364 "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF]
367 { V{ "a" "a" "a" } } [
368 "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF]
372 "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF]
373 "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ x ]] EBNF] =
376 { V{ "a" "a" "a" } } [
377 "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF]
381 "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF]
382 "aaa" [EBNF a=('a')* b=(a):x => [[ x ]] EBNF] =
386 "number=(digit)+:n 'a'" 'ebnf' (parse) remaining>> length zero?
390 "number=(digit)+ 'a'" 'ebnf' (parse) remaining>> length zero?
394 "number=digit+ 'a'" 'ebnf' (parse) remaining>> length zero?
398 "number=digit+:n 'a'" 'ebnf' (parse) remaining>> length zero?
402 "foo=(name):n !(keyword) => [[ n ]]" 'rule' parse
403 "foo=name:n !(keyword) => [[ n ]]" 'rule' parse =
407 "foo=!(keyword) (name):n => [[ n ]]" 'rule' parse
408 "foo=!(keyword) name:n => [[ n ]]" 'rule' parse =
418 foo=<foreign parser1 foo> 'b'
422 foo=<foreign parser1> 'c'
426 foo=<foreign any-char> 'd'
441 { V{ CHAR: a "d" } } [
446 "USING: kernel peg.ebnf ; \"a\\n\" [EBNF foo='a' '\n' => [[ drop \"\n\" ]] EBNF] drop" eval( -- )
450 "USING: peg.ebnf ; <EBNF foo='a' foo='b' EBNF>" eval( -- ) drop
454 #! Rule lookup occurs in a namespace. This causes an incorrect duplicate rule
455 #! if a var in a namespace is set. This unit test is to remind me to fix this.
456 [ "fail" "foo" set "foo='a'" 'ebnf' parse transform drop t ] with-scope
460 { V{ "a" CHAR: b } } [
461 "ab" [EBNF tokenizer=default foo="a" . EBNF]
464 TUPLE: ast-number value ;
470 SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]]
471 MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]]
472 Space = " " | "\t" | "\r" | "\n" | SingleLineComment | MultiLineComment
473 Spaces = Space* => [[ ignore ]]
474 Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]]
475 | Digits => [[ >string string>number ast-number boa ]]
476 Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";"
477 | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">="
478 | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-="
479 | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&="
480 | "&&" | "||=" | "||" | "." | "!"
481 Tok = Spaces (Number | Special )
484 { V{ CHAR: 1 T{ ast-number f 23 } ";" CHAR: x } } [
485 "123;x" [EBNF bar = .
486 tokenizer = <foreign a-tokenizer Tok> foo=.
487 tokenizer=default baz=.
488 main = bar foo foo baz
492 { V{ CHAR: 5 "+" CHAR: 2 } } [
497 spaces=space* => [[ ignore ]]
498 tokenizer=spaces (number | operator)
503 { V{ CHAR: 5 "+" CHAR: 2 } } [
508 spaces=space* => [[ ignore ]]
509 tokenizer=spaces (number | operator)
515 "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF]
519 "\\" [EBNF foo="\\" EBNF]
522 [ "USE: peg.ebnf [EBNF EBNF]" eval( -- ) ] must-fail
524 [ """USE: peg.ebnf [EBNF
529 error>> [ redefined-rule? ] [ name>> "lol" = ] bi and