1 ! Copyright (C) 2007 Chris Double.
2 ! See http://factorcode.org/license.txt for BSD license.
4 USING: kernel tools.test peg peg.ebnf peg.ebnf.private words
5 math math.parser sequences accessors peg.parsers parser
6 namespaces arrays strings eval unicode multiline ;
9 { T{ ebnf-non-terminal f "abc" } } [
10 "abc" non-terminal-parser parse
13 { T{ ebnf-terminal f "55" } } [
14 "'55'" terminal-parser parse
21 V{ T{ ebnf-terminal f "1" } T{ ebnf-terminal f "2" } }
25 "digit = '1' | '2'" rule-parser parse
32 V{ T{ ebnf-terminal f "1" } T{ ebnf-terminal f "2" } }
36 "digit = '1' '2'" rule-parser parse
43 V{ T{ ebnf-non-terminal f "one" } T{ ebnf-non-terminal f "two" } }
45 T{ ebnf-non-terminal f "three" }
49 "one two | three" choice-parser parse
55 T{ ebnf-non-terminal f "one" }
58 V{ T{ ebnf-non-terminal f "two" } T{ ebnf-non-terminal f "three" } }
64 "one {two | three}" choice-parser parse
70 T{ ebnf-non-terminal f "one" }
75 V{ T{ ebnf-non-terminal f "two" } T{ ebnf-non-terminal f "three" } }
77 T{ ebnf-non-terminal f "four" }
84 "one ((two | three) four)*" choice-parser parse
90 T{ ebnf-non-terminal f "one" }
95 V{ T{ ebnf-non-terminal f "two" } T{ ebnf-non-terminal f "three" } }
97 T{ ebnf-non-terminal f "four" }
104 "one ((two | three) four)~" choice-parser parse
110 T{ ebnf-non-terminal f "one" }
111 T{ ebnf-optional f T{ ebnf-non-terminal f "two" } }
112 T{ ebnf-non-terminal f "three" }
116 "one ( two )? three" choice-parser parse
120 "\"foo\"" identifier-parser parse
124 "'foo'" identifier-parser parse
128 "\"\\\"\"" identifier-parser parse
132 "\"\\\\\"" identifier-parser parse
136 "'\\x41\\u{greek-capital-letter-sigma}\\u01D11E'" identifier-parser parse
140 "foo" non-terminal-parser parse symbol>>
144 "foo]" non-terminal-parser parse symbol>>
148 "ab" EBNF[=[ foo='a' 'b' ]=]
152 "ab" EBNF[=[ foo=('a')[[ drop 1 ]] 'b' ]=]
156 "ab" EBNF[=[ foo=('a') [[ drop 1 ]] ('b') [[ drop 2 ]] ]=]
160 "A" EBNF[=[ foo=[A-Z] ]=]
164 "Z" EBNF[=[ foo=[A-Z] ]=]
168 "0" EBNF[=[ foo=[A-Z] ]=]
172 "0" EBNF[=[ foo=[^A-Z] ]=]
176 "A" EBNF[=[ foo=[^A-Z] ]=]
180 "Z" EBNF[=[ foo=[^A-Z] ]=]
183 { V{ "1" "+" "foo" } } [
184 "1+1" EBNF[=[ foo='1' '+' '1' [[ drop "foo" ]] ]=]
188 "1+1" EBNF[=[ foo='1' '+' '1' => [[ drop "foo" ]] ]=]
192 "1+1" EBNF[=[ foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] ]=]
196 "1-1" EBNF[=[ foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] ]=]
200 "4+2" EBNF[=[ num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] ]=]
204 "4+2" EBNF[=[ foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] ]=]
208 { 1 2 3 4 } EBNF[=[ num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num ]=]
212 { "a" 2 3 4 } EBNF[=[ num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num ]=]
216 { 1 2 "a" 4 } EBNF[=[ num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num ]=]
220 "ab" EBNF[=[ -=" " | "\t" | "\n" foo="a" - "b" ]=]
223 { V{ "a" " " "b" } } [
224 "a b" EBNF[=[ -=" " | "\t" | "\n" foo="a" - "b" ]=]
227 { V{ "a" "\t" "b" } } [
228 "a\tb" EBNF[=[ -=" " | "\t" | "\n" foo="a" - "b" ]=]
231 { V{ "a" "\n" "b" } } [
232 "a\nb" EBNF[=[ -=" " | "\t" | "\n" foo="a" - "b" ]=]
236 "ab" EBNF[=[ -=" " | "\t" | "\n" foo="a" (-)? "b" ]=]
239 { V{ "a" " " "b" } } [
240 "a b" EBNF[=[ -=" " | "\t" | "\n" foo="a" (-)? "b" ]=]
244 { V{ "a" "\t" "b" } } [
245 "a\tb" EBNF[=[ -=" " | "\t" | "\n" foo="a" (-)? "b" ]=]
248 { V{ "a" "\n" "b" } } [
249 "a\nb" EBNF[=[ -=" " | "\t" | "\n" foo="a" (-)? "b" ]=]
253 "ab" EBNF[=[ -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" ]=]
257 "a\tb" EBNF[=[ -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" ]=]
261 "a\nb" EBNF[=[ -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" ]=]
265 "axb" EBNF[=[ -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" ]=]
268 { V{ V{ 49 } "+" V{ 49 } } } [
269 ! Test direct left recursion.
270 ! Using packrat, so first part of expr fails, causing 2nd choice to be used
271 "1+1" EBNF[=[ num=([0-9])+ expr=expr "+" num | num ]=]
274 { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [
275 ! Test direct left recursion.
276 ! Using packrat, so first part of expr fails, causing 2nd choice to be used
277 "1+1+1" EBNF[=[ num=([0-9])+ expr=expr "+" num | num ]=]
280 { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [
281 ! Test indirect left recursion.
282 ! Using packrat, so first part of expr fails, causing 2nd choice to be used
283 "1+1+1" EBNF[=[ num=([0-9])+ x=expr expr=x "+" num | num ]=]
287 "abcd='9' | ('8'):x => [[ x ]]" ebnf-parser (parse) remaining>> empty?
291 Primary = PrimaryNoNewArray
292 PrimaryNoNewArray = ClassInstanceCreationExpression
297 ClassInstanceCreationExpression = "new" ClassOrInterfaceType "(" ")"
298 | Primary "." "new" Identifier "(" ")"
299 MethodInvocation = Primary "." MethodName "(" ")"
301 FieldAccess = Primary "." Identifier
302 | "super" "." Identifier
303 ArrayAccess = Primary "[" Expression "]"
304 | ExpressionName "[" Expression "]"
305 ClassOrInterfaceType = ClassName | InterfaceTypeName
306 ClassName = "C" | "D"
307 InterfaceTypeName = "I" | "J"
308 Identifier = "x" | "y" | ClassOrInterfaceType
309 MethodName = "m" | "n"
310 ExpressionName = Identifier
311 Expression = "i" | "j"
319 { V{ "this" "." "x" } } [
323 { V{ V{ "this" "." "x" } "." "y" } } [
327 { V{ V{ "this" "." "x" } "." "m" "(" ")" } } [
331 { V{ V{ V{ "x" "[" "i" "]" } "[" "j" "]" } "." "y" } } [
335 { V{ V{ "a" "b" } "c" } } [
336 "abc" EBNF[=[ a="a" "b" foo=(a "c") ]=]
340 "abc" EBNF[=[ a="a" "b"~ foo=(a "c") ]=]
343 { V{ V{ "a" V{ "b" "b" } } "c" } } [
344 "abbc" EBNF[=[ a=("a" "b"*) foo=(a "c") ]=]
348 "abc" EBNF[=[ a=("a" ("b")~) foo=(a "c") ]=]
352 "abc" EBNF[=[ a=("a" "b"~) foo=(a "c") ]=]
356 "abc" EBNF[=[ a=("a" "b")~ foo=(a "c") ]=]
359 { V{ V{ "a" "b" } "c" } } [
360 "abc" EBNF[=[ a="a" "b" foo={a "c"} ]=]
363 { V{ V{ "a" "b" } "c" } } [
364 "abc" EBNF[=[ a="a" "b" foo=a "c" ]=]
368 "a bc" EBNF[=[ a="a" "b" foo=(a "c") ]=]
372 "a bc" EBNF[=[ a="a" "b" foo=a "c" ]=]
376 "a bc" EBNF[=[ a="a" "b" foo={a "c"} ]=]
380 "ab c" EBNF[=[ a="a" "b" foo=a "c" ]=]
383 { V{ V{ "a" "b" } "c" } } [
384 "ab c" EBNF[=[ a="a" "b" foo={a "c"} ]=]
388 "ab c" EBNF[=[ a="a" "b" foo=(a "c") ]=]
392 "a b c" EBNF[=[ a="a" "b" foo=a "c" ]=]
396 "a b c" EBNF[=[ a="a" "b" foo=(a "c") ]=]
400 "a b c" EBNF[=[ a="a" "b" foo={a "c"} ]=]
403 { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
404 "ab cab c" EBNF[=[ a="a" "b" foo={a "c"}* ]=]
408 "ab cab c" EBNF[=[ a="a" "b" foo=(a "c")* ]=]
411 { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
412 "ab c ab c" EBNF[=[ a="a" "b" foo={a "c"}* ]=]
415 { V{ V{ "a" "c" } V{ "a" "c" } } } [
416 "ab c ab c" EBNF[=[ a="a" "b"~ foo={a "c"}* ]=]
420 "ab c ab c" EBNF[=[ a="a" "b" foo=(a "c")* ]=]
424 "ab c ab c" EBNF[=[ a="a" "b" foo=(a "c")* ]=]
427 { V{ "a" "a" "a" } } [
428 "aaa" EBNF[=[ a=('a')* b=!('b') a:x => [[ x ]] ]=]
432 "aaa" EBNF[=[ a=('a')* b=!('b') a:x => [[ x ]] ]=]
433 "aaa" EBNF[=[ a=('a')* b=!('b') (a):x => [[ x ]] ]=] =
436 { V{ "a" "a" "a" } } [
437 "aaa" EBNF[=[ a=('a')* b=a:x => [[ x ]] ]=]
441 "aaa" EBNF[=[ a=('a')* b=a:x => [[ x ]] ]=]
442 "aaa" EBNF[=[ a=('a')* b=(a):x => [[ x ]] ]=] =
446 "number=(digit)+:n 'a'" ebnf-parser (parse) remaining>> length zero?
450 "number=(digit)+ 'a'" ebnf-parser (parse) remaining>> length zero?
454 "number=digit+ 'a'" ebnf-parser (parse) remaining>> length zero?
458 "number=digit+:n 'a'" ebnf-parser (parse) remaining>> length zero?
462 "foo=(name):n !(keyword) => [[ n ]]" rule-parser parse
463 "foo=name:n !(keyword) => [[ n ]]" rule-parser parse =
467 "foo=!(keyword) (name):n => [[ n ]]" rule-parser parse
468 "foo=!(keyword) name:n => [[ n ]]" rule-parser parse =
478 foo=<foreign parser1 foo> 'b'
482 foo=<foreign parser1> 'c'
486 foo=<foreign any-char> 'd'
501 { V{ CHAR: a "d" } } [
506 "USING: kernel peg.ebnf ; \"a\\n\" EBNF[=[ foo='a' '\n' => [[ drop \"\n\" ]] ]=] drop" eval( -- )
510 "USING: peg.ebnf ; EBNF[=[ foo='a' foo='b' ]=]" eval( -- ) drop
514 ! Rule lookup occurs in a namespace. This causes an incorrect duplicate rule
515 ! if a var in a namespace is set. This unit test is to remind me to fix this.
516 [ "fail" "foo" set "foo='a'" ebnf-parser parse transform drop t ] with-scope
520 { V{ "a" CHAR: b } } [
521 "ab" EBNF[=[ tokenizer=default foo="a" . ]=]
524 TUPLE: ast-number value ;
526 EBNF: a-tokenizer [=[
530 SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]]
531 MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]]
532 Space = " " | "\t" | "\r" | "\n" | SingleLineComment | MultiLineComment
533 Spaces = Space* => [[ ignore ]]
534 Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array "" concat-as string>number ast-number boa ]]
535 | Digits => [[ >string string>number ast-number boa ]]
536 Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";"
537 | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">="
538 | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-="
539 | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&="
540 | "&&" | "||=" | "||" | "." | "!"
541 Tok = Spaces (Number | Special )
544 { V{ CHAR: 1 T{ ast-number f 23 } ";" CHAR: x } } [
545 "123;x" EBNF[=[ bar = .
546 tokenizer = <foreign a-tokenizer Tok> foo=.
547 tokenizer=default baz=.
548 main = bar foo foo baz
552 { V{ CHAR: 5 "+" CHAR: 2 } } [
557 spaces=space* => [[ ignore ]]
558 tokenizer=spaces (number | operator)
563 { V{ CHAR: 5 "+" CHAR: 2 } } [
568 spaces=space* => [[ ignore ]]
569 tokenizer=spaces (number | operator)
575 "++--" EBNF[=[ tokenizer=("++" | "--") main="++" ]=]
579 "\\" EBNF[=[ foo="\\" ]=]
582 [ "USE: peg.ebnf EBNF[=[ ]=]" eval( -- ) ] must-fail
584 [ "USE: peg.ebnf EBNF[=[
589 error>> [ redefined-rule? ] [ name>> "lol" = ] bi and
595 EBNF: foo [=[ Bar = "a":a1 "a":a2 => [[ a1 a2 2array ]] ]=]
602 EBNF: foo2 [=[ Bar = "a":a-1 "a":a-2 => [[ a-1 a-2 2array ]] ]=]
607 EBNF: parse-til-right-bracket [=[
610 "abc]" parse-til-right-bracket >string
613 ! Doesn't match anything, don't run it.
615 EBNF: parse-empty-range [=[