Fix peg.ebnf tests. Handle \ in EBNF

diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor
index 2269af6625d854933fb83e11b69d53fe6bcf6797..a2807d20dba75e7355f85dc1d101af58294c6936 100644 (file)
--- a/extra/peg/ebnf/ebnf-tests.factor
+++ b/extra/peg/ebnf/ebnf-tests.factor
@@ -132,21 +132,21 @@ IN: peg.ebnf.tests
   "Z" [EBNF foo=[A-Z] EBNF] call ast>> 
 ] unit-test
 
-{ f } [
+[
   "0" [EBNF foo=[A-Z] EBNF] call  
-] unit-test
+] must-fail
 
 { CHAR: 0 } [
   "0" [EBNF foo=[^A-Z] EBNF] call ast>> 
 ] unit-test
 
-{ f } [
+[
   "A" [EBNF foo=[^A-Z] EBNF] call  
-] unit-test
+] must-fail
 
-{ f } [
+[
   "Z" [EBNF foo=[^A-Z] EBNF] call  
-] unit-test
+] must-fail
 
 { V{ "1" "+" "foo" } } [
   "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF] call ast>>
@@ -176,17 +176,17 @@ IN: peg.ebnf.tests
   { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   { "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call 
-] unit-test
+] must-fail
 
 { 3 } [
   { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   "ab" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call 
-] unit-test
+] must-fail
 
 { V{ "a" " " "b" } } [
   "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call ast>>
@@ -229,9 +229,9 @@ IN: peg.ebnf.tests
   "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   "axb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call 
-] unit-test
+] must-fail
 
 { V{ V{ 49 } "+" V{ 49 } } } [ 
   #! Test direct left recursion. 
@@ -314,41 +314,41 @@ main = Primary
   "abc" [EBNF a="a" "b" foo=a "c" EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   "a bc" [EBNF a="a" "b" foo=(a "c") EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a bc" [EBNF a="a" "b" foo=a "c" EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a bc" [EBNF a="a" "b" foo={a "c"} EBNF] call
-] unit-test
+] must-fail
 
-{ f } [
+[
   "ab c" [EBNF a="a" "b" foo=a "c" EBNF] call 
-] unit-test
+] must-fail
 
 { V{ V{ "a" "b" } "c" } } [
   "ab c" [EBNF a="a" "b" foo={a "c"} EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   "ab c" [EBNF a="a" "b" foo=(a "c") EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a b c" [EBNF a="a" "b" foo=a "c" EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a b c" [EBNF a="a" "b" foo=(a "c") EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a b c" [EBNF a="a" "b" foo={a "c"} EBNF] call 
-] unit-test
+] must-fail
 
 { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
   "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF] call ast>>
@@ -515,4 +515,8 @@ Tok                = Spaces (Number | Special )
 
 { "++" } [
   "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>>
+] unit-test
+
+{ "\\" } [
+  "\\" [EBNF foo="\\" EBNF] call ast>>
 ] unit-test
\ No newline at end of file

diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor
index 3d48665c8c23611c279a77eeb9b3507750d4da93..610cffd2732cbe8291392a999870b11d7272a2e7 100644 (file)
--- a/extra/peg/ebnf/ebnf.factor
+++ b/extra/peg/ebnf/ebnf.factor
@@ -99,6 +99,7 @@ PEG: escaper ( string -- ast )
     "\\t" token [ drop "\t" ] action ,\r
     "\\n" token [ drop "\n" ] action ,\r
     "\\r" token [ drop "\r" ] action ,\r
+    "\\\\" token [ drop "\\" ] action ,\r
   ] choice* any-char-parser 2array choice repeat0 ;\r
 \r
 : replace-escapes ( string -- string )\r

diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor
index 825c8f03d11e36afe1712d67535bb0ea86c85fc8..256e478571e19525451ec90bad68456a5d4a4243 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer.factor
@@ -58,7 +58,14 @@ Str                =   '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]]
                      | '"' StringChars2:cs '"' => [[ cs ast-string boa ]]
                      | "'" StringChars3:cs "'" => [[ cs ast-string boa ]]
 RegExpFlags        = NameRest*
-RegExpBody         = (!("/" | "\n" | "\r") .)* => [[ >string ]]
+NonTerminator      = !("\n" | "\r") .
+BackslashSequence  = "\\" NonTerminator
+RegExpFirstChar    =   !("*" | "\\" | "/") NonTerminator
+                     | BackslashSequence
+RegExpChar         =   !("\\" | "/") NonTerminator
+                     | BackslashSequence
+RegExpChars        = RegExpChar*
+RegExpBody         = RegExpFirstChar RegExpChars
 RegExp             = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]]
 Special            =   "("   | ")"   | "{"   | "}"   | "["   | "]"   | ","   | ";"
                      | "?"   | ":"   | "!==" | "!="  | "===" | "=="  | "="   | ">="