peg.javascript: add some other escapes since we're doing this. :-)

diff --git a/extra/peg/javascript/tokenizer/tokenizer-tests.factor b/extra/peg/javascript/tokenizer/tokenizer-tests.factor
index f3c5fc726efb0449391995008a64bb71bd13310e..1e9df1cf52b519de68166ee7614c421f1f018d84 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer-tests.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer-tests.factor
@@ -25,5 +25,9 @@ IN: peg.javascript.tokenizer.tests
 ] unit-test
 
 {
-  V{ T{ ast-string { value "abc\"def\"" } } }
-} [ "\"abc\\\"def\\\"\"" tokenize-javascript ] unit-test
\ No newline at end of file
+    V{ T{ ast-string { value "abc\"def\"" } } }
+} [ "\"abc\\\"def\\\"\"" tokenize-javascript ] unit-test
+
+{
+    V{ T{ ast-string { value "\b\f\n\r\t\v'\"\\" } } }
+} [ "\"\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\"" tokenize-javascript ] unit-test

diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor
index 1a027e99e6ce42d2a50819c65dc989eb6e4c9b78..8dcbc70e851a697e9b85bf5c4f6f6570d1f2c1eb 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer.factor
@@ -48,10 +48,15 @@ Name              = !(Keyword) iName  => [[ ast-name boa ]]
 Number            =   Digits:ws '.' Digits:fs => [[ ws "." fs 3array "" concat-as string>number ast-number boa ]]
                     | Digits => [[ >string string>number ast-number boa ]]
 
-EscapeChar        =   "\\n"  => [[ CHAR: \n ]]
+EscapeChar        =   "\\b"  => [[ CHAR: \b ]]
+                    | "\\f"  => [[ CHAR: \f ]]
+                    | "\\n"  => [[ CHAR: \n ]]
                     | "\\r"  => [[ CHAR: \r ]]
                     | "\\t"  => [[ CHAR: \t ]]
-                    | "\\\"" => [[ CHAR: \" ]]
+                    | "\\v"  => [[ CHAR: \v ]]
+                    | "\\'"  => [[ CHAR: '  ]]
+                    | "\\\"" => [[ CHAR: "  ]]
+                    | "\\\\" => [[ CHAR: \\ ]]
 StringChars1       = (EscapeChar | !('"""') .)* => [[ >string ]]
 StringChars2       = (EscapeChar | !('"') .)* => [[ >string ]]
 StringChars3       = (EscapeChar | !("'") .)* => [[ >string ]]