peg.javascript.tokenizer: allow escaped quotes in strings

Previously, the tokenizer didn't understand
escaped quotes in string literals. Also added was
a test to ensure the escaping works.

diff --git a/extra/peg/javascript/tokenizer/tokenizer-tests.factor b/extra/peg/javascript/tokenizer/tokenizer-tests.factor
index 9d5a9fb2f702425fbe2d950e0a68a8a1d423c7d6..f3c5fc726efb0449391995008a64bb71bd13310e 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer-tests.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer-tests.factor
@@ -23,3 +23,7 @@ IN: peg.javascript.tokenizer.tests
 { V{ T{ ast-regexp f "<(w+)[^>]*?)/>" "g" } } } [
   "/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript
 ] unit-test
+
+{
+  V{ T{ ast-string { value "abc\"def\"" } } }
+} [ "\"abc\\\"def\\\"\"" tokenize-javascript ] unit-test
\ No newline at end of file

diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor
index d6bf057232b5b3d80bf24d5d5dfb531465cc0d88..1a027e99e6ce42d2a50819c65dc989eb6e4c9b78 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer.factor
@@ -8,7 +8,7 @@ IN: peg.javascript.tokenizer
 
 USE: prettyprint
 
-EBNF: tokenize-javascript 
+EBNF: tokenize-javascript
 Letter            = [a-zA-Z]
 Digit             = [0-9]
 Digits            = Digit+
@@ -43,14 +43,15 @@ Keyword           =  ("break"
                     | "var"
                     | "void"
                     | "while"
-                    | "with") !(NameRest) 
+                    | "with") !(NameRest)
 Name              = !(Keyword) iName  => [[ ast-name boa ]]
 Number            =   Digits:ws '.' Digits:fs => [[ ws "." fs 3array "" concat-as string>number ast-number boa ]]
-                    | Digits => [[ >string string>number ast-number boa ]]  
+                    | Digits => [[ >string string>number ast-number boa ]]
 
-EscapeChar        =   "\\n" => [[ 10 ]]
-                    | "\\r" => [[ 13 ]]
-                    | "\\t" => [[ 9 ]]
+EscapeChar        =   "\\n"  => [[ CHAR: \n ]]
+                    | "\\r"  => [[ CHAR: \r ]]
+                    | "\\t"  => [[ CHAR: \t ]]
+                    | "\\\"" => [[ CHAR: \" ]]
 StringChars1       = (EscapeChar | !('"""') .)* => [[ >string ]]
 StringChars2       = (EscapeChar | !('"') .)* => [[ >string ]]
 StringChars3       = (EscapeChar | !("'") .)* => [[ >string ]]