RegExp fix for javascript tokenizer

diff --git a/extra/peg/javascript/tokenizer/tokenizer-tests.factor b/extra/peg/javascript/tokenizer/tokenizer-tests.factor
index 509ff4a0fed2f19e7c0c97801462ca476b2a62a4..a61125d08c6e2ba0921b807501faff90746fde87 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer-tests.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer-tests.factor
@@ -21,3 +21,7 @@ IN: peg.javascript.tokenizer.tests
 } [
   "123; 'hello'; foo(x);" tokenize-javascript ast>>
 ] unit-test
+
+{ V{ T{ ast-regexp f "<(w+)[^>]*?)/>" "g" } } } [
+  "/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript ast>>
+] unit-test
\ No newline at end of file

diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor
index 256e478571e19525451ec90bad68456a5d4a4243..f65b0b2ad69d71e17bd49b83cf824623b3688f59 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer.factor
@@ -57,15 +57,15 @@ StringChars3       = (EscapeChar | !("'") .)* => [[ >string ]]
 Str                =   '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]]
                      | '"' StringChars2:cs '"' => [[ cs ast-string boa ]]
                      | "'" StringChars3:cs "'" => [[ cs ast-string boa ]]
-RegExpFlags        = NameRest*
+RegExpFlags        = NameRest* => [[ >string ]]
 NonTerminator      = !("\n" | "\r") .
-BackslashSequence  = "\\" NonTerminator
+BackslashSequence  = "\\" NonTerminator => [[ second ]]
 RegExpFirstChar    =   !("*" | "\\" | "/") NonTerminator
                      | BackslashSequence
 RegExpChar         =   !("\\" | "/") NonTerminator
                      | BackslashSequence
 RegExpChars        = RegExpChar*
-RegExpBody         = RegExpFirstChar RegExpChars
+RegExpBody         = RegExpFirstChar RegExpChars => [[ first2 swap prefix >string ]]
 RegExp             = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]]
 Special            =   "("   | ")"   | "{"   | "}"   | "["   | "]"   | ","   | ";"
                      | "?"   | ":"   | "!==" | "!="  | "===" | "=="  | "="   | ">="