Fix terminal parser with non-default tokenizers

diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor
index a90fa984361ffe53b97057b5c13b73e1e0febfe8..2269af6625d854933fb83e11b69d53fe6bcf6797 100644 (file)
--- a/extra/peg/ebnf/ebnf-tests.factor
+++ b/extra/peg/ebnf/ebnf-tests.factor
@@ -511,4 +511,8 @@ Tok                = Spaces (Number | Special )
           tokenizer=spaces (number | operator) 
           main= . . . 
         EBNF] call ast>> 
+] unit-test
+
+{ "++" } [
+  "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>>
 ] unit-test
\ No newline at end of file

diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor
index 564b376b29df92b60fa20d4ae1ca65f228f7a592..47255341788f3a4e2c2a17567bfe86c08e6bf7a9 100644 (file)
--- a/extra/peg/ebnf/ebnf.factor
+++ b/extra/peg/ebnf/ebnf.factor
@@ -15,13 +15,14 @@ TUPLE: tokenizer any one many ;
 \r
 : default-tokenizer ( -- tokenizer )\r
   T{ tokenizer f \r
-    [ [ drop t ] satisfy ]\r
+    [ any-char ]\r
     [ token ]\r
-    [ [ = ] curry satisfy ]\r
+    [ [ = ] curry any-char swap semantic ]\r
   } ;\r
 \r
 : parser-tokenizer ( parser -- tokenizer )\r
-  1quotation [ [ = ] curry satisfy ] dup tokenizer boa ;\r
+  [ 1quotation ] keep\r
+  [ swap [ = ] curry semantic ] curry dup tokenizer boa ;\r
 \r
 : rule-tokenizer ( name word -- tokenizer )\r
   rule parser-tokenizer ;\r

diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor
index d27a06e9d22cf486e695853aa60062aae3bbbcb1..791f63c56bfdd8c173896a992417ab3f60e99851 100644 (file)
--- a/extra/peg/javascript/javascript.factor
+++ b/extra/peg/javascript/javascript.factor
@@ -5,7 +5,7 @@ IN: peg.javascript
 
 : parse-javascript ( string -- ast )
   tokenize-javascript [
-    ast>> parse-javascript [
+    ast>> javascript [
       ast>>
     ] [
       "Unable to parse JavaScript" throw

diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor
index b19bb314bbd1053f7e98439a2003f5cdad530286..fd0e27b6d4c13034c17e61c98afa1043392b3028 100644 (file)
--- a/extra/peg/javascript/parser/parser-tests.factor
+++ b/extra/peg/javascript/parser/parser-tests.factor
@@ -1,11 +1,11 @@
 ! Copyright (C) 2008 Chris Double.
 ! See http://factorcode.org/license.txt for BSD license.
 !
-USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer  
-       peg.javascript.parser accessors multiline sequences math ;
+USING: kernel tools.test peg peg.javascript.ast peg.javascript.parser 
+       accessors multiline sequences math ;
 IN: peg.javascript.parser.tests
 
-\ parse-javascript must-infer
+\ javascript must-infer
 
 {
   T{
@@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests
       }
   }
 } [
-  "123; 'hello'; foo(x);" tokenize-javascript ast>> parse-javascript ast>>
+  "123; 'hello'; foo(x);" javascript ast>>
 ] unit-test
 
 { t } [ 
 <"
 var x=5
 var y=10
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 
 
@@ -41,7 +41,7 @@ function foldl(f, initial, seq) {
      initial = f(initial, seq[i]);
    return initial;
 }
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 
 { t } [ 
@@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) {
     r.length = this.length - index;
     return r;
 }
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 

diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor
index 1a074090f325608f70a5650d65d77fcc56a333a9..0239ce882cc88a05abe2d051f35f539b3b27e255 100644 (file)
--- a/extra/peg/javascript/parser/parser.factor
+++ b/extra/peg/javascript/parser/parser.factor
@@ -1,12 +1,13 @@
 ! Copyright (C) 2008 Chris Double.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: kernel accessors peg peg.ebnf peg.javascript.ast ;
+USING: kernel accessors peg peg.ebnf peg.javascript.ast peg.javascript.tokenizer ;
 IN: peg.javascript.parser
 
 #! Grammar for JavaScript. Based on OMeta-JS example from:
 #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler 
 
-EBNF: parse-javascript
+EBNF: javascript
+tokenizer         = <foreign tokenize-javascript Tok>
 End               = !(.)
 Space             = " " | "\t" | "\n" 
 Spaces            = Space* => [[ ignore ]]