Remove EBNF: and [EBNF EBNF] +tokenizer syntax

replaced with tokenizer support in ebnf grammar itself.

diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor
index 25889fe44d70763e9b6068a93d56e1a6e2b7bde9..564b376b29df92b60fa20d4ae1ca65f228f7a592 100644 (file)
--- a/extra/peg/ebnf/ebnf.factor
+++ b/extra/peg/ebnf/ebnf.factor
@@ -520,20 +520,10 @@ M: ebnf-non-terminal (transform) ( ast -- parser )
   parse-result-ast transform dup dup parser [ main swap at compile ] with-variable\r
   [ compiled-parse ] curry [ with-scope ] curry ;\r
 \r
-: [EBNF \r
-  scan {\r
-    { "+" [ scan-word execute "" swap ] }\r
-    [ " " append default-tokenizer ]\r
-  } case \ tokenizer set-global\r
-  [ "EBNF]" parse-multiline-string ] [ drop "" ] recover append ebnf>quot nip parsed \r
-  reset-tokenizer ; parsing\r
+: [EBNF "EBNF]" reset-tokenizer parse-multiline-string ebnf>quot nip parsed reset-tokenizer ; parsing\r
 \r
 : EBNF: \r
-  CREATE-WORD scan {\r
-    { "+" [ scan-word execute "" swap ] }\r
-    [ " " append default-tokenizer ]\r
-  } case \ tokenizer set-global\r
-  dupd [ ";EBNF" parse-multiline-string ] [ drop "" ] recover append \r
+  reset-tokenizer CREATE-WORD dup ";EBNF" parse-multiline-string  \r
   ebnf>quot swapd 1 1 <effect> define-declared "ebnf-parser" set-word-prop \r
   reset-tokenizer ; parsing\r
 \r

diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor
index 23a4b4f7f0682a7caea5700d775552f78faa363b..d27a06e9d22cf486e695853aa60062aae3bbbcb1 100644 (file)
--- a/extra/peg/javascript/javascript.factor
+++ b/extra/peg/javascript/javascript.factor
@@ -4,8 +4,8 @@ USING: kernel accessors peg.javascript.tokenizer peg.javascript.parser ;
 IN: peg.javascript
 
 : parse-javascript ( string -- ast )
-  tokenizer [
-    ast>> javascript [
+  tokenize-javascript [
+    ast>> parse-javascript [
       ast>>
     ] [
       "Unable to parse JavaScript" throw

diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor
index d911a27285ef064c4507c0a7a31836bc1d6ee5c6..b19bb314bbd1053f7e98439a2003f5cdad530286 100644 (file)
--- a/extra/peg/javascript/parser/parser-tests.factor
+++ b/extra/peg/javascript/parser/parser-tests.factor
@@ -5,7 +5,7 @@ USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer
        peg.javascript.parser accessors multiline sequences math ;
 IN: peg.javascript.parser.tests
 
-\ javascript must-infer
+\ parse-javascript must-infer
 
 {
   T{
@@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests
       }
   }
 } [
-  "123; 'hello'; foo(x);" tokenizer ast>> javascript ast>>
+  "123; 'hello'; foo(x);" tokenize-javascript ast>> parse-javascript ast>>
 ] unit-test
 
 { t } [ 
 <"
 var x=5
 var y=10
-"> tokenizer ast>> javascript remaining>> length zero?
+"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
 ] unit-test
 
 
@@ -41,7 +41,7 @@ function foldl(f, initial, seq) {
      initial = f(initial, seq[i]);
    return initial;
 }
-"> tokenizer ast>> javascript remaining>> length zero?
+"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
 ] unit-test
 
 { t } [ 
@@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) {
     r.length = this.length - index;
     return r;
 }
-"> tokenizer ast>> javascript remaining>> length zero?
+"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
 ] unit-test
 

diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor
index 45aa0f022ca0c76cbcf601e66952dfbaaf7bd8a6..1a074090f325608f70a5650d65d77fcc56a333a9 100644 (file)
--- a/extra/peg/javascript/parser/parser.factor
+++ b/extra/peg/javascript/parser/parser.factor
@@ -6,7 +6,7 @@ IN: peg.javascript.parser
 #! Grammar for JavaScript. Based on OMeta-JS example from:
 #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler 
 
-EBNF: javascript
+EBNF: parse-javascript
 End               = !(.)
 Space             = " " | "\t" | "\n" 
 Spaces            = Space* => [[ ignore ]]

diff --git a/extra/peg/javascript/tokenizer/tokenizer-tests.factor b/extra/peg/javascript/tokenizer/tokenizer-tests.factor
index 1300b3c9c7145b92dbc2e0f803d3ec864d6035c1..509ff4a0fed2f19e7c0c97801462ca476b2a62a4 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer-tests.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer-tests.factor
@@ -4,7 +4,7 @@
 USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer accessors ;
 IN: peg.javascript.tokenizer.tests
 
-\ tokenizer must-infer
+\ tokenize-javascript must-infer
 
 {
   V{
@@ -19,5 +19,5 @@ IN: peg.javascript.tokenizer.tests
     ";"
   }    
 } [
-  "123; 'hello'; foo(x);" tokenizer ast>>
+  "123; 'hello'; foo(x);" tokenize-javascript ast>>
 ] unit-test

diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor
index 5bf6a639fa2f03c5da8c8dc5b9cff1f2f43fc259..b72173f956fa79946b14928a0f53e04fc70db8b5 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer.factor
@@ -8,7 +8,7 @@ IN: peg.javascript.tokenizer
 
 USE: prettyprint
 
-EBNF: javascript-tokenizer 
+EBNF: tokenize-javascript 
 Letter            = [a-zA-Z]
 Digit             = [0-9]
 Digits            = Digit+