Provide ability to plug in tokenizers to ebnf parsers

diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor
index 8f36218b6179c8a4086472dc10a9a3b5c961df2a..921ba7be67911da541c166687b787f9d8528eec1 100644 (file)
--- a/extra/peg/ebnf/ebnf.factor
+++ b/extra/peg/ebnf/ebnf.factor
@@ -11,6 +11,31 @@ IN: peg.ebnf
   #! Given an EBNF word produced from EBNF: return the EBNF rule\r
   "ebnf-parser" word-prop at ;\r
 \r
+TUPLE: tokenizer any one many ;\r
+\r
+: default-tokenizer ( -- tokenizer )\r
+  T{ tokenizer f \r
+    [ [ drop t ] satisfy ]\r
+    [ token ]\r
+    [ [ = ] curry satisfy ]\r
+  } ;\r
+\r
+: parser-tokenizer ( parser -- tokenizer )\r
+  1quotation [ [ = ] curry satisfy ] dup tokenizer boa ;\r
+\r
+: rule-tokenizer ( name word -- tokenizer )\r
+  rule parser-tokenizer ;\r
+\r
+: tokenizer ( -- word )\r
+  \ tokenizer get [ default-tokenizer ] unless* ;\r
+\r
+: reset-tokenizer ( -- )\r
+  default-tokenizer \ tokenizer set-global ;\r
+\r
+: TOKENIZER: \r
+  scan search [ "Tokenizer not found" throw ] unless*\r
+  execute \ tokenizer set-global ; parsing\r
+\r
 TUPLE: ebnf-non-terminal symbol ;\r
 TUPLE: ebnf-terminal symbol ;\r
 TUPLE: ebnf-foreign word rule ;\r
@@ -344,7 +369,7 @@ M: ebnf-choice (transform) ( ast -- parser )
   options>> [ (transform) ] map choice ;\r
 \r
 M: ebnf-any-character (transform) ( ast -- parser )\r
-  drop any-char ;\r
+  drop [ tokenizer any>> call ] box ;\r
 \r
 M: ebnf-range (transform) ( ast -- parser )\r
   pattern>> range-pattern ;\r
@@ -435,7 +460,7 @@ M: ebnf-var (transform) ( ast -- parser )
   parser>> (transform) ;\r
 \r
 M: ebnf-terminal (transform) ( ast -- parser )\r
-  symbol>> [ token ] keep [ = ] curry satisfy 2choice ;\r
+  symbol>> [ tokenizer one>> call ] curry box ;\r
 \r
 M: ebnf-foreign (transform) ( ast -- parser )\r
   dup word>> search\r
@@ -476,10 +501,22 @@ M: ebnf-non-terminal (transform) ( ast -- parser )
   parse-result-ast transform dup dup parser [ main swap at compile ] with-variable\r
   [ compiled-parse ] curry [ with-scope ] curry ;\r
 \r
-: [EBNF "EBNF]" parse-multiline-string ebnf>quot nip parsed ; parsing\r
+: [EBNF \r
+  scan {\r
+    { "+" [ scan-word execute "" swap ] }\r
+    [ " " append default-tokenizer ]\r
+  } case \ tokenizer [\r
+    [ "EBNF]" parse-multiline-string ] [ drop "" ] recover append ebnf>quot nip parsed \r
+  ] with-variable ; parsing\r
 \r
 : EBNF: \r
-  CREATE-WORD dup \r
-  ";EBNF" parse-multiline-string \r
-  ebnf>quot swapd 1 1 <effect> define-declared "ebnf-parser" set-word-prop ; parsing\r
+  CREATE-WORD scan {\r
+    { "+" [ scan-word execute "" swap ] }\r
+    [ " " append default-tokenizer ]\r
+  } case \ tokenizer [\r
+    dupd [ ";EBNF" parse-multiline-string ] [ drop "" ] recover append \r
+    ebnf>quot swapd 1 1 <effect> define-declared "ebnf-parser" set-word-prop \r
+  ] with-variable ; parsing\r
+\r
+\r
 \r

diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor
index a1cff8a3dbae241e355a273063e2419dd8cdbea4..5bf6a639fa2f03c5da8c8dc5b9cff1f2f43fc259 100644 (file)
--- a/extra/peg/javascript/tokenizer/tokenizer.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer.factor
@@ -8,7 +8,7 @@ IN: peg.javascript.tokenizer
 
 USE: prettyprint
 
-EBNF: tokenizer 
+EBNF: javascript-tokenizer 
 Letter            = [a-zA-Z]
 Digit             = [0-9]
 Digits            = Digit+