peg.ebnf: add rule~ syntax to discard a matching subrule from the AST; document ...

diff --git a/basis/peg/ebnf/ebnf-docs.factor b/basis/peg/ebnf/ebnf-docs.factor
index b03ec455269306fe07b4e984d0d6e9d25ed6683e..26d8a3d0b67c8175b2a410d18de2c4a8165a30ac 100644 (file)
--- a/basis/peg/ebnf/ebnf-docs.factor
+++ b/basis/peg/ebnf/ebnf-docs.factor
@@ -94,6 +94,28 @@ ARTICLE: "peg.ebnf.sequence" "Sequence"
 }
 ;
 
+ARTICLE: "peg.ebnf.grouping" "Group"
+"Any sequence of rules may be grouped using parentheses (" { $snippet "()" } "). "
+"The parenthesized sequence can then be modified as a group. Parentheses also "
+"delimit sets of choices separated by pipe (|) characters."
+$nl
+"A group can also be delimited with curly braces (" { $snippet "{}" } "), in "
+"which case an implicit optional whitespace-matching rule will be inserted between "
+"rules sequenced within the braces."
+{ $examples
+    { $example
+       "USING: prettyprint peg.ebnf ;"
+       "\"abcca\" [EBNF rule=\"a\" (\"b\" | \"c\")* \"a\" EBNF] ."
+       "V{ \"a\" V{ \"b\" \"c\" \"c\" } \"a\" }"
+    }
+    { $example
+       "USING: prettyprint peg.ebnf ;"
+       "\"ab  c\nd \" [EBNF rule={\"a\" \"b\" \"c\" \"d\"} EBNF] ."
+       "V{ \"a\" \"b\" \"c\" \"d\" }"
+    }
+}
+;
+
 ARTICLE: "peg.ebnf.choice" "Choice"
 "Any rule element separated by a pipe character (|) is considered a choice. Choices "
 "are matched against the input stream in order. If a match succeeds then the remaining "
@@ -117,6 +139,18 @@ ARTICLE: "peg.ebnf.choice" "Choice"
 }
 ;
 
+ARTICLE: "peg.ebnf.ignore" "Ignore"
+"Any rule element followed by a tilde (~) will be matched, and its results "
+"discarded from the AST."
+{ $examples
+    { $example
+       "USING: prettyprint peg.ebnf ;"
+       "\"abc\" [EBNF rule=\"a\" \"b\"~ \"c\" EBNF] ."
+       "V{ \"a\" \"c\" }"
+    }
+}
+;
+
 ARTICLE: "peg.ebnf.option" "Option"
 "Any rule element followed by a question mark (?) is considered optional. The "
 "rule is tested against the input. If it succeeds the result is stored in the AST. "
@@ -452,7 +486,9 @@ ARTICLE: "peg.ebnf" "EBNF"
 { $subsections "peg.ebnf.strings"
 "peg.ebnf.any"
 "peg.ebnf.sequence"
+"peg.ebnf.grouping"
 "peg.ebnf.choice"
+"peg.ebnf.ignore"
 "peg.ebnf.option"
 "peg.ebnf.one-or-more"
 "peg.ebnf.zero-or-more"

diff --git a/basis/peg/ebnf/ebnf-tests.factor b/basis/peg/ebnf/ebnf-tests.factor
index 897746a9c963d7c8c66d9753100345cdf273e231..f267f69255a6a4219858981a690b26e01e55c6c0 100644 (file)
--- a/basis/peg/ebnf/ebnf-tests.factor
+++ b/basis/peg/ebnf/ebnf-tests.factor
@@ -84,6 +84,26 @@ IN: peg.ebnf.tests
   "one ((two | three) four)*" 'choice' parse
 ] unit-test
 
+{
+  T{ ebnf-sequence f
+     V{ 
+       T{ ebnf-non-terminal f "one" }
+       T{ ebnf-ignore f
+          T{ ebnf-sequence f
+             V{
+                T{ ebnf-choice f
+                   V{ T{ ebnf-non-terminal f "two" } T{ ebnf-non-terminal f "three" } }
+                }
+                T{ ebnf-non-terminal f "four" }
+             }
+          }
+        }
+     }
+  } 
+} [
+  "one ((two | three) four)~" 'choice' parse
+] unit-test
+
 {
   T{ ebnf-sequence f
      V{ 
@@ -304,6 +324,26 @@ main = Primary
   "abc" [EBNF a="a" "b" foo=(a "c") EBNF]
 ] unit-test
 
+{ V{ "a" "c" } } [
+  "abc" [EBNF a="a" "b"~ foo=(a "c") EBNF]
+] unit-test
+
+{ V{ V{ "a" V{ "b" "b" } } "c" } } [
+  "abbc" [EBNF a=("a" "b"*) foo=(a "c") EBNF]
+] unit-test
+
+{ V{ "a" "c" } } [
+  "abc" [EBNF a=("a" ("b")~) foo=(a "c") EBNF]
+] unit-test
+
+{ V{ "a" "c" } } [
+  "abc" [EBNF a=("a" "b"~) foo=(a "c") EBNF]
+] unit-test
+
+{ "c" } [
+  "abc" [EBNF a=("a" "b")~ foo=(a "c") EBNF]
+] unit-test
+
 { V{ V{ "a" "b" } "c" } } [
   "abc" [EBNF a="a" "b" foo={a "c"} EBNF]
 ] unit-test
@@ -360,6 +400,14 @@ main = Primary
   "ab c ab c" [EBNF a="a" "b" foo={a "c"}* EBNF]
 ] unit-test
 
+{ V{ V{ "a" "c" } V{ "a" "c" } } } [
+  "ab c ab c" [EBNF a="a" "b"~ foo={a "c"}* EBNF]
+] unit-test
+
+{ V{ } } [
+  "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF]
+] unit-test
+
 { V{ } } [
   "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF]
 ] unit-test

diff --git a/basis/peg/ebnf/ebnf.factor b/basis/peg/ebnf/ebnf.factor
index 045b0a588d375051b8912a7331b7a25a6f2b5396..eac9c21fd68c2992a75a6f0a36481b7071ee5506 100644 (file)
--- a/basis/peg/ebnf/ebnf.factor
+++ b/basis/peg/ebnf/ebnf.factor
@@ -61,6 +61,7 @@ TUPLE: ebnf-ensure group ;
 TUPLE: ebnf-ensure-not group ;\r
 TUPLE: ebnf-choice options ;\r
 TUPLE: ebnf-sequence elements ;\r
+TUPLE: ebnf-ignore group ;\r
 TUPLE: ebnf-repeat0 group ;\r
 TUPLE: ebnf-repeat1 group ;\r
 TUPLE: ebnf-optional group ;\r
@@ -81,6 +82,7 @@ C: <ebnf-ensure> ebnf-ensure
 C: <ebnf-ensure-not> ebnf-ensure-not\r
 C: <ebnf-choice> ebnf-choice\r
 C: <ebnf-sequence> ebnf-sequence\r
+C: <ebnf-ignore> ebnf-ignore\r
 C: <ebnf-repeat0> ebnf-repeat0\r
 C: <ebnf-repeat1> ebnf-repeat1\r
 C: <ebnf-optional> ebnf-optional\r
@@ -215,6 +217,7 @@ PEG: escaper ( string -- ast )
         'range-parser' ,\r
         'any-character' ,\r
       ] choice* \r
+      [ dup , "~" token hide , ] seq* [ first <ebnf-ignore> ] action ,\r
       [ dup , "*" token hide , ] seq* [ first <ebnf-repeat0> ] action ,\r
       [ dup , "+" token hide , ] seq* [ first <ebnf-repeat1> ] action ,\r
       [ dup , "?[" token ensure-not , "?" token hide , ] seq* [ first <ebnf-optional> ] action ,\r
@@ -257,11 +260,15 @@ DEFER: 'choice'
 : 'group' ( -- parser )\r
   #! A grouping with no suffix. Used for precedence.\r
   [ ] [\r
+    "~" token sp ensure-not ,\r
     "*" token sp ensure-not ,\r
     "+" token sp ensure-not ,\r
     "?" token sp ensure-not ,\r
   ] seq* hide grouped ; \r
 \r
+: 'ignore' ( -- parser )\r
+  [ <ebnf-ignore> ] "~" syntax grouped ;\r
+\r
 : 'repeat0' ( -- parser )\r
   [ <ebnf-repeat0> ] "*" syntax grouped ;\r
 \r
@@ -305,6 +312,7 @@ DEFER: 'choice'
       'ensure' sp ,\r
       'element' sp ,\r
       'group' sp , \r
+      'ignore' sp ,\r
       'repeat0' sp ,\r
       'repeat1' sp ,\r
       'optional' sp , \r
@@ -425,6 +433,9 @@ M: ebnf-ensure (transform) ( ast -- parser )
 M: ebnf-ensure-not (transform) ( ast -- parser )\r
   transform-group ensure-not ;\r
 \r
+M: ebnf-ignore (transform) ( ast -- parser )\r
+  transform-group [ drop ignore ] action ;\r
+\r
 M: ebnf-repeat0 (transform) ( ast -- parser )\r
   transform-group repeat0 ;\r
 \r
@@ -532,7 +543,7 @@ M: ebnf-non-terminal (transform) ( ast -- parser )
     dup remaining>> [ blank? ] trim [\r
       [ \r
         "Unable to fully parse EBNF. Left to parse was: " %\r
-        remaining>> % \r
+        % \r
       ] "" make throw\r
     ] unless-empty\r
   ] [\r