Merge branch 'lexer-parsing-word-errors' of git://factorcode.org/git/factor into...

[factor.git] / core / lexer / lexer.factor

diff --git a/core/lexer/lexer.factor b/core/lexer/lexer.factor
index 3b0348aa1016a868e2138799ecf24376d1913cac..e03cae74db80444f77ee4ae5b1d9398eb43d56f6 100644 (file)
--- a/core/lexer/lexer.factor
+++ b/core/lexer/lexer.factor
@@ -97,15 +97,17 @@ PREDICATE: unexpected-eof < unexpected
     [ unexpected-eof ]
     if* ;
 
-: (parse-tokens) ( accum end -- accum )
-    scan 2dup = [
-        2drop
-    ] [
-        [ pick push (parse-tokens) ] [ unexpected-eof ] if*
-    ] if ;
+: (each-token) ( end quot -- pred quot )
+    [ [ [ scan dup ] ] dip [ = not ] curry [ [ f ] if* ] curry compose ] dip ; inline
+
+: each-token ( end quot -- )
+    (each-token) while drop ; inline
+
+: map-tokens ( end quot -- seq )
+    (each-token) produce nip ; inline
 
 : parse-tokens ( end -- seq )
-    100 <vector> swap (parse-tokens) >array ;
+    [ ] map-tokens ;
 
 TUPLE: lexer-error line column line-text parsing-words error ;