lexer: change to parse string syntax consistently.

diff --git a/basis/html/templates/fhtml/fhtml.factor b/basis/html/templates/fhtml/fhtml.factor
index d76193bd3577f1e60eea6c1418f65fe21a3d94ac..768bc41ea422e7861a8e88e95297efc0f1d927ba 100644 (file)
--- a/basis/html/templates/fhtml/fhtml.factor
+++ b/basis/html/templates/fhtml/fhtml.factor
@@ -2,8 +2,9 @@
 ! Copyright (C) 2006, 2010 Slava Pestov
 ! See http://factorcode.org/license.txt for BSD license.
 USING: accessors combinators compiler.units html.templates io
-io.encodings.utf8 io.files kernel lexer math namespaces parser
-parser.notes quotations sequences splitting vocabs.parser ;
+io.encodings.utf8 io.files kernel lexer lexer.private math
+namespaces parser parser.notes quotations sequences splitting
+vocabs.parser ;
 IN: html.templates.fhtml
 
 ! We use a custom lexer so that %> ends a token even if not
@@ -15,11 +16,8 @@ TUPLE: template-lexer < lexer ;
 
 M: template-lexer skip-word
     [
-        {
-            { [ 2dup nth CHAR: \" = ] [ drop 1 + ] }
-            { [ 2dup swap tail-slice "%>" head? ] [ drop 2 + ] }
-            [ f skip ]
-        } cond
+        2dup swap tail-slice "%>" head?
+        [ drop 2 + ] [ (skip-word) ] if
     ] change-lexer-column ;
 
 DEFER: <% delimiter

diff --git a/core/lexer/lexer.factor b/core/lexer/lexer.factor
index 2bdd39b6cace31002c5cc866d5a75eae332a03f6..f576d9d3b825c402a6c2861cff2aa9ecc35724fa 100644 (file)
--- a/core/lexer/lexer.factor
+++ b/core/lexer/lexer.factor
@@ -51,11 +51,6 @@ ERROR: unexpected want got ;
 : forbid-tab ( c -- c )
     [ CHAR: \t eq? [ "[space]" "[tab]" unexpected ] when ] keep ; inline
 
-: skip ( i seq ? -- n )
-    over length [
-        [ swap forbid-tab CHAR: \s eq? xor ] curry find-from drop
-    ] dip or ; inline
-
 <PRIVATE
 
 : shebang? ( lexer -- lexer ? )
@@ -65,6 +60,14 @@ ERROR: unexpected want got ;
         ] [ f ] if
     ] [ f ] if ; inline
 
+: (skip-blank) ( col line -- newcol )
+    [ [ forbid-tab CHAR: \s eq? not ] find-from drop ]
+    [ length or ] bi ;
+
+: (skip-word) ( col line -- newcol )
+    [ [ forbid-tab " \"" member-eq? ] find-from CHAR: \" eq? [ 1 + ] when ]
+    [ length or ] bi ;
+
 PRIVATE>
 
 GENERIC: skip-blank ( lexer -- )
@@ -73,15 +76,13 @@ M: lexer skip-blank
     shebang? [
         [ nip length ] change-lexer-column
     ] [
-        [ t skip ] change-lexer-column
+        [ (skip-blank) ] change-lexer-column
     ] if ;
 
 GENERIC: skip-word ( lexer -- )
 
 M: lexer skip-word
-    [
-        2dup nth CHAR: \" eq? [ drop 1 + ] [ f skip ] if
-    ] change-lexer-column ;
+    [ (skip-word) ] change-lexer-column ;
 
 : still-parsing? ( lexer -- ? )
     lexer check-instance [ line>> ] [ text>> length ] bi <= ;