factor: trim using lists

[factor.git] / extra / robots / robots.factor

diff --git a/extra/robots/robots.factor b/extra/robots/robots.factor
index 6802fe0eda7245ae9074f4909aef976e71ab8e7f..6e61420d43ec9f3bf7f0700d9b7792802d41f6d2 100644 (file)
--- a/extra/robots/robots.factor
+++ b/extra/robots/robots.factor
@@ -1,10 +1,9 @@
 ! Copyright (C) 2009 Doug Coleman.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: accessors arrays assocs calendar.format combinators
-combinators.short-circuit fry globs http.client kernel make
-math.parser multiline namespaces present regexp
-regexp.combinators sequences sets splitting splitting.monotonic
-unicode.case unicode.categories urls ;
+USING: accessors arrays assocs calendar.parser combinators
+combinators.short-circuit globs http.client kernel math.parser
+namespaces present regexp regexp.combinators sequences splitting
+splitting.monotonic unicode urls ;
 IN: robots
 
 ! visit-time is GMT, request-rate is pages/second
@@ -33,10 +32,10 @@ visit-time request-rate crawl-delay unknowns ;
     >robots.txt-url http-get nip ;
 
 : normalize-robots.txt ( string -- sitemaps seq )
-    string-lines
-    [ [ blank? ] trim ] map
+    split-lines
+    [ [ unicode:blank? ] trim ] map
     [ "#" head? ] reject harvest
-    [ ":" split1 [ [ blank? ] trim ] bi@ [ >lower ] dip  ] { } map>assoc
+    [ ":" split1 [ [ unicode:blank? ] trim ] bi@ [ >lower ] dip  ] { } map>assoc
     [ first "sitemap" = ] partition [ values ] dip
     [
         {
@@ -64,7 +63,7 @@ visit-time request-rate crawl-delay unknowns ;
         { "crawl-delay" [ string>number >>crawl-delay ] }
         { "request-rate" [ string>number >>request-rate ] }
         {
-            "visit-time" [ "-" split1 [ hhmm>timestamp ] bi@ 2array
+            "visit-time" [ "-" split1 [ hhmm>duration ] bi@ 2array
             >>visit-time
         ] }
         [ pick unknowns>> push-at ]