! Copyright (C) 2008 Doug Coleman.
! See http://factorcode.org/license.txt for BSD license.
-USING: assocs html.parser kernel math sequences strings ascii
-arrays generalizations shuffle namespaces make
-splitting http accessors io combinators http.client urls
-urls.encoding fry prettyprint sets ;
+USING: accessors assocs assocs.extras combinators
+combinators.short-circuit html.parser http.client io kernel math
+math.statistics sequences sets splitting unicode urls
+urls.encoding ;
IN: html.parser.analyzer
-TUPLE: link attributes clickable ;
-
-: scrape-html ( url -- headers vector )
+: scrape-html ( url -- response vector )
http-get parse-html ;
+: attribute ( tag string -- obj/f )
+ swap attributes>> at ;
+
+: attribute* ( tag string -- obj ? )
+ swap attributes>> at* ;
+
+: attribute? ( tag string -- ? )
+ swap attributes>> key? ;
+
: find-all ( seq quot -- alist )
- [ <enum> >alist ] [ '[ second @ ] ] bi* filter ; inline
+ [ <enumerated> >alist ] [ '[ second @ ] ] bi* filter ; inline
+
+: loopn-index ( n quot -- )
+ [ <iota> ] [ '[ @ not ] ] bi* find 2drop ; inline
+
+: loopn ( n quot -- )
+ [ drop ] prepose loopn-index ; inline
+
+: html-class? ( tag string -- ? )
+ swap "class" attribute [ blank? ] split-when member? ;
+
+: html-id? ( tag string -- ? )
+ swap "id" attribute = ;
+
+ERROR: undefined-find-nth m n seq quot ;
+
+: check-trivial-find ( m n seq quot -- m n seq quot )
+ pick 0 = [ undefined-find-nth ] when ; inline
-: find-nth ( seq quot n -- i elt )
- [ <enum> >alist ] 2dip -rot
- '[ _ [ second @ ] find-from rot drop swap 1 + ]
- [ f 0 ] 2dip times drop first2 ; inline
+: find-nth-from ( m n seq quot -- i/f elt/f )
+ check-trivial-find [ f ] 3dip '[
+ drop _ _ find-from [ dup [ 1 + ] when ] dip over
+ ] loopn [ dup [ 1 - ] when ] dip ; inline
+
+: find-nth ( n seq quot -- i/f elt/f )
+ [ 0 ] 3dip find-nth-from ; inline
+
+: find-last-nth-from ( m n seq quot -- i/f elt/f )
+ check-trivial-find [ f ] 3dip '[
+ drop _ _ find-last-from [ dup [ 1 - ] when ] dip over
+ ] loopn [ dup [ 1 + ] when ] dip ; inline
+
+: find-last-nth ( n seq quot -- i/f elt/f )
+ [ [ nip length 1 - ] [ ] 2bi ] dip find-last-nth-from ; inline
: find-first-name ( vector string -- i/f tag/f )
>lower '[ name>> _ = ] find ; inline
-: find-matching-close ( vector string -- i/f tag/f )
+: stack-find ( seq quot: ( elt -- 1/0/-1 ) -- i/f )
+ map cum-sum 0 swap index ; inline
+
+: tag-classifier ( string -- quot )
>lower
- '[ [ name>> _ = ] [ closing?>> ] bi and ] find ; inline
+ '[ dup name>> _ = [ closing?>> -1 1 ? ] [ drop 0 ] if ] ; inline
: find-between* ( vector i/f tag/f -- vector )
over integer? [
[ tail-slice ] [ name>> ] bi*
- dupd find-matching-close drop dup [ 1 + ] when
- [ head ] [ first ] if*
+ dupd tag-classifier stack-find [ 1 + ] [ 1 ] if*
+ head
] [
3drop V{ } clone
] if ; inline
] map ;
: find-by-id ( vector id -- vector' elt/f )
- '[ attributes>> "id" at _ = ] find ;
-
+ '[ _ html-id? ] find ;
+
: find-by-class ( vector id -- vector' elt/f )
- '[ attributes>> "class" at _ = ] find ;
+ '[ _ html-class? ] find ;
: find-by-name ( vector string -- vector elt/f )
>lower '[ name>> _ = ] find ;
: find-by-id-between ( vector string -- vector' )
- dupd
- '[ attributes>> "id" swap at _ = ] find find-between* ;
-
+ '[ _ html-id? ] dupd find find-between* ;
+
: find-by-class-between ( vector string -- vector' )
- dupd
- '[ attributes>> "class" swap at _ = ] find find-between* ;
-
+ '[ _ html-class? ] dupd find find-between* ;
+
: find-by-class-id-between ( vector class id -- vector' )
'[
- [ attributes>> "class" swap at _ = ]
- [ attributes>> "id" swap at _ = ] bi and
+ [ _ html-class? ] [ _ html-id? ] bi and
] dupd find find-between* ;
-: find-by-attribute-key ( vector key -- vector' elt/? )
- >lower
- [ attributes>> at _ = ] filter sift ;
+: find-by-attribute-key ( vector key -- vector' )
+ >lower '[ _ attribute? ] filter sift ;
: find-by-attribute-key-value ( vector value key -- vector' )
- >lower
- [ attributes>> at over = ] with filter nip
- sift ;
+ >lower swap '[ _ attribute _ = ] filter sift ;
: find-first-attribute-key-value ( vector value key -- i/f tag/f )
- >lower
- [ attributes>> at over = ] with find rot drop ;
-
-: tag-link ( tag -- link/f )
- attributes>> [ "href" swap at ] [ f ] if* ;
+ >lower swap '[ _ attribute _ = ] find ;
: find-links ( vector -- vector' )
- [ [ name>> "a" = ] [ attributes>> "href" swap at ] bi and ]
+ [ { [ name>> "a" = ] [ "href" attribute ] } 1&& ]
find-between-all ;
-: <link> ( vector -- link )
- [ first attributes>> ]
- [ [ name>> { text "img" } member? ] filter ] bi
- link boa ;
-
-: link. ( vector -- )
- [ attributes>> "href" swap at write nl ]
- [ clickable>> [ bl bl text>> print ] each nl ] bi ;
+: find-images ( vector -- vector' )
+ [ { [ name>> "img" = ] [ "src" attribute ] } 1&& ] filter sift
+ [ "src" attribute ] map ;
: find-by-text ( seq quot -- tag )
[ dup name>> text = ] prepose find drop ; inline
: find-opening-tags-by-name ( name seq -- seq )
- [ [ name>> = ] [ closing?>> not ] bi and ] with find-all ;
+ [ { [ name>> = ] [ closing?>> not ] } 1&& ] with find-all ;
: href-contains? ( str tag -- ? )
- attributes>> "href" swap at* [ subseq? ] [ 2drop f ] if ;
+ "href" attribute* [ swap subseq-of? ] [ 2drop f ] if ;
: find-hrefs ( vector -- vector' )
- find-links
- [ [
- [ name>> "a" = ]
- [ attributes>> "href" swap key? ] bi and ] filter
- ] map sift
- [ [ attributes>> "href" swap at ] map ] map concat
- [ >url ] map ;
+ [ { [ name>> "a" = ] [ "href" attribute? ] } 1&& ] filter sift
+ [ "href" attribute >url ] map ;
: find-frame-links ( vector -- vector' )
- [ name>> "frame" = ] find-between-all
- [ [ attributes>> "src" swap at ] map sift ] map concat sift
- [ >url ] map ;
+ [ { [ name>> "frame" = ] [ "src" attribute? ] } 1&& ] filter sift
+ [ "src" attribute >url ] map ;
+
+: find-script-links ( vector -- vector' )
+ [ { [ name>> "script" = ] [ "src" attribute? ] } 1&& ] filter sift
+ [ "src" attribute >url ] map ;
: find-all-links ( vector -- vector' )
- [ find-hrefs ] [ find-frame-links ] bi append prune ;
+ [ find-hrefs ] [ find-frame-links ] [ find-script-links ] tri union union ;
: find-forms ( vector -- vector' )
"form" over find-opening-tags-by-name
[ [ name>> { "form" "input" } member? ] filter ] map ;
: find-html-objects ( vector string -- vector' )
- dupd find-opening-tags-by-name
- [ first2 find-between* ] curry map ;
+ over find-opening-tags-by-name
+ [ first2 find-between* ] with map ;
: form-action ( vector -- string )
- [ name>> "form" = ] find nip
- attributes>> "action" swap at ;
+ [ name>> "form" = ] find nip "action" attribute ;
: hidden-form-values ( vector -- strings )
- [ attributes>> "type" swap at "hidden" = ] filter ;
+ [ "type" attribute "hidden" = ] filter ;
: input. ( tag -- )
- dup name>> print
- attributes>>
+ [ name>> print ] [ attributes>> ] bi
[ bl bl bl bl [ write "=" write ] [ write bl ] bi* nl ] assoc-each ;
: form. ( vector -- )
- [ closing?>> not ] filter
+ [ closing?>> ] reject
[
{
{ [ dup name>> "form" = ]
- [ "form action: " write attributes>> "action" swap at print ] }
+ [ "form action: " write "action" attribute print ] }
{ [ dup name>> "input" = ] [ input. ] }
[ drop ]
} cond
: query>assoc* ( str -- hash )
"?" split1 nip query>assoc ;
-
-: html-class? ( tag string -- ? )
- swap attributes>> "class" swap at = ;
-
-: html-id? ( tag string -- ? )
- swap attributes>> "id" swap at = ;
: opening-tag? ( tag -- ? )
closing?>> not ;
+
+TUPLE: link attributes clickable ;
+
+: <link> ( vector -- link )
+ [ first attributes>> ]
+ [ [ name>> { text "img" } member? ] filter ] bi
+ link boa ;
+
+: link. ( vector -- )
+ [ "href" attribute write nl ]
+ [ clickable>> [ bl bl text>> print ] each nl ] bi ;
+
+: find-classes-named ( seq name -- seq' )
+ dupd
+ '[ attributes>> "class" of _ = ] find-all
+ [ find-between ] kv-with { } assoc>map ;
+
+: find-classes-named* ( seq name -- seq' )
+ dupd
+ '[ attributes>> "class" of _ = ] find-all
+ [ find-between* ] kv-with { } assoc>map ;
projects / factor.git / blobdiff