1 ! Copyright (C) 2008 Doug Coleman.
2 ! See http://factorcode.org/license.txt for BSD license.
3 USING: assocs html.parser kernel math sequences strings ascii
4 arrays generalizations shuffle unicode.case namespaces make
5 splitting http accessors io combinators http.client urls
7 IN: html.parser.analyzer
9 TUPLE: link attributes clickable ;
11 : scrape-html ( url -- headers vector )
14 : find-all ( seq quot -- alist )
15 [ <enum> >alist ] [ '[ second @ ] ] bi* filter ; inline
17 : find-nth ( seq quot n -- i elt )
18 [ <enum> >alist ] 2dip -rot
19 '[ _ [ second @ ] find-from rot drop swap 1+ ]
20 [ f 0 ] 2dip times drop first2 ; inline
22 : find-first-name ( str vector -- i/f tag/f )
23 [ >lower ] dip [ name>> = ] with find ; inline
25 : find-matching-close ( str vector -- i/f tag/f )
27 [ [ name>> = ] [ closing?>> ] bi and ] with find ; inline
29 : find-between* ( i/f tag/f vector -- vector )
33 [ find-matching-close drop dup [ 1+ ] when ] keep
34 swap [ head ] [ first ] if*
39 : find-between ( i/f tag/f vector -- vector )
40 find-between* dup length 3 >= [
41 [ rest-slice but-last-slice ] keep like
44 : find-between-first ( string vector -- vector' )
45 [ find-first-name ] keep find-between ; inline
47 : find-between-all ( vector quot -- seq )
48 [ [ [ closing?>> not ] bi and ] curry find-all ] curry
49 [ [ >r first2 r> find-between* ] curry map ] bi ; inline
52 : remove-blank-text ( vector -- vector' )
55 [ text>> [ blank? ] all? not ] [ drop t ] if
58 : trim-text ( vector -- vector' )
61 [ [ [ blank? ] trim ] change-text ] when
64 : find-by-id ( id vector -- vector )
65 [ attributes>> "id" swap at = ] with filter ;
67 : find-by-class ( id vector -- vector )
68 [ attributes>> "class" swap at = ] with filter ;
70 : find-by-name ( str vector -- vector )
71 [ >lower ] dip [ name>> = ] with filter ;
73 : find-by-attribute-key ( key vector -- vector )
75 [ attributes>> at ] with filter
78 : find-by-attribute-key-value ( value key vector -- vector )
80 [ attributes>> at over = ] with filter nip
83 : find-first-attribute-key-value ( value key vector -- i/f tag/f )
85 [ attributes>> at over = ] with find rot drop ;
87 : tag-link ( tag -- link/f )
88 attributes>> [ "href" swap at ] [ f ] if* ;
90 : find-links ( vector -- vector' )
91 [ [ name>> "a" = ] [ attributes>> "href" swap at ] bi and ]
94 : <link> ( vector -- link )
95 [ first attributes>> ]
96 [ [ name>> { text "img" } member? ] filter ] bi
100 [ attributes>> "href" swap at write nl ]
101 [ clickable>> [ bl bl text>> print ] each nl ] bi ;
103 : find-by-text ( seq quot -- tag )
104 [ dup name>> text = ] prepose find drop ;
106 : find-opening-tags-by-name ( name seq -- seq )
107 [ [ name>> = ] [ closing?>> not ] bi and ] with find-all ;
109 : href-contains? ( str tag -- ? )
110 attributes>> "href" swap at* [ subseq? ] [ 2drop f ] if ;
112 : find-hrefs ( vector -- vector' )
116 [ attributes>> "href" swap key? ] bi and ] filter
117 ] map sift [ [ attributes>> "href" swap at ] map ] map concat ;
119 : find-forms ( vector -- vector' )
120 "form" over find-opening-tags-by-name
121 swap [ >r first2 r> find-between* ] curry map
122 [ [ name>> { "form" "input" } member? ] filter ] map ;
124 : find-html-objects ( string vector -- vector' )
125 [ find-opening-tags-by-name ] keep
126 [ [ first2 ] dip find-between* ] curry map ;
128 : form-action ( vector -- string )
129 [ name>> "form" = ] find nip
130 attributes>> "action" swap at ;
132 : hidden-form-values ( vector -- strings )
133 [ attributes>> "type" swap at "hidden" = ] filter ;
138 [ bl bl bl bl [ write "=" write ] [ write bl ] bi* nl ] assoc-each ;
140 : form. ( vector -- )
141 [ closing?>> not ] filter
144 { [ dup name>> "form" = ]
145 [ "form action: " write attributes>> "action" swap at print ] }
146 { [ dup name>> "input" = ] [ input. ] }
151 : query>assoc* ( str -- hash )
152 "?" split1 nip query>assoc ;