1 ! Copyright (C) 2008 Doug Coleman.
2 ! See http://factorcode.org/license.txt for BSD license.
3 USING: assocs html.parser kernel math sequences strings ascii
4 arrays generalizations shuffle namespaces make
5 splitting http accessors io combinators http.client urls
6 urls.encoding fry prettyprint sets combinators.short-circuit ;
7 IN: html.parser.analyzer
9 TUPLE: link attributes clickable ;
11 : scrape-html ( url -- headers vector )
14 : find-all ( seq quot -- alist )
15 [ <enum> >alist ] [ '[ second @ ] ] bi* filter ; inline
17 : find-nth ( seq quot n -- i elt )
18 [ <enum> >alist ] 2dip -rot
19 '[ _ [ second @ ] find-from rot drop swap 1 + ]
20 [ f 0 ] 2dip times drop first2 ; inline
22 : find-first-name ( vector string -- i/f tag/f )
23 >lower '[ name>> _ = ] find ; inline
25 : find-matching-close ( vector string -- i/f tag/f )
27 '[ [ name>> _ = ] [ closing?>> ] bi and ] find ; inline
29 : find-between* ( vector i/f tag/f -- vector )
31 [ tail-slice ] [ name>> ] bi*
32 dupd find-matching-close drop 0 or 1 + head
37 : find-between ( vector i/f tag/f -- vector )
38 find-between* dup length 3 >= [
39 [ rest-slice but-last-slice ] keep like
42 : find-between-first ( vector string -- vector' )
43 dupd find-first-name find-between ; inline
45 : find-between-all ( vector quot -- seq )
47 '[ _ [ closing?>> not ] bi and ] find-all
48 [ first2 find-between* ] with map ; inline
50 : remove-blank-text ( vector -- vector' )
53 [ text>> [ blank? ] all? not ] [ drop t ] if
56 : trim-text ( vector -- vector' )
59 [ [ [ blank? ] trim ] change-text ] when
62 : find-by-id ( vector id -- vector' elt/f )
63 '[ attributes>> "id" swap at _ = ] find ;
65 : find-by-class ( vector id -- vector' elt/f )
66 '[ attributes>> "class" swap at _ = ] find ;
68 : find-by-name ( vector string -- vector elt/f )
69 >lower '[ name>> _ = ] find ;
71 : find-by-id-between ( vector string -- vector' )
73 '[ attributes>> "id" swap at _ = ] find find-between* ;
75 : find-by-class-between ( vector string -- vector' )
77 '[ attributes>> "class" swap at _ = ] find find-between* ;
79 : find-by-class-id-between ( vector class id -- vector' )
81 [ attributes>> "class" swap at _ = ]
82 [ attributes>> "id" swap at _ = ] bi and
83 ] dupd find find-between* ;
85 : find-by-attribute-key ( vector key -- vector' elt/? )
87 [ attributes>> at _ = ] filter sift ;
89 : find-by-attribute-key-value ( vector value key -- vector' )
91 [ attributes>> at over = ] with filter nip
94 : find-first-attribute-key-value ( vector value key -- i/f tag/f )
96 [ attributes>> at over = ] with find rot drop ;
98 : tag-link ( tag -- link/f )
99 attributes>> [ "href" swap at ] [ f ] if* ;
101 : find-links ( vector -- vector' )
102 [ [ name>> "a" = ] [ attributes>> "href" swap at ] bi and ]
105 : find-images ( vector -- vector' )
109 [ attributes>> "src" swap at ]
112 values [ attributes>> "src" swap at ] map ;
114 : <link> ( vector -- link )
115 [ first attributes>> ]
116 [ [ name>> { text "img" } member? ] filter ] bi
119 : link. ( vector -- )
120 [ attributes>> "href" swap at write nl ]
121 [ clickable>> [ bl bl text>> print ] each nl ] bi ;
123 : find-by-text ( seq quot -- tag )
124 [ dup name>> text = ] prepose find drop ; inline
126 : find-opening-tags-by-name ( name seq -- seq )
127 [ [ name>> = ] [ closing?>> not ] bi and ] with find-all ;
129 : href-contains? ( str tag -- ? )
130 attributes>> "href" swap at* [ subseq? ] [ 2drop f ] if ;
132 : find-hrefs ( vector -- vector' )
136 [ attributes>> "href" swap key? ] bi and ] filter
138 [ [ attributes>> "href" swap at ] map ] map concat
141 : find-frame-links ( vector -- vector' )
142 [ name>> "frame" = ] find-between-all
143 [ [ attributes>> "src" swap at ] map sift ] map concat sift
146 : find-all-links ( vector -- vector' )
147 [ find-hrefs ] [ find-frame-links ] bi union ;
149 : find-forms ( vector -- vector' )
150 "form" over find-opening-tags-by-name
151 swap [ [ first2 ] dip find-between* ] curry map
152 [ [ name>> { "form" "input" } member? ] filter ] map ;
154 : find-html-objects ( vector string -- vector' )
155 dupd find-opening-tags-by-name
156 [ first2 find-between* ] curry map ;
158 : form-action ( vector -- string )
159 [ name>> "form" = ] find nip
160 attributes>> "action" swap at ;
162 : hidden-form-values ( vector -- strings )
163 [ attributes>> "type" swap at "hidden" = ] filter ;
168 [ bl bl bl bl [ write "=" write ] [ write bl ] bi* nl ] assoc-each ;
170 : form. ( vector -- )
171 [ closing?>> not ] filter
174 { [ dup name>> "form" = ]
175 [ "form action: " write attributes>> "action" swap at print ] }
176 { [ dup name>> "input" = ] [ input. ] }
181 : query>assoc* ( str -- hash )
182 "?" split1 nip query>assoc ;
184 : html-class? ( tag string -- ? )
185 swap attributes>> "class" swap at = ;
187 : html-id? ( tag string -- ? )
188 swap attributes>> "id" swap at = ;
190 : opening-tag? ( tag -- ? )