1 ! Copyright (C) 2009 Slava Pestov.
2 ! See http://factorcode.org/license.txt for BSD license.
3 USING: accessors arrays assocs combinators html kernel math
4 math.statistics namespaces sequences sorting urls xml.syntax ;
7 SYMBOL: network-failures
11 : record-broken-page ( url spider-result -- )
12 headers>> [ code>> ] [ message>> ] bi 2array 2array
15 : record-page-timings ( url spider-result -- )
16 fetched-in>> 2array timings get push ;
18 : record-network-failure ( url -- )
19 network-failures get push ;
21 : process-result ( url spider-result -- )
23 { f [ record-network-failure ] }
25 dup headers>> code>> 200 =
26 [ record-page-timings ] [ record-broken-page ] if
37 : process-timings ( -- )
38 timings get sort-values
39 [ slowest bound tail* reverse slowest-pages set ]
42 [ mean 1000000 /f mean-time set ]
43 [ median 1000000 /f median-time set ]
44 [ std 1000000 /f time-std set ] tri
48 : process-results ( results -- )
49 V{ } clone network-failures set
50 V{ } clone broken-pages set
51 V{ } clone timings set
52 [ process-result ] assoc-each
55 : info-table ( alist -- html )
57 first2 dupd 1000000 /f
59 <tr><td><a href=<->><-></a></td><td><-> seconds</td></tr>
61 ] map [XML <table border="1"><-></table> XML] ;
63 : report-broken-pages ( -- html )
64 broken-pages get info-table ;
66 : report-network-failures ( -- html )
67 network-failures get [
68 dup [XML <li><a href=<->><-></a></li> XML]
69 ] map [XML <ul><-></ul> XML] ;
71 : slowest-pages-table ( -- html )
72 slowest-pages get info-table ;
74 : timing-summary-table ( -- html )
80 <tr><th>Mean</th><td><-> seconds</td></tr>
81 <tr><th>Median</th><td><-> seconds</td></tr>
82 <tr><th>Standard deviation</th><td><-> seconds</td></tr>
86 : report-timings ( -- html )
90 <h3>Slowest pages</h3>
97 : generate-report ( -- html )
100 report-network-failures
103 <h1>Spider report</h1>
104 URL: <a href=<->><-></a>
106 <h2>Broken pages</h2>
109 <h2>Network failures</h2>
116 : spider-report ( spider -- html )
117 [ "Spider report" f ] dip
120 [ spidered>> process-results ] bi