1 ! Copyright (C) 2009 Slava Pestov.
2 ! See http://factorcode.org/license.txt for BSD license.
3 USING: accessors arrays assocs combinators kernel math
4 math.statistics namespaces sequences sorting xml.syntax
8 SYMBOL: network-failures
12 : record-broken-page ( url spider-result -- )
13 headers>> [ code>> ] [ message>> ] bi 2array 2array
16 : record-page-timings ( url spider-result -- )
17 fetched-in>> 2array timings get push ;
19 : record-network-failure ( url -- )
20 network-failures get push ;
22 : process-result ( url spider-result -- )
24 { f [ record-network-failure ] }
26 dup headers>> code>> 200 =
27 [ record-page-timings ] [ record-broken-page ] if
38 : process-timings ( -- )
39 timings get sort-values
40 [ slowest short tail* reverse slowest-pages set ]
43 [ mean 1000000 /f mean-time set ]
44 [ median 1000000 /f median-time set ]
45 [ std 1000000 /f time-std set ] tri
49 : process-results ( results -- )
50 V{ } clone network-failures set
51 V{ } clone broken-pages set
52 V{ } clone timings set
53 [ process-result ] assoc-each
56 : info-table ( alist -- html )
58 first2 dupd 1000000 /f
60 <tr><td><a href=<->><-></a></td><td><-> seconds</td></tr>
62 ] map [XML <table border="1"><-></table> XML] ;
64 : report-broken-pages ( -- html )
65 broken-pages get info-table ;
67 : report-network-failures ( -- html )
68 network-failures get [
69 dup [XML <li><a href=<->><-></a></li> XML]
70 ] map [XML <ul><-></ul> XML] ;
72 : slowest-pages-table ( -- html )
73 slowest-pages get info-table ;
75 : timing-summary-table ( -- html )
81 <tr><th>Mean</th><td><-> seconds</td></tr>
82 <tr><th>Median</th><td><-> seconds</td></tr>
83 <tr><th>Standard deviation</th><td><-> seconds</td></tr>
87 : report-timings ( -- html )
91 <h3>Slowest pages</h3>
98 : generate-report ( -- html )
101 report-network-failures
104 <h1>Spider report</h1>
105 URL: <a href=<->><-></a>
107 <h2>Broken pages</h2>
110 <h2>Network failures</h2>
117 : spider-report ( spider -- html )
118 [ "Spider report" f ] dip
121 [ spidered>> process-results ] bi