1 ! Copyright (C) 2009 Doug Coleman.
2 ! See http://factorcode.org/license.txt for BSD license.
3 USING: calendar io.encodings.utf8 io.files robots tools.test
7 { "http://www.chiplist.com/sitemap.txt" }
10 { user-agents V{ "*" } }
16 URL" /ChipList2/scripts/"
17 URL" /ChipList2/styles/"
20 URL" /advertisements/"
21 URL" /ChipList2/advertisements/"
23 URL" /ChipList2/graphics/"
28 T{ duration { hour 2 } }
29 T{ duration { hour 5 } }
37 { user-agents V{ "UbiCrawler" } }
39 { disallows V{ URL" /" } }
43 { user-agents V{ "DOC" } }
45 { disallows V{ URL" /" } }
49 { user-agents V{ "Zao" } }
51 { disallows V{ URL" /" } }
55 { user-agents V{ "sitecheck.internetseer.com" } }
57 { disallows V{ URL" /" } }
61 { user-agents V{ "Zealbot" } }
63 { disallows V{ URL" /" } }
67 { user-agents V{ "MSIECrawler" } }
69 { disallows V{ URL" /" } }
73 { user-agents V{ "SiteSnagger" } }
75 { disallows V{ URL" /" } }
79 { user-agents V{ "WebStripper" } }
81 { disallows V{ URL" /" } }
85 { user-agents V{ "WebCopier" } }
87 { disallows V{ URL" /" } }
91 { user-agents V{ "Fetch" } }
93 { disallows V{ URL" /" } }
97 { user-agents V{ "Offline Explorer" } }
99 { disallows V{ URL" /" } }
103 { user-agents V{ "Teleport" } }
105 { disallows V{ URL" /" } }
109 { user-agents V{ "TeleportPro" } }
111 { disallows V{ URL" /" } }
115 { user-agents V{ "WebZIP" } }
117 { disallows V{ URL" /" } }
121 { user-agents V{ "linko" } }
123 { disallows V{ URL" /" } }
127 { user-agents V{ "HTTrack" } }
129 { disallows V{ URL" /" } }
133 { user-agents V{ "Microsoft.URL.Control" } }
135 { disallows V{ URL" /" } }
139 { user-agents V{ "Xenu" } }
141 { disallows V{ URL" /" } }
145 { user-agents V{ "larbin" } }
147 { disallows V{ URL" /" } }
151 { user-agents V{ "libwww" } }
153 { disallows V{ URL" /" } }
157 { user-agents V{ "ZyBORG" } }
159 { disallows V{ URL" /" } }
163 { user-agents V{ "Download Ninja" } }
165 { disallows V{ URL" /" } }
169 { user-agents V{ "wget" } }
171 { disallows V{ URL" /" } }
175 { user-agents V{ "grub-client" } }
177 { disallows V{ URL" /" } }
181 { user-agents V{ "k2spider" } }
183 { disallows V{ URL" /" } }
187 { user-agents V{ "NPBot" } }
189 { disallows V{ URL" /" } }
193 { user-agents V{ "WebReaper" } }
195 { disallows V{ URL" /" } }
225 "exactseek-pagereaper"
246 "Kolinka Forum Search"
249 "Lincoln State Web Browser"
253 "MapoftheInternet.com"
265 "OpenIntelligenceData"
266 "Oracle Enterprise Search"
269 "PeerFactor 404 crawler"
276 "QuickFinder Crawler"
277 "Radiation Retriever"
284 "Seeker.lookseek.com"
288 "snap.com beta crawler"
300 "T-H-U-N-D-E-R-S-T-O-N-E"
330 { disallows V{ URL" /" } }
334 } [ "vocab:robots/robots.txt" utf8 file-contents parse-robots.txt ] unit-test