tools.test urls ;
IN: http.client.tests
-{ "foo.txt" } [ "http://www.paulgraham.com/foo.txt" download-name ] unit-test
-{ "foo.txt" } [ "http://www.arc.com/foo.txt?xxx" download-name ] unit-test
-{ "foo.txt" } [ "http://www.arc.com/foo.txt/" download-name ] unit-test
-{ "www.arc.com" } [ "http://www.arc.com////" download-name ] unit-test
-
{
T{ request
{ url T{ url { protocol "http" } { host "www.apple.com" } { port 80 } { path "/index.html" } } }
! Copyright (C) 2024 Doug Coleman.
! See https://factorcode.org/license.txt for BSD license.
USING: accessors calendar checksums combinators.short-circuit
-http.client io io.backend io.directories io.encodings.binary
-io.files io.files.info io.files.unique io.pathnames kernel math
+http.client io io.directories io.encodings.binary io.files
+io.files.info io.files.unique io.pathnames kernel math
math.order math.parser present sequences shuffle splitting ;
+
IN: http.download
: file-too-old-or-not-exists? ( file duration -- ? )
: download-once-to ( url file -- path )
dup file-exists? [ nip ] [ download-to ] if ;
+: download-once ( url -- path )
+ dup download-name download-once-to ;
+
: download-outdated-to ( url file duration -- path )
2dup delete-when-old [ drop download-to ] [ drop nip ] if ;
-USING: destructors http http.server http.server.requests http.client
-http.client.private tools.test multiline fry io.streams.string io.crlf
-io.encodings.utf8 io.encodings.latin1 io.encodings.binary io.encodings.string
-io.encodings.ascii kernel arrays splitting sequences assocs io.sockets db
-db.sqlite make continuations urls hashtables accessors namespaces xml.data
-random combinators.short-circuit literals ;
+USING: accessors combinators.short-circuit continuations db
+db.sqlite db.tuples destructors furnace furnace.actions
+furnace.alloy furnace.auth furnace.auth.login
+furnace.conversations furnace.db furnace.redirection
+furnace.sessions html.components html.forms http http.client
+http.client.private http.download http.server
+http.server.dispatchers http.server.redirection
+http.server.requests http.server.responses http.server.static io
+io.crlf io.directories io.encodings.ascii io.encodings.binary
+io.encodings.utf8 io.files io.files.temp io.servers io.sockets
+io.streams.string kernel literals locals make multiline
+namespaces random sequences splitting threads tools.test urls
+validators xml xml.data xml.traversal ;
IN: http.tests
{ "text/plain" "UTF-8" } [ "text/plain" parse-content-type ] unit-test
] unit-test
! Live-fire exercise
-USING: http.server.static furnace.sessions furnace.alloy
-furnace.actions furnace.auth furnace.auth.login furnace.db
-io.servers io.files io.files.temp io.directories io
-threads
-http.server.responses http.server.redirection furnace.redirection
-http.server.dispatchers db.tuples ;
: add-quit-action ( responder -- responder )
<action>
] with-threaded-server
] with-variable ; inline
-USING: locals ;
-
:: test-with-db-persistence ( db-persistence quot -- )
db-persistence [
quot test-with-dispatcher
] test-with-db-persistence
-USING: html.components html.forms
-xml xml.traversal validators
-furnace furnace.conversations ;
-
SYMBOL: a
: test-a ( xml -- value )
! Check that download throws errors (reported by Chris Double)
[
[
- "http://localhost/tweet_my_twat" add-addr download
+ "http://localhost/tweet_my_twat" add-addr download drop
] with-temp-directory
] must-fail
! Copyright (C) 2008 Slava Pestov.
! See https://factorcode.org/license.txt for BSD license.
USING: accessors arrays assocs combinators combinators.smart csv
-grouping http.client interval-maps io.encodings.ascii io.files
+grouping http.download interval-maps io.encodings.ascii io.files
io.files.temp io.launcher io.pathnames ip-parser kernel math
math.parser sequences splitting strings ;
IN: geo-ip
! Copyright (C) 2010 Doug Coleman.
! See https://factorcode.org/license.txt for BSD license.
-USING: assocs combinators grouping http.client io io.files.temp
-io.files.unique json kernel make sequences urls ;
+USING: assocs combinators grouping http.client http.download io
+io.files.temp io.files.unique json kernel make sequences urls ;
IN: google.translate
: translate-tts ( text -- file )
"https://translate.google.com/translate_tts?tl=en" >url
swap "q" set-query-param [
- "" ".mp3" unique-file [ download-to ] keep
+ "" ".mp3" unique-file download-to
] with-temp-directory ;
! Example:
! See https://factorcode.org/license.txt for BSD license
USING: accessors arrays ascii assocs byte-arrays combinators
-combinators.short-circuit concurrency.combinators csv grouping
-http.client images images.viewer io io.directories
-io.encodings.binary io.encodings.utf8 io.files io.launcher
-io.pathnames kernel math math.parser namespaces sequences
-splitting ui.gadgets.panes ;
+concurrency.combinators csv grouping http.download images
+images.viewer io io.directories io.encodings.binary
+io.encodings.utf8 io.files io.launcher io.pathnames kernel math
+math.parser namespaces sequences splitting ui.gadgets.panes ;
+
IN: machine-learning.data-sets
TUPLE: data-set
[ >>targets ] [ >>target-names ] bi*
"linnerud.rst" load-file >>description ;
-: download-to-directory ( url directory -- )
- dup make-directories
- [
- dup { [ download-name file-exists? ] [ file-stem file-exists? ] } 1|| [
- drop
- ] [
- download
- ] if
- ] with-directory ;
-
: gzip-decompress-file ( path -- )
{ "gzip" "-d" } swap suffix try-process ;
output-stream get stream-nl
] each ;
+CONSTANT: datasets-path "resource:datasets/"
+
: load-mnist ( -- data-set )
- "resource:datasets" dup make-directories [
+ datasets-path dup make-directories [
{
- "https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz"
- "https://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz"
- "https://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz"
- "https://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz"
+ "https://github.com/golbin/TensorFlow-MNIST/raw/master/mnist/data/train-images-idx3-ubyte.gz"
+ "https://github.com/golbin/TensorFlow-MNIST/raw/master/mnist/data/train-labels-idx1-ubyte.gz"
+ "https://github.com/golbin/TensorFlow-MNIST/raw/master/mnist/data/t10k-images-idx3-ubyte.gz"
+ "https://github.com/golbin/TensorFlow-MNIST/raw/master/mnist/data/t10k-labels-idx1-ubyte.gz"
}
- [ [ "resource:datasets/" download-to-directory ] parallel-each ]
+ [ [ download-once ] parallel-each ]
[ [ dup file-stem file-exists? [ drop ] [ file-name gzip-decompress-file ] if ] each ]
[ [ file-stem binary file-contents ] map ] tri
first4 {
! Copyright (c) 2023 John Benediktsson.
! See https://factorcode.org/license.txt for BSD license.
-USING: anagrams assocs combinators.short-circuit http.client
+USING: anagrams assocs combinators.short-circuit http.download
io.encodings.utf8 io.files io.files.temp kernel math
math.combinatorics math.functions math.order math.parser
project-euler.common ranges sequences splitting ;
! Copyright (c) 2012 Anonymous
! See https://factorcode.org/license.txt for BSD license.
-USING: accessors http.client io io.encodings.ascii io.files
+USING: accessors http.download io io.encodings.ascii io.files
io.files.temp kernel math math.parser sequences
splitting urls ;
IN: rosetta-code.text-processing.max-licenses
projects / factor.git / commitdiff