From 1a083937fb43db3c978df1960d41ae257ff449aa Mon Sep 17 00:00:00 2001 From: Doug Coleman Date: Mon, 25 Mar 2024 09:50:52 -0500 Subject: [PATCH] http.download: fix for new changes --- basis/http/client/client-tests.factor | 5 --- basis/http/download/download.factor | 8 +++-- basis/http/http-tests.factor | 33 ++++++++---------- extra/geo-ip/geo-ip.factor | 2 +- extra/google/translate/translate.factor | 6 ++-- .../data-sets/data-sets.factor | 34 +++++++------------ extra/project-euler/098/098.factor | 2 +- .../max-licenses/max-licenses.factor | 2 +- 8 files changed, 39 insertions(+), 53 deletions(-) diff --git a/basis/http/client/client-tests.factor b/basis/http/client/client-tests.factor index 6ccf080b06..a89d5107b1 100644 --- a/basis/http/client/client-tests.factor +++ b/basis/http/client/client-tests.factor @@ -3,11 +3,6 @@ io.streams.string kernel namespaces sequences splitting tools.test urls ; IN: http.client.tests -{ "foo.txt" } [ "http://www.paulgraham.com/foo.txt" download-name ] unit-test -{ "foo.txt" } [ "http://www.arc.com/foo.txt?xxx" download-name ] unit-test -{ "foo.txt" } [ "http://www.arc.com/foo.txt/" download-name ] unit-test -{ "www.arc.com" } [ "http://www.arc.com////" download-name ] unit-test - { T{ request { url T{ url { protocol "http" } { host "www.apple.com" } { port 80 } { path "/index.html" } } } diff --git a/basis/http/download/download.factor b/basis/http/download/download.factor index 1f54064206..df9fca6e59 100644 --- a/basis/http/download/download.factor +++ b/basis/http/download/download.factor @@ -1,9 +1,10 @@ ! Copyright (C) 2024 Doug Coleman. ! See https://factorcode.org/license.txt for BSD license. USING: accessors calendar checksums combinators.short-circuit -http.client io io.backend io.directories io.encodings.binary -io.files io.files.info io.files.unique io.pathnames kernel math +http.client io io.directories io.encodings.binary io.files +io.files.info io.files.unique io.pathnames kernel math math.order math.parser present sequences shuffle splitting ; + IN: http.download : file-too-old-or-not-exists? ( file duration -- ? ) @@ -87,6 +88,9 @@ PRIVATE> : download-once-to ( url file -- path ) dup file-exists? [ nip ] [ download-to ] if ; +: download-once ( url -- path ) + dup download-name download-once-to ; + : download-outdated-to ( url file duration -- path ) 2dup delete-when-old [ drop download-to ] [ drop nip ] if ; diff --git a/basis/http/http-tests.factor b/basis/http/http-tests.factor index ca9db152d4..42c84ac3a7 100644 --- a/basis/http/http-tests.factor +++ b/basis/http/http-tests.factor @@ -1,9 +1,16 @@ -USING: destructors http http.server http.server.requests http.client -http.client.private tools.test multiline fry io.streams.string io.crlf -io.encodings.utf8 io.encodings.latin1 io.encodings.binary io.encodings.string -io.encodings.ascii kernel arrays splitting sequences assocs io.sockets db -db.sqlite make continuations urls hashtables accessors namespaces xml.data -random combinators.short-circuit literals ; +USING: accessors combinators.short-circuit continuations db +db.sqlite db.tuples destructors furnace furnace.actions +furnace.alloy furnace.auth furnace.auth.login +furnace.conversations furnace.db furnace.redirection +furnace.sessions html.components html.forms http http.client +http.client.private http.download http.server +http.server.dispatchers http.server.redirection +http.server.requests http.server.responses http.server.static io +io.crlf io.directories io.encodings.ascii io.encodings.binary +io.encodings.utf8 io.files io.files.temp io.servers io.sockets +io.streams.string kernel literals locals make multiline +namespaces random sequences splitting threads tools.test urls +validators xml xml.data xml.traversal ; IN: http.tests { "text/plain" "UTF-8" } [ "text/plain" parse-content-type ] unit-test @@ -216,12 +223,6 @@ Set-Cookie: oo="bar; a=b"; comment="your mom"; httponly=yes ] unit-test ! Live-fire exercise -USING: http.server.static furnace.sessions furnace.alloy -furnace.actions furnace.auth furnace.auth.login furnace.db -io.servers io.files io.files.temp io.directories io -threads -http.server.responses http.server.redirection furnace.redirection -http.server.dispatchers db.tuples ; : add-quit-action ( responder -- responder ) @@ -255,8 +256,6 @@ http.server.dispatchers db.tuples ; ] with-threaded-server ] with-variable ; inline -USING: locals ; - :: test-with-db-persistence ( db-persistence quot -- ) db-persistence [ quot test-with-dispatcher @@ -344,10 +343,6 @@ test-db [ ] test-with-db-persistence -USING: html.components html.forms -xml xml.traversal validators -furnace furnace.conversations ; - SYMBOL: a : test-a ( xml -- value ) @@ -442,7 +437,7 @@ test-db [ ! Check that download throws errors (reported by Chris Double) [ [ - "http://localhost/tweet_my_twat" add-addr download + "http://localhost/tweet_my_twat" add-addr download drop ] with-temp-directory ] must-fail diff --git a/extra/geo-ip/geo-ip.factor b/extra/geo-ip/geo-ip.factor index 4e8cfac746..20e000f7d9 100644 --- a/extra/geo-ip/geo-ip.factor +++ b/extra/geo-ip/geo-ip.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2008 Slava Pestov. ! See https://factorcode.org/license.txt for BSD license. USING: accessors arrays assocs combinators combinators.smart csv -grouping http.client interval-maps io.encodings.ascii io.files +grouping http.download interval-maps io.encodings.ascii io.files io.files.temp io.launcher io.pathnames ip-parser kernel math math.parser sequences splitting strings ; IN: geo-ip diff --git a/extra/google/translate/translate.factor b/extra/google/translate/translate.factor index 260d20dfed..a8ddafa31f 100644 --- a/extra/google/translate/translate.factor +++ b/extra/google/translate/translate.factor @@ -1,8 +1,8 @@ ! Copyright (C) 2010 Doug Coleman. ! See https://factorcode.org/license.txt for BSD license. -USING: assocs combinators grouping http.client io io.files.temp -io.files.unique json kernel make sequences urls ; +USING: assocs combinators grouping http.client http.download io +io.files.temp io.files.unique json kernel make sequences urls ; IN: google.translate @@ -55,7 +55,7 @@ TUPLE: response-error response error ; : translate-tts ( text -- file ) "https://translate.google.com/translate_tts?tl=en" >url swap "q" set-query-param [ - "" ".mp3" unique-file [ download-to ] keep + "" ".mp3" unique-file download-to ] with-temp-directory ; ! Example: diff --git a/extra/machine-learning/data-sets/data-sets.factor b/extra/machine-learning/data-sets/data-sets.factor index d01cf51a2c..8de5211251 100644 --- a/extra/machine-learning/data-sets/data-sets.factor +++ b/extra/machine-learning/data-sets/data-sets.factor @@ -2,11 +2,11 @@ ! See https://factorcode.org/license.txt for BSD license USING: accessors arrays ascii assocs byte-arrays combinators -combinators.short-circuit concurrency.combinators csv grouping -http.client images images.viewer io io.directories -io.encodings.binary io.encodings.utf8 io.files io.launcher -io.pathnames kernel math math.parser namespaces sequences -splitting ui.gadgets.panes ; +concurrency.combinators csv grouping http.download images +images.viewer io io.directories io.encodings.binary +io.encodings.utf8 io.files io.launcher io.pathnames kernel math +math.parser namespaces sequences splitting ui.gadgets.panes ; + IN: machine-learning.data-sets TUPLE: data-set @@ -65,16 +65,6 @@ PRIVATE> [ >>targets ] [ >>target-names ] bi* "linnerud.rst" load-file >>description ; -: download-to-directory ( url directory -- ) - dup make-directories - [ - dup { [ download-name file-exists? ] [ file-stem file-exists? ] } 1|| [ - drop - ] [ - download - ] if - ] with-directory ; - : gzip-decompress-file ( path -- ) { "gzip" "-d" } swap suffix try-process ; @@ -98,15 +88,17 @@ PRIVATE> output-stream get stream-nl ] each ; +CONSTANT: datasets-path "resource:datasets/" + : load-mnist ( -- data-set ) - "resource:datasets" dup make-directories [ + datasets-path dup make-directories [ { - "https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz" - "https://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz" - "https://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz" - "https://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz" + "https://github.com/golbin/TensorFlow-MNIST/raw/master/mnist/data/train-images-idx3-ubyte.gz" + "https://github.com/golbin/TensorFlow-MNIST/raw/master/mnist/data/train-labels-idx1-ubyte.gz" + "https://github.com/golbin/TensorFlow-MNIST/raw/master/mnist/data/t10k-images-idx3-ubyte.gz" + "https://github.com/golbin/TensorFlow-MNIST/raw/master/mnist/data/t10k-labels-idx1-ubyte.gz" } - [ [ "resource:datasets/" download-to-directory ] parallel-each ] + [ [ download-once ] parallel-each ] [ [ dup file-stem file-exists? [ drop ] [ file-name gzip-decompress-file ] if ] each ] [ [ file-stem binary file-contents ] map ] tri first4 { diff --git a/extra/project-euler/098/098.factor b/extra/project-euler/098/098.factor index 72c332a1f5..4a97d916a9 100644 --- a/extra/project-euler/098/098.factor +++ b/extra/project-euler/098/098.factor @@ -1,7 +1,7 @@ ! Copyright (c) 2023 John Benediktsson. ! See https://factorcode.org/license.txt for BSD license. -USING: anagrams assocs combinators.short-circuit http.client +USING: anagrams assocs combinators.short-circuit http.download io.encodings.utf8 io.files io.files.temp kernel math math.combinatorics math.functions math.order math.parser project-euler.common ranges sequences splitting ; diff --git a/extra/rosetta-code/text-processing/max-licenses/max-licenses.factor b/extra/rosetta-code/text-processing/max-licenses/max-licenses.factor index 238d596753..2110ad19cd 100644 --- a/extra/rosetta-code/text-processing/max-licenses/max-licenses.factor +++ b/extra/rosetta-code/text-processing/max-licenses/max-licenses.factor @@ -1,6 +1,6 @@ ! Copyright (c) 2012 Anonymous ! See https://factorcode.org/license.txt for BSD license. -USING: accessors http.client io io.encodings.ascii io.files +USING: accessors http.download io io.encodings.ascii io.files io.files.temp kernel math math.parser sequences splitting urls ; IN: rosetta-code.text-processing.max-licenses -- 2.34.1