From 722d92fbfdeff55645f4148d3016798afe4fd433 Mon Sep 17 00:00:00 2001 From: Doug Coleman Date: Sun, 24 Mar 2024 23:46:53 -0500 Subject: [PATCH] http.download: ?download-to -> download-to-once Hopefully better name for a word that caches the download instead of downloading every time. --- basis/fixups/fixups.factor | 1 + basis/http/download/download-docs.factor | 9 +++++++++ basis/http/download/download.factor | 10 +++++----- basis/unicode/breaks/breaks-tests.factor | 4 ++-- basis/unicode/collation/collation-tests.factor | 2 +- basis/unicode/normalize/normalize-tests.factor | 2 +- extra/bunny/model/model.factor | 2 +- extra/geo-ip/geo-ip.factor | 2 +- extra/gpu/demos/bunny/bunny.factor | 2 +- extra/macho/macho-tests.factor | 4 ++-- extra/project-euler/098/098.factor | 2 +- .../anagrams-deranged/anagrams-deranged.factor | 2 +- extra/rosetta-code/ordered-words/ordered-words.factor | 2 +- .../text-processing/max-licenses/max-licenses.factor | 6 ++---- extra/scryfall/scryfall.factor | 4 ++-- extra/spelling/spelling.factor | 2 +- extra/tldr/tldr.factor | 2 +- extra/youtube/youtube.factor | 8 ++++---- 18 files changed, 37 insertions(+), 29 deletions(-) diff --git a/basis/fixups/fixups.factor b/basis/fixups/fixups.factor index abe96e3f90..9ec5d6b285 100644 --- a/basis/fixups/fixups.factor +++ b/basis/fixups/fixups.factor @@ -74,6 +74,7 @@ CONSTANT: word-renames { { "assoc-all-value?" { "all-values?" "0.100" } } { "assoc-any-key?" { "any-key?" "0.100" } } { "assoc-any-value?" { "any-value?" "0.100" } } + { "?download-to" { "download-once-to" "0.100" } } } : compute-assoc-fixups ( continuation name assoc -- seq ) diff --git a/basis/http/download/download-docs.factor b/basis/http/download/download-docs.factor index 72dc7b2d9d..76f3a19dc0 100644 --- a/basis/http/download/download-docs.factor +++ b/basis/http/download/download-docs.factor @@ -7,11 +7,19 @@ IN: http.download HELP: download { $values { "url" { $or url string } } { "path" "a pathname string" } } { $description "Downloads the contents of the URL to a file in the " { $link current-directory } " having the same file name and returns the pathname." } +{ $notes "Use this to download the file every time." } { $errors "Throws an error if the HTTP request fails." } ; HELP: download-to { $values { "url" { $or url string } } { "file" "a pathname string" } { "path" "a pathname string" } } { $description "Downloads the contents of the URL to a file with the given pathname and returns the pathname." } +{ $notes "Use this to download the file every time." } +{ $errors "Throws an error if the HTTP request fails." } ; + +HELP: download-once-to +{ $values { "url" { $or url string } } { "file" "a pathname string" } { "path" "a pathname string" } } +{ $description "If the file exists on disk, returns that pathname without downloading anything. Otherwise, downloads the contents of the URL to a file with the given pathname and returns the pathname." } +{ $notes "Use this if the contents of the URL are not expected to change." } { $errors "Throws an error if the HTTP request fails." } ; @@ -22,6 +30,7 @@ ARTICLE: "http.download" "HTTP Download Utilities" { $subsections download download-to + download-once-to } ; diff --git a/basis/http/download/download.factor b/basis/http/download/download.factor index 59a71199ac..1f54064206 100644 --- a/basis/http/download/download.factor +++ b/basis/http/download/download.factor @@ -76,16 +76,16 @@ IN: http.download : download-temporary-name ( url -- prefix suffix ) [ "temp." ".temp" ] dip download-name prepend ; -: download-file-to ( url file -- path ) +PRIVATE> + +: download-to ( url file -- path ) [ [ download-temporary-name binary ] keep '[ _ http-write-request ] with-unique-file-writer ] dip [ move-file ] keep ; -PRIVATE> - -: download-to ( url file -- path ) - dup file-exists? [ nip ] [ download-file-to ] if ; +: download-once-to ( url file -- path ) + dup file-exists? [ nip ] [ download-to ] if ; : download-outdated-to ( url file duration -- path ) 2dup delete-when-old [ drop download-to ] [ drop nip ] if ; diff --git a/basis/unicode/breaks/breaks-tests.factor b/basis/unicode/breaks/breaks-tests.factor index 2fef15c601..801b486ad8 100644 --- a/basis/unicode/breaks/breaks-tests.factor +++ b/basis/unicode/breaks/breaks-tests.factor @@ -21,11 +21,11 @@ IN: unicode.breaks.tests : grapheme-break-test ( -- filename ) "https://downloads.factorcode.org/misc/UCD/15.1.0/auxiliary/GraphemeBreakTest.txt" - "GraphemeBreakTest-15.1.0.txt" cache-file download-to ; + "GraphemeBreakTest-15.1.0.txt" cache-file download-once-to ; : word-break-test ( -- filename ) "https://downloads.factorcode.org/misc/UCD/15.1.0/auxiliary/WordBreakTest.txt" - "WordBreakTest-15.1.0.txt" cache-file download-to ; + "WordBreakTest-15.1.0.txt" cache-file download-once-to ; : parse-test-file ( file-name -- tests ) utf8 file-lines diff --git a/basis/unicode/collation/collation-tests.factor b/basis/unicode/collation/collation-tests.factor index 2cf37e1b1e..6e985c2bab 100644 --- a/basis/unicode/collation/collation-tests.factor +++ b/basis/unicode/collation/collation-tests.factor @@ -19,7 +19,7 @@ IN: unicode.collation.tests : collation-test-lines ( -- lines ) "https://downloads.factorcode.org/misc/UCA/15.1.0/CollationTest_SHIFTED.txt" - "CollationTest_SHIFTED_15.1.0.txt" cache-file download-to + "CollationTest_SHIFTED_15.1.0.txt" cache-file download-once-to utf8 file-lines [ "#" head? ] reject harvest ; : parse-collation-test-shifted ( -- lines ) diff --git a/basis/unicode/normalize/normalize-tests.factor b/basis/unicode/normalize/normalize-tests.factor index 3d72ad99e5..edfc27dba8 100644 --- a/basis/unicode/normalize/normalize-tests.factor +++ b/basis/unicode/normalize/normalize-tests.factor @@ -27,7 +27,7 @@ IN: unicode.normalize.tests ! Could use simple-flat-file after some cleanup : parse-normalization-tests ( -- tests ) "https://downloads.factorcode.org/misc/UCD/15.1.0/NormalizationTest.txt" - "NormalizationTest-15.1.0.txt" cache-file download-to + "NormalizationTest-15.1.0.txt" cache-file download-once-to utf8 file-lines [ "#" head? ] reject [ "@" head? ] split*-when 2 [ first2 [ first ] dip 2array ] map diff --git a/extra/bunny/model/model.factor b/extra/bunny/model/model.factor index fb64d7193d..dd2003a4c9 100644 --- a/extra/bunny/model/model.factor +++ b/extra/bunny/model/model.factor @@ -46,7 +46,7 @@ CONSTANT: model-url "https://downloads.factorcode.org/misc/bun_zipper.ply" : download-bunny ( -- path ) - model-url model-path download-to ; + model-url model-path download-once-to ; :: (draw-triangle) ( ns vs triple -- ) triple [| elt | diff --git a/extra/geo-ip/geo-ip.factor b/extra/geo-ip/geo-ip.factor index 890969fca4..4e8cfac746 100644 --- a/extra/geo-ip/geo-ip.factor +++ b/extra/geo-ip/geo-ip.factor @@ -12,7 +12,7 @@ CONSTANT: db-url "https://software77.net/geo-ip/?DL=1" : download-db ( -- path ) db-path dup file-exists? [ - db-url over ".gz" append download-to + db-url over ".gz" append download-once-to { "gunzip" } over ".gz" append absolute-path suffix try-process ] unless ; diff --git a/extra/gpu/demos/bunny/bunny.factor b/extra/gpu/demos/bunny/bunny.factor index 97bde0ad87..f5963e714e 100644 --- a/extra/gpu/demos/bunny/bunny.factor +++ b/extra/gpu/demos/bunny/bunny.factor @@ -149,7 +149,7 @@ CONSTANT: bunny-model-url "https://downloads.factorcode.org/misc/bun_zipper.ply" : download-bunny ( -- path ) - bunny-model-url bunny-model-path download-to ; + bunny-model-url bunny-model-path download-once-to ; : get-bunny-data ( bunny-state -- ) download-bunny bunny-data diff --git a/extra/macho/macho-tests.factor b/extra/macho/macho-tests.factor index 48626fcaf3..5bcfa60378 100644 --- a/extra/macho/macho-tests.factor +++ b/extra/macho/macho-tests.factor @@ -24,11 +24,11 @@ STRING: validation-output : a.macho ( -- path ) URL" https://downloads.factorcode.org/misc/a.macho" - "a.macho" cache-file download-to ; + "a.macho" cache-file download-once-to ; : a2.macho ( -- path ) URL" https://downloads.factorcode.org/misc/a2.macho" - "a2.macho" cache-file download-to ; + "a2.macho" cache-file download-once-to ; cpu ppc? [ { $ validation-output } diff --git a/extra/project-euler/098/098.factor b/extra/project-euler/098/098.factor index 27db0b8cf5..72c332a1f5 100644 --- a/extra/project-euler/098/098.factor +++ b/extra/project-euler/098/098.factor @@ -42,7 +42,7 @@ IN: project-euler.098 : wordlist ( -- seq ) "https://projecteuler.net/project/resources/p098_words.txt" - "p098_words.txt" temp-file download-to + "p098_words.txt" temp-file download-once-to utf8 file-contents "," split [ rest-slice but-last ] map ; : squarelist ( n -- seq ) diff --git a/extra/rosetta-code/anagrams-deranged/anagrams-deranged.factor b/extra/rosetta-code/anagrams-deranged/anagrams-deranged.factor index d65f06a8a9..53db81bb00 100644 --- a/extra/rosetta-code/anagrams-deranged/anagrams-deranged.factor +++ b/extra/rosetta-code/anagrams-deranged/anagrams-deranged.factor @@ -42,7 +42,7 @@ IN: rosettacode.anagrams-deranged : default-word-list ( -- path ) URL" https://raw.githubusercontent.com/quinnj/Rosetta-Julia/master/unixdict.txt" - "unixdict.txt" temp-file download-to ; + "unixdict.txt" temp-file download-once-to ; : longest-deranged-anagrams ( -- anagrams ) default-word-list (longest-deranged-anagrams) ; diff --git a/extra/rosetta-code/ordered-words/ordered-words.factor b/extra/rosetta-code/ordered-words/ordered-words.factor index 0ce7c4319e..8c37abfc36 100644 --- a/extra/rosetta-code/ordered-words/ordered-words.factor +++ b/extra/rosetta-code/ordered-words/ordered-words.factor @@ -19,7 +19,7 @@ IN: rosetta-code.ordered-words MEMO: word-list ( -- seq ) URL" https://raw.githubusercontent.com/quinnj/Rosetta-Julia/master/unixdict.txt" "unixdict.txt" temp-file - download-to utf8 file-lines ; + download-once-to utf8 file-lines ; : ordered-word? ( word -- ? ) >lower [ <= ] monotonic? ; diff --git a/extra/rosetta-code/text-processing/max-licenses/max-licenses.factor b/extra/rosetta-code/text-processing/max-licenses/max-licenses.factor index 9456ea799f..238d596753 100644 --- a/extra/rosetta-code/text-processing/max-licenses/max-licenses.factor +++ b/extra/rosetta-code/text-processing/max-licenses/max-licenses.factor @@ -62,10 +62,8 @@ TUPLE: maxlicense max-count current-count times ; : process ( max line -- max ) split-line inc-current-count update-time ; MEMO: mlijobs ( -- lines ) - "mlijobs.txt" temp-file dup file-exists? [ - URL" https://rosettacode.org/resources/mlijobs.txt" - over download-to - ] unless ascii file-lines ; + URL" https://raw.githubusercontent.com/def-/nim-unsorted/master/mlijobs.txt" + "mlijobs.txt" temp-file download-once-to ascii file-lines ; PRIVATE> diff --git a/extra/scryfall/scryfall.factor b/extra/scryfall/scryfall.factor index 0e019a507a..58cd66d52d 100644 --- a/extra/scryfall/scryfall.factor +++ b/extra/scryfall/scryfall.factor @@ -53,7 +53,7 @@ CONSTANT: scryfall-images-path "resource:scryfall-images/" ensure-scryfall-images-directory normal-images [ dup scryfall>local dup delete-when-zero-size - [ download-to ] [ nip ] if + [ download-once-to ] [ nip ] if ] map [ load-image ] map ; @@ -61,7 +61,7 @@ CONSTANT: scryfall-images-path "resource:scryfall-images/" ensure-scryfall-images-directory small-images [ dup scryfall>local dup delete-when-zero-size - [ download-to ] [ nip ] if + [ download-once-to ] [ nip ] if ] map [ load-image ] map ; diff --git a/extra/spelling/spelling.factor b/extra/spelling/spelling.factor index d48bffdc84..f3c512f9cd 100644 --- a/extra/spelling/spelling.factor +++ b/extra/spelling/spelling.factor @@ -58,7 +58,7 @@ CONSTANT: ALPHABET "abcdefghijklmnopqrstuvwxyz" MEMO: default-dictionary ( -- counts ) URL" https://norvig.com/big.txt" "big.txt" temp-file - download-to load-dictionary ; + download-once-to load-dictionary ; : (correct) ( word dictionary -- word/f ) corrections ?first ; diff --git a/extra/tldr/tldr.factor b/extra/tldr/tldr.factor index 5fc08b8af5..47560add8b 100644 --- a/extra/tldr/tldr.factor +++ b/extra/tldr/tldr.factor @@ -26,7 +26,7 @@ CONSTANT: tldr-zip URL" https://tldr-pages.github.io/assets/tldr.zip" : download-tldr ( -- ) "tldr" cache-file dup make-directory [ - tldr-zip "tldr.zip" download-to drop + tldr-zip "tldr.zip" download-once-to drop { "unzip" "tldr.zip" } try-process ] with-directory ; diff --git a/extra/youtube/youtube.factor b/extra/youtube/youtube.factor index 6d1d549c7f..261165dbb4 100644 --- a/extra/youtube/youtube.factor +++ b/extra/youtube/youtube.factor @@ -1,8 +1,8 @@ ! Copyright (C) 2013 John Benediktsson ! See https://factorcode.org/license.txt for BSD license -USING: assocs http.client kernel make math.order sequences -splitting urls urls.encoding ; +USING: assocs http.client http.download kernel math.order +sequences splitting urls urls.encoding ; IN: youtube @@ -66,12 +66,12 @@ CONSTANT: video-info-url URL" https://www.youtube.com/get_video_info" : downloadable? ( video-info -- ? ) "use_cipher_signature" of "False" = ; -: download-video ( video-id -- ) +: download-video ( video-id -- path ) get-video-info [ downloadable? [ "Video is encrypted." throw ] unless ] [ video-formats [ "type" of "video/mp4" head? ] find nip video-download-url ] [ - "title" of sanitize ".mp4" append download-to + "title" of sanitize ".mp4" append download-once-to ] tri ; -- 2.34.1