env/bin/fetch.sh

76 lines
1.6 KiB
Bash
Raw Normal View History

2019-09-11 04:11:40 +02:00
#!/bin/sh
2019-12-08 19:14:46 +01:00
# fetch
# Written in 2019-2022 by Lucas
2019-09-11 04:11:40 +02:00
# CC0 1.0 Universal/Public domain - No rights reserved
#
# To the extent possible under law, the author(s) have dedicated all
# copyright and related and neighboring rights to this software to the
# public domain worldwide. This software is distributed without any
# warranty. You should have received a copy of the CC0 Public Domain
# Dedication along with this software. If not, see
# <http://creativecommons.org/publicdomain/zero/1.0/>.
2019-12-08 19:14:46 +01:00
usage()
{
printf "Usage: %s [-T] URL [URL ...]\n" "${0##*/}" >&2
2019-09-11 04:11:40 +02:00
exit 1
}
fetch_cmd()
{
# ftp(1) is chatty if stdin is a terminal. It prints its operations to
# stdout if "-o output" is used. In that case, we'll need to redirect
# stdout to the tty we're being called from, in order to be able to
# pipe the output to another command.
if tty=$(tty) && [ X"$tty" != X"not a tty" ]; then
$torsocks ftp "$@" >"$tty"
else
$torsocks ftp "$@"
fi
}
2019-09-11 04:11:40 +02:00
torsocks=torsocks
while getopts T flag; do
case $flag in
T) torsocks=
;;
*) usage
;;
esac
done
2020-05-03 16:47:27 +02:00
shift $((OPTIND - 1))
2019-12-08 19:14:46 +01:00
[ $# -gt 0 ] || usage
2019-09-11 04:11:40 +02:00
: ${ARCHIVE_BASEDIR:=~/tmp/archive}
mkdir -p "$ARCHIVE_BASEDIR"
rc=0
for url; do
sha=$(sha256 -qs "$url")
2019-09-11 04:11:40 +02:00
t=$sha
h0=${t%${t#??}}
t=${t#??}
h1=${t%${t#??}}
t=${t#??}
ht=$t
outdir=$ARCHIVE_BASEDIR/$h0/$h1/$ht
if [ -f "$outdir/file" ]; then
2020-05-03 16:47:27 +02:00
printf "%s: already fetched %s\n" "${0##*/}" "$url" >&2
printf "%s\n" "$outdir/file"
2019-09-11 04:11:40 +02:00
continue
fi
mkdir -p "$outdir"
{
2020-08-22 02:01:38 +02:00
fetch_cmd -o "$outdir/file" "$url" &&
printf "%s=%s\n" \
url "$url" date "$(date +%Y-%m-%d)" >"$outdir/meta" &&
2020-08-22 02:01:38 +02:00
printf "%s\n" "$outdir/file"
2019-09-11 04:11:40 +02:00
} || rc=1
done
exit $rc