63 lines
1.4 KiB
Bash
63 lines
1.4 KiB
Bash
|
#!/bin/sh
|
||
|
|
# env
|
||
|
|
# Written in 2019 by Lucas
|
||
|
|
# CC0 1.0 Universal/Public domain - No rights reserved
|
||
|
|
#
|
||
|
|
# To the extent possible under law, the author(s) have dedicated all
|
||
|
|
# copyright and related and neighboring rights to this software to the
|
||
|
|
# public domain worldwide. This software is distributed without any
|
||
|
|
# warranty. You should have received a copy of the CC0 Public Domain
|
||
|
|
# Dedication along with this software. If not, see
|
||
|
|
# <http://creativecommons.org/publicdomain/zero/1.0/>.
|
||
|
|
|
||
|
|
usage() {
|
||
|
|
printf "Usage: %s [-T] URL [URLs...]\n" "${0##*/}">&2
|
||
|
|
exit 1
|
||
|
|
}
|
||
|
|
|
||
|
|
torsocks=torsocks
|
||
|
|
while getopts T flag; do
|
||
|
|
case $flag in
|
||
|
|
T) torsocks=
|
||
|
|
;;
|
||
|
|
*) usage
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
shift $(($OPTIND - 1))
|
||
|
|
if [ $# -eq 0 ]; then
|
||
|
|
usage
|
||
|
|
fi
|
||
|
|
|
||
|
|
: ${ARCHIVE_BASEDIR:=~/tmp/archive}
|
||
|
|
mkdir -p "$ARCHIVE_BASEDIR"
|
||
|
|
|
||
|
|
everything=$ARCHIVE_BASEDIR/everything
|
||
|
|
test -f "$everything" || touch "$everything"
|
||
|
|
|
||
|
|
dir=$ARCHIVE_BASEDIR/$(date +%Y/%m/%d)
|
||
|
|
mkdir -p "$dir" || exit 1
|
||
|
|
|
||
|
|
rc=0
|
||
|
|
for url; do
|
||
|
|
sha=$(printf "%s" "$url" | sha256)
|
||
|
|
outdir=$dir/$sha
|
||
|
|
|
||
|
|
if grep -q "^$url\$" "$everything"; then
|
||
|
|
printf "%s: \"%s\" already fetched.\n" "${0##*/}" "$url" >&2
|
||
|
|
printf "%s\n" "$outdir/file"
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
|
||
|
|
mkdir -p "$outdir"
|
||
|
|
|
||
|
|
{
|
||
|
|
$torsocks ftp -o "$outdir/file" "$url" \
|
||
|
|
&& printf "%s\n" "$url" >"$outdir/url" \
|
||
|
|
&& printf "%s\n" "$name" >"$outdir/name" \
|
||
|
|
&& printf "%s\n" "$url" >>"$everything" \
|
||
|
|
&& printf "%s\n" "$outdir/file"
|
||
|
|
} || rc=1
|
||
|
|
done
|
||
|
|
exit $rc
|