env/bin/fetch.sh

#!/bin/sh
# fetch
# Written in 2019-2020 by Lucas
# CC0 1.0 Universal/Public domain - No rights reserved
#
# To the extent possible under law, the author(s) have dedicated all
# copyright and related and neighboring rights to this software to the
# public domain worldwide. This software is distributed without any
# warranty. You should have received a copy of the CC0 Public Domain
# Dedication along with this software. If not, see
# <http://creativecommons.org/publicdomain/zero/1.0/>.

usage()
{
	printf "Usage: %s [-T] URL [URL ...]\n" "${0##*/}" >&2
	exit 1
}

fetch_cmd()
{
	if tty=$(tty) && [ "$tty" != "not a tty" ]; then
		$torsocks ftp "$@" >"$tty"
	else
		$torsocks ftp "$@"
	fi
}

find_dir()
{
	find "$ARCHIVE_BASEDIR" -type d -name "$1"
}

torsocks=torsocks
while getopts T flag; do
	case $flag in
	T)	torsocks=
		;;
	*)	usage
		;;
	esac
done
shift $((OPTIND - 1))
[ $# -gt 0 ] || usage

: ${ARCHIVE_BASEDIR:=~/tmp/archive}
mkdir -p "$ARCHIVE_BASEDIR"

everything=$ARCHIVE_BASEDIR/everything
touch "$everything"

dir=$ARCHIVE_BASEDIR/$(date +%Y/%m/%d)
mkdir -p "$dir" || exit 1

rc=0
for url; do
	sha=$(printf "%s" "$url" | sha256)
	outdir=$dir/$sha

	if grep -q "^$url\$" "$everything"; then
		printf "%s: already fetched %s\n" "${0##*/}" "$url" >&2
		printf "%s/file\n" "$(find_dir "$sha")"
		continue
	fi

	mkdir -p "$outdir"

	{
		fetch_cmd -o "$outdir/file" "$url" &&
		    printf "%s\n" "$url" >"$outdir/url" &&
		    printf "%s\n" "$name" >"$outdir/name" &&
		    printf "%s\n" "$url" >>"$everything" &&
		    printf "%s\n" "$outdir/file"
	} || rc=1
done
exit $rc