commit 98b94536f785e9f881cc0a21a216812d1fdf4325 Author: notwa Date: Mon Jul 29 00:45:45 2013 -0700 diff --git a/boorufind b/boorufind new file mode 100644 index 0000000..bac9476 --- /dev/null +++ b/boorufind @@ -0,0 +1,44 @@ +#!/bin/zsh +q=$1 +url='http://danbooru.donmai.us/posts.xml' +de=--data-urlencode + +curl=(curl -sS -m 32 --connect-timeout 8 --retry 3 --retry-delay 1) + +die() { + echo -E "$@" >&2 + exit 1 +} + +push() { + if [ -z $md5 ] || [ -z $ext ] || [ -z $size ]; then + print missing value >&2 + return + fi + local fn=$md5.$ext + local s="$(find . -name $fn 2>/dev/null)" + [ -n "$s" ] && print $s || print \#$fn + md5= ext= size= +} + +page=1 +while; do + once=0 + ${curl[@]} -G -d page=search -d limit=100 -d page=$page $de tags=$q $url \ + | xml2 2>/dev/null \ + | while IFS== read -r tree val; do + [ $tree = /posts/post ] && push && continue + [ ${tree:0:12} = /posts/post/ ] && tree=${tree:12} || continue + once=1 + + case $tree in + (md5) md5=$val ;; + (file-ext) ext=$val ;; + (file-size) size=$val ;; + esac + done + [[ ${PIPESTATUS[1]} -eq 0 ]] || die "curl failed on page $page" + [[ $once -eq 0 ]] && exit 0 # no posts left, well done + push + let page++ +done diff --git a/boorugrab b/boorugrab new file mode 100644 index 0000000..95bd4a8 --- /dev/null +++ b/boorugrab @@ -0,0 +1,18 @@ +#!/bin/zsh +dir=$1 +url='http://danbooru.donmai.us/data/' + +curl=(curl -sS -m 900 --connect-timeout 8 --retry 3 --retry-delay 1) + +mkdir -p $dir +while read -r; do + if [[ ${REPLY[1]} = "#" ]]; then + md5e=${REPLY:1} + ${curl[@]} $url$md5e > $dir/$md5e || rm -f $dir/$md5e + else + md5e=${REPLY##*/} + [ -s $dir/$md5e ] && continue + [ $REPLY -ef $dir/$md5e ] || cp $REPLY $dir + fi +done +# zsh builtin md5 sum?