rewrite filtering method; per-action completion logging

meow.sh

@@ -2,10 +2,12 @@
 SEP=$'\t'
 curl=(curl -sS -m 32 --connect-timeout 8 --retry 3 --retry-delay 1)
 
+URL_SEARCH='http://www.nyaa.se/'
+URL_DOWNLOAD='http://www.nyaa.se/?page=download&tid='
+
 # all timestamps are given in seconds since the epoch
 declare -A searchquery
 declare -A searchregex
-declare -A searchtime # last seen release
 
 die() {
     echo -E "$@" >&2
@@ -13,8 +15,7 @@ die() {
 }
 
 retrieve() {
-    ${curl[@]} -G --data-urlencode "term=[$1]" -d page=rss \
+    ${curl[@]} -G --data-urlencode "term=[$1]" -d page=rss "$URL_SEARCH"
-      "http://www.nyaa.se/"
 }
 
 nullcheck() { # {query}
@@ -43,48 +44,46 @@ watch() { # {search query} [regex...]
     done
 }
 
-touchquery() { # {search query} {timestamp}
-    nullcheck "$1"
-    local gs="$(sanitize<<<"$1")"
-    searchtime[$gs]="$2"
-}
-
 search() {
     nullcheck "$1"
     retrieve "$1" | tr -d '\r\n'"$SEP" | splittags item | scrape
     [ ${PIPESTATUS[0]} = 0 ] || die "Failed to search for $1"
 }
 
-searchfilter() { # key regex [timestamp]
+searchfilter() { # database regex [timestamp]
-    while IFS=$SEP read -r title etc; do
+    while read -r; do
-        grep -P "$2" <<< "$title" >/dev/null && echo -E "$title$SEP$etc"
+        IFS=$SEP read -r time tid title <<< "$REPLY"
-    done < db.txt
+        [ "$time" -gt "${3:-0}" ] \
-    [ ${PIPESTATUS[0]} = 0 ] || exit 1
+        && grep -qP "$2" <<< "$title" \
+        && echo -E "$REPLY"
+    done < "$1"
 }
 
-cleanup() {
+runfilter() { # {action} [database]
-    local gs= v=
+    declare -A already
-    for gs in "${!searchtime[@]}"; do
+    local action="${1:-echo}"
-        v="${searchtime[$gs]}"
+    local mark="$action.txt"
-        echo -E "touchquery $gs $v" >> times.sh
+    local db="${2:-db.txt}"
-        [ -e "$gs.xml" ] && rm "$gs.xml"
+
+    touch "$mark"
+    while IFS=$SEP read -r tid time; do
+        already["$tid"]="$time"
+    done < "$mark"
+
+    now="$(date +%s)"
+    for regex in "${searchregex[@]}"; do
+        while IFS=$SEP read -r time tid title; do
+            [ -n "${already[$tid]}" ] \
+            || $action $time $tid "$title" \
+            || break
+            already[$tid]="$now"
+        done < <(searchfilter "$db" "${regex:1}")
     done
-    exit ${1:-1}
-}
 
-runfilter() {
+    rm "$mark"
-    local query= regex= timestamp= res= _= recent=
+    for tid in "${!already[@]}"; do
-    query="${searchquery[$1]}"
+        echo "$tid$SEP${already[$tid]}" >> "$mark"
-    regex="${searchregex[$1]:1}" # exclude first | character
+    done
-    timestamp="${searchtime[$1]}"
-    res="$(searchfilter "$query" "$regex" "$timestamp")"
-    [ $? = 0 ] || return $?
-    IFS=$SEP read -r _ _ recent <<< "$res"
-    [ -n "$recent" ] && {
-        searchtime[$1]="$recent"
-        echo -E "$res"
-    }
-    return 0
 }
 
 runsearch() { # [database]
@@ -94,17 +93,11 @@ runsearch() { # [database]
     for q in "${!searchquery[@]}"; do
         search "${searchquery[$q]}" \
         | while IFS=$SEP read -r title torrent time; do
-            echo -E "$time$SEP$q$SEP$title$SEP$torrent"
+            local tid="${torrent##*=}"
+            echo -E "$time$SEP$tid$SEP$title"
         done
     done | sort -n -- "$db" - | uniq > $tmp
     # TODO: don't accidentally overwrite $db with something blank/incomplete
     #       maybe check if filesize has decreased and die if so
     mv $tmp "$db"
 }
-
-runall() {
-    trap cleanup INT
-    local ret=0 gs=
-    for gs in "${!searchregex[@]}"; do runfilter "$gs" || ret=1; done
-    cleanup $ret
-}

run

@@ -13,34 +13,27 @@ while getopts 'aph' opt; do
 done
 
 prettify() {
-    echo -E "$1"$'\n'"dl:"$'\t'"$2"$'\n'"at:"$'\t'"$(date -d @"$3")"
+    echo -E "$3"$'\n'"dl:"$'\t'"$URL_DOWNLOAD$2"$'\n'"at:"$'\t'"$(date -d @"$1")"
 }
 
-declare -a torrents
 addtorrent() {
-    torrents+=("$2")
+    transmission-remote -a "$URL_DOWNLOAD$2"
-}
-
-runactions() {
-    [ ${#actions} = 0 ] && {
-        cat
-        return
-    }
-    while IFS=$SEP read -r title torrent time; do
-        for a in "${actions[@]}"; do
-            "$a" "$title" "$torrent" "$time"
-        done
-    done
-    for t in "${torrents[@]}"; do transmission-remote -a "$t"; done
 }
 
 . "$SRCDIR/meow.sh"
 . config.sh
 
-#[ -e times.sh ] && { . times.sh; mv times.sh times.sh.old; }
-#runall | runactions
-#exit ${PIPESTATUS[0]}
-
 runsearch
 
+if [ "${#actions}" -eq 0 ]; then
+    # TODO: abstract
+    for regex in "${searchregex[@]}"; do
+        searchfilter db.txt "${regex:1}"
+    done
+else
+    for a in "${actions[@]}"; do
+        runfilter "$a"
+    done
+fi
+
 exit 0