fix bugs, update RevSpace config

conf/kiekus-revspace-tz71.conf

@@ -1,7 +1,8 @@
 devtype=blk
-devargs=sd*
+devargs=sd*[0-9]
+devsuffix=/DCIM/105_PANA
 destproto=lftp
-dest=sftp://kiekjes@mediaserver/photos
+dest=sftp://wipcamera:unused@mediaserver//photos/wip
 
 rpi_hub_port=2
 camera_sleep_delay=120

kieklist-file

@@ -3,7 +3,12 @@ set -e
 
 for dir; do
     # disregard the 'total' header and symlink references
-    ls -apl "$dir" | sed -ne '2,$p' | sed -e 's/ -> .*$//g' | while read -r mode nlinks user group size month day yearortime filename; do
-        rp-make "$filename" "size=$size" "mode=$mode" "user=$user" "group=$group" "timestamp=$(date -r "$dir/$filename" "+%Y-%m-%dT%H:%I:%S")"
+    ls -apl "$dir" | grep -v '^total' | sed -e 's/ -> .*$//g' | while read -r mode nlinks user group size month day yearortime filename; do
+        statfile="$dir/$filename"
+        case "$filename" in
+        */);;
+        */*) statfile="$filename"; filename="$(basename "$filename")";;
+        esac
+        rp-make "$filename" "size=$size" "mode=$mode" "user=$user" "group=$group" "timestamp=$(date -r "$statfile" "+%Y-%m-%dT%H:%I:%S")"
     done
 done

kieklist-lftp

@@ -2,7 +2,7 @@
 set -e
 
 for target; do
-    lftp -c "connect $target; cls -l --time-style='%Y-%m-%dT%H:%I:%S'" | while read -r mode nlinks user group size datetime filename; do
+    lftp -c "connect $target; cls -al --time-style='%Y-%m-%dT%H:%I:%S'" | while read -r mode nlinks user group size datetime filename; do
         rp-make "$filename" "size=$size" "mode=$mode" "user=$user" "group=$group" "timestamp=$datetime"
     done
 done

kiekpull-file

@@ -2,5 +2,5 @@
 set -e
 basedir="$1"; shift
 [ -d "$basedir" ] || exit 1
-[ "$#" -gt 1 ] || set -- .
+[ "$#" -ge 1 ] || set -- .
 (cd "$basedir" && tar -cz -- "$@")

kiekpush-lftp

@@ -10,6 +10,7 @@ directory="${1#$server/}"
 
 (cd "$tempdir"; tar -xz)
 while read -r filename; do
+    [ -n "$filename" ] || continue
     lftp -c "connect $server && mkdir -pf $directory && cd $directory && lcd $tempdir && put $filename"
     printf "%s\n" "$filename"
 done <<EOF

kiekus

@@ -20,6 +20,7 @@ eval "$(kiekconf \
     ${cfgfile:+-c "$cfgfile"} \
     devtype=blk \
     devargs= \
+    devsuffix= \
     destproto=lftp \
     dest="$dest" \
     statedir=/tmp/kiekstate.$$ \
@@ -31,34 +32,48 @@ eval "$(kiekconf \
 
 
 iso8601_to_touch() {
-    printf "%s\n" | sed -E -e 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2}).*/\1\2\3\4\5.\6/g'
+    printf "%s\n" "$1" | sed -E -e 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2}).*/\1\2\3\4\5.\6/g'
 }
 
 mkdir -p "$statedir"
-eval "kiekmon-$devtype $devargs" | rp-get . protocol rm_command | while read -r device && read -r srcproto && read -r rmcmd; do (
-    trap 'set +e; code=$?; [ -z "$posthook" ] || eval "$posthook"; eval "$rmcmd"; exit $code' EXIT INT TERM
+eval "kiekmon-$devtype $devargs" | while read -r line; do (
+    read -r rmcmd <<EOF
+$(printf "%s\n" "$line" | rp-get rm_command)
+EOF
+    trap 'exit $?' INT TERM ERR
+    trap 'code=$?; set +e; [ -z "$posthook" ] || eval "$posthook"; eval "$rmcmd"; exit $code' EXIT
+    { read -r srcproto; read -r device; } <<EOF
+$(printf "%s\n" "$line" | rp-get protocol)
+$(printf "%s\n" "$line" | rp-get .)
+EOF
     [ -z "$prehook" ] || eval "$prehook"
 
     # determine destination directory and last upload timestamp
     destdir="$dest/$(date +"%Y%m%d")"
     kiekpull-$destproto "$dest" "$logfilename" | kiekpush-file "$statedir" || true
     if [ -f "$statedir/$logfilename" ]; then
-        lastts=$(date -r "$statedir/$logfilename" +"%Y-%m-%d%H:%I:%S")
+        lastts=$(date -r "$statedir/$logfilename" +"%Y-%m-%dT%H:%I:%S")
     else
         # best-effort
-        lastts="$(kieklist-$destproto "$dest" | rp-filter '.= | endswith("/")' | rp-sort -r . | rp-get timestamp)"
+        lastts="$(kieklist-$destproto "$dest" | rp-filter '.= | endswith("/")' | rp-filter '.= != "../"' | rp-sort -r . | rp-get timestamp | head -n 1)"
     fi
 
     # get source file listing and filter it
-    filenames="$(kieklist-$srcproto "$device" | rp-filter "timestamp= > $lastts" | rp-get .)"
+    files="$(kieklist-$srcproto "$device$devsuffix" | rp-filter "timestamp= > \"$lastts\"")"
+    filenames="$(printf "%s\n" "$files" | rp-get . | grep -v -e '^./$' -e '^../$')"
     [ -f "$statedir/$logfilename" ] && filenames=$(printf "%s\n" "$filenames" | grep -vF -f "$statedir/$logfilename")
 
     # pull from source and push into destination
-    kiekpull-$srcproto "$device" $filenames | kiekpush-$destproto "$destdir" | while read -r line; do
-        printf "%s\n" "$line" >> "$statedir/$logfilename"
-    done
+    if [ -n "$filenames" ]; then
+        lastts="$(printf "%s\n" "$files" | rp-get timestamp | sort -r | head -n 1)"
+        kiekpull-$srcproto "$device$devsuffix" $filenames | kiekpush-$destproto "$destdir" | while read -r line; do
+            printf "%s\n" "$line" >> "$statedir/$logfilename"
+        done
+    fi
 
     # update logfile timestamp, upload logfile
-    touch -t $(iso8601_to_touch "$lastts") "$logfilename"
-    kiekpull-file "$statedir" "$logfilename" | kiekpush-$destproto "$dest"
+    if [ -n "$filenames" ] || ! [ -f "$statedir/$logfilename" ]; then
+        touch -t $(iso8601_to_touch "$lastts") "$statedir/$logfilename"
+        kiekpull-file "$statedir" "$logfilename" | kiekpush-$destproto "$dest"
+    fi
 ) || true; done