This commit is contained in:
Shiz 2022-08-14 01:45:57 +02:00
commit 18efa3a277
25 changed files with 670 additions and 0 deletions

View File

@ -0,0 +1,8 @@
devtype=blk
devargs=sd*
destproto=lftp
dest=sftp://kiekjes@mediaserver/photos
rpi_hub_port=2
camera_sleep_delay=120
posthook="nohup uhubctl -a cycle -d $(($camera_sleep_delay + 8)) -l $rpi_hub_port >/dev/null &"

32
kiekconf Executable file
View File

@ -0,0 +1,32 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] -c CONFIG NAME=DEFAULT..."
}
while getopts hc: opt; do
case "$opt" in
c) __cfgfile="$OPTARG";;
h) echo "source variables from configuration file without clobbering the environment"; usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND - 1))
[ "$#" -ge 1 ] || { usage; exit 255; }
(
# clear old out exports by un-exporting them manually
while read -r var; do
eval "__tmp=\"\$$var\"; unset $var; $var=\"\$__tmp\""
done <<EOI
$(export -p | sed -e 's/^export //g' | cut -d= -f1)
EOI
# introduce our config variables as new exports
for item; do
eval "export ${item%%=*}='${item#*=}'"
done
# load config file
[ -z "$__cfgfile" ] || . "$__cfgfile"
export -p
) | sed -e '/=/!d' -e 's/^export //g'

9
kieklist-file Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
set -e
for dir; do
# disregard the 'total' header and symlink references
ls -apl "$dir" | sed -ne '2,$p' | sed -e 's/ -> .*$//g' | while read -r mode nlinks user group size month day yearortime filename; do
rp-make "$filename" "size=$size" "mode=$mode" "user=$user" "group=$group" "timestamp=$(date -r "$dir/$filename" "+%Y-%m-%dT%H:%I:%S")"
done
done

8
kieklist-lftp Executable file
View File

@ -0,0 +1,8 @@
#!/bin/sh
set -e
for target; do
lftp -c "connect $target; cls -l --time-style='%Y-%m-%dT%H:%I:%S'" | while read -r mode nlinks user group size datetime filename; do
rp-make "$filename" "size=$size" "mode=$mode" "user=$user" "group=$group" "timestamp=$datetime"
done
done

11
kieklist-ptp Executable file
View File

@ -0,0 +1,11 @@
#!/bin/sh
set -e
for sysdir; do
busnum=$(cat "$sysdir/busnum")
devnum=$(cat "$sysdir/devnum")
rp-make ./
ptpcam --bus=$busnum --dev=$devnum -L | sed -ne '/^Handler:/,$p' | sed -ne '2,$p' | while read -r handler size date time filename; do
rp-make "$filename" "size=$size" timestamp="${date}T$time"
done
done

72
kiekmon-blk Executable file
View File

@ -0,0 +1,72 @@
#!/bin/sh
set -e
action=watch
blkpattern=
blkname=
blkuuid=
blkfs=
mntprefix=/run/kiekblk
usage() {
echo "usage: $0 [-h] [-r | -w [-n PARTNAME] [-u PARTUUID] [-t FSTYPE] [-m MNTPREFIX]] PATTERN" >&2
}
while getopts hwrn:u:t:m: opt; do
case "$opt" in
w) action=watch;;
r) action=remove;;
d) devpattern="$OPTARG";;
n) blkname="$OPTARG";;
u) blkuuid="$OPTARG";;
t) blkfs="$OPTARG";;
m) mntprefix="$OPTARG";;
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND - 1))
blkpattern="${1:-*}"
case "$action" in
watch)
inotifywait -e create,delete -m /dev | {
devices=
while read -r _ action devname; do
case "$devname" in
$blkpattern);;
*) continue;;
esac
case "$action" in
CREATE)
eval $(blkid /dev/$devname | cut -d: -f2-)
if [ -n "$blkname" ] && [ "$blkname" != "$NAME" ]; then
continue
fi
if [ -n "$blkuuid" ] && [ "$blkuuid" != "$UUID" ] && [ "$blkuuid" != "$PARTUUID" ]; then
continue
fi
if [ -n "$blkfs" ] && [ "$blkfs" != "$FSTYPE" ]; then
continue
fi
mkdir -p "$mntprefix/$devname"
if mount ${blkfs:+-t "$blkfs"} /dev/"$devname" "$mntprefix/$devname"; then
devices="$(printf "%s\n%s\n" "$devices" "$devname")"
rp-make "$mntprefix/$devname" protocol=file rm_command="$0 -r '$mntprefix/$devname'"
else
rmdir "$mntprefix/$devname"
fi
;;
DELETE)
devices=$(printf "%s\n" "$devices" | grep -v "^$devname\$")
;;
esac
done
}
;;
remove)
umount "$blkpattern"
rmdir "$blkpattern"
;;
esac

61
kiekmon-mtp Executable file
View File

@ -0,0 +1,61 @@
#!/bin/sh
set -e
USB_CLASS_STILL_IMAGE=6
USB_SUBCLASS_STILL_IMAGE_DEVICE=1
USB_PROTOCOL_STILL_IMAGE_DEVICE=1
device=
usbbid=
usbpid=
usbvid=
mntprefix=/run/kiekmtp
usage() {
echo "usage: $0 [-h] [-r DEVICE | -w [-b BUSID] [-v VENDORID] [-p PRODUCTID] [-m MNTPREFIX]]" >&2
}
while getopts hwr:b:v:p:m: opt; do
case "$opt" in
w) action=watch;;
r) action=remove; device="$OPTARG";;
b) usbbid="$OPTARG";;
v) usbvid="$(printf "%04x" 0x$OPTARG)";;
p) usbpid="$(printf "%04x" 0x$OPTARG)";;
m) mntprefix="$OPTARG";;
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND - 1))
case "$action" in
watch)
kiekmon-usb \
${usbbid:+-b $usbbid} \
${usbvid:+-v $usbvid} \
${usbpid:+-p $usbpid} | \
-c $USB_CLASS_STILL_IMAGE -C $USB_SUBCLASS_STILL_IMAGE_DEVICE -P $USB_PROTOCOL_STILL_IMAGE_DEVICE | \
rp-get . rm_command | \
while read -r sysdir && read -r rmcmd; do (
busnum=$(cat "$sysdir/busnum")
devnum=$(cat "$sysdir/devnum")
devvid=$(cat "$sysdir/idVendor")
devpid=$(cat "$sysdir/idProduct")
mkdir -p "$mntprefix/$busnum-$devnum"
if afp-mtp-mount -D $devvid:$devpid "$mntprefix/$busnum-$devnum"; then
rp-make "$mntprefix/$busnum-$devnum" protocol=file rm_command="$0 -r '$mntprefix/$devname@$rmcmd'"
else
rmdir "$mntprefix/$devname"
fi
) || true; done
;;
remove)
printf "%s\n" "$device" | sed -e 's/@/ /' | read -r mntdir usbrmcmd
fusermount -u "$mntdir"
rmdir "$mntdir"
$usbrmcmd
;;
esac

58
kiekmon-ptp Executable file
View File

@ -0,0 +1,58 @@
#!/bin/sh
set -e
device=
usbbid=
usbpid=
usbvid=
usage() {
echo "usage: $0 [-h] [-r DEVICE | -w [-b BUSID] [-v VENDORID] [-p PRODUCTID] [PATTERN]]" >&2
}
while getopts hwr:b:v:p: opt; do
case "$opt" in
w) action=watch;;
r) action=remove; device="$OPTARG";;
b) usbbid="$OPTARG";;
v) usbvid="$(printf "%04x" 0x$OPTARG)";;
p) usbpid="$(printf "%04x" 0x$OPTARG)";;
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND - 1))
pattern="$1"
case "$action" in
watch)
kiekmon-usb -w \
${usbbid:+-b $usbbid} \
${usbvid:+-v $usbvid} \
${usbpid:+-p $usbpid} \
-c 6 -C 1 -P 1 | \
rp-get . rm_command | \
while read -r sysdir && read -r rmcmd; do (
trap "$rmcmd" INT TERM
busnum=$(cat "$sysdir/busnum")
devnum=$(cat "$sysdir/devnum")
info=$(ptpcam --bus=$busnum --dev=$devnum -i)
if [ -z "$info" ]; then
$rmcmd
exit
fi
if [ -n "$pattern" ]; then
model="$(printf "%s\n" "$info" | grep '^Model:' | cut -d ' ' -f2-)"
case "$model" in
$pattern) ;;
*) "$rmcmd"; exit;
esac
fi
rp-make "$sysdir" protocol=ptp rm_command="$rmcmd"
) || true; done
;;
remove)
kiekmonitor-usb -r "$device"
;;
esac

64
kiekmon-usb Executable file
View File

@ -0,0 +1,64 @@
#!/bin/sh
set -e
action=watch
device=
usbbid=
usbvid=
usbpid=
usbcls=
usbsubcls=
usbproto=
usage() {
echo "usage: $0 [-h] [-r DEVICE | -w [-b BUSID] [-v VENDORID] [-p PRODUCTID] [-c CLASS] [-C SUBCLASS] [-P PROTOCOL]]" >&2
}
while getopts hwrb:v:p:c:C:P: opt; do
case "$opt" in
w) action=watch;;
r) action=remove; device="$OPTARG";;
b) usbbid="$OPTARG";;
v) usbvid="$(printf "%04x" 0x$OPTARG)";;
p) usbpid="$(printf "%04x" 0x$OPTARG)";;
c) usbcls="$(printf "%02x" $OPTARG)";;
C) usbsubcls="$(printf "%02x" $OPTARG)";;
P) usbproto="$(printf "%02x" $OPTARG)";;
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND - 1))
case "$action" in
watch)
inotifywait -e create -m -r /dev/bus/usb | {
while read -r basedir action devname; do (
bus="${basedir#/dev/bus/usb}"
bus="${bus#/}"
[ -n "$bus" ] || continue
busnum="$(echo "$bus" | sed -e 's/^0+//')"
devnum="$(echo "$devname" | sed -e 's/^0+//')"
[ -z "$usbbid" ] || [ "$busnum" -eq "$usbbid" ] || continue
# ref: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/ABI/stable/sysfs-bus-usb?h=v5.19#n63
sysdir="$(echo /sys/bus/usb/devices/$busnum-$devnum:*)"
devvid="$(cat "$sysdir/idVendor" | tr '[A-Z]' '[a-z]')"
[ -z "$usbvid" ] || [ "$devvid" = "$usbvid" ] || continue
devpid="$(cat "$sysdir/idProduct" | tr '[A-Z]' '[a-z]')"
[ -z "$usbpid" ] || [ "$devpid" = "$usbpid" ] || continue
devcls="$(cat "$sysdir/bDeviceClass" | tr '[A-Z]' '[a-z]')"
[ -z "$usbcls" ] || [ "$devcls" = "$usbcls" ] || continue
debsubcls="$(cat "$sysdir/bDeviceSubClass" | tr '[A-Z]' '[a-z]')"
[ -z "$usbsubcls" ] || [ "$devsubcls" = "$usbsubcls" ] || continue
devproto="$(cat "$sysdir/bDeviceProtocol" | tr '[A-Z]' '[a-z]')"
[ -z "$usbproto" ] || [ "$devproto" = "$usbproto" ] || continue
rp-make "$sysdir" protocol=usb rm_command="$0 -r '$sysdir'"
) || true; done
}
;;
remove)
# todo
;;
esac

6
kiekpull-file Executable file
View File

@ -0,0 +1,6 @@
#!/bin/sh
set -e
basedir="$1"; shift
[ -d "$basedir" ] || exit 1
[ "$#" -gt 1 ] || set -- .
(cd "$basedir" && tar -cz -- "$@")

12
kiekpull-lftp Executable file
View File

@ -0,0 +1,12 @@
#!/bin/sh
set -e
# no built-in tar support, write to a temporary directory first
tempdir=/tmp/kiekpull.$$
mkdir -p "$tempdir"
trap 'rm -r "$tempdir"' EXIT INT TERM
# use printf to built a nice lftp cmdline in one go, fetch, and tar!
server="$1"; shift
lftp -c "connect $server && lcd $tempdir $(printf " && get %s" "$@")"
(cd "$tempdir" && tar -cz -- "$@")

27
kiekpull-ptp Executable file
View File

@ -0,0 +1,27 @@
#!/bin/sh
set -e
sysdir="$1"; shift
[ -d "$basedir" ] || exit 1
# no built-in tar support, write to a temporary directory first
tempdir=/tmp/kiekpull.$$
mkdir -p "$tempdir"
trap 'rm -r "$tempdir"' EXIT INT TERM
# read file listing, find files of interest, and fetch them
busnum=$(cat "$sysdir/busnum")
devnum=$(cat "$sysdir/devnum")
files="$(printf "%s\n" "$@")"
while read -r handler size date time filename; do
if printf "%s\n" "$files" | grep -q "^$filename\$"; then
(cd "$tempdir" && ptpcam --bus=$busnum --dev=$devnum --get-file="${handler%:}" --filename="$filename")
files="$(printf "%s\n" "$files" | grep -v "^$filename\$")"
fi
done <<EOF
$(ptpcam --bus=$busnum --dev=$devnum -L | sed -ne '/^Handler:/,$p' | sed -ne '2,$p')
EOF
# managed to get all the files?
[ -z "$files" ]
# tar it up!
(cd "$tempdir" && tar -cz -- "$@")

4
kiekpush-file Executable file
View File

@ -0,0 +1,4 @@
#!/bin/sh
set -e
[ -d "$1" ] || mkdir -p "$1"
(cd "$1" && tar -vxz | sed -e 's#^./##g')

17
kiekpush-lftp Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
set -e
tempdir=/tmp/kiekpush.$$
mkdir -p "$tempdir"
trap 'rm -r "$tempdir"' EXIT INT TERM
server="$(printf "%s\n" "$1" | cut -d/ -f1-3)"
directory="${1#$server/}"
(cd "$tempdir"; tar -xz)
while read -r filename; do
lftp -c "connect $server && mkdir -pf $directory && cd $directory && lcd $tempdir && put $filename"
printf "%s\n" "$filename"
done <<EOF
$(ls -1 "$tempdir")
EOF

64
kiekus Executable file
View File

@ -0,0 +1,64 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] -c CONFIG | DEST"
}
while getopts hc: opt; do
case "$opt" in
c) cfgfile="$OPTARG";;
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND - 1))
dest="$1"; shift || true
# read configuration
eval "$(kiekconf \
${cfgfile:+-c "$cfgfile"} \
devtype=blk \
devargs= \
destproto=lftp \
dest="$dest" \
statedir=/tmp/kiekstate.$$ \
logfilename="$(basename "$0".log)" \
prehook= \
posthook= \
)"
[ -n "$dest" ] || { echo "kiekus: error: no dest specified in config or arguments"; usage; exit 255; }
iso8601_to_touch() {
printf "%s\n" | sed -E -e 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2}).*/\1\2\3\4\5.\6/g'
}
mkdir -p "$statedir"
eval "kiekmon-$devtype $devargs" | rp-get . protocol rm_command | while read -r device && read -r srcproto && read -r rmcmd; do (
trap 'set +e; code=$?; [ -z "$posthook" ] || eval "$posthook"; eval "$rmcmd"; exit $code' EXIT INT TERM
[ -z "$prehook" ] || eval "$prehook"
# determine destination directory and last upload timestamp
destdir="$dest/$(date +"%Y%m%d")"
kiekpull-$destproto "$dest" "$logfilename" | kiekpush-file "$statedir" || true
if [ -f "$statedir/$logfilename" ]; then
lastts=$(date -r "$statedir/$logfilename" +"%Y-%m-%d%H:%I:%S")
else
# best-effort
lastts="$(kieklist-$destproto "$dest" | rp-filter '.= | endswith("/")' | rp-sort -r . | rp-get timestamp)"
fi
# get source file listing and filter it
filenames="$(kieklist-$srcproto "$device" | rp-filter "timestamp= > $lastts" | rp-get .)"
[ -f "$statedir/$logfilename" ] && filenames=$(printf "%s\n" "$filenames" | grep -vF -f "$statedir/$logfilename")
# pull from source and push into destination
kiekpull-$srcproto "$device" $filenames | kiekpush-$destproto "$destdir" | while read -r line; do
printf "%s\n" "$line" >> "$statedir/$logfilename"
done
# update logfile timestamp, upload logfile
touch -t $(iso8601_to_touch "$lastts") "$logfilename"
kiekpull-file "$statedir" "$logfilename" | kiekpush-$destproto "$dest"
) || true; done

21
rp-delete Executable file
View File

@ -0,0 +1,21 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] [NAME...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
cleared=
for arg; do
[ -n "$cleared" ] || { set -- .; cleared=yes; }
set -- "$@" '|' "del(.[\"$arg\"])"
done
exec jq -M -c "$@"

20
rp-filter Executable file
View File

@ -0,0 +1,20 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] [NAME=FUNC...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
cond='false'
for arg; do
cond="$cond or (.[\"${arg%%=*}\"] ${arg#*=})"
done
exec jq -M -c "select($cond)"

20
rp-get Executable file
View File

@ -0,0 +1,20 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] [NAME...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
select=".[\"$1\"]"; shift
for arg; do
select="$select, .[\"$arg\"] // \"\""
done
exec jq -M -r "$select"

16
rp-make Executable file
View File

@ -0,0 +1,16 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] [VALUE | NAME=VALUE...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
echo '{}' | rp-set -- "$@"

21
rp-rename Executable file
View File

@ -0,0 +1,21 @@
#!/bin/sh
usage() {
echo "usage: $0 [-h] [NAME=NEWNAME...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
pipe=.
for arg; do
old="${arg%%=*}"
new="${arg#*=}"
pipe="$pipe | .[\"$new\"] = .[\"$old\"] | del(.[\"$old\"])"
done
exec jq -M -c "$pipe"

22
rp-replace Executable file
View File

@ -0,0 +1,22 @@
#!/bin/sh
usage() {
echo "usage: $0 [-h] [NAME=EXPR...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
tee=
cleared=
for replacement; do
[ -n "$cleared" ] || { set --; cleared=yes; }
tee="$tee 'rp-get ${replacement%%=*}'"
set -- "$@" "read -r value; rp-set ${replacement%%=*}=\"${replacement#*=}\""
done
eval "rp-tee $tee" | rp-untee "$@"

28
rp-set Executable file
View File

@ -0,0 +1,28 @@
#!/bin/sh
usage() {
echo "usage: $0 [-h] [VALUE | NAME=VALUE...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
cleared=
obj=
i=1
for arg; do
[ -n "$cleared" ] || { set --; cleared=yes; }
case "$arg" in
*=*) key="${arg%%=*}"; value="${arg#*=}";;
*) key=.; value="$arg";;
esac
obj="$obj\"$key\": \$v$i,"
set -- "$@" --arg v$i "$value"
i=$(($i + 1))
done
exec jq -M -c "$@" ". + {$obj}"

26
rp-sort Executable file
View File

@ -0,0 +1,26 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] [-r] [NAME...]" >&2
}
reverse=
while getopts hr opt; do
case "$opt" in
r) reverse=yes;;
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
spec="[.[\"$1\"]"; shift
for arg; do
spec="$spec, .[\"$arg\"]"
done
spec="$spec]"
cmd="sort_by($spec)"
[ -z "$reverse" ] || cmd="$cmd | reverse"
exec jq -M -s -c "$cmd[]"

21
rp-tee Executable file
View File

@ -0,0 +1,21 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] [CMD...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
while read -r line; do
printf "%s\n" "$line"
for cmd; do (
printf "%s\n" "$line" | eval "$cmd"
) done
done

22
rp-untee Executable file
View File

@ -0,0 +1,22 @@
#!/bin/sh
set -e
usage() {
echo "usage: $0 [-h] [CMD...]" >&2
}
while getopts h opt; do
case "$opt" in
h) usage; exit 0;;
?) usage; exit 255;;
esac
done
shift $(($OPTIND-1))
while read -r line; do
for cmd; do
read -r value
line="$(printf "%s\n%s\n" "$value" "$line" | eval "$cmd")"
done
printf "%s\n" "$line"
done