Hatarashi Hako now provides Prometheus metrics; old updates

This commit is contained in:
2023-11-17 16:50:20 +02:00
parent 2d9f26a558
commit 6b1790171d
3 changed files with 25 additions and 46 deletions

View File

@@ -21,7 +21,7 @@ flag_error()
help() help()
{ {
echo "$arg0 - show how long a process has been running" echo "$arg0 - Smartly update container stack"
echo " {-s|--stack} [name] -- Run update process just for specified stack (ex: vikunja)" echo " {-s|--stack} [name] -- Run update process just for specified stack (ex: vikunja)"
echo " {-v|--verbose|verbose} -- Print debug messages" echo " {-v|--verbose|verbose} -- Print debug messages"
echo " {-h|--help} -- Print this help message and exit" echo " {-h|--help} -- Print this help message and exit"
@@ -69,9 +69,11 @@ declare -a CONTAINER_PATHS
declare -a OUTDATED_CONTAINERS declare -a OUTDATED_CONTAINERS
declare -a FAILED_CONTAINERS declare -a FAILED_CONTAINERS
declare -a UPDATED_CONTAINERS declare -a UPDATED_CONTAINERS
PROM_FILE="/var/Red-Vol/Media/Containers/prometheus/data/hatarashi-hako.prom"
#Remove log from last runs, if present #Remove log from last runs, if present
if [[ -f /tmp/docker-updated ]]; then if [[ -f $PROM_FILE ]]; then
rm /tmp/docker-updated rm $PROM_FILE
fi fi
#Start counting how many containers fail; appears in final mail subject. #Start counting how many containers fail; appears in final mail subject.
OUTDATED_COUNT="0" OUTDATED_COUNT="0"
@@ -112,7 +114,7 @@ for container_path in ${CONTAINER_PATHS[@]}; do
if [[ -z $local_image ]]; then if [[ -z $local_image ]]; then
error "Error fetching local image checksum for container $container_name!" error "Error fetching local image checksum for container $container_name!"
#The script will complain about failed containers later on #The script will complain about failed containers later on
FAILED_CONTAINERS+=("(local) $container_name") echo "container_updated{name=\"$container_name\"} -1" >> $PROM_FILE
continue 2 continue 2
else else
debug "Local SHA256 for $container_image is" "$local_image" debug "Local SHA256 for $container_image is" "$local_image"
@@ -127,7 +129,7 @@ for container_path in ${CONTAINER_PATHS[@]}; do
#Now, if you still don't have an image after the second try, something's fuckey. #Now, if you still don't have an image after the second try, something's fuckey.
if [[ -z $remote_image ]]; then if [[ -z $remote_image ]]; then
error "Error fetching remote image checksum for container" "$container_name!" error "Error fetching remote image checksum for container" "$container_name!"
FAILED_CONTAINERS+=("(remote) $container_name") echo "container_updated{name=\"$container_name\"} -1" >> $PROM_FILE
continue 2 continue 2
else else
debug "Remote SHA256 for $container_image is" "$remote_image" debug "Remote SHA256 for $container_image is" "$remote_image"
@@ -141,24 +143,18 @@ for container_path in ${CONTAINER_PATHS[@]}; do
service=$(cat "$container_path/docker-compose.yml" | grep -B1 "image: $container_image" | head -1 | sed -e 's/^[ \t]*//' -e 's/://g' | grep -v 'container_name') service=$(cat "$container_path/docker-compose.yml" | grep -B1 "image: $container_image" | head -1 | sed -e 's/^[ \t]*//' -e 's/://g' | grep -v 'container_name')
# get container service name (1 line above image) ^ print service name^ ^ omit tabs and : and ^omit container_name # get container service name (1 line above image) ^ print service name^ ^ omit tabs and : and ^omit container_name
info "Attempting to update service" "$service" info "Attempting to update service" "$service"
if docker-compose pull $service; then if docker compose pull $service; then
info "Pulled latest image for" "$container_name" info "Pulled latest image for" "$container_name"
if docker-compose up -d --remove-orphans; then if docker compose up -d --remove-orphans; then
ok "$container_stack" "has been updated sucessfully!" ok "$container_stack" "has been updated sucessfully!"
UPDATED_CONTAINERS+=("($container_stack) $container_name") echo "container_updated{name=\"$container_name\"} 1" >> $PROM_FILE
else else
error "Failed to update" "$container_name!" error "Failed to update" "$container_name!"
FAILED_CONTAINERS+=("($container_stack|update) $container_name") echo "container_updated{name=\"$container_name\"} 0" >> $PROM_FILE
#Add to array for mail report
OUTDATED_CONTAINERS+=("($container_stack) $container_name")
export OUTDATED_COUNT=$(($OUTDATED_COUNT+1))
fi fi
else else
error "Failed to pull image for" "$container_name!" error "Failed to pull image for" "$container_name!"
FAILED_CONTAINERS+=("($container_stack|pull) $container_name") echo "container_updated{name=\"$container_name\"} 0" >> $PROM_FILE
#Add to array for mail report
OUTDATED_CONTAINERS+=("($container_stack) $container_name")
export OUTDATED_COUNT=$(($OUTDATED_COUNT+1))
fi fi
fi fi
#If you found an image tag, reset it before moving on to another container #If you found an image tag, reset it before moving on to another container
@@ -167,20 +163,3 @@ for container_path in ${CONTAINER_PATHS[@]}; do
cd $LOCAL_DIR cd $LOCAL_DIR
done done
echo "All done!" echo "All done!"
#Iterate over the arrays and output results to a file; if needed, it will be neatly mailed.
if [[ ${#OUTDATED_CONTAINERS[@]} != 0 ]] || [[ ${#FAILED_CONTAINERS[@]} != 0 ]] || [[ ${#UPDATED_CONTAINERS[@]} != 0 ]]; then
if [[ ${#UPDATED_CONTAINERS[@]} != 0 ]]; then
printf "The following containers have updated succefully:\n" > /tmp/docker-updated
printf "%s\n" "${UPDATED_CONTAINERS[@]}" >> /tmp/docker-updated
fi
if [[ ${#OUTDATED_CONTAINERS[@]} != 0 ]]; then
printf "The following containers are out of date:\n" >> /tmp/docker-updated
printf "%s\n" "${OUTDATED_CONTAINERS[@]}" >> /tmp/docker-updated
fi
if [[ ${#FAILED_CONTAINERS[@]} != 0 ]]; then
printf "The following containers failed the update process:\n" >> /tmp/docker-updated
printf "%s\n" "${FAILED_CONTAINERS[@]}" >> /tmp/docker-updated
fi
cat /tmp/docker-updated | mail -s "Docker containers update report - `date +"%B %d %Y"`" -r "Takahe@pukeko.xyz" matanhorovitz@protonmail.com
rm /tmp/docker-updated
fi

View File

@@ -141,8 +141,17 @@ clean(){
fi fi
fi fi
} }
#If cleaning was forced, or if it's the first of this month - clean.
if [[ -n $CLEAN ]] || [[ $(date +%d) == "1" ]]; then
check Gerbil-TK
clean Gerbil-TK
check Pukeko-XYZ-Containers
clean Pukeko-XYZ-Containers
check Pukeko-XYZ-Cloud
clean Pukeko-XYZ-Cloud
#If a specific repository was requested, back it up; otherwise, back them all up. #If a specific repository was requested, back it up; otherwise, back them all up.
if [[ -n $REPOSITORY ]] && [[ -z $CLEAN ]]; then elif [[ -n $REPOSITORY ]] && [[ -z $CLEAN ]]; then
case "$REPOSITORY" in case "$REPOSITORY" in
(Gerbil-TK) (Gerbil-TK)
backup Gerbil-TK /var/Red-Vol/Media/Pictures/ backup Gerbil-TK /var/Red-Vol/Media/Pictures/
@@ -179,14 +188,6 @@ elif [[ -n $REPOSITORY ]] && [[ -n $CLEAN ]]; then
(*) (*)
help;; help;;
esac esac
#If cleaning was forced, or if it's the first of this month - clean.
elif [[ -n $CLEAN ]] || [[ $(date +%d) == "1" ]]; then
check Gerbil-TK
clean Gerbil-TK
check Pukeko-XYZ-Containers
clean Pukeko-XYZ-Containers
check Pukeko-XYZ-Cloud
clean Pukeko-XYZ-Cloud
fi fi
#Wrap up this run's log and report nicely #Wrap up this run's log and report nicely
echo "All done; have a nice day!" echo "All done; have a nice day!"

View File

@@ -1,7 +1,6 @@
#!/bin/bash #!/bin/bash
#Script to backup all important environment files #Script to backup all important environment files
source /etc/environment source /home/shmick/Scripts/Okiru
source $SCRIPTS/Okiru
logging "ConfigBackup" logging "ConfigBackup"
export CONF_DIR="/home/shmick/Config" export CONF_DIR="/home/shmick/Config"
if [[ ! -d $CONF_DIR ]]; then if [[ ! -d $CONF_DIR ]]; then
@@ -48,10 +47,10 @@ conf_sync /etc/profile
for conf in $(ls /etc/snapper/configs/* 2> /dev/null); do for conf in $(ls /etc/snapper/configs/* 2> /dev/null); do
conf_sync $conf conf_sync $conf
done done
for conf in $(ls /etc/systemd/system/murchison*); do for conf in $(ls /etc/systemd/system/Murchison*); do
conf_sync $conf conf_sync $conf
done done
for conf in $(ls /etc/systemd/system/kumonoboru*); do for conf in $(ls /etc/systemd/system/Kumonoboru*); do
conf_sync $conf conf_sync $conf
done done
#Fix permissions #Fix permissions