From 3ad26474a6c7d6ab3ad582df4566744326e23c71 Mon Sep 17 00:00:00 2001 From: shmick Date: Sat, 18 Nov 2023 13:20:28 +0200 Subject: [PATCH] Remove split HatarashiHako and Kumonoboru --- HatarashiHako | 165 ------------------------------------------ Kumonoboru | 194 -------------------------------------------------- 2 files changed, 359 deletions(-) delete mode 100755 HatarashiHako delete mode 100755 Kumonoboru diff --git a/HatarashiHako b/HatarashiHako deleted file mode 100755 index f5844fc..0000000 --- a/HatarashiHako +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash -##Script to update Docker container images occasionally and alert when update is done. -source /root/.bash_profile -arg0=$(basename "$0") -#This is where containers live -CONTAINER_DIR="/var/Red-Vol/Media/Containers/" -#Show help if arguments are misused -usage() -{ - exec 1>2 # Send standard output to standard error - help - exit 1 -} - -flag_error() -{ - echo -e "$arg0: $*." >&2 - help - exit 1 -} - -help() -{ - echo "$arg0 - Smartly update container stack" - echo " {-s|--stack} [name] -- Run update process just for specified stack (ex: vikunja)" - echo " {-v|--verbose|verbose} -- Print debug messages" - echo " {-h|--help} -- Print this help message and exit" - exit 0 -} -#Pass arguments to the script -flags() -{ - while test $# -gt 0 - do - case "$1" in - #If a stack is specified, run the process for that stack only - (-s|--stack) - shift - if [[ -d $CONTAINER_DIR/$1 ]]; then - export CONTAINER_PATHS="$CONTAINER_DIR/$1" - else - CONTAINER_PATHS="$(find /var/Red-Vol/Media/Containers/ -maxdepth 1 -type d -name "*$1*" | head -1)" - if [[ -n $CONTAINER_PATHS ]]; then - export CONTAINER_PATHS - fi - fi - shift;; - (-v|--verbose|verbose) - export VERBOSE="1" -# Okiru looks for^ this variable - shift;; - (-h|--help) - help;; - (*) help;; - esac - done -} -flags "$@" - -if [[ -n $verbose ]]; then - source /home/shmick/Scripts/Okiru "$verbose" -else - source /home/shmick/Scripts/Okiru -fi -source /etc/environment -#Remember where you are to change back to later -LOCAL_DIR=`pwd` -declare -a CONTAINER_PATHS -declare -a OUTDATED_CONTAINERS -declare -a FAILED_CONTAINERS -declare -a UPDATED_CONTAINERS - -PROM_FILE="/var/Red-Vol/Media/Containers/prometheus/data/hatarashi-hako.prom" -#Remove log from last runs, if present -if [[ -f $PROM_FILE ]]; then - rm $PROM_FILE -fi -#Start counting how many containers fail; appears in final mail subject. -OUTDATED_COUNT="0" -#Check if path is already set by user specified stack; otherwise, find all containers. -if [[ -z $CONTAINER_PATHS ]]; then - CONTAINER_PATHS=$(find $CONTAINER_DIR -maxdepth 2 -type f -name docker-compose.yml ! -path '*Archive*' | xargs dirname ) -fi -#Find containers in ^ base dir ^ in base container path ^ by finding compose files ^ (not here) ^ and getting their directory name. -for container_path in ${CONTAINER_PATHS[@]}; do - cd $container_path - debug "Working on container directory" "$container_path" - container_stack=$(basename $container_path) - info "Working on stack" "$container_stack" - #It's deadly to update tagless database images; this line is safe because it only catches tagged images. - container_images="$(cat $container_path/docker-compose.yml | grep -E "image: ([a-z]+)((/)|(:))([a-z]+)?(:)?([a-z0-9].*$)?" | awk '{print $2}')" -# search for a pattern of something:something with optional :tag print ^ image name - for container_image in $container_images; do - debug "$container_stack has image" "$container_image" - debug "echo $container_image | awk -F/ '{print \$2}' | sed 's/\:.*//'" - container_name="$(echo $container_image | awk -F/ '{print $2}' | sed "s/\:.*//")" -# remove everything after the : ^ - if [[ -z $container_name ]]; then #&& [[ -n $(echo $container_image | grep -Ev 'postgres|mariadb') ]]; then - export container_name="$container_image" - fi - debug "$container_image has name" "$container_name" - if [[ -n $(echo $container_image | grep -E "(.*:[a-z0-9].*$)") ]]; then -# check if there is a :tag present ^ - image_tag=":$(echo $container_image | awk -F: '{print $NF}')" -# !! Add : ^ before image !! so it is only added to later commands if there is an image at all - debug "$container_image has tag" "$image_tag" - export container_image=$(echo $container_image | awk -F: '{print $1}') -# If the container does have a tag, keep the base name ^ without it (before the :) - export container_name=$(echo $container_name | awk -F: '{print $1}') - fi - debug "Fetching local image checksum with:" "docker inspect \"$container_image$image_tag\" | grep -Eo \"($container_image@)?sha256:([0-9a-zA-Z].*)(\\\")\" | sed -e 's/\"//g' | awk -F@ '{print \$2}" - local_image=$(docker inspect "$container_image$image_tag" | grep -Eo "($container_image@)?sha256:([0-9a-zA-Z].*)(\")" | sed -e 's/"//g' -e 's/\s+//g' | awk -F@ '{print $2}') -# remember, this bit ^ is empty without an image ^ this is the main image checksum remove ^ " and whitespace and^ get the checksum after the @ - if [[ -z $local_image ]]; then - error "Error fetching local image checksum for container $container_name!" - #The script will complain about failed containers later on - echo "container_updated{name=\"$container_name\"} -1" >> $PROM_FILE - continue 2 - else - debug "Local SHA256 for $container_image is" "$local_image" - fi - debug "Fetching remote image with:" "skopeo inspect --creds \"dkd6:Vulthuryol569\" docker://docker.io/$container_image$image_tag | grep Digest | head -1 | grep -Eo 'sha256:([0-9a-zA-Z].*)(\")' | sed -e 's/\"//g'" - #Use Skopeo, a Red Hat tool, with my Docker Hub account to register the remote image checksum - remote_image=$(skopeo inspect --creds "dkd6:Vulthuryol569" docker://docker.io/$container_image$image_tag | grep Digest | head -1 | grep -Eo 'sha256:([0-9a-zA-Z].*)(")' | sed -e 's/"//g' -e 's/\s+//g' ) - #Sometimes; Docker hub hangs up; try again if you failed - if [[ -z $remote_image ]]; then - remote_image=$(skopeo inspect --creds "dkd6:Vulthuryol569" docker://docker.io/$container_image$image_tag | grep Digest | head -1 | grep -Eo 'sha256:([0-9a-zA-Z].*)(")' | sed -e 's/"//g') - fi - #Now, if you still don't have an image after the second try, something's fuckey. - if [[ -z $remote_image ]]; then - error "Error fetching remote image checksum for container" "$container_name!" - echo "container_updated{name=\"$container_name\"} -1" >> $PROM_FILE - continue 2 - else - debug "Remote SHA256 for $container_image is" "$remote_image" - fi - #If we have both checksums, compare them; they should be identical, or the container is outdated. - if [[ -n $local_image ]] && [[ -n $remote_image ]] && [[ "$local_image" =~ "$remote_image" ]]; then - ok "$container_name" "is up to date!" - else - warn "$container_name" "is out of date!" - debug "cat \"$container_path/docker-compose.yml\" | grep -B1 \"image: $container_image\" | head -1 | sed -e 's/^[ \t]*//' -e 's/://g' | awk '{print \$NF}')" - service=$(cat "$container_path/docker-compose.yml" | grep -B1 "image: $container_image" | head -1 | sed -e 's/^[ \t]*//' -e 's/://g' | grep -v 'container_name') - # get container service name (1 line above image) ^ print service name^ ^ omit tabs and : and ^omit container_name - info "Attempting to update service" "$service" - if docker compose pull $service; then - info "Pulled latest image for" "$container_name" - if docker compose up -d --remove-orphans; then - ok "$container_stack" "has been updated sucessfully!" - echo "container_updated{name=\"$container_name\"} 1" >> $PROM_FILE - else - error "Failed to update" "$container_name!" - echo "container_updated{name=\"$container_name\"} 0" >> $PROM_FILE - fi - else - error "Failed to pull image for" "$container_name!" - echo "container_updated{name=\"$container_name\"} 0" >> $PROM_FILE - fi - fi - #If you found an image tag, reset it before moving on to another container - image_tag="" - done - cd $LOCAL_DIR -done -echo "All done!" diff --git a/Kumonoboru b/Kumonoboru deleted file mode 100755 index 8523c20..0000000 --- a/Kumonoboru +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash -##Simple script to run Restic backups -source /root/.bash_profile -source /etc/environment -export HOME=/root/ -arg0=$(basename "$0") -#Show help if arguments are misused -usage() -{ - exec 1>2 # Send standard output to standard error - help - exit 1 -} - -flag_error() -{ - echo -e "$arg0: $*." >&2 - help - exit 1 -} - -help() -{ - echo "$arg0 - Back up important location to the B2 cloud using Restic." - echo " {-c|--clean} -- Force prune of the remote repositories" - echo " {-r|--repository} repository -- Only backup the specified repository." - echo " {-l|--limit} #[Kbps] -- Limit upload & download speed" - echo " {-v|--verbose} -- Print debug messages" - echo " {-h|--help} -- Print this help message and exit" - echo "Available repositories:" - echo "Gerbil-TK Photos (path: /var/Red-Vol/Media/Pictures)" - echo "Pukeko-XYZ-Containers Containers (path: /var/Red-Vol/Media/Containers)" - echo "Pukeko-XYZ-Cloud Data from all devices (path: /var/Red-Vol/Media/Cloud)" - exit 0 -} -#Pass arguments to the script -flags() -{ - #This is utterly useless - if [[ $# == "0" ]]; then - : - fi - while test $# -gt 0 - do - case "$1" in - (-c|--clean) - export CLEAN="1" - shift;; - (-r|--repository) - shift - export REPOSITORY="$1" - shift;; - (-l|--limit) - shift - export BWLIMIT="$1" - shift;; - (-v|--verbose) - shift - export VERBOSE="$1" - shift;; - (-h|--help) - help;; - (*) help;; - esac - done -} -flags "$@" -source /home/shmick/Scripts/Okiru "$VERBOSE" -#Defaults -if [[ -z $BWLIMIT ]]; then - export BWLIMIT="0" -else - debug "Bandwidth will be limited to" "$BWLIMIT Kbps" -fi -if [[ -n $CLEAN ]]; then - debug "Cleaning will take place per request." -fi -if [[ -n $REPOSITORY ]]; then - debug "Will only process repository" "$1" -fi -#Safety function; accepts repository to check -safety(){ - REPOSITORY="$1" - info "Checking if repository is in use - " "$REPOSITORY" - #Check no other Restic process is using this repository; Free unnecessary locks, if present - if [[ -n $(ps aux | grep restic | grep "$REPOSITORY") ]]; then - warn "Repository is in use - ignoring" - return 1 -# ^ If there's a restic process holding the repository, leave it alone. - else - info "Repository is not in use - unlocking" - restic -q -r b2:$REPOSITORY unlock -# ^ If a lock exists but no process, the repository is safe and should be unlocked. - fi -} -#Backup function; accepts repository and path to backup -backup(){ - REPOSITORY="$1" - REPOSITORY_PATH="$2" - if safety "$REPOSITORY"; then - #Run the backup - info "Backing up repository" "$REPOSITORY" - if restic --cache-dir="$RESTIC_CACHE_DIR" -r b2:"$REPOSITORY" backup "$REPOSITORY_PATH" --limit-upload="$BWLIMIT" --limit-download="$BWLIMIT"; then - ok "$REPOSITORY_PATH" "completed upload to $REPOSITORY." - else - error "$REPOSITORY failed to upload path" "$REPOSITORY_PATH" - fi - fi -} -check(){ - REPOSITORY="$1" - PRUNE="$2" - debug "Checking integrity (prune: $PRUNE) of repository" "$REPOSITORY" -## ^ This variable will have value if repo is already clean, indicating -#+ This is a post backup check. - if [[ -n $PRUNE ]]; then - warn "This repository has been cleaned already; will not clean again." - fi - if safety "$REPOSITORY"; then - info "Checking repository health - " "$REPOSITORY" - if restic -r b2:"$REPOSITORY" check --limit-upload="$BWLIMIT" --limit-download="$BWLIMIT"; then - ok "Repository passed integrity check - " "$REPOSITORY" - info "Current snapshots:" - restic -r b2:"$REPOSITORY" snapshots | tee -a $LOG - else - error "Repository failed integrity check - " "$REPOSITORY" - fi - fi -} -clean(){ - REPOSITORY="$1" - if safety "$REPOSITORY"; then - info "Cleaning repository" "$REPOSITORY" - if restic -r b2:$REPOSITORY forget --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --prune --limit-upload="$BWLIMIT" --limit-download="$BWLIMIT"; then - ok "Repository is trim - " "$REPOSITORY" - debug "Running post clean check..." - check "$REPOSITORY" "1" -# Marks repository as cleaned already ^ so it won't passed to this function again. - else - error "Failed to prune repository" "$REPOSITORY" - fi - fi -} - -#If cleaning was forced, or if it's the first of this month - clean. -if [[ -n $CLEAN ]] || [[ $(date +%d) == "1" ]]; then - check Gerbil-TK - clean Gerbil-TK - check Pukeko-XYZ-Containers - clean Pukeko-XYZ-Containers - check Pukeko-XYZ-Cloud - clean Pukeko-XYZ-Cloud -#If a specific repository was requested, back it up; otherwise, back them all up. -elif [[ -n $REPOSITORY ]] && [[ -z $CLEAN ]]; then - case "$REPOSITORY" in - (Gerbil-TK) - backup Gerbil-TK /var/Red-Vol/Media/Pictures/ - ;; - (Pukeko-XYZ-Containers) - backup Pukeko-XYZ-Containers /var/Red-Vol/Media/Containers/ - ;; - (Pukeko-XYZ-Cloud) - backup Pukeko-XYZ-Cloud /var/Red-Vol/Media/Cloud/ - ;; - (*) - help;; - esac -#If cleaning was not forced, backup the repositories -elif [[ -z $CLEAN ]]; then - backup Gerbil-TK /var/Red-Vol/Media/Pictures/ - backup Pukeko-XYZ-Containers /var/Red-Vol/Media/Containers/ - backup Pukeko-XYZ-Cloud /var/Red-Vol/Media/Cloud/ -#If a specific repository was requested to be cleaned, clean it -elif [[ -n $REPOSITORY ]] && [[ -n $CLEAN ]]; then - case "$REPOSITORY" in - (Gerbil-TK) - check Gerbil-TK - clean Gerbil-TK - ;; - (Pukeko-XYZ-Containers) - check Pukeko-XYZ-Containers - clean Pukeko-XYZ-Containers - ;; - (Pukeko-XYZ-Cloud) - check Pukeko-XYZ-Cloud - clean Pukeko-XYZ-Cloud - ;; - (*) - help;; - esac -fi -#Wrap up this run's log and report nicely -echo "All done; have a nice day!" -exit 0