| 
														
															@@ -99,7 +99,7 @@ MAIL_BIN="/usr/bin/mail" 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 TERSE=2 
														 | 
														
														 | 
														
															 TERSE=2 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   
														 | 
														
														 | 
														
															   
														 | 
													
												
											
												
													
														| 
														 | 
														
															 ## INTERNAL TEMP VARS ## 
														 | 
														
														 | 
														
															 ## INTERNAL TEMP VARS ## 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-EMAIL_SUBJECT_PREFIX="[`hostname`] SnapRAID - " 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+EMAIL_SUBJECT_PREFIX="[`hostname`] SnapRAID -" 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 TMP_OUTPUT="/tmp/snapRAID.out" 
														 | 
														
														 | 
														
															 TMP_OUTPUT="/tmp/snapRAID.out" 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 SYNC_WARN_FILE="/tmp/snapRAID.warnCount" 
														 | 
														
														 | 
														
															 SYNC_WARN_FILE="/tmp/snapRAID.warnCount" 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 SYNC_WARN_COUNT="" 
														 | 
														
														 | 
														
															 SYNC_WARN_COUNT="" 
														 | 
													
												
											
										
											
												
													
														 | 
														
															@@ -154,7 +154,15 @@ if [ -z "$DEL_COUNT" -o -z "$ADD_COUNT" -o -z "$MOVE_COUNT" -o -z "$COPY_COUNT" 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   # failed to get one or more of the count values, lets report to user and exit with error code 
														 | 
														
														 | 
														
															   # failed to get one or more of the count values, lets report to user and exit with error code 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   echo "[`date`] ERROR - failed to get one or more count values. Unable to proceed. Exiting script." 
														 | 
														
														 | 
														
															   echo "[`date`] ERROR - failed to get one or more count values. Unable to proceed. Exiting script." 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   if [ $EMAIL_ADDRESS ]; then 
														 | 
														
														 | 
														
															   if [ $EMAIL_ADDRESS ]; then 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    $MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX WARNING - Unable to proceed with SYNC/SCRUB job(s). Check DIFF job output inside." "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+#    $MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX WARNING - Unable to proceed with SYNC/SCRUB job(s). Check DIFF job output inside." "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    curl \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Title: $EMAIL_SUBJECT_PREFIX WARNING" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Priority: urgent" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Tags: warning" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -d "Unable to proceed with SYNC/SCRUB job(s). Check DIFF job output inside 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+$(cat $TMP_OUTPUT)" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      https://ntfy.warchildstory.com/{{ server_notifications_topic }} 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   fi 
														 | 
														
														 | 
														
															   fi 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   exit 1; 
														 | 
														
														 | 
														
															   exit 1; 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 fi 
														 | 
														
														 | 
														
															 fi 
														 | 
													
												
											
										
											
												
													
														 | 
														
															@@ -270,31 +278,59 @@ fi 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 #echo "Spinning down disks..." >> $TMP_OUTPUT 
														 | 
														
														 | 
														
															 #echo "Spinning down disks..." >> $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 #$SNAPRAID_BIN down 
														 | 
														
														 | 
														
															 #$SNAPRAID_BIN down 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   
														 | 
														
														 | 
														
															   
														 | 
													
												
											
												
													
														| 
														 | 
														
															-# all jobs done, let's send output to user if configured 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-if [ $EMAIL_ADDRESS ]; then 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-  echo "[`date`] Email address is set. Sending email report to <$EMAIL_ADDRESS>" 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-  # check if deleted count exceeded threshold 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-  if [ $DEL_COUNT -gt $DEL_THRESHOLD -a $DO_SYNC -eq 0 ]; then 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    # YES, lets inform user with an appropriate subject line 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    $MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX WARNING $SYNC_WARN_COUNT - Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD)" "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-  elif [ -z "${JOBS_DONE##*"SYNC"*}" -a -z "$(grep -w "SYNC_JOB-" $TMP_OUTPUT)" ]; then 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    # Sync ran but did not complete successfully so lets warn the user 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    $MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX WARNING - SYNC job ran but did not complete successfully" "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-  elif [ -z "${JOBS_DONE##*"SCRUB"*}" -a -z "$(grep -w "SCRUB_JOB-" $TMP_OUTPUT)" ]; then 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    # Scrub ran but did not complete successfully so lets warn the user 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    $MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX WARNING - SCRUB job ran but did not complete successfully" "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-  else 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    # OPTIONALLY, let's reduce the amount of status lines in output. 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    if [ $TERSE -gt 1 ]; then 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-      # Report progress in interval of tens % 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-      sed -i '$!N; /^\([0-9]\).*\n\1.*$/!P; D'  $TMP_OUTPUT 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-      sed -i '/^[1-8]%.*$/d'  $TMP_OUTPUT 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    elif [ $TERSE -gt 0 ]; then 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-      # Report progress in interval of ones % 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-      sed -i '$!N; /^\([0-9]*\)%.*\n\1.*$/!P; D'  $TMP_OUTPUT 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    fi 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															-    $MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX INFO - $JOBS_DONE Jobs COMPLETED" "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
														
														 | 
														
															 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+# check if deleted count exceeded threshold 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+if [ $DEL_COUNT -gt $DEL_THRESHOLD -a $DO_SYNC -eq 0 ]; then 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  # YES, lets inform user with an appropriate subject line 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  #$MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX WARNING $SYNC_WARN_COUNT - Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD)" "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    curl \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Title: $EMAIL_SUBJECT_PREFIX WARNING" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Priority: urgent" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Tags: warning" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -d "$SYNC_WARN_COUNT - Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD) 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+$(cat $TMP_OUTPUT)" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      https://ntfy.warchildstory.com/{{ server_notifications_topic }} 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+elif [ -z "${JOBS_DONE##*"SYNC"*}" -a -z "$(grep -w "SYNC_JOB-" $TMP_OUTPUT)" ]; then 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  # Sync ran but did not complete successfully so lets warn the user 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  #$MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX WARNING - SYNC job ran but did not complete successfully" "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    curl \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Title: $EMAIL_SUBJECT_PREFIX WARNING" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Priority: urgent" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Tags: warning" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -d "SYNC job ran but did not complete successfully 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+$(cat $TMP_OUTPUT)" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      https://ntfy.warchildstory.com/{{ server_notifications_topic }} 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+elif [ -z "${JOBS_DONE##*"SCRUB"*}" -a -z "$(grep -w "SCRUB_JOB-" $TMP_OUTPUT)" ]; then 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  # Scrub ran but did not complete successfully so lets warn the user 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  #$MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX WARNING - SCRUB job ran but did not complete successfully" "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    curl \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Title: $EMAIL_SUBJECT_PREFIX WARNING" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Priority: urgent" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Tags: warning" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -d "SCRUB job ran but did not complete successfully 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+$(cat $TMP_OUTPUT)" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      https://ntfy.warchildstory.com/{{ server_notifications_topic }} 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+else 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  # OPTIONALLY, let's reduce the amount of status lines in output. 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  if [ $TERSE -gt 1 ]; then 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    # Report progress in interval of tens % 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    sed -i '$!N; /^\([0-9]\).*\n\1.*$/!P; D'  $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    sed -i '/^[1-8]%.*$/d'  $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  elif [ $TERSE -gt 0 ]; then 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    # Report progress in interval of ones % 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    sed -i '$!N; /^\([0-9]*\)%.*\n\1.*$/!P; D'  $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   fi 
														 | 
														
														 | 
														
															   fi 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+  #$MAIL_BIN -s "$EMAIL_SUBJECT_PREFIX INFO - $JOBS_DONE Jobs COMPLETED" "$EMAIL_ADDRESS" < $TMP_OUTPUT 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+    curl \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Title: $EMAIL_SUBJECT_PREFIX INFO" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Priority: default" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -H "Tags: white_check_mark" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      -d "$JOBS_DONE Jobs COMPLETED 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+$(cat $TMP_OUTPUT)" \ 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 
														 | 
														
														 | 
														
															+      https://ntfy.warchildstory.com/{{ server_notifications_topic }} 
														 | 
													
												
											
												
													
														| 
														 | 
														
															 fi 
														 | 
														
														 | 
														
															 fi 
														 | 
													
												
											
												
													
														| 
														 | 
														
															   
														 | 
														
														 | 
														
															   
														 | 
													
												
											
												
													
														| 
														 | 
														
															 echo "[`date`] All jobs ended." 
														 | 
														
														 | 
														
															 echo "[`date`] All jobs ended." 
														 |