@ -11,76 +11,152 @@ commandaction="Monitoring"
moduleselfname = " $( basename " $( readlink -f " ${ BASH_SOURCE [0] } " ) " ) " moduleselfname = " $( basename " $( readlink -f " ${ BASH_SOURCE [0] } " ) " ) "
fn_firstcommand_setfn_firstcommand_set
fn_monitor_check_lockfile( ) { fn_monitor_check_monitoring( ) {
# Monitor does not run if lockfile is not found. if [ -f " ${ lockdir } / ${ selfname } .lock " ] ; then
if [ ! -f " ${ lockdir } / ${ selfname } .lock " ] ; then # Part of migration to v23.5.0. #4296
rm -f " ${ lockdir : ? } / ${ selfname } .lock "
date '+%s' > " ${ lockdir : ? } / ${ selfname } -monitoring.lock "
elif [ ! -f " ${ lockdir } / ${ selfname } -monitoring.lock " ] ; then
# Monitor does not run if lockfile is not found.
fn_print_dots "Checking lockfile: " fn_print_dots "Checking lockfile: "
fn_print_checking_eol fn_print_checking_eol
fn_script_log_info "Checking lockfile: CHECKING" fn_script_log_info "Checking lockfile: CHECKING"
fn_monitor_check_update
fn_monitor_check_backup
fn_monitor_check_debug
fn_print_error "Checking lockfile: No lockfile found: " fn_print_error "Checking lockfile: No lockfile found: "
fn_print_error_eol_nl fn_print_error_eol_nl
fn_script_log_error "Checking lockfile: No lockfile found: ERROR" fn_script_log_error "Checking lockfile: No lockfile found: ERROR"
echo -e " * Start ${ selfname } to run monitor. " echo -e " * Start ${ selfname } to run monitor. "
core_exit.sh core_exit.sh
fi fi
}
# Fix if lockfile is not unix time or contains letters fn_monitor_check_install( ) {
if [ -f " ${ lockdir } / ${ selfname } .lock " ] && [ [ " $( head -n 1 " ${ lockdir } / ${ selfname } .lock " ) " = ~ [ A-Za-z] ] ] ; then if [ " $( pgrep -fc -u " ${ USER } " " /bin/bash ./ ${ selfname } install " ) " != "0" ] || [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } i " ) " != "0" ] || [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } auto-install " ) " != "0" ] || [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } ai " ) " != "0" ] ; then
date '+%s' > " ${ lockdir } / ${ selfname } .lock " fn_print_dots "Checking installer: "
echo " ${ version } " >> " ${ lockdir } / ${ selfname } .lock " fn_print_checking_eol
echo " ${ port } " >> " ${ lockdir } / ${ selfname } .lock " fn_script_log_info "Checking installer: CHECKING"
fn_print_info "Checking installer: LinuxGSM is installing: "
fn_print_info_eol_nl
fn_script_log_pass "Checking installer: LinuxGSM is installing"
core_exit.sh
fi fi
} }
fn_monitor_check_backup( ) { fn_monitor_check_debug( ) {
# Monitor will check if backup is running. if [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } debug " ) " != "0" ] || [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } d " ) " != "0" ] ; then
if [ " $( pgrep " ${ selfname } backup " | wc -l) " != "0" ] || [ " $( pgrep " ${ selfname } b " | wc -l) " != "0" ] ; then fn_print_dots "Checking debug: "
fn_print_info "Checking lockfile: LinuxGSM is currently running a backup: " fn_print_checking_eol
fn_print_info_eol fn_print_info "Checking debug: Debug is running: "
fn_script_log_info "Checking lockfile: LinuxGSM is currently running a backup" fn_print_info_eol_nl
fn_script_log_pass "Checking debug: Debug is running"
core_exit.sh core_exit.sh
fi fi
} }
fn_monitor_check_debug( ) { fn_monitor_check_starting( ) {
# Monitor will check if backup is running. # Remove stale lockfile.
if [ " $( pgrep -fc " ${ selfname } backup " ) " != "0" ] || [ " $( pgrep -fc " ${ selfname } b " ) " != "0" ] ; then if [ -f " ${ lockdir } / ${ selfname } -starting.lock " ] ; then
fn_print_info "Checking lockfile: LinuxGSM is currently in debug mode: " if [ " $( find " ${ lockdir } / ${ selfname } -starting.lock " -mmin +5) " ] ; then
fn_print_info_eol fn_print_dots "Checking start: "
fn_script_log_pass "Checking lockfile: LinuxGSM is currently in debug mode" fn_print_checking_eol
fn_print_warn "Checking start: Removing stale lockfile: "
fn_print_warn_eol_nl
fn_script_log_warn "Checking start: Removing stale lockfile"
rm -f " ${ lockdir : ? } / ${ selfname } -starting.lock "
fi
fi
if [ -f " ${ lockdir } / ${ selfname } -starting.lock " ] && [ [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } start " ) " != "0" || " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } s " ) " != "0" ] ] ; then
fn_print_dots "Checking start: "
fn_print_checking_eol
fn_print_info "Checking start: LinuxGSM is starting: "
fn_print_info_eol_nl
fn_script_log_info "Checking backup: LinuxGSM is starting"
core_exit.sh core_exit.sh
fi fi
} }
fn_monitor_check_install( ) { fn_monitor_check_stopping( ) {
# Monitor will check if update is running. # Remove stale lockfile.
if [ " $( pgrep -fc " ${ selfname } install " ) " != "0" ] || [ " $( pgrep -fc " ${ selfname } i " ) " != "0" ] || [ " $( pgrep -fc " ${ selfname } auto-install " ) " != "0" ] || [ " $( pgrep -fc " ${ selfname } ai " ) " != "0" ] ; then if [ -f " ${ lockdir } / ${ selfname } -stopping.lock " ] ; then
fn_print_dots "Checking for installer: " if [ " $( find " ${ lockdir } / ${ selfname } -stopping.lock " -mmin +5) " ] ; then
fn_print_dots "Checking stop: "
fn_print_checking_eol
fn_print_warn "Checking stop: Removing stale lockfile: "
fn_print_warn_eol_nl
fn_script_log_warn "Checking stop: Removing stale lockfile"
rm -f " ${ lockdir : ? } / ${ selfname } -stopping.lock "
fi
fi
if [ -f " ${ lockdir } / ${ selfname } -stopping.lock " ] && [ [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } stop " ) " != "0" || " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } s " ) " != "0" ] ] ; then
fn_print_dots "Checking stop: "
fn_print_checking_eol
fn_print_info "Checking stop: LinuxGSM is stopping: "
fn_print_info_eol_nl
fn_script_log_info "Checking backup: LinuxGSM is stopping"
core_exit.sh
fi
}
fn_monitor_check_backup( ) {
# Remove stale lockfile.
if [ -f " ${ lockdir } /backup.lock " ] ; then
if [ " $( find " ${ lockdir } /backup.lock " -mmin +60) " ] ; then
fn_print_dots "Checking backup: "
fn_print_checking_eol
fn_print_warn "Checking backup: Removing stale lockfile: "
fn_print_warn_eol
fn_script_log_warn "Checking backup: Removing stale lockfile"
rm -f " ${ lockdir : ? } /backup.lock "
fi
fi
if [ -f " ${ lockdir } /backup.lock " ] && [ [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } backup " ) " != "0" || " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } b " ) " != "0" ] ] ; then
fn_print_dots "Checking backup: "
fn_print_checking_eol fn_print_checking_eol
fn_script_log_info "Checking for installer: CHECKING" fn_print_info "Checking backup: Backup is running: "
fn_print_info "Checking for installer: LinuxGSM is currently installing: " fn_print_info_eol_nl
fn_print_info_eol fn_script_log_info "Checking backup: Backup is running"
fn_script_log_pass "Checking for installer: LinuxGSM is currently installing"
core_exit.sh core_exit.sh
fi fi
} }
fn_monitor_check_update( ) { fn_monitor_check_update( ) {
# Specific check for docker. Will ignore the command watch -n 1800 ./csgoserver update # Remove stale lockfile.
monitorps = 0 if [ -f " ${ lockdir } /update.lock " ] ; then
if [ " $( pgrep -fc " n* ${ selfname } update " ) " != "0" ] ; then if [ " $( find " ${ lockdir } /update.lock " -mmin +15) " ] ; then
monitorps = " $(( monitorps - 1 )) " fn_print_dots "Checking update: "
fn_print_checking_eol
fn_print_warn "Checking update: Removing stale lockfile: "
fn_print_warn_eol_nl
fn_script_log_warn "Checking update: Removing stale lockfile"
rm -f " ${ lockdir : ? } /update.lock "
fi
fi fi
# Monitor will check if an update is running.
if [ " $( pgrep -fc " ${ selfname } update " ) " != "0" ] || [ " $( pgrep -fc " ${ selfname } u " ) " != "0" ] || [ " $( pgrep -fc " ${ selfname } validate " ) " != "0" ] || [ " $( pgrep -fc " ${ selfname } v " ) " != "0" ] ; then if [ -f " ${ lockdir } /update.lock " ] && [ [ " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } update " ) " != "0" || " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } validate " ) " != "0" || " $( pgrep -fcx -u " ${ USER } " " /bin/bash ./ ${ selfname } v " ) " != "0" || " $( pgrep -fc force-update " ${ USER } " " /bin/bash ./ ${ selfname } fu " ) " != "0" ] ] ; then
monitorps = " $(( monitorps + 2 )) " fn_print_dots "Checking update: "
if [ " ${ monitorps } " != "0" ] ; then fn_print_checking_eol
fn_print_info_nl "Checking lockfile: LinuxGSM is currently updating: " fn_print_info "Checking update: LinuxGSM is updating the game server: "
fn_print_info_eol fn_print_info_eol_nl
fn_script_log_pass "Checking lockfile: LinuxGSM is currently updating" fn_script_log_pass "Checking update: LinuxGSM is updating the game server"
core_exit.sh
fi
}
# Source engine games may display a messages to indicate the server needs restarting.
fn_monitor_check_update_source( ) {
if [ -f " ${ consolelogdir } / ${ selfname } -console.log " ] && [ " ${ engine } " = = "source" ] ; then
if grep -q "Your server needs to be restarted in order to receive the latest update." " ${ consolelogdir } / ${ selfname } -console.log " ; then
fn_print_dots "Checking update: "
fn_print_checking_eol
fn_script_log_info "Checking update: CHECKING"
fn_print_ok "Checking update: "
fn_print_ok_eol_nl
fn_script_log_info " Checking update: Monitor is restarting ${ selfname } to apply update "
alert = "update-restart"
alert.sh
command_restart.sh
core_exit.sh core_exit.sh
fi fi
fi fi
@ -90,8 +166,40 @@ fn_monitor_check_session() {
fn_print_dots "Checking session: " fn_print_dots "Checking session: "
fn_print_checking_eol fn_print_checking_eol
fn_script_log_info "Checking session: CHECKING" fn_script_log_info "Checking session: CHECKING"
# uses status var from check_status.sh # Tmux session width and height needs to be reviewed as may no longer be required.
if [ " ${ status } " != "0" ] ; then sessionwidth = "80"
sessionheight = "23"
# Check for PIDS with identical tmux sessions running.
if [ " $( pgrep -fcx " tmux -L ${ socketname } new-session -d -x ${ sessionwidth } -y ${ sessionheight } -s ${ sessionname } " ) " -ge "2" ] ; then
fn_print_error "Checking session: There are PIDS with identical tmux sessions running: "
fn_print_error_eol_nl
fn_script_log_error "Checking session: ERROR"
fn_script_log_error "Checking session: There are PIDS with identical tmux sessions running"
fn_script_log_error " Checking session: Killing all tmux sessions with the socketname name ${ socketname } and session name ${ sessionname } "
pkill -f " tmux -L ${ socketname } new-session -d -x ${ sessionwidth } -y ${ sessionheight } -s ${ sessionname } "
command_restart.sh
core_exit.sh
# Check for tmux pids with the same tmux session and socket names. This will reduce issues with migration to release v23.5.0. #4296
elif [ " $( pgrep -fc -u " ${ USER } " " tmux -L ${ sessionname } new-session -d -x ${ sessionwidth } -y ${ sessionheight } -s ${ sessionname } " ) " != "0" ] ; then
fn_print_error "Checking session: PIDS with the same tmux session and socket names are running: "
fn_print_error_eol_nl
fn_script_log_error "Checking session: ERROR"
fn_script_log_error "Checking session: PIDS with the same tmux session and socket names are running"
fn_script_log_error " Checking session: Killing session with the socketname name ${ sessionname } and session name ${ sessionname } "
pkill -f " tmux -L ${ sessionname } new-session -d -x ${ sessionwidth } -y ${ sessionheight } -s ${ sessionname } "
command_restart.sh
core_exit.sh
# Check for tmux pids that are using the old type of tmux session. This will reduce issues with migration to release v23.5.0. #4296
elif [ " $( pgrep -fc -u " ${ USER } " " tmux new-session -d -x ${ sessionwidth } -y ${ sessionheight } -s ${ sessionname } " ) " != "0" ] ; then
fn_print_error "Checking session: PIDS with old type tmux session are running: "
fn_print_error_eol_nl
fn_script_log_error "Checking session: ERROR"
fn_script_log_error "Checking session: PIDS with old type tmux session are running"
fn_script_log_error " Checking session: Killing session with the session name ${ sessionname } "
pkill -f " tmux new-session -d -x ${ sessionwidth } -y ${ sessionheight } -s ${ sessionname } "
command_restart.sh
core_exit.sh
elif [ " ${ status } " != "0" ] ; then
fn_print_ok "Checking session: " fn_print_ok "Checking session: "
fn_print_ok_eol_nl fn_print_ok_eol_nl
fn_script_log_pass "Checking session: OK" fn_script_log_pass "Checking session: OK"
@ -107,8 +215,8 @@ fn_monitor_check_session() {
fi fi
} }
# Monitor will check queryport is set before continuing.
fn_monitor_check_queryport( ) { fn_monitor_check_queryport( ) {
# Monitor will check queryport is set before continuing.
if [ -z " ${ queryport } " ] || [ " ${ queryport } " = = "0" ] ; then if [ -z " ${ queryport } " ] || [ " ${ queryport } " = = "0" ] ; then
fn_print_dots "Checking port: " fn_print_dots "Checking port: "
fn_print_checking_eol fn_print_checking_eol
@ -133,7 +241,7 @@ fn_query_gsquery() {
} }
fn_query_tcp( ) { fn_query_tcp( ) {
bash -c ' exec 3<> /dev/tcp/'${ queryip } ${ queryport } '' & 1 bash -c " ${ queryip } ${ queryport } ' " & 1
querystatus = " $? " querystatus = " $? "
} }
@ -147,12 +255,12 @@ fn_monitor_query() {
fn_print_querying_eol fn_print_querying_eol
fn_script_log_info " Querying port: ${ querymethod } : ${ queryip } : ${ queryport } : ${ queryattempt } : QUERYING " fn_script_log_info " Querying port: ${ querymethod } : ${ queryip } : ${ queryport } : ${ queryattempt } : QUERYING "
# querydelay # querydelay
if [ " $( head -n 1 " ${ lockdir } / ${ selfname } .lock " ) " -gt " $( date "+%s" -d " ${ querydelay } mins ago " ) " ] ; then if [ " $( head -n 1 " ${ lockdir } / ${ selfname } -started .lock" ) " -gt " $( date "+%s" -d " ${ querydelay } mins ago " ) " ] ; then
fn_print_ok " Querying port: ${ querymethod } : ${ ip } : ${ queryport } : ${ totalseconds } / ${ queryattempt } : " fn_print_ok " Querying port: ${ querymethod } : ${ ip } : ${ queryport } : ${ totalseconds } / ${ queryattempt } : "
fn_print_delay_eol_nl fn_print_delay_eol_nl
fn_script_log_info " Querying port: ${ querymethod } : ${ ip } : ${ queryport } : ${ queryattempt } : DELAY " fn_script_log_info " Querying port: ${ querymethod } : ${ ip } : ${ queryport } : ${ queryattempt } : DELAY "
fn_script_log_info " Query bypassed: ${ gameservername } started less than ${ querydelay } minutes ago " fn_script_log_info " Query bypassed: ${ gameservername } started less than ${ querydelay } minutes ago "
fn_script_log_info " Server started: $( date -d @$( head -n 1 " ${ lockdir } / ${ selfname } .lock " ) ) " fn_script_log_info " Server started: $( date -d " $( head -n 1 " ${ lockdir } / ${ selfname } -started .lock" ) " ) "
fn_script_log_info " Current time: $( date) " fn_script_log_info " Current time: $( date) "
monitorpass = 1 monitorpass = 1
core_exit.sh core_exit.sh
@ -271,8 +379,15 @@ core_logs.sh
info_game.shinfo_game.sh
# query pre-checks # query pre-checks
fn_monitor_check_lockfilefn_monitor_check_update_source
fn_monitor_check_update
fn_monitor_check_backup
fn_monitor_check_debug
fn_monitor_check_monitoring
fn_monitor_check_starting
fn_monitor_check_stopping
fn_monitor_check_sessionfn_monitor_check_session
# Monitor will not continue if session only check. # Monitor will not continue if session only check.
if [ " ${ querymode } " != "1" ] ; then if [ " ${ querymode } " != "1" ] ; then
fn_monitor_check_queryport fn_monitor_check_queryport
Daniel Gibbs
2 years ago