gameservergame-servergame-servershacktoberfestdedicated-game-serversgamelinuxgsmserverbashgaminglinuxmultiplayer-game-servershell
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
287 lines
9.9 KiB
287 lines
9.9 KiB
#!/bin/bash
|
|
# LinuxGSM command_monitor.sh module
|
|
# Author: Daniel Gibbs
|
|
# Contributors: http://linuxgsm.com/contrib
|
|
# Website: https://linuxgsm.com
|
|
# Description: Monitors server by checking for running processes
|
|
# then passes to gamedig and gsquery.
|
|
|
|
commandname="MONITOR"
|
|
commandaction="Monitoring"
|
|
moduleselfname="$(basename "$(readlink -f "${BASH_SOURCE[0]}")")"
|
|
fn_firstcommand_set
|
|
|
|
fn_monitor_check_lockfile() {
|
|
# Monitor does not run if lockfile is not found.
|
|
if [ ! -f "${lockdir}/${selfname}.lock" ]; then
|
|
fn_print_dots "Checking lockfile: "
|
|
fn_print_checking_eol
|
|
fn_script_log_info "Checking lockfile: CHECKING"
|
|
fn_monitor_check_update
|
|
fn_monitor_check_backup
|
|
fn_monitor_check_debug
|
|
fn_print_error "Checking lockfile: No lockfile found: "
|
|
fn_print_error_eol_nl
|
|
fn_script_log_error "Checking lockfile: No lockfile found: ERROR"
|
|
echo -e "* Start ${selfname} to run monitor."
|
|
core_exit.sh
|
|
fi
|
|
|
|
# Fix if lockfile is not unix time or contains letters
|
|
if [ -f "${lockdir}/${selfname}.lock" ] && [[ "$(head -n 1 "${lockdir}/${selfname}.lock")" =~ [A-Za-z] ]]; then
|
|
date '+%s' > "${lockdir}/${selfname}.lock"
|
|
echo "${version}" >> "${lockdir}/${selfname}.lock"
|
|
echo "${port}" >> "${lockdir}/${selfname}.lock"
|
|
fi
|
|
}
|
|
|
|
fn_monitor_check_backup() {
|
|
# Monitor will check if backup is running.
|
|
if [ "$(pgrep "${selfname} backup" | wc -l)" != "0" ] || [ "$(pgrep "${selfname} b" | wc -l)" != "0" ]; then
|
|
fn_print_info_nl "Checking lockfile: LinuxGSM is currently running a backup: "
|
|
fn_print_info_eol
|
|
fn_script_log_info "Checking lockfile: LinuxGSM is currently running a backup"
|
|
core_exit.sh
|
|
fi
|
|
}
|
|
|
|
fn_monitor_check_debug() {
|
|
# Monitor will check if backup is running.
|
|
if [ "$(pgrep -fc "${selfname} backup")" != "0" ] || [ "$(pgrep -fc "${selfname} b")" != "0" ]; then
|
|
fn_print_info_nl "Checking lockfile: LinuxGSM is currently in debug mode: "
|
|
fn_print_info_eol
|
|
fn_script_log_pass "Checking lockfile: LinuxGSM is currently in debug mode"
|
|
core_exit.sh
|
|
fi
|
|
}
|
|
|
|
fn_monitor_check_install() {
|
|
# Monitor will check if update is running.
|
|
if [ "$(pgrep -fc "${selfname} install")" != "0" ] || [ "$(pgrep -fc "${selfname} i")" != "0" ] || [ "$(pgrep -fc "${selfname} auto-install")" != "0" ] || [ "$(pgrep -fc "${selfname} ai")" != "0" ]; then
|
|
fn_print_dots "Checking for installer: "
|
|
fn_print_checking_eol
|
|
fn_script_log_info "Checking for installer: CHECKING"
|
|
fn_print_info_nl "Checking for installer: LinuxGSM is currently installing: "
|
|
fn_print_info_eol
|
|
fn_script_log_pass "Checking for installer: LinuxGSM is currently installing"
|
|
core_exit.sh
|
|
fi
|
|
}
|
|
|
|
fn_monitor_check_update() {
|
|
# Specific check for docker. Will ignore the command watch -n 1800 ./csgoserver update
|
|
monitorps=0
|
|
if [ "$(pgrep -fc "n*${selfname} update")" != "0" ]; then
|
|
monitorps="$((monitorps - 1))"
|
|
fi
|
|
# Monitor will check if an update is running.
|
|
if [ "$(pgrep -fc "${selfname} update")" != "0" ] || [ "$(pgrep -fc "${selfname} u")" != "0" ] || [ "$(pgrep -fc "${selfname} validate")" != "0" ] || [ "$(pgrep -fc "${selfname} v")" != "0" ]; then
|
|
monitorps="$((monitorps + 2))"
|
|
if [ "${monitorps}" != "0" ]; then
|
|
fn_print_info_nl "Checking lockfile: LinuxGSM is currently updating: "
|
|
fn_print_info_eol
|
|
fn_script_log_pass "Checking lockfile: LinuxGSM is currently updating"
|
|
core_exit.sh
|
|
fi
|
|
fi
|
|
}
|
|
|
|
fn_monitor_check_session() {
|
|
fn_print_dots "Checking session: "
|
|
fn_print_checking_eol
|
|
fn_script_log_info "Checking session: CHECKING"
|
|
# uses status var from check_status.sh
|
|
if [ "${status}" != "0" ]; then
|
|
fn_print_ok "Checking session: "
|
|
fn_print_ok_eol_nl
|
|
fn_script_log_pass "Checking session: OK"
|
|
else
|
|
fn_print_error "Checking session: "
|
|
fn_print_fail_eol_nl
|
|
fn_script_log_fatal "Checking session: FAIL"
|
|
alert="restart"
|
|
alert.sh
|
|
fn_script_log_info "Checking session: Monitor is restarting ${selfname}"
|
|
command_restart.sh
|
|
core_exit.sh
|
|
fi
|
|
}
|
|
|
|
fn_monitor_check_queryport() {
|
|
# Monitor will check queryport is set before continuing.
|
|
if [ -z "${queryport}" ] || [ "${queryport}" == "0" ]; then
|
|
fn_print_dots "Checking port: "
|
|
fn_print_checking_eol
|
|
fn_script_log_info "Checking port: CHECKING"
|
|
if [ -n "${rconenabled}" ] && [ "${rconenabled}" != "true" ] && [ "${shortname}" == "av" ]; then
|
|
fn_print_warn "Checking port: Unable to query, rcon is not enabled"
|
|
fn_script_log_warn "Checking port: Unable to query, rcon is not enabled"
|
|
else
|
|
fn_print_error "Checking port: Unable to query, queryport is not set"
|
|
fn_script_log_error "Checking port: Unable to query, queryport is not set"
|
|
fi
|
|
core_exit.sh
|
|
fi
|
|
}
|
|
|
|
fn_query_gsquery() {
|
|
if [ ! -f "${modulesdir}/query_gsquery.py" ]; then
|
|
fn_fetch_file_github "lgsm/modules" "query_gsquery.py" "${modulesdir}" "chmodx" "norun" "noforce" "nohash"
|
|
fi
|
|
"${modulesdir}"/query_gsquery.py -a "${queryip}" -p "${queryport}" -e "${querytype}" > /dev/null 2>&1
|
|
querystatus="$?"
|
|
}
|
|
|
|
fn_query_tcp() {
|
|
bash -c 'exec 3<> /dev/tcp/'${queryip}'/'${queryport}'' > /dev/null 2>&1
|
|
querystatus="$?"
|
|
}
|
|
|
|
fn_monitor_query() {
|
|
# Will loop and query up to 5 times every 15 seconds.
|
|
# Query will wait up to 60 seconds to confirm server is down as server can become non-responsive during map changes.
|
|
totalseconds=0
|
|
for queryattempt in {1..5}; do
|
|
for queryip in "${queryips[@]}"; do
|
|
fn_print_dots "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}: "
|
|
fn_print_querying_eol
|
|
fn_script_log_info "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : QUERYING"
|
|
# querydelay
|
|
if [ "$(head -n 1 "${lockdir}/${selfname}.lock")" -gt "$(date "+%s" -d "${querydelay} mins ago")" ]; then
|
|
fn_print_ok "Querying port: ${querymethod}: ${ip}:${queryport} : ${totalseconds}/${queryattempt}: "
|
|
fn_print_delay_eol_nl
|
|
fn_script_log_info "Querying port: ${querymethod}: ${ip}:${queryport} : ${queryattempt} : DELAY"
|
|
fn_script_log_info "Query bypassed: ${gameservername} started less than ${querydelay} minutes ago"
|
|
fn_script_log_info "Server started: $(date -d @$(head -n 1 "${lockdir}/${selfname}.lock"))"
|
|
fn_script_log_info "Current time: $(date)"
|
|
monitorpass=1
|
|
core_exit.sh
|
|
# will use query method selected in fn_monitor_loop
|
|
# gamedig
|
|
elif [ "${querymethod}" == "gamedig" ]; then
|
|
query_gamedig.sh
|
|
# gsquery
|
|
elif [ "${querymethod}" == "gsquery" ]; then
|
|
fn_query_gsquery
|
|
#tcp query
|
|
elif [ "${querymethod}" == "tcp" ]; then
|
|
fn_query_tcp
|
|
fi
|
|
|
|
if [ "${querystatus}" == "0" ]; then
|
|
# Server query OK.
|
|
fn_print_ok "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}: "
|
|
fn_print_ok_eol_nl
|
|
fn_script_log_pass "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : OK"
|
|
monitorpass=1
|
|
if [ "${querystatus}" == "0" ]; then
|
|
# Add query data to log.
|
|
if [ "${gdname}" ]; then
|
|
fn_script_log_info "Server name: ${gdname}"
|
|
fi
|
|
if [ "${gdplayers}" ]; then
|
|
fn_script_log_info "Players: ${gdplayers}/${gdmaxplayers}"
|
|
fi
|
|
if [ "${gdbots}" ]; then
|
|
fn_script_log_info "Bots: ${gdbots}"
|
|
fi
|
|
if [ "${gdmap}" ]; then
|
|
fn_script_log_info "Map: ${gdmap}"
|
|
fi
|
|
if [ "${gdgamemode}" ]; then
|
|
fn_script_log_info "Game Mode: ${gdgamemode}"
|
|
fi
|
|
|
|
# send LinuxGSM stats if monitor is OK.
|
|
if [ "${stats}" == "on" ] || [ "${stats}" == "y" ]; then
|
|
info_stats.sh
|
|
fi
|
|
fi
|
|
core_exit.sh
|
|
else
|
|
# Server query FAIL.
|
|
fn_print_fail "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}: "
|
|
fn_print_fail_eol
|
|
fn_script_log_warn "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : FAIL"
|
|
# Monitor will try gamedig (if supported) for first 30s then gsquery before restarting.
|
|
# gsquery will fail if longer than 60s
|
|
if [ "${totalseconds}" -ge "59" ]; then
|
|
# Monitor will FAIL if over 60s and trigger gane server reboot.
|
|
fn_print_fail "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}: "
|
|
fn_print_fail_eol_nl
|
|
fn_script_log_warn "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : FAIL"
|
|
# Send alert if enabled.
|
|
alert="restartquery"
|
|
alert.sh
|
|
command_restart.sh
|
|
fn_firstcommand_reset
|
|
core_exit.sh
|
|
fi
|
|
fi
|
|
done
|
|
# Second counter will wait for 15s before breaking loop.
|
|
for seconds in {1..15}; do
|
|
fn_print_fail "Querying port: ${querymethod}: ${ip}:${queryport} : ${totalseconds}/${queryattempt} : ${cyan}WAIT${default}"
|
|
sleep 0.5
|
|
totalseconds=$((totalseconds + 1))
|
|
if [ "${seconds}" == "15" ]; then
|
|
break
|
|
fi
|
|
done
|
|
done
|
|
}
|
|
|
|
fn_monitor_loop() {
|
|
# loop though query methods selected by querymode.
|
|
totalseconds=0
|
|
if [ "${querymode}" == "2" ]; then
|
|
local query_methods_array=(gamedig gsquery)
|
|
elif [ "${querymode}" == "3" ]; then
|
|
local query_methods_array=(gamedig)
|
|
elif [ "${querymode}" == "4" ]; then
|
|
local query_methods_array=(gsquery)
|
|
elif [ "${querymode}" == "5" ]; then
|
|
local query_methods_array=(tcp)
|
|
fi
|
|
for querymethod in "${query_methods_array[@]}"; do
|
|
# Will check if gamedig is installed and bypass if not.
|
|
if [ "${querymethod}" == "gamedig" ]; then
|
|
if [ "$(command -v gamedig 2> /dev/null)" ] && [ "$(command -v jq 2> /dev/null)" ]; then
|
|
if [ -z "${monitorpass}" ]; then
|
|
fn_monitor_query
|
|
fi
|
|
else
|
|
fn_script_log_info "gamedig is not installed"
|
|
fn_script_log_info "https://docs.linuxgsm.com/requirements/gamedig"
|
|
fi
|
|
else
|
|
# will not query if query already passed.
|
|
if [ -z "${monitorpass}" ]; then
|
|
fn_monitor_query
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
monitorflag=1
|
|
# Dont do any monitoring or checks if installer is running.
|
|
fn_monitor_check_install
|
|
check.sh
|
|
core_logs.sh
|
|
info_game.sh
|
|
|
|
# query pre-checks
|
|
fn_monitor_check_lockfile
|
|
fn_monitor_check_session
|
|
# Monitor will not continue if session only check.
|
|
if [ "${querymode}" != "1" ]; then
|
|
fn_monitor_check_queryport
|
|
|
|
# Add a querydelay of 1 min if var missing.
|
|
if [ -z "${querydelay}" ]; then
|
|
querydelay="1"
|
|
fi
|
|
|
|
fn_monitor_loop
|
|
fi
|
|
core_exit.sh
|
|
|