import paramiko
import schedule
import time
import os
import sys
from flask import Flask, jsonify, render_template_string
from threading import Thread
import logging
from datetime import datetime, timedelta
app = Flask(__name__)
vps_status = {}
# 设置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.StreamHandler(sys.stderr)
]
)
logger = logging.getLogger()
def get_vps_configs():
configs = []
index = 1
while True:
hostname = os.environ.get(f'HOSTNAME_{index}')
if not hostname:
break
username = os.environ.get(f'USERNAME_{index}')
password = os.environ.get(f'PASSWORD_{index}')
script_paths = []
script_index = 1
while True:
script_path = os.environ.get(f'SCRIPT_PATHS_{index}_{script_index}')
if not script_path:
break
script_paths.append(script_path.strip())
script_index += 1
for script_path in script_paths:
config = {
'index': index,
'hostname': hostname,
'username': username,
'password': password,
'script_path': script_path
}
configs.append(config)
index += 1
return configs
def check_and_run_script(config):
logger.info(f"Checking VPS {config['index']}: {config['hostname']} - {config['script_path']}")
client = None
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(
hostname=config['hostname'],
username=config['username'],
password=config['password'],
port=22
)
script_path = config['script_path']
script_name = os.path.basename(script_path)
key = f"{config['hostname']}:{script_name}"
# 获取上次保存的 PID（如果有）
last_pid = vps_status.get(key, {}).get('pid', None)
# 检查进程是否在运行
if last_pid:
check_command = f"ps -p {last_pid} -o pid,etime,args"
else:
check_command = f"ps aux | grep '{script_path}' | grep -v grep"
stdin, stdout, stderr = client.exec_command(check_command)
output = stdout.read().decode('utf-8').strip()
if output and (last_pid or script_path in output):
parts = output.split()
if last_pid:
pid = last_pid
runtime = parts[1] if len(parts) > 1 else "Unknown"
else:
pid = parts[1] if len(parts) > 1 else "Unknown"
runtime = parts[9] if len(parts) > 9 else "Unknown"
status = "Running"
logger.info(f"Script {script_name} is running. PID: {pid}, Runtime: {runtime}")
else:
logger.info(f"Script {script_name} not running. Attempting to restart.")
restart_command = f"nohup /bin/sh {script_path} > /dev/null 2>&1 & echo $!"
stdin, stdout, stderr = client.exec_command(restart_command)
new_pid = stdout.read().decode('utf-8').strip()
if new_pid.isdigit():
pid = new_pid
runtime = "Just started"
status = "Restarted"
logger.info(f"Script {script_name} restarted. New PID: {pid}")
else:
pid = "N/A"
runtime = "N/A"
status = "Restart Failed"
logger.error(f"Failed to restart script {script_name}")
vps_status[key] = {
'index': config['index'],
'status': status,
'last_check': time.strftime('%Y-%m-%d %H:%M:%S'),
'username': config['username'],
'script_name': script_name,
'runtime': runtime,
'pid': pid
}
except Exception as e:
logger.error(f"Error occurred while checking VPS {config['index']} - {config['hostname']} - {script_name}: {str(e)}")
key = f"{config['hostname']}:{script_name}"
vps_status[key] = {
'index': config['index'],
'status': f"Error: {str(e)}",
'last_check': time.strftime('%Y-%m-%d %H:%M:%S'),
'username': config['username'],
'script_name': script_name,
'runtime': "N/A",
'pid': "N/A"
}
finally:
if client:
client.close()
def check_all_vps():
logger.info("Starting VPS check")
vps_configs = get_vps_configs()
for config in vps_configs:
check_and_run_script(config)
# 创建表格头
table = "+---------+-----------------------+------------------+----------+-------------------------+----------+----------+-------+\n"
table += "| Index | Hostname | Script Name | Status | Last Check | Username | Runtime | PID |\n"
table += "+---------+-----------------------+------------------+----------+-------------------------+----------+----------+-------+\n"
# 添加每个VPS的状态
for key, status in vps_status.items():
hostname, script_name = key.split(':')
table += "| {:<7} | {:<21} | {:<16} | {:<8} | {:<23} | {:<8} | {:<8} | {:<5} |\n".format(
status['index'],
hostname[:21],
script_name[:16],
status['status'][:8],
status['last_check'],
status['username'][:8],
status['runtime'][:8],
status['pid'][:5]
)
table += "+---------+-----------------------+------------------+----------+-------------------------+----------+----------+-------+\n"
logger.info("\n" + table)
@app.route('/')
def index():
html = '''
VPS Status Overview
| Index |
Hostname |
Script Name |
Status |
Last Check |
Username |
Runtime |
PID |
{% for key, data in vps_status.items() %}
| {{ data.index }} |
{{ key.split(':')[0] }} |
{{ data.script_name }} |
{{ data.status }} |
{{ data.last_check }} |
{{ data.username }} |
{{ data.runtime }} |
{{ data.pid }} |
{% endfor %}
'''
return render_template_string(html, vps_status=vps_status)
@app.route('/status/')
def vps_status_detail(key):
if key in vps_status:
return jsonify(vps_status[key])
else:
return jsonify({"error": "VPS or script not found"}), 404
@app.route('/health')
def health_check():
return jsonify({"status": "healthy", "uptime": time.time() - start_time}), 200
def run_flask():
app.run(host='0.0.0.0', port=8080)
def main():
global start_time
start_time = time.time()
logger.info("===== VPS monitoring script is starting =====")
flask_thread = Thread(target=run_flask)
flask_thread.start()
logger.info("Flask server started in background")
logger.info("Running initial VPS check")
check_all_vps()
schedule.every(15).minutes.do(check_all_vps)
logger.info("Scheduled VPS check every 15 minutes")
logger.info("===== VPS monitoring script is running =====")
heartbeat_count = 0
while True:
schedule.run_pending()
time.sleep(60)
heartbeat_count += 1
if heartbeat_count % 5 == 0: # 每5分钟输出一次心跳信息
logger.info(f"Heartbeat: Script is still running. Uptime: {heartbeat_count} minutes")
if __name__ == "__main__":
main()