网络质量监控与自动化故障转移：MTR、延迟探测与代理自愈方案

# 安装 MTR
# macOS
brew install mtr

# Linux (Debian/Ubuntu)
apt install mtr-tiny

# Windows: 从 https://winmtr.net/download/ 下载

# 基础用法：持续探测到目标服务器的路由和延迟
mtr -rwc 50 你的VPS_IP

# 参数说明：
# -r: 输出报告模式
# -c 50: 发送 50 个 ICMP 包
# -w: 使用 WHOIS 解析 IP 归属
# -p: 高亮显示丢包

# 输出示例：
#                         My traceroute  [v0.92]
# VPS-IP (VPS-IP)
# Host                                           Loss%   Snt   Last   Avg  Best  Wrst StDev
# 1. _gateway                                     0.0%    50    1.2    1.8    0.8    3.2    0.5
# 2. 10.0.0.1                                     0.0%    50    2.1    2.4    1.9    4.1    0.4
# 3. 72.14.215.85                                 0.0%    50   12.3   15.7   11.2   28.4    3.8
# 4. VPS公网IP                                    0.0%    50   45.2   48.9   43.1   62.3    4.2

# 安装
# macOS
brew install iperf3

# Linux
apt install iperf3

# VPS 上启动 iperf3 服务端（端口 5201）
iperf3 -s -p 5201 -f M

# 客户端测试带宽（从本地电脑测试到 VPS）
iperf3 -c VPS_IP -p 5201 -t 10 -f M

# 参数说明：
# -s: 服务端模式
# -c: 客户端模式，后接服务端 IP
# -t 10: 测试持续 10 秒
# -f M: 以 Mbps 为单位显示结果
# -R: 反向测试（服务端下载，客户端上传）
# -P 4: 并行 4 个连接

# 输出示例：
# [ ID] Interval           Transfer     Bitrate
# [  5]   0.00-10.00  sec  112 MBytes  112 Mbits/sec

#!/bin/bash
# latency_check.sh - 多节点延迟探测

NODES=(
  "VPS1|1.2.3.4"
  "VPS2|5.6.7.8"
  "VPS3|9.10.11.12"
)

echo "========== 节点延迟探测 $(date +'%Y-%m-%d %H:%M:%S') =========="
echo ""

for node in "${NODES[@]}"; do
  IFS='|' read -r name ip <<< "$node"

  # 探测 5 次，计算平均值
  result=$(ping -c 5 -q "$ip" 2>/dev/null | tail -1)
  loss=$(echo "$result" | awk -F',' '{print $3}' | awk '{print $1}')
  avg=$(echo "$result" | awk -F'/' '{print $5}' | awk -F'.' '{print $1}')

  # 判断状态
  if [ "$loss" = "0.0%" ]; then
    if [ -n "$avg" ] && [ "$avg" -lt 200 ]; then
      status="✅ 优秀"
    elif [ -n "$avg" ] && [ "$avg" -lt 400 ]; then
      status="⚠️ 一般"
    else
      status="🔴 延迟高"
    fi
  else
    status="❌ 丢包 $loss"
  fi

  printf "%-15s | %-8s | %s\n" "$name" "${avg}ms" "$status"
done

# 在每个 VPS 上安装 node_exporter
wget https://github.com/prometheus/node_exporter/releases/latest/download/node_exporter-1.6.1.linux-amd64.tar.gz
tar -xzvf node_exporter-1.6.1.linux-amd64.tar.gz
cd node_exporter-1.6.1.linux-amd64
sudo cp node_exporter /usr/local/bin/

# 创建 systemd 服务
sudo cat > /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Prometheus Node Exporter
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/node_exporter
Restart=on-failure
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now node_exporter

# 验证
curl http://localhost:9100/metrics

#!/bin/bash
# proxy_exporter.sh - 代理节点健康指标

# 配置
CHECK_TIMEOUT=5
NODES=(
  "node1|1.2.3.4|443"
  "node2|5.6.7.8|443"
)

# 输出 Prometheus 格式指标
echo '# HELP proxy_node_up 节点是否在线'
echo '# TYPE proxy_node_up gauge'

for node in "${NODES[@]}"; do
  IFS='|' read -r name ip port <<< "$node"

  # 检测端口连通性
  if timeout "$CHECK_TIMEOUT" bash -c "echo >/dev/tcp/$ip/$port" 2>/dev/null; then
    status=1
  else
    status=0
  fi

  echo "proxy_node_up{name=\"$name\",ip=\"$ip\",port=\"$port\"} $status"
done

# 检测延迟
echo '# HELP proxy_node_latency_ms 节点延迟（毫秒）'
echo '# TYPE proxy_node_latency_ms gauge'

for node in "${NODES[@]}"; do
  IFS='|' read -r name ip port <<< "$node"

  # 使用 curl 测量延迟
  latency=$(curl -o /dev/null -s -w '%{time_total}' \
    --connect-timeout 3 \
    "https://$ip:$port" 2>/dev/null)

  if [ -n "$latency" ]; then
    latency_ms=$(echo "$latency * 1000" | bc)
  else
    latency_ms=0
  fi

  echo "proxy_node_latency_ms{name=\"$name\",ip=\"$ip\"} $latency_ms"
done

# /etc/prometheus/prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - alertmanager:9093

rule_files:
  - /etc/prometheus/rules/*.yml

scrape_configs:
  # Prometheus 自身
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # 节点 Exporter
  - job_name: 'node'
    static_configs:
      - targets:
          - 'node1:9100'
          - 'node2:9100'
          - 'node3:9100'

  # 代理指标 Exporter
  - job_name: 'proxy'
    static_configs:
      - targets:
          - 'node1:9200'
          - 'node2:9200'

# Clash 配置中的自动切换策略

proxy-groups:
  # 方案 A: 自动选最快（定期测试延迟）
  - name: "⚡ 自动选择"
    type: url-test
    proxies:
      - 节点1
      - 节点2
      - 节点3
    url: http://www.gstatic.com/generate_204
    interval: 300    # 每 5 分钟测试一次
    tolerance: 50    # 延迟容忍度（毫秒）
    lazy: true       # 只在需要时测试

  # 方案 B: 故障转移（按顺序尝试，可用则切换回来）
  - name: "🔄 故障转移"
    type: fallback
    proxies:
      - 主节点
      - 备用节点1
      - 备用节点2
    url: http://www.gstatic.com/generate_204
    interval: 300

  # 方案 C: 负载均衡（轮询或一致性哈希）
  - name: "⚖️ 负载均衡"
    type: load-balance
    proxies:
      - 节点1
      - 节点2
      - 节点3
    url: http://www.gstatic.com/generate_204
    interval: 300
    strategy: consistent-hashing  # 一致性哈希，同一域名走同一节点

#!/bin/bash
# auto_switch.sh - 代理节点自动切换脚本
# 配合 systemd timer 或 cron 使用

# ============== 配置区域 ==============
PRIMARY_NODE="1.2.3.4"          # 主节点 IP
SECONDARY_NODE="5.6.7.8"        # 备用节点 IP
PRIMARY_LABEL="primary"         # 用于日志标识
SECONDARY_LABEL="secondary"
CHECK_URL="http://www.gstatic.com/generate_204"
MAX_LATENCY=500                 # 最大允许延迟（毫秒）
MAX_LOSS=5                       # 最大允许丢包率（%）
CHECK_COUNT=5                    # 每次检测的包数量
LOG_FILE="/var/log/proxy_switch.log"

# ============== 辅助函数 ==============
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

check_node() {
    local ip=$1
    local result

    # 检测丢包和延迟
    result=$(ping -c "$CHECK_COUNT" -q "$ip" 2>&1)
    local loss=$(echo "$result" | grep -oP '\d+(?=% packet loss)' || echo "100")
    local avg=$(echo "$result" | grep -oP 'rtt min/avg/max/mdev = [\d.]+/([\d.]+)' | grep -oP '(?<=avg/max/mdev = )[\d.]+' | head -1)

    # 转换延迟为整数（毫秒）
    if [ -n "$avg" ]; then
        avg_ms=$(printf "%.0f" "$avg" 2>/dev/null || echo "999")
    else
        avg_ms=999
    fi

    echo "$loss $avg_ms"
}

# ============== 主逻辑 ==============
log "========== 开始检测 =========="

# 检查主节点
read -r primary_loss primary_latency <<< "$(check_node "$PRIMARY_NODE")"
log "主节点 $PRIMARY_NODE: 丢包=${primary_loss}%, 延迟=${primary_latency}ms"

# 判断是否需要切换
should_switch=false

if [ "$primary_loss" -gt "$MAX_LOSS" ]; then
    log "⚠️ 主节点丢包率超过阈值 ($primary_loss% > $MAX_LOSS%)"
    should_switch=true
fi

if [ "$primary_latency" -gt "$MAX_LATENCY" ]; then
    log "⚠️ 主节点延迟超过阈值 (${primary_latency}ms > ${MAX_LATENCY}ms)"
    should_switch=true
fi

if [ "$primary_loss" -eq 100 ]; then
    log "🔴 主节点完全不可达"
    should_switch=true
fi

# 如果需要切换，检测备用节点
if [ "$should_switch" = true ]; then
    log "🔄 尝试切换到备用节点..."

    read -r secondary_loss secondary_latency <<< "$(check_node "$SECONDARY_NODE")"
    log "备用节点 $SECONDARY_NODE: 丢包=${secondary_loss}%, 延迟=${secondary_latency}ms"

    if [ "$secondary_loss" -lt "$MAX_LOSS" ] && [ "$secondary_latency" -lt "$MAX_LATENCY" ]; then
        log "✅ 备用节点可用，执行切换"

        # 调用切换脚本（根据你的实际情况修改）
        # 例如：更新 sing-box 配置、重启服务等
        # /opt/scripts/switch_to_backup.sh

        # 发送通知（可选）
        # curl -s -X POST "https://notify.example.com/send" \
        #   -d "text=主节点故障，已自动切换到备用节点"
    else
        log "❌ 备用节点也不可用，切换失败！"
        # 发送紧急告警
        # curl -s -X POST "https://notify.example.com/alert" \
        #   -d "text=所有节点均不可用！"
    fi
else
    log "✅ 主节点状态正常，无需切换"
fi

log "========== 检测完成 =========="

# 安装 Keepalived
apt install -y keepalived

# /etc/keepalived/keepalived.conf
global_defs {
    router_id proxy_vip
    vrrp_version 3
}

vrrp_instance v_proxy {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    nopreempt                    # 非抢占模式

    virtual_ipaddress {
        192.168.1.100/24       # 虚拟 IP（客户端连接此 IP）
    }

    track_script {
        check_proxy
    }

    notify_master /opt/scripts/become_master.sh
    notify_backup /opt/scripts/become_backup.sh
    notify_fault /opt/scripts/fault.sh
}

vrrp_script check_proxy {
    script "/opt/scripts/check_proxy.sh"
    interval 5
    fall 2
    rise 2
}

#!/bin/bash
# dns_health_check.sh - 更新 DNS 记录实现故障转移
# 需要域名托管在支持 API 更新的 DNS 服务商

DNS_API="https://api.cloudflare.com/client/v4"
AUTH_EMAIL="your@email.com"
AUTH_KEY="your_api_key"
ZONE_ID="your_zone_id"
RECORD_NAME="proxy.example.com"

# 检测节点状态
check_and_update() {
    local node_ip=$1
    local node_name=$2

    # 测试节点是否可达
    if ping -c 3 -W 5 "$node_ip" > /dev/null 2>&1; then
        echo "✅ $node_name ($node_ip) 在线"

        # 如果当前 DNS 指向该节点，无需更新
        current_ip=$(curl -s "$DNS_API/zones/$ZONE_ID/dns_records" \
            -H "X-Auth-Email: $AUTH_EMAIL" \
            -H "X-Auth-Key: $AUTH_KEY" \
            -H "Content-Type: application/json" \
            | jq -r ".result[] | select(.name == \"$RECORD_NAME\") | .content")

        if [ "$current_ip" != "$node_ip" ]; then
            # 更新 DNS 记录
            record_id=$(curl -s "$DNS_API/zones/$ZONE_ID/dns_records" \
                -H "X-Auth-Email: $AUTH_EMAIL" \
                -H "X-Auth-Key: $AUTH_KEY" \
                | jq -r ".result[] | select(.name == \"$RECORD_NAME\") | .id")

            curl -X PUT "$DNS_API/zones/$ZONE_ID/dns_records/$record_id" \
                -H "X-Auth-Email: $AUTH_EMAIL" \
                -H "X-Auth-Key: $AUTH_KEY" \
                -H "Content-Type: application/json" \
                -d "{\"type\":\"A\",\"name\":\"$RECORD_NAME\",\"content\":\"$node_ip\"}"

            echo "🔄 DNS 已更新: $RECORD_NAME -> $node_ip"
        fi
    else
        echo "❌ $node_name ($node_ip) 不通"
    fi
}

# 检查所有节点，按顺序选择第一个可用的
NODES=(
    "1.2.3.4|节点1"
    "5.6.7.8|节点2"
    "9.10.11.12|节点3"
)

for entry in "${NODES[@]}"; do
    IFS='|' read -r ip name <<< "$entry"
    check_and_update "$ip" "$name" && break
done

# /etc/prometheus/rules/proxy_alerts.yml
groups:
  - name: proxy_alerts
    rules:
      # 节点离线告警
      - alert: ProxyNodeDown
        expr: proxy_node_up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "代理节点 {{ $labels.name }} ({{ $labels.ip }}) 离线"
          description: "节点已离线超过 2 分钟，当前状态: {{ $value }}"

      # 延迟过高告警
      - alert: ProxyNodeHighLatency
        expr: proxy_node_latency_ms > 300
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "代理节点 {{ $labels.name }} 延迟过高"
          description: "节点延迟持续超过 300ms，当前: {{ $value }}ms"

      # 丢包告警
      - alert: ProxyNodePacketLoss
        expr: rate(proxy_node_packets_lost_total[5m]) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "代理节点 {{ $labels.name }} 存在丢包"
          description: "丢包率: {{ $value | humanizePercentage }}"

# /etc/alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 12h
  receiver: 'default-receiver'

receivers:
  - name: 'default-receiver'
    # 邮件通知
    email_configs:
      - to: your@email.com
        send_resolved: true
        smarthost: smtp.gmail.com:587
        auth_username: your@email.com
        auth_password: your_app_password

    # 钉钉通知（需要 webhook）
    webhook_configs:
      - url: http://your-dingtalk-webhook/api/send

    # 企业微信通知
    webhook_configs:
      - url: http://your-wecom-webhook/api/send

抑制规则:
inhibit_rules:
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['alertname', 'instance']

#!/bin/bash
# daily_health_check.sh - 每日代理节点巡检
# 建议配合 cron 每天早上执行：0 9 * * * /opt/scripts/daily_health_check.sh

LOG_DIR="/var/log/health_check"
REPORT_FILE="$LOG_DIR/report_$(date +%Y%m%d).log"
ALERT_EMAIL="your@email.com"

mkdir -p "$LOG_DIR"

log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$REPORT_FILE"
}

log "========== 每日巡检报告 $(date +'%Y-%m-%d') =========="

# 1. 服务状态检查
log "【1. 服务状态】"
for service in xray sing-box hysteria-server; do
    if systemctl is-active --quiet "$service" 2>/dev/null; then
        log "  ✅ $service: 运行中"
    else
        log "  ❌ $service: 未运行"
    fi
done

# 2. 端口监听检查
log "【2. 端口监听】"
for port in 443 80 8080; do
    if ss -tulnp | grep -q ":$port "; then
        log "  ✅ 端口 $port: 监听中"
    else
        log "  ⚠️  端口 $port: 未监听"
    fi
done

# 3. 证书有效期检查
log "【3. TLS 证书】"
cert_file="/etc/ssl/server.crt"
if [ -f "$cert_file" ]; then
    expiry_date=$(openssl x509 -in "$cert_file" -noout -enddate | cut -d= -f2)
    expiry_epoch=$(date -d "$expiry_date" +%s)
    now_epoch=$(date +%s)
    days_left=$(( ($expiry_epoch - $now_epoch) / 86400 ))

    if [ $days_left -lt 30 ]; then
        log "  🔴 证书将在 $days_left 天后过期！需要续期！"
    elif [ $days_left -lt 90 ]; then
        log "  ⚠️  证书将在 $days_left 天后过期，建议近期续期"
    else
        log "  ✅ 证书有效期: $expiry_date ($days_left 天)"
    fi
fi

# 4. 资源使用率检查
log "【4. 资源使用】"
cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | sed 's/%us,//')
mem_usage=$(free | grep Mem | awk '{printf "%.1f", $3/$2 * 100}')
disk_usage=$(df -h / | tail -1 | awk '{print $5}' | sed 's/%//')

log "  CPU: ${cpu_usage}%, 内存: ${mem_usage}%, 磁盘: ${disk_usage}%"

if (( $(echo "$cpu_usage > 80" | bc -l) )); then
    log "  ⚠️  CPU 使用率过高"
fi

if (( $(echo "$mem_usage > 90" | bc -l) )); then
    log "  ⚠️  内存使用率过高"
fi

if [ "$disk_usage" -gt 85 ]; then
    log "  ⚠️  磁盘使用率超过 85%"
fi

# 5. 带宽使用检查（如果有 vnstat）
log "【5. 带宽使用】"
if command -v vnstat &> /dev/null; then
    today=$(vnstat | grep "today" | awk '{print $5, $6}')
    log "  今日流量: $today"
fi

# 6. 日志错误检查
log "【6. 最近错误】"
journalctl -p err -n 10 --no-pager >> "$REPORT_FILE" 2>&1

# 7. 生成总结
log ""
log "========== 巡检完成 =========="

# 发送报告邮件（如果有异常）
if grep -q "❌\|🔴\|⚠️" "$REPORT_FILE"; then
    log "⚠️  检测到异常，发送告警..."
    mail -s "[代理监控] 每日巡检异常报告 $(date +%Y%m%d)" "$ALERT_EMAIL" < "$REPORT_FILE"
fi

# /etc/systemd/system/daily_health_check.service
[Unit]
Description=每日代理节点巡检
After=network.target

[Service]
Type=oneshot
ExecStart=/opt/scripts/daily_health_check.sh
User=root

[Install]
WantedBy=multi-user.target

# /etc/systemd/system/daily_health_check.timer
[Unit]
Description=每日代理节点巡检定时器

[Timer]
OnCalendar=*-*-* 09:00:00
Persistent=true

[Install]
WantedBy=timers.target

systemctl enable --now daily_health_check.timer
systemctl list-timers --all | grep daily_health_check