- 三个新事件（默认关闭）：公网服务端掉线、公网 agent 不可达、配置漂移；连续失败阈值与漂移提醒间隔均可配，达标与恢复各报一次 - 避免误报：只在「用户要服务端运行」时判定掉线；ConnectionState=unknown 按健康处理；没配 agent 的服务端不参与可达性判定 - internal/monitor 改 Source 接口（Routes/Servers/Agents/Drift），frp 适配在 internal/app/monitor_source.go；「连续 N 次」抽成共用 failureTransition，通知一律在锁外调用 - 通知图标分类补齐；通知方式标签改「Webhook / IM 机器人」；新增 6 个设置项；补 serverDown/serverLabel/stateOr 测试 - agent 无改动，AgentVersion 保持 0.12.2
This commit is contained in:
+221
-29
@@ -1,4 +1,5 @@
|
||||
// Package monitor 周期巡检「资源占用」与「隧道连通性」这两类必须持续观察才会出现的问题。
|
||||
// Package monitor 周期巡检几类必须持续观察才会出现的问题：主机资源占用、隧道连通性、
|
||||
// 公网服务端掉线、agent 不可达、配置漂移。
|
||||
// 事件型通知（DDNS 失败、证书到期）由各自模块在发生时直接发出，不需要巡检。
|
||||
//
|
||||
// 判定在这里，通知只走 notify.Service：开关、阈值、冷却都读设置项，
|
||||
@@ -18,27 +19,63 @@ import (
|
||||
"github.com/fonu/fonu/internal/sysinfo"
|
||||
)
|
||||
|
||||
// RouteStatus 是巡检关心的隧道状态；由调用方注入（避免 monitor 依赖 frp 包）。
|
||||
// RouteStatus 是巡检关心的隧道状态。
|
||||
type RouteStatus struct {
|
||||
Domain string
|
||||
TunnelOK bool
|
||||
Detail string
|
||||
}
|
||||
|
||||
// ServerStatus 是巡检关心的公网服务端状态：只有 DesiredState 表示「用户要它运行」时才判定掉线。
|
||||
type ServerStatus struct {
|
||||
ID int64
|
||||
Name string
|
||||
DesiredState string
|
||||
ProcessState string
|
||||
ConnectionState string
|
||||
Detail string
|
||||
}
|
||||
|
||||
// AgentStatus 是某台服务端上 fonu-agent 的可达性；没配 agent 的服务端不会出现在这里。
|
||||
type AgentStatus struct {
|
||||
ID int64
|
||||
Name string
|
||||
Reachable bool
|
||||
Detail string
|
||||
}
|
||||
|
||||
// DriftStatus 是某台服务端的部署漂移摘要，Count 为 0 表示没有漂移。
|
||||
type DriftStatus struct {
|
||||
ID int64
|
||||
Name string
|
||||
Count int
|
||||
Items []string
|
||||
}
|
||||
|
||||
// Source 提供巡检需要的外部状态；由 app 侧把 frp 服务适配过来，monitor 因此不依赖 frp 包。
|
||||
type Source interface {
|
||||
Routes(ctx context.Context) ([]RouteStatus, error)
|
||||
Servers(ctx context.Context) ([]ServerStatus, error)
|
||||
Agents(ctx context.Context) ([]AgentStatus, error)
|
||||
Drift(ctx context.Context) ([]DriftStatus, error)
|
||||
}
|
||||
|
||||
// Checker 每轮都重新读设置项：改阈值或开关不需要重启。
|
||||
type Checker struct {
|
||||
settings *settings.Store
|
||||
notify *notify.Service
|
||||
logger *slog.Logger
|
||||
routes func(ctx context.Context) ([]RouteStatus, error)
|
||||
source Source
|
||||
|
||||
mu sync.Mutex
|
||||
cpuPrev sysinfo.CPUSample
|
||||
resourceAlerts map[string]time.Time
|
||||
tunnelFailures map[string]int
|
||||
failures map[string]int
|
||||
driftAlerted map[int64]bool
|
||||
driftNotified map[int64]time.Time
|
||||
}
|
||||
|
||||
func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.Logger, routes func(context.Context) ([]RouteStatus, error)) *Checker {
|
||||
func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.Logger, source Source) *Checker {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
@@ -46,9 +83,11 @@ func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.
|
||||
settings: settingsStore,
|
||||
notify: notifySvc,
|
||||
logger: logger,
|
||||
routes: routes,
|
||||
source: source,
|
||||
resourceAlerts: map[string]time.Time{},
|
||||
tunnelFailures: map[string]int{},
|
||||
failures: map[string]int{},
|
||||
driftAlerted: map[int64]bool{},
|
||||
driftNotified: map[int64]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,7 +97,10 @@ func (c *Checker) Run(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
c.checkResources(ctx)
|
||||
c.checkTunnels(ctx)
|
||||
c.checkRoutes(ctx)
|
||||
c.checkServers(ctx)
|
||||
c.checkAgents(ctx)
|
||||
c.checkDrift(ctx)
|
||||
}
|
||||
|
||||
func (c *Checker) checkResources(ctx context.Context) {
|
||||
@@ -118,8 +160,8 @@ func (c *Checker) allowResourceAlert(metric string, now time.Time) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Checker) checkTunnels(ctx context.Context) {
|
||||
if c.routes == nil {
|
||||
func (c *Checker) checkRoutes(ctx context.Context) {
|
||||
if c.source == nil {
|
||||
return
|
||||
}
|
||||
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnTunnelDown); !enabled {
|
||||
@@ -130,7 +172,7 @@ func (c *Checker) checkTunnels(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
routes, err := c.routes(ctx)
|
||||
routes, err := c.source.Routes(ctx)
|
||||
if err != nil {
|
||||
c.logger.Warn("巡检隧道状态失败", "module", "MONITOR", "error", err.Error())
|
||||
return
|
||||
@@ -139,44 +181,194 @@ func (c *Checker) checkTunnels(ctx context.Context) {
|
||||
if strings.TrimSpace(route.Domain) == "" {
|
||||
continue
|
||||
}
|
||||
c.mu.Lock()
|
||||
failures, event := tunnelTransition(c.tunnelFailures[route.Domain], route.TunnelOK, limit)
|
||||
c.tunnelFailures[route.Domain] = failures
|
||||
c.mu.Unlock()
|
||||
|
||||
_, event := c.track("route:"+route.Domain, route.TunnelOK, limit)
|
||||
switch event {
|
||||
case tunnelEventDown:
|
||||
case failureEventDown:
|
||||
c.notify.Alert(ctx, notify.EventTunnelDown, "隧道连续不通",
|
||||
fmt.Sprintf("%s 连续 %d 次巡检未在 frps 上注册：%s", route.Domain, limit, detailOr(route.Detail)))
|
||||
case tunnelEventRecovered:
|
||||
case failureEventRecovered:
|
||||
c.notify.Alert(ctx, notify.EventTunnelDown, "隧道已恢复",
|
||||
fmt.Sprintf("%s 已重新在 frps 上注册", route.Domain))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type tunnelEvent string
|
||||
// checkServers 公网服务端掉线：进程没在跑，或隧道客户端连不上 frps。
|
||||
// 只对「用户要它运行」的服务端判定，手动停掉的不报。
|
||||
func (c *Checker) checkServers(ctx context.Context) {
|
||||
if c.source == nil {
|
||||
return
|
||||
}
|
||||
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnFrpsDown); !enabled {
|
||||
return
|
||||
}
|
||||
limit, _ := c.settings.GetInt(ctx, settings.KeyNotifyFrpsFailThreshold)
|
||||
if limit <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
servers, err := c.source.Servers(ctx)
|
||||
if err != nil {
|
||||
c.logger.Warn("巡检服务端状态失败", "module", "MONITOR", "error", err.Error())
|
||||
return
|
||||
}
|
||||
for _, server := range servers {
|
||||
label := serverLabel(server.ID, server.Name)
|
||||
_, event := c.track(fmt.Sprintf("server:%d", server.ID), !serverDown(server), limit)
|
||||
switch event {
|
||||
case failureEventDown:
|
||||
c.notify.Alert(ctx, notify.EventFrpsServerDown, "公网服务端掉线",
|
||||
fmt.Sprintf("%s 连续 %d 次巡检未运行或未连接（进程 %s / 连接 %s）：%s",
|
||||
label, limit, stateOr(server.ProcessState), stateOr(server.ConnectionState), detailOr(server.Detail)))
|
||||
case failureEventRecovered:
|
||||
c.notify.Alert(ctx, notify.EventFrpsServerDown, "公网服务端已恢复",
|
||||
fmt.Sprintf("%s 已重新运行并连接", label))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkAgents agent 不可达：VPS 重启后 agent 没起来是最常见的成因。
|
||||
func (c *Checker) checkAgents(ctx context.Context) {
|
||||
if c.source == nil {
|
||||
return
|
||||
}
|
||||
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnAgentUnreachable); !enabled {
|
||||
return
|
||||
}
|
||||
limit, _ := c.settings.GetInt(ctx, settings.KeyNotifyAgentFailThreshold)
|
||||
if limit <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
agents, err := c.source.Agents(ctx)
|
||||
if err != nil {
|
||||
c.logger.Warn("巡检 agent 可达性失败", "module", "MONITOR", "error", err.Error())
|
||||
return
|
||||
}
|
||||
for _, agent := range agents {
|
||||
label := serverLabel(agent.ID, agent.Name)
|
||||
_, event := c.track(fmt.Sprintf("agent:%d", agent.ID), agent.Reachable, limit)
|
||||
switch event {
|
||||
case failureEventDown:
|
||||
c.notify.Alert(ctx, notify.EventAgentUnreachable, "公网 agent 不可达",
|
||||
fmt.Sprintf("%s 的 fonu-agent 连续 %d 次无响应：%s", label, limit, detailOr(agent.Detail)))
|
||||
case failureEventRecovered:
|
||||
c.notify.Alert(ctx, notify.EventAgentUnreachable, "公网 agent 已恢复",
|
||||
fmt.Sprintf("%s 的 fonu-agent 已恢复响应", label))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkDrift 配置漂移：VPS 上的 vhost 与 Fonu 记录不一致（被手工改过、或 agent 重装后没重部署）。
|
||||
// 它不是「连续 N 次」型故障，因此用冷却时间限流：首次发现即报，之后每 cooldown 提醒一次，漂移清空时补一条恢复。
|
||||
func (c *Checker) checkDrift(ctx context.Context) {
|
||||
if c.source == nil {
|
||||
return
|
||||
}
|
||||
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnRouteDrift); !enabled {
|
||||
return
|
||||
}
|
||||
cooldownHours, _ := c.settings.GetInt(ctx, settings.KeyNotifyDriftCooldownHours)
|
||||
if cooldownHours <= 0 {
|
||||
cooldownHours = notify.DefaultDriftCooldownHours
|
||||
}
|
||||
cooldown := time.Duration(cooldownHours) * time.Hour
|
||||
|
||||
reports, err := c.source.Drift(ctx)
|
||||
if err != nil {
|
||||
c.logger.Warn("巡检配置漂移失败", "module", "MONITOR", "error", err.Error())
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
for _, report := range reports {
|
||||
c.mu.Lock()
|
||||
alerted := c.driftAlerted[report.ID]
|
||||
last := c.driftNotified[report.ID]
|
||||
if report.Count == 0 {
|
||||
delete(c.driftAlerted, report.ID)
|
||||
delete(c.driftNotified, report.ID)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
label := serverLabel(report.ID, report.Name)
|
||||
if report.Count == 0 {
|
||||
if alerted {
|
||||
c.notify.Alert(ctx, notify.EventRouteDrift, "配置漂移已消除",
|
||||
fmt.Sprintf("%s 上的 vhost 已与 Fonu 记录一致", label))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if alerted && !last.IsZero() && now.Sub(last) < cooldown {
|
||||
continue
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.driftAlerted[report.ID] = true
|
||||
c.driftNotified[report.ID] = now
|
||||
c.mu.Unlock()
|
||||
c.notify.Alert(ctx, notify.EventRouteDrift, "公网反代配置漂移",
|
||||
fmt.Sprintf("%s 有 %d 项与 Fonu 记录不一致：%s", label, report.Count, strings.Join(report.Items, "、")))
|
||||
}
|
||||
}
|
||||
|
||||
// track 推进某个实体的连续失败计数，返回本轮要发的事件。
|
||||
// 锁只包住 map 读写：notify 一律在锁外调用。
|
||||
func (c *Checker) track(key string, healthy bool, limit int) (int, failureEvent) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
failures, event := failureTransition(c.failures[key], healthy, limit)
|
||||
c.failures[key] = failures
|
||||
return failures, event
|
||||
}
|
||||
|
||||
type failureEvent string
|
||||
|
||||
const (
|
||||
tunnelEventNone tunnelEvent = ""
|
||||
tunnelEventDown tunnelEvent = "down"
|
||||
tunnelEventRecovered tunnelEvent = "recovered"
|
||||
failureEventNone failureEvent = ""
|
||||
failureEventDown failureEvent = "down"
|
||||
failureEventRecovered failureEvent = "recovered"
|
||||
)
|
||||
|
||||
// tunnelTransition 推进某个域名的连续失败计数，返回新计数与本轮要发的事件。
|
||||
// failureTransition 推进某个实体的连续失败计数，返回新计数与本轮要发的事件。
|
||||
// 只在「刚好达到阈值」与「从失败中恢复」两个时刻发，避免每轮巡检都刷通知。
|
||||
func tunnelTransition(previous int, tunnelOK bool, limit int) (int, tunnelEvent) {
|
||||
if tunnelOK {
|
||||
func failureTransition(previous int, healthy bool, limit int) (int, failureEvent) {
|
||||
if healthy {
|
||||
if previous >= limit {
|
||||
return 0, tunnelEventRecovered
|
||||
return 0, failureEventRecovered
|
||||
}
|
||||
return 0, tunnelEventNone
|
||||
return 0, failureEventNone
|
||||
}
|
||||
next := previous + 1
|
||||
if next == limit {
|
||||
return next, tunnelEventDown
|
||||
return next, failureEventDown
|
||||
}
|
||||
return next, tunnelEventNone
|
||||
return next, failureEventNone
|
||||
}
|
||||
|
||||
// serverDown 判定服务端是否处于「用户要它运行、但它没在跑或没连上」。
|
||||
// ConnectionState 为 unknown 时按健康处理：状态源（manager+log）偶尔读不到不代表掉线，避免误报。
|
||||
func serverDown(status ServerStatus) bool {
|
||||
if status.DesiredState != "running" && status.DesiredState != "starting" {
|
||||
return false
|
||||
}
|
||||
if status.ProcessState == "stopped" || status.ProcessState == "error" {
|
||||
return true
|
||||
}
|
||||
return status.ConnectionState == "disconnected" || status.ConnectionState == "error"
|
||||
}
|
||||
|
||||
func serverLabel(id int64, name string) string {
|
||||
if trimmed := strings.TrimSpace(name); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
return fmt.Sprintf("服务端 #%d", id)
|
||||
}
|
||||
|
||||
// stateOr 让通知里的状态字段不会因为空串而变成「（进程 / 连接 ）」。
|
||||
func stateOr(state string) string {
|
||||
if strings.TrimSpace(state) == "" {
|
||||
return "未知"
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func percent(used, total uint64) float64 {
|
||||
|
||||
Reference in New Issue
Block a user