mrlee/Havline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(notify): 告警补服务端掉线/Agent 不可达/配置漂移，巡检改 Source 接口（v1.2.10）

Browse Source 
- 三个新事件（默认关闭）：公网服务端掉线、公网 agent 不可达、配置漂移；连续失败阈值与漂移提醒间隔均可配，达标与恢复各报一次

- 避免误报：只在「用户要服务端运行」时判定掉线；ConnectionState=unknown 按健康处理；没配 agent 的服务端不参与可达性判定

- internal/monitor 改 Source 接口（Routes/Servers/Agents/Drift），frp 适配在 internal/app/monitor_source.go；「连续 N 次」抽成共用 failureTransition，通知一律在锁外调用

- 通知图标分类补齐；通知方式标签改「Webhook / IM 机器人」；新增 6 个设置项；补 serverDown/serverLabel/stateOr 测试

- agent 无改动，AgentVersion 保持 0.12.2
This commit is contained in:
mrlee
2026-09-25 09:54:48 +08:00
parent 862442e77f
commit c8264bf280
17 changed files with 606 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/monitor/monitor.go
+221 -29
View File
@@ -1,4 +1,5 @@
// Package monitor 周期巡检「资源占用」与「隧道连通性」这两类必须持续观察才会出现的问题。
// Package monitor 周期巡检几类必须持续观察才会出现的问题：主机资源占用、隧道连通性、
// 公网服务端掉线、agent 不可达、配置漂移。
// 事件型通知（DDNS 失败、证书到期）由各自模块在发生时直接发出，不需要巡检。
//
// 判定在这里，通知只走 notify.Service：开关、阈值、冷却都读设置项，
@@ -18,27 +19,63 @@ import (
"github.com/fonu/fonu/internal/sysinfo"
)
// RouteStatus 是巡检关心的隧道状态；由调用方注入（避免 monitor 依赖 frp 包）。
// RouteStatus 是巡检关心的隧道状态。
type RouteStatus struct {
Domain string
TunnelOK bool
Detail string
}
// ServerStatus 是巡检关心的公网服务端状态：只有 DesiredState 表示「用户要它运行」时才判定掉线。
type ServerStatus struct {
ID int64
Name string
DesiredState string
ProcessState string
ConnectionState string
Detail string
}
// AgentStatus 是某台服务端上 fonu-agent 的可达性；没配 agent 的服务端不会出现在这里。
type AgentStatus struct {
ID int64
Name string
Reachable bool
Detail string
}
// DriftStatus 是某台服务端的部署漂移摘要，Count 为 0 表示没有漂移。
type DriftStatus struct {
ID int64
Name string
Count int
Items []string
}
// Source 提供巡检需要的外部状态；由 app 侧把 frp 服务适配过来，monitor 因此不依赖 frp 包。
type Source interface {
Routes(ctx context.Context) ([]RouteStatus, error)
Servers(ctx context.Context) ([]ServerStatus, error)
Agents(ctx context.Context) ([]AgentStatus, error)
Drift(ctx context.Context) ([]DriftStatus, error)
}
// Checker 每轮都重新读设置项：改阈值或开关不需要重启。
type Checker struct {
settings *settings.Store
notify *notify.Service
logger *slog.Logger
routes func(ctx context.Context) ([]RouteStatus, error)
source Source
mu sync.Mutex
cpuPrev sysinfo.CPUSample
resourceAlerts map[string]time.Time
tunnelFailures map[string]int
failures map[string]int
driftAlerted map[int64]bool
driftNotified map[int64]time.Time
}
func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.Logger, routes func(context.Context) ([]RouteStatus, error)) *Checker {
func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.Logger, source Source) *Checker {
if logger == nil {
logger = slog.Default()
}
@@ -46,9 +83,11 @@ func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.
settings: settingsStore,
notify: notifySvc,
logger: logger,
routes: routes,
source: source,
resourceAlerts: map[string]time.Time{},
tunnelFailures: map[string]int{},
failures: map[string]int{},
driftAlerted: map[int64]bool{},
driftNotified: map[int64]time.Time{},
}
}
@@ -58,7 +97,10 @@ func (c *Checker) Run(ctx context.Context) {
return
}
c.checkResources(ctx)
c.checkTunnels(ctx)
c.checkRoutes(ctx)
c.checkServers(ctx)
c.checkAgents(ctx)
c.checkDrift(ctx)
}
func (c *Checker) checkResources(ctx context.Context) {
@@ -118,8 +160,8 @@ func (c *Checker) allowResourceAlert(metric string, now time.Time) bool {
return true
}
func (c *Checker) checkTunnels(ctx context.Context) {
if c.routes == nil {
func (c *Checker) checkRoutes(ctx context.Context) {
if c.source == nil {
return
}
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnTunnelDown); !enabled {
@@ -130,7 +172,7 @@ func (c *Checker) checkTunnels(ctx context.Context) {
return
}
routes, err := c.routes(ctx)
routes, err := c.source.Routes(ctx)
if err != nil {
c.logger.Warn("巡检隧道状态失败", "module", "MONITOR", "error", err.Error())
return
@@ -139,44 +181,194 @@ func (c *Checker) checkTunnels(ctx context.Context) {
if strings.TrimSpace(route.Domain) == "" {
continue
}
c.mu.Lock()
failures, event := tunnelTransition(c.tunnelFailures[route.Domain], route.TunnelOK, limit)
c.tunnelFailures[route.Domain] = failures
c.mu.Unlock()
_, event := c.track("route:"+route.Domain, route.TunnelOK, limit)
switch event {
case tunnelEventDown:
case failureEventDown:
c.notify.Alert(ctx, notify.EventTunnelDown, "隧道连续不通",
fmt.Sprintf("%s 连续 %d 次巡检未在 frps 上注册：%s", route.Domain, limit, detailOr(route.Detail)))
case tunnelEventRecovered:
case failureEventRecovered:
c.notify.Alert(ctx, notify.EventTunnelDown, "隧道已恢复",
fmt.Sprintf("%s 已重新在 frps 上注册", route.Domain))
}
}
}
type tunnelEvent string
// checkServers 公网服务端掉线：进程没在跑，或隧道客户端连不上 frps。
// 只对「用户要它运行」的服务端判定，手动停掉的不报。
func (c *Checker) checkServers(ctx context.Context) {
if c.source == nil {
return
}
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnFrpsDown); !enabled {
return
}
limit, _ := c.settings.GetInt(ctx, settings.KeyNotifyFrpsFailThreshold)
if limit <= 0 {
return
}
servers, err := c.source.Servers(ctx)
if err != nil {
c.logger.Warn("巡检服务端状态失败", "module", "MONITOR", "error", err.Error())
return
}
for _, server := range servers {
label := serverLabel(server.ID, server.Name)
_, event := c.track(fmt.Sprintf("server:%d", server.ID), !serverDown(server), limit)
switch event {
case failureEventDown:
c.notify.Alert(ctx, notify.EventFrpsServerDown, "公网服务端掉线",
fmt.Sprintf("%s 连续 %d 次巡检未运行或未连接（进程 %s / 连接 %s）：%s",
label, limit, stateOr(server.ProcessState), stateOr(server.ConnectionState), detailOr(server.Detail)))
case failureEventRecovered:
c.notify.Alert(ctx, notify.EventFrpsServerDown, "公网服务端已恢复",
fmt.Sprintf("%s 已重新运行并连接", label))
}
}
}
// checkAgents agent 不可达：VPS 重启后 agent 没起来是最常见的成因。
func (c *Checker) checkAgents(ctx context.Context) {
if c.source == nil {
return
}
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnAgentUnreachable); !enabled {
return
}
limit, _ := c.settings.GetInt(ctx, settings.KeyNotifyAgentFailThreshold)
if limit <= 0 {
return
}
agents, err := c.source.Agents(ctx)
if err != nil {
c.logger.Warn("巡检 agent 可达性失败", "module", "MONITOR", "error", err.Error())
return
}
for _, agent := range agents {
label := serverLabel(agent.ID, agent.Name)
_, event := c.track(fmt.Sprintf("agent:%d", agent.ID), agent.Reachable, limit)
switch event {
case failureEventDown:
c.notify.Alert(ctx, notify.EventAgentUnreachable, "公网 agent 不可达",
fmt.Sprintf("%s 的 fonu-agent 连续 %d 次无响应：%s", label, limit, detailOr(agent.Detail)))
case failureEventRecovered:
c.notify.Alert(ctx, notify.EventAgentUnreachable, "公网 agent 已恢复",
fmt.Sprintf("%s 的 fonu-agent 已恢复响应", label))
}
}
}
// checkDrift 配置漂移：VPS 上的 vhost 与 Fonu 记录不一致（被手工改过、或 agent 重装后没重部署）。
// 它不是「连续 N 次」型故障，因此用冷却时间限流：首次发现即报，之后每 cooldown 提醒一次，漂移清空时补一条恢复。
func (c *Checker) checkDrift(ctx context.Context) {
if c.source == nil {
return
}
if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnRouteDrift); !enabled {
return
}
cooldownHours, _ := c.settings.GetInt(ctx, settings.KeyNotifyDriftCooldownHours)
if cooldownHours <= 0 {
cooldownHours = notify.DefaultDriftCooldownHours
}
cooldown := time.Duration(cooldownHours) * time.Hour
reports, err := c.source.Drift(ctx)
if err != nil {
c.logger.Warn("巡检配置漂移失败", "module", "MONITOR", "error", err.Error())
return
}
now := time.Now()
for _, report := range reports {
c.mu.Lock()
alerted := c.driftAlerted[report.ID]
last := c.driftNotified[report.ID]
if report.Count == 0 {
delete(c.driftAlerted, report.ID)
delete(c.driftNotified, report.ID)
}
c.mu.Unlock()
label := serverLabel(report.ID, report.Name)
if report.Count == 0 {
if alerted {
c.notify.Alert(ctx, notify.EventRouteDrift, "配置漂移已消除",
fmt.Sprintf("%s 上的 vhost 已与 Fonu 记录一致", label))
}
continue
}
if alerted && !last.IsZero() && now.Sub(last) < cooldown {
continue
}
c.mu.Lock()
c.driftAlerted[report.ID] = true
c.driftNotified[report.ID] = now
c.mu.Unlock()
c.notify.Alert(ctx, notify.EventRouteDrift, "公网反代配置漂移",
fmt.Sprintf("%s 有 %d 项与 Fonu 记录不一致：%s", label, report.Count, strings.Join(report.Items, "、")))
}
}
// track 推进某个实体的连续失败计数，返回本轮要发的事件。
// 锁只包住 map 读写：notify 一律在锁外调用。
func (c *Checker) track(key string, healthy bool, limit int) (int, failureEvent) {
c.mu.Lock()
defer c.mu.Unlock()
failures, event := failureTransition(c.failures[key], healthy, limit)
c.failures[key] = failures
return failures, event
}
type failureEvent string
const (
tunnelEventNone tunnelEvent = ""
tunnelEventDown tunnelEvent = "down"
tunnelEventRecovered tunnelEvent = "recovered"
failureEventNone failureEvent = ""
failureEventDown failureEvent = "down"
failureEventRecovered failureEvent = "recovered"
)
// tunnelTransition 推进某个域名的连续失败计数，返回新计数与本轮要发的事件。
// failureTransition 推进某个实体的连续失败计数，返回新计数与本轮要发的事件。
// 只在「刚好达到阈值」与「从失败中恢复」两个时刻发，避免每轮巡检都刷通知。
func tunnelTransition(previous int, tunnelOK bool, limit int) (int, tunnelEvent) {
if tunnelOK {
func failureTransition(previous int, healthy bool, limit int) (int, failureEvent) {
if healthy {
if previous >= limit {
return 0, tunnelEventRecovered
return 0, failureEventRecovered
}
return 0, tunnelEventNone
return 0, failureEventNone
}
next := previous + 1
if next == limit {
return next, tunnelEventDown
return next, failureEventDown
}
return next, tunnelEventNone
return next, failureEventNone
}
// serverDown 判定服务端是否处于「用户要它运行、但它没在跑或没连上」。
// ConnectionState 为 unknown 时按健康处理：状态源（manager+log）偶尔读不到不代表掉线，避免误报。
func serverDown(status ServerStatus) bool {
if status.DesiredState != "running" && status.DesiredState != "starting" {
return false
}
if status.ProcessState == "stopped" || status.ProcessState == "error" {
return true
}
return status.ConnectionState == "disconnected" || status.ConnectionState == "error"
}
func serverLabel(id int64, name string) string {
if trimmed := strings.TrimSpace(name); trimmed != "" {
return trimmed
}
return fmt.Sprintf("服务端 #%d", id)
}
// stateOr 让通知里的状态字段不会因为空串而变成「（进程 / 连接 ）」。
func stateOr(state string) string {
if strings.TrimSpace(state) == "" {
return "未知"
}
return state
}
func percent(used, total uint64) float64 {
internal/monitor/monitor_test.go
+54 -16
View File
@@ -2,20 +2,20 @@ package monitor
import "testing"
func TestTunnelTransitionAlertsOncePerOutage(t *testing.T) {
func TestFailureTransitionAlertsOncePerOutage(t *testing.T) {
const limit = 3
failures := 0
var events []tunnelEvent
var events []failureEvent
// 连续三轮不通：只在第 3 轮（刚好达到阈值）发一次
for i := 0; i < 5; i++ {
var event tunnelEvent
failures, event = tunnelTransition(failures, false, limit)
if event != tunnelEventNone {
var event failureEvent
failures, event = failureTransition(failures, false, limit)
if event != failureEventNone {
events = append(events, event)
}
}
if len(events) != 1 || events[0] != tunnelEventDown {
if len(events) != 1 || events[0] != failureEventDown {
t.Fatalf("连续不通应只告警一次，实际 %v", events)
}
if failures != 5 {
@@ -23,31 +23,31 @@ func TestTunnelTransitionAlertsOncePerOutage(t *testing.T) {
}
// 恢复：发一次恢复通知并清零
failures, event := tunnelTransition(failures, true, limit)
if event != tunnelEventRecovered || failures != 0 {
failures, event := failureTransition(failures, true, limit)
if event != failureEventRecovered || failures != 0 {
t.Fatalf("恢复应通知一次并清零：event=%v failures=%d", event, failures)
}
// 恢复后再次不通：还能再报一次
failures, event = tunnelTransition(failures, false, limit)
if event != tunnelEventNone || failures != 1 {
failures, event = failureTransition(failures, false, limit)
if event != failureEventNone || failures != 1 {
t.Fatalf("恢复后重新计数：event=%v failures=%d", event, failures)
}
failures, event = tunnelTransition(failures, false, limit)
if event != tunnelEventNone {
failures, event = failureTransition(failures, false, limit)
if event != failureEventNone {
t.Fatalf("未达阈值不应告警：%v", event)
}
if _, event = tunnelTransition(failures, false, limit); event != tunnelEventDown {
if _, event = failureTransition(failures, false, limit); event != failureEventDown {
t.Fatalf("第二次中断达到阈值应再告警一次：%v", event)
}
}
func TestTunnelTransitionBelowThresholdIsSilent(t *testing.T) {
func TestFailureTransitionBelowThresholdIsSilent(t *testing.T) {
// 单次抖动（未达阈值）不打扰用户，也不算作恢复
if failures, event := tunnelTransition(0, true, 3); failures != 0 || event != tunnelEventNone {
if failures, event := failureTransition(0, true, 3); failures != 0 || event != failureEventNone {
t.Fatalf("健康状态不应产生事件：%d %v", failures, event)
}
if failures, event := tunnelTransition(1, true, 3); failures != 0 || event != tunnelEventNone {
if failures, event := failureTransition(1, true, 3); failures != 0 || event != failureEventNone {
t.Fatalf("未达阈值的恢复不应通知：%d %v", failures, event)
}
}
@@ -69,3 +69,41 @@ func TestDetailOrFallsBack(t *testing.T) {
t.Fatalf("非空详情应原样返回，实际 %q", got)
}
}
func TestServerDownOnlyWhenUserWantsItRunning(t *testing.T) {
// 用户手动停掉的不算掉线
stopped := ServerStatus{DesiredState: "stopped", ProcessState: "stopped", ConnectionState: "disconnected"}
if serverDown(stopped) {
t.Fatal("用户手动停止的服务端不应判为掉线")
}
// 期望运行但进程没在跑
if !serverDown(ServerStatus{DesiredState: "running", ProcessState: "stopped"}) {
t.Fatal("期望运行但进程未运行应判为掉线")
}
// 进程在跑但连不上 frps
if !serverDown(ServerStatus{DesiredState: "running", ProcessState: "running", ConnectionState: "disconnected"}) {
t.Fatal("连接断开应判为掉线")
}
// 状态源读不到连接状态时不报警：manager+log 偶尔读不到不代表掉线
if serverDown(ServerStatus{DesiredState: "running", ProcessState: "running", ConnectionState: "unknown"}) {
t.Fatal("连接状态未知时不应误报")
}
}
func TestServerLabelFallsBackToID(t *testing.T) {
if got := serverLabel(3, " "); got != "服务端 #3" {
t.Fatalf("无名服务端应回落到编号，实际 %q", got)
}
if got := serverLabel(3, "VPS-香港"); got != "VPS-香港" {
t.Fatalf("有名字应直接用名字，实际 %q", got)
}
}
func TestStateOrFallsBackToPlaceholder(t *testing.T) {
if got := stateOr(""); got != "未知" {
t.Fatalf("空状态应回落到占位文案，实际 %q", got)
}
if got := stateOr("error"); got != "error" {
t.Fatalf("非空状态应原样返回，实际 %q", got)
}
}