diff --git a/CHANGELOG.md b/CHANGELOG.md index c3ca8ce..3984041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,34 @@ All notable changes to this project will be documented in this file. +## [v1.2.10] - 2026-09-25 + +### 新增 + +- **告警事件新增三项巡检**（默认关闭，设置页「告警事件 → 系统」）： + - **公网服务端掉线**：服务端进程没在跑、或隧道连不上 frps 时通知，恢复时再通知一次；只对「用户要它运行」的服务端判定，手动停止的不报 + - **公网 agent 不可达**：VPS 上的 fonu-agent 连续无响应（重启后没起来最常见），恢复时再通知一次；没配 agent 的服务端不参与判定 + - **配置漂移**：VPS 上的 vhost 与 Fonu 记录不一致（被手工改过、或 agent 重装后没重部署）时通知，漂移清空时补一条恢复 +- 通知图标与分类补齐：📈 系统（资源占用）、🔌🛑📡 隧道（隧道 / 服务端 / agent）、🧭 配置（漂移） +- 通知方式的 `Webhook` 改名为「Webhook / IM 机器人」：企业微信、钉钉、飞书机器人挂在 Webhook 预设下，原标签没说明白 + +### 改进 + +- 巡检改为 Source 接口（Routes / Servers / Agents / Drift），`internal/app/monitor_source.go` 负责把 frp 服务适配进来，`internal/monitor` 因此不依赖 frp 包 +- 「连续 N 次失败」抽成共用的 `failureTransition`（原 `tunnelTransition` 泛化）：三条新巡检与原有隧道巡检共用「刚好达标报一次、恢复报一次」；漂移不是连续失败型，单独用冷却（默认 24 小时，可配） +- 避免误报的三条约定：服务端只在「用户要它运行」时判定；`ConnectionState` 为 `unknown` 按健康处理；没配 agent 的服务端不参与可达性判定 +- 开关关闭时连数据采集都跳过（关掉的功能不该有后台开销）；所有 `notify.Alert` 一律在锁外调用 +- 新增设置项：`notify_on_frps_down` / `notify_on_agent_unreachable` / `notify_on_route_drift`，以及 `notify_frps_fail_threshold`(2) / `notify_agent_fail_threshold`(3) / `notify_drift_cooldown_hours`(24) + +### 测试 + +- `internal/monitor/monitor_test.go` 增加 `serverDown`（手动停止不报、进程未运行或连接断开要报、连接状态未知不误报）、`serverLabel`、`stateOr` 用例 + +### 变更 + +- 版本号提升到 1.2.10（版本单一来源仍是根目录 `VERSION`）；`web/package.json` 同步对齐 +- agent 无改动，`AgentVersion` 保持 **0.12.2**，本轮不需要在 VPS 上重新升级 Agent + ## [v1.2.9] - 2026-09-24 ### 新增 diff --git a/VERSION b/VERSION index 9d4f823..963ed7c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.9 +1.2.10 diff --git a/docs/feature-benchmark-2026-09-24.md b/docs/feature-benchmark-2026-09-24.md index 863f4af..4646b86 100644 --- a/docs/feature-benchmark-2026-09-24.md +++ b/docs/feature-benchmark-2026-09-24.md @@ -1,6 +1,6 @@ # 同类项目对标与增补建议 -> 日期：2026-09-24　对象：Fonu 1.0.2（VERSION 1.2.9） +> 日期：2026-09-24　对象：Fonu 1.0.2（VERSION 1.2.10） > 方法：检索 GitHub 上与本项目同域的开源项目（隧道/反代管理面板、自托管监控、Nginx 面板、DDNS 面板），逐项对照 Fonu 现状给出「可优化 / 可增补」。 > 可信度：对标项目的能力来自其首页 / README 摘要与检索片段，**未逐个深读源码**；Fonu 现状来自本仓库代码与 2026-09-24 的全项目审查（`docs/project-audit-2026-09-24.md`）。 @@ -266,6 +266,20 @@ 2. **VPS 上点「升级 Agent」** 才会装上 0.12.2，公网反代页的版本历史才会出现（否则提示「需 ≥ 0.12.2」）。 3. 已随本轮提交：`CHANGELOG.md` 新增 v1.2.9 段，`VERSION` 与 `web/package.json` 同步到 1.2.9。 +## 十、A 档告警落地（2026-09-25） + +> 承接 §三-P0-4 的阈值告警：把「隧道不通」这一族故障的三个常见成因补齐。**纯应用侧改动，agent 无改动**（`AgentVersion` 保持 0.12.2，无需在 VPS 上重新升级）。 + +| 新事件 | 判定 | 默认 | 限流 | +| --- | --- | --- | --- | +| 公网服务端掉线 `frps_server_down` | `DesiredState` 为 running/starting 时，进程 stopped/error 或连接 disconnected/error | 关闭 · 连续 2 次 | 达标报一次 + 恢复报一次 | +| 公网 agent 不可达 `agent_unreachable` | agent 地址与令牌都可用，但 `GET /api/v1/ping` 连续失败 | 关闭 · 连续 3 次 | 同上 | +| 配置漂移 `route_drift` | `RouteDrift` 报告有 items（未部署 / 孤儿配置 / 未生效 / 内容不一致） | 关闭 · 24 小时 | 首次即报 + 每 24h 重复 + 清空报恢复 | + +**避免误报的三条约定**：① 服务端只在「用户要它运行」时判定，手动停掉的不报；② `ConnectionState=unknown` 按健康处理（状态源 `manager+log` 偶尔读不到不代表掉线）；③ 没配 agent 的服务端不参与可达性判定。 + +**结构变化**：`internal/monitor` 的注入方式从「一个 routes 闭包」改成 `Source` 接口（Routes / Servers / Agents / Drift），frp 侧适配放在新文件 `internal/app/monitor_source.go`——monitor 依旧不依赖 frp 包，新增数据源只需扩接口。三条新检查与原有隧道巡检共用 `failureTransition`（原 `tunnelTransition` 泛化），漂移因不是「连续失败」型而单独用冷却；`notify.Alert` 一律在锁外调用。 + --- > 说明：本文件是**建议清单**，未改动任何代码；实施时按本仓库既有约定（每步验证 `go build/vet/test` + `vue-tsc/vite build`、agent 行为变更递增 `AgentVersion`、提交同批更新 `CHANGELOG` 与 `VERSION`）。 diff --git a/internal/api/settings_handler.go b/internal/api/settings_handler.go index b7e84b0..45df554 100644 --- a/internal/api/settings_handler.go +++ b/internal/api/settings_handler.go @@ -74,6 +74,12 @@ func (h *SettingsHandler) Put(w http.ResponseWriter, r *http.Request) { settings.KeyNotifyMemoryThreshold: true, settings.KeyNotifyDiskThreshold: true, settings.KeyNotifyTunnelFailThreshold: true, + settings.KeyNotifyOnFrpsDown: true, + settings.KeyNotifyOnAgentUnreachable: true, + settings.KeyNotifyOnRouteDrift: true, + settings.KeyNotifyFrpsFailThreshold: true, + settings.KeyNotifyAgentFailThreshold: true, + settings.KeyNotifyDriftCooldownHours: true, settings.KeyTrustedProxy: true, settings.KeyGlobalIPBlacklist: true, settings.KeyGlobalIPWhitelist: true, @@ -151,6 +157,12 @@ func isNotifySettingKey(key string) bool { settings.KeyNotifyMemoryThreshold, settings.KeyNotifyDiskThreshold, settings.KeyNotifyTunnelFailThreshold, + settings.KeyNotifyOnFrpsDown, + settings.KeyNotifyOnAgentUnreachable, + settings.KeyNotifyOnRouteDrift, + settings.KeyNotifyFrpsFailThreshold, + settings.KeyNotifyAgentFailThreshold, + settings.KeyNotifyDriftCooldownHours, settings.KeyNotifyWebhookURL: return true default: diff --git a/internal/app/app.go b/internal/app/app.go index 641b87c..8f044b4 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -139,26 +139,8 @@ func New(cfg config.Config, staticFS fs.FS, migrationsDir string) (*App, error) if chinaHours <= 0 { chinaHours = 24 } - // 资源与隧道巡检：间隔比其它任务短，「连续 N 次不通」需要足够采样次数才能判定 - monitorChecker := monitor.New(settingsStore, notifySvc, logger, func(ctx context.Context) ([]monitor.RouteStatus, error) { - servers, err := frpMultiSvc.ListServers(ctx) - if err != nil { - return nil, err - } - routes := make([]monitor.RouteStatus, 0, len(servers)) - for _, server := range servers { - // 单台服务端探测失败不影响其它服务端：它本轮跳过，等下一轮 - health, err := frpMultiSvc.RouteHealth(ctx, server.ID) - if err != nil { - logger.Warn("巡检跳过服务端", "module", "MONITOR", "server_id", server.ID, "error", err.Error()) - continue - } - for _, item := range health { - routes = append(routes, monitor.RouteStatus{Domain: item.Domain, TunnelOK: item.TunnelOK, Detail: item.TunnelDetail}) - } - } - return routes, nil - }) + // 巡检间隔比其它任务短：「连续 N 次不通」需要足够采样次数才能判定 + monitorChecker := monitor.New(settingsStore, notifySvc, logger, monitorSource{frp: frpMultiSvc, logger: logger}) sched := scheduler.New( scheduler.Job{ Name: "ddns", diff --git a/internal/app/monitor_source.go b/internal/app/monitor_source.go new file mode 100644 index 0000000..f69ad5d --- /dev/null +++ b/internal/app/monitor_source.go @@ -0,0 +1,130 @@ +package app + +import ( + "context" + "fmt" + "log/slog" + + "github.com/fonu/fonu/internal/frp" + "github.com/fonu/fonu/internal/monitor" +) + +// monitorSource 把 frp 服务适配成 monitor.Source：monitor 只关心「状态快照」， +// 怎么从 frp / agent 取是这里的事，两边因此互不依赖。 +type monitorSource struct { + frp *frp.Service + logger *slog.Logger +} + +func (s monitorSource) Routes(ctx context.Context) ([]monitor.RouteStatus, error) { + servers, err := s.enabledServers(ctx) + if err != nil { + return nil, err + } + routes := make([]monitor.RouteStatus, 0, len(servers)) + for _, server := range servers { + // 单台服务端探测失败不影响其它服务端：它本轮跳过，等下一轮 + health, err := s.frp.RouteHealth(ctx, server.ID) + if err != nil { + s.logger.Warn("巡检跳过服务端隧道", "module", "MONITOR", "server_id", server.ID, "error", err.Error()) + continue + } + for _, item := range health { + routes = append(routes, monitor.RouteStatus{Domain: item.Domain, TunnelOK: item.TunnelOK, Detail: item.TunnelDetail}) + } + } + return routes, nil +} + +func (s monitorSource) Servers(ctx context.Context) ([]monitor.ServerStatus, error) { + servers, err := s.enabledServers(ctx) + if err != nil { + return nil, err + } + out := make([]monitor.ServerStatus, 0, len(servers)) + for _, server := range servers { + out = append(out, monitor.ServerStatus{ + ID: server.ID, + Name: server.Name, + DesiredState: server.DesiredState, + ProcessState: server.ProcessState, + ConnectionState: server.ConnectionState, + Detail: server.LastError, + }) + } + return out, nil +} + +func (s monitorSource) Agents(ctx context.Context) ([]monitor.AgentStatus, error) { + servers, err := s.enabledServers(ctx) + if err != nil { + return nil, err + } + out := make([]monitor.AgentStatus, 0, len(servers)) + for _, server := range servers { + // AgentClient 只在「地址与令牌都可用」时才返回 ok：没配 agent 的服务端不参与可达性判定 + client, ok := s.frp.AgentClient(ctx, server.ID) + if !ok { + continue + } + status := monitor.AgentStatus{ID: server.ID, Name: server.Name, Reachable: true} + if err := client.Ping(ctx); err != nil { + status.Reachable = false + status.Detail = err.Error() + } + out = append(out, status) + } + return out, nil +} + +func (s monitorSource) Drift(ctx context.Context) ([]monitor.DriftStatus, error) { + servers, err := s.enabledServers(ctx) + if err != nil { + return nil, err + } + out := make([]monitor.DriftStatus, 0, len(servers)) + for _, server := range servers { + report, err := s.frp.RouteDrift(ctx, server.ID) + if err != nil { + s.logger.Warn("巡检跳过漂移检测", "module", "MONITOR", "server_id", server.ID, "error", err.Error()) + continue + } + status := monitor.DriftStatus{ID: server.ID, Name: server.Name, Count: len(report.Items)} + for _, item := range report.Items { + status.Items = append(status.Items, fmt.Sprintf("%s %s", item.Domain, driftKindText(item.Kind))) + } + out = append(out, status) + } + return out, nil +} + +func (s monitorSource) enabledServers(ctx context.Context) ([]frp.Server, error) { + servers, err := s.frp.ListServers(ctx) + if err != nil { + return nil, err + } + enabled := make([]frp.Server, 0, len(servers)) + for _, server := range servers { + // 停用中的服务端是用户主动关掉的，不该参与任何告警判定 + if server.Enabled { + enabled = append(enabled, server) + } + } + return enabled, nil +} + +// driftKindText 与公网反代页漂移清单的文案保持一致，通知里才不用回页面去对照。 +func driftKindText(kind string) string { + switch kind { + case frp.RouteDriftMissing: + return "未部署" + case frp.RouteDriftOrphan: + return "孤儿配置" + case frp.RouteDriftNotLoaded: + return "未生效" + case frp.RouteDriftContentMismatch: + return "内容不一致" + default: + return kind + } +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 2efdb03..f76f5ae 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -1,4 +1,5 @@ -// Package monitor 周期巡检「资源占用」与「隧道连通性」这两类必须持续观察才会出现的问题。 +// Package monitor 周期巡检几类必须持续观察才会出现的问题：主机资源占用、隧道连通性、 +// 公网服务端掉线、agent 不可达、配置漂移。 // 事件型通知（DDNS 失败、证书到期）由各自模块在发生时直接发出，不需要巡检。 // // 判定在这里，通知只走 notify.Service：开关、阈值、冷却都读设置项， @@ -18,27 +19,63 @@ import ( "github.com/fonu/fonu/internal/sysinfo" ) -// RouteStatus 是巡检关心的隧道状态；由调用方注入（避免 monitor 依赖 frp 包）。 +// RouteStatus 是巡检关心的隧道状态。 type RouteStatus struct { Domain string TunnelOK bool Detail string } +// ServerStatus 是巡检关心的公网服务端状态：只有 DesiredState 表示「用户要它运行」时才判定掉线。 +type ServerStatus struct { + ID int64 + Name string + DesiredState string + ProcessState string + ConnectionState string + Detail string +} + +// AgentStatus 是某台服务端上 fonu-agent 的可达性；没配 agent 的服务端不会出现在这里。 +type AgentStatus struct { + ID int64 + Name string + Reachable bool + Detail string +} + +// DriftStatus 是某台服务端的部署漂移摘要，Count 为 0 表示没有漂移。 +type DriftStatus struct { + ID int64 + Name string + Count int + Items []string +} + +// Source 提供巡检需要的外部状态；由 app 侧把 frp 服务适配过来，monitor 因此不依赖 frp 包。 +type Source interface { + Routes(ctx context.Context) ([]RouteStatus, error) + Servers(ctx context.Context) ([]ServerStatus, error) + Agents(ctx context.Context) ([]AgentStatus, error) + Drift(ctx context.Context) ([]DriftStatus, error) +} + // Checker 每轮都重新读设置项：改阈值或开关不需要重启。 type Checker struct { settings *settings.Store notify *notify.Service logger *slog.Logger - routes func(ctx context.Context) ([]RouteStatus, error) + source Source mu sync.Mutex cpuPrev sysinfo.CPUSample resourceAlerts map[string]time.Time - tunnelFailures map[string]int + failures map[string]int + driftAlerted map[int64]bool + driftNotified map[int64]time.Time } -func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.Logger, routes func(context.Context) ([]RouteStatus, error)) *Checker { +func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog.Logger, source Source) *Checker { if logger == nil { logger = slog.Default() } @@ -46,9 +83,11 @@ func New(settingsStore *settings.Store, notifySvc *notify.Service, logger *slog. settings: settingsStore, notify: notifySvc, logger: logger, - routes: routes, + source: source, resourceAlerts: map[string]time.Time{}, - tunnelFailures: map[string]int{}, + failures: map[string]int{}, + driftAlerted: map[int64]bool{}, + driftNotified: map[int64]time.Time{}, } } @@ -58,7 +97,10 @@ func (c *Checker) Run(ctx context.Context) { return } c.checkResources(ctx) - c.checkTunnels(ctx) + c.checkRoutes(ctx) + c.checkServers(ctx) + c.checkAgents(ctx) + c.checkDrift(ctx) } func (c *Checker) checkResources(ctx context.Context) { @@ -118,8 +160,8 @@ func (c *Checker) allowResourceAlert(metric string, now time.Time) bool { return true } -func (c *Checker) checkTunnels(ctx context.Context) { - if c.routes == nil { +func (c *Checker) checkRoutes(ctx context.Context) { + if c.source == nil { return } if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnTunnelDown); !enabled { @@ -130,7 +172,7 @@ func (c *Checker) checkTunnels(ctx context.Context) { return } - routes, err := c.routes(ctx) + routes, err := c.source.Routes(ctx) if err != nil { c.logger.Warn("巡检隧道状态失败", "module", "MONITOR", "error", err.Error()) return @@ -139,44 +181,194 @@ func (c *Checker) checkTunnels(ctx context.Context) { if strings.TrimSpace(route.Domain) == "" { continue } - c.mu.Lock() - failures, event := tunnelTransition(c.tunnelFailures[route.Domain], route.TunnelOK, limit) - c.tunnelFailures[route.Domain] = failures - c.mu.Unlock() - + _, event := c.track("route:"+route.Domain, route.TunnelOK, limit) switch event { - case tunnelEventDown: + case failureEventDown: c.notify.Alert(ctx, notify.EventTunnelDown, "隧道连续不通", fmt.Sprintf("%s 连续 %d 次巡检未在 frps 上注册：%s", route.Domain, limit, detailOr(route.Detail))) - case tunnelEventRecovered: + case failureEventRecovered: c.notify.Alert(ctx, notify.EventTunnelDown, "隧道已恢复", fmt.Sprintf("%s 已重新在 frps 上注册", route.Domain)) } } } -type tunnelEvent string +// checkServers 公网服务端掉线：进程没在跑，或隧道客户端连不上 frps。 +// 只对「用户要它运行」的服务端判定，手动停掉的不报。 +func (c *Checker) checkServers(ctx context.Context) { + if c.source == nil { + return + } + if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnFrpsDown); !enabled { + return + } + limit, _ := c.settings.GetInt(ctx, settings.KeyNotifyFrpsFailThreshold) + if limit <= 0 { + return + } + + servers, err := c.source.Servers(ctx) + if err != nil { + c.logger.Warn("巡检服务端状态失败", "module", "MONITOR", "error", err.Error()) + return + } + for _, server := range servers { + label := serverLabel(server.ID, server.Name) + _, event := c.track(fmt.Sprintf("server:%d", server.ID), !serverDown(server), limit) + switch event { + case failureEventDown: + c.notify.Alert(ctx, notify.EventFrpsServerDown, "公网服务端掉线", + fmt.Sprintf("%s 连续 %d 次巡检未运行或未连接（进程 %s / 连接 %s）：%s", + label, limit, stateOr(server.ProcessState), stateOr(server.ConnectionState), detailOr(server.Detail))) + case failureEventRecovered: + c.notify.Alert(ctx, notify.EventFrpsServerDown, "公网服务端已恢复", + fmt.Sprintf("%s 已重新运行并连接", label)) + } + } +} + +// checkAgents agent 不可达：VPS 重启后 agent 没起来是最常见的成因。 +func (c *Checker) checkAgents(ctx context.Context) { + if c.source == nil { + return + } + if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnAgentUnreachable); !enabled { + return + } + limit, _ := c.settings.GetInt(ctx, settings.KeyNotifyAgentFailThreshold) + if limit <= 0 { + return + } + + agents, err := c.source.Agents(ctx) + if err != nil { + c.logger.Warn("巡检 agent 可达性失败", "module", "MONITOR", "error", err.Error()) + return + } + for _, agent := range agents { + label := serverLabel(agent.ID, agent.Name) + _, event := c.track(fmt.Sprintf("agent:%d", agent.ID), agent.Reachable, limit) + switch event { + case failureEventDown: + c.notify.Alert(ctx, notify.EventAgentUnreachable, "公网 agent 不可达", + fmt.Sprintf("%s 的 fonu-agent 连续 %d 次无响应：%s", label, limit, detailOr(agent.Detail))) + case failureEventRecovered: + c.notify.Alert(ctx, notify.EventAgentUnreachable, "公网 agent 已恢复", + fmt.Sprintf("%s 的 fonu-agent 已恢复响应", label)) + } + } +} + +// checkDrift 配置漂移：VPS 上的 vhost 与 Fonu 记录不一致（被手工改过、或 agent 重装后没重部署）。 +// 它不是「连续 N 次」型故障，因此用冷却时间限流：首次发现即报，之后每 cooldown 提醒一次，漂移清空时补一条恢复。 +func (c *Checker) checkDrift(ctx context.Context) { + if c.source == nil { + return + } + if enabled, _ := c.settings.GetBool(ctx, settings.KeyNotifyOnRouteDrift); !enabled { + return + } + cooldownHours, _ := c.settings.GetInt(ctx, settings.KeyNotifyDriftCooldownHours) + if cooldownHours <= 0 { + cooldownHours = notify.DefaultDriftCooldownHours + } + cooldown := time.Duration(cooldownHours) * time.Hour + + reports, err := c.source.Drift(ctx) + if err != nil { + c.logger.Warn("巡检配置漂移失败", "module", "MONITOR", "error", err.Error()) + return + } + now := time.Now() + for _, report := range reports { + c.mu.Lock() + alerted := c.driftAlerted[report.ID] + last := c.driftNotified[report.ID] + if report.Count == 0 { + delete(c.driftAlerted, report.ID) + delete(c.driftNotified, report.ID) + } + c.mu.Unlock() + + label := serverLabel(report.ID, report.Name) + if report.Count == 0 { + if alerted { + c.notify.Alert(ctx, notify.EventRouteDrift, "配置漂移已消除", + fmt.Sprintf("%s 上的 vhost 已与 Fonu 记录一致", label)) + } + continue + } + if alerted && !last.IsZero() && now.Sub(last) < cooldown { + continue + } + c.mu.Lock() + c.driftAlerted[report.ID] = true + c.driftNotified[report.ID] = now + c.mu.Unlock() + c.notify.Alert(ctx, notify.EventRouteDrift, "公网反代配置漂移", + fmt.Sprintf("%s 有 %d 项与 Fonu 记录不一致：%s", label, report.Count, strings.Join(report.Items, "、"))) + } +} + +// track 推进某个实体的连续失败计数，返回本轮要发的事件。 +// 锁只包住 map 读写：notify 一律在锁外调用。 +func (c *Checker) track(key string, healthy bool, limit int) (int, failureEvent) { + c.mu.Lock() + defer c.mu.Unlock() + failures, event := failureTransition(c.failures[key], healthy, limit) + c.failures[key] = failures + return failures, event +} + +type failureEvent string const ( - tunnelEventNone tunnelEvent = "" - tunnelEventDown tunnelEvent = "down" - tunnelEventRecovered tunnelEvent = "recovered" + failureEventNone failureEvent = "" + failureEventDown failureEvent = "down" + failureEventRecovered failureEvent = "recovered" ) -// tunnelTransition 推进某个域名的连续失败计数，返回新计数与本轮要发的事件。 +// failureTransition 推进某个实体的连续失败计数，返回新计数与本轮要发的事件。 // 只在「刚好达到阈值」与「从失败中恢复」两个时刻发，避免每轮巡检都刷通知。 -func tunnelTransition(previous int, tunnelOK bool, limit int) (int, tunnelEvent) { - if tunnelOK { +func failureTransition(previous int, healthy bool, limit int) (int, failureEvent) { + if healthy { if previous >= limit { - return 0, tunnelEventRecovered + return 0, failureEventRecovered } - return 0, tunnelEventNone + return 0, failureEventNone } next := previous + 1 if next == limit { - return next, tunnelEventDown + return next, failureEventDown } - return next, tunnelEventNone + return next, failureEventNone +} + +// serverDown 判定服务端是否处于「用户要它运行、但它没在跑或没连上」。 +// ConnectionState 为 unknown 时按健康处理：状态源（manager+log）偶尔读不到不代表掉线，避免误报。 +func serverDown(status ServerStatus) bool { + if status.DesiredState != "running" && status.DesiredState != "starting" { + return false + } + if status.ProcessState == "stopped" || status.ProcessState == "error" { + return true + } + return status.ConnectionState == "disconnected" || status.ConnectionState == "error" +} + +func serverLabel(id int64, name string) string { + if trimmed := strings.TrimSpace(name); trimmed != "" { + return trimmed + } + return fmt.Sprintf("服务端 #%d", id) +} + +// stateOr 让通知里的状态字段不会因为空串而变成「（进程 / 连接 ）」。 +func stateOr(state string) string { + if strings.TrimSpace(state) == "" { + return "未知" + } + return state } func percent(used, total uint64) float64 { diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index 87c023e..f37fa7c 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -2,20 +2,20 @@ package monitor import "testing" -func TestTunnelTransitionAlertsOncePerOutage(t *testing.T) { +func TestFailureTransitionAlertsOncePerOutage(t *testing.T) { const limit = 3 failures := 0 - var events []tunnelEvent + var events []failureEvent // 连续三轮不通：只在第 3 轮（刚好达到阈值）发一次 for i := 0; i < 5; i++ { - var event tunnelEvent - failures, event = tunnelTransition(failures, false, limit) - if event != tunnelEventNone { + var event failureEvent + failures, event = failureTransition(failures, false, limit) + if event != failureEventNone { events = append(events, event) } } - if len(events) != 1 || events[0] != tunnelEventDown { + if len(events) != 1 || events[0] != failureEventDown { t.Fatalf("连续不通应只告警一次，实际 %v", events) } if failures != 5 { @@ -23,31 +23,31 @@ func TestTunnelTransitionAlertsOncePerOutage(t *testing.T) { } // 恢复：发一次恢复通知并清零 - failures, event := tunnelTransition(failures, true, limit) - if event != tunnelEventRecovered || failures != 0 { + failures, event := failureTransition(failures, true, limit) + if event != failureEventRecovered || failures != 0 { t.Fatalf("恢复应通知一次并清零：event=%v failures=%d", event, failures) } // 恢复后再次不通：还能再报一次 - failures, event = tunnelTransition(failures, false, limit) - if event != tunnelEventNone || failures != 1 { + failures, event = failureTransition(failures, false, limit) + if event != failureEventNone || failures != 1 { t.Fatalf("恢复后重新计数：event=%v failures=%d", event, failures) } - failures, event = tunnelTransition(failures, false, limit) - if event != tunnelEventNone { + failures, event = failureTransition(failures, false, limit) + if event != failureEventNone { t.Fatalf("未达阈值不应告警：%v", event) } - if _, event = tunnelTransition(failures, false, limit); event != tunnelEventDown { + if _, event = failureTransition(failures, false, limit); event != failureEventDown { t.Fatalf("第二次中断达到阈值应再告警一次：%v", event) } } -func TestTunnelTransitionBelowThresholdIsSilent(t *testing.T) { +func TestFailureTransitionBelowThresholdIsSilent(t *testing.T) { // 单次抖动（未达阈值）不打扰用户，也不算作恢复 - if failures, event := tunnelTransition(0, true, 3); failures != 0 || event != tunnelEventNone { + if failures, event := failureTransition(0, true, 3); failures != 0 || event != failureEventNone { t.Fatalf("健康状态不应产生事件：%d %v", failures, event) } - if failures, event := tunnelTransition(1, true, 3); failures != 0 || event != tunnelEventNone { + if failures, event := failureTransition(1, true, 3); failures != 0 || event != failureEventNone { t.Fatalf("未达阈值的恢复不应通知：%d %v", failures, event) } } @@ -69,3 +69,41 @@ func TestDetailOrFallsBack(t *testing.T) { t.Fatalf("非空详情应原样返回，实际 %q", got) } } + +func TestServerDownOnlyWhenUserWantsItRunning(t *testing.T) { + // 用户手动停掉的不算掉线 + stopped := ServerStatus{DesiredState: "stopped", ProcessState: "stopped", ConnectionState: "disconnected"} + if serverDown(stopped) { + t.Fatal("用户手动停止的服务端不应判为掉线") + } + // 期望运行但进程没在跑 + if !serverDown(ServerStatus{DesiredState: "running", ProcessState: "stopped"}) { + t.Fatal("期望运行但进程未运行应判为掉线") + } + // 进程在跑但连不上 frps + if !serverDown(ServerStatus{DesiredState: "running", ProcessState: "running", ConnectionState: "disconnected"}) { + t.Fatal("连接断开应判为掉线") + } + // 状态源读不到连接状态时不报警：manager+log 偶尔读不到不代表掉线 + if serverDown(ServerStatus{DesiredState: "running", ProcessState: "running", ConnectionState: "unknown"}) { + t.Fatal("连接状态未知时不应误报") + } +} + +func TestServerLabelFallsBackToID(t *testing.T) { + if got := serverLabel(3, " "); got != "服务端 #3" { + t.Fatalf("无名服务端应回落到编号，实际 %q", got) + } + if got := serverLabel(3, "VPS-香港"); got != "VPS-香港" { + t.Fatalf("有名字应直接用名字，实际 %q", got) + } +} + +func TestStateOrFallsBackToPlaceholder(t *testing.T) { + if got := stateOr(""); got != "未知" { + t.Fatalf("空状态应回落到占位文案，实际 %q", got) + } + if got := stateOr("error"); got != "error" { + t.Fatalf("非空状态应原样返回，实际 %q", got) + } +} diff --git a/internal/notify/config.go b/internal/notify/config.go index e985aee..90db3d1 100644 --- a/internal/notify/config.go +++ b/internal/notify/config.go @@ -11,6 +11,9 @@ const ( EventNginxReloadFailure = "nginx_reload_failure" EventResourceThreshold = "resource_threshold" EventTunnelDown = "tunnel_down" + EventFrpsServerDown = "frps_server_down" + EventAgentUnreachable = "agent_unreachable" + EventRouteDrift = "route_drift" MaskedSecret = "********" @@ -29,6 +32,9 @@ const ( DefaultMemoryThreshold = 90 DefaultDiskThreshold = 85 DefaultTunnelFailThreshold = 3 + DefaultFrpsFailThreshold = 2 + DefaultAgentFailThreshold = 3 + DefaultDriftCooldownHours = 24 // DefaultResourceAlertCooldown 是资源类告警的静默期：持续高占用时每半小时提醒一次， // 期间恢复正常会清掉冷却，再次超标可立即再报。 DefaultResourceAlertCooldown = 30 * 60 // seconds @@ -113,6 +119,12 @@ type Config struct { MemoryThreshold int `json:"memory_threshold"` DiskThreshold int `json:"disk_threshold"` TunnelFailThreshold int `json:"tunnel_fail_threshold"` + OnFrpsDown bool `json:"on_frps_down"` + OnAgentUnreachable bool `json:"on_agent_unreachable"` + OnRouteDrift bool `json:"on_route_drift"` + FrpsFailThreshold int `json:"frps_fail_threshold"` + AgentFailThreshold int `json:"agent_fail_threshold"` + DriftCooldownHours int `json:"drift_cooldown_hours"` } type RuntimeConfig struct { @@ -148,6 +160,12 @@ type SaveInput struct { MemoryThreshold int DiskThreshold int TunnelFailThreshold int + OnFrpsDown bool + OnAgentUnreachable bool + OnRouteDrift bool + FrpsFailThreshold int + AgentFailThreshold int + DriftCooldownHours int } type TestInput struct { diff --git a/internal/notify/format.go b/internal/notify/format.go index 5a1c7b8..72d956f 100644 --- a/internal/notify/format.go +++ b/internal/notify/format.go @@ -86,6 +86,16 @@ func eventMetaFor(event string) eventMeta { return eventMeta{"🔐", "安全"} case EventNginxReloadFailure: return eventMeta{"⚙️", "系统"} + case EventResourceThreshold: + return eventMeta{"📈", "系统"} + case EventTunnelDown: + return eventMeta{"🔌", "隧道"} + case EventFrpsServerDown: + return eventMeta{"🛑", "隧道"} + case EventAgentUnreachable: + return eventMeta{"📡", "隧道"} + case EventRouteDrift: + return eventMeta{"🧭", "配置"} case "test": return eventMeta{"🔔", "系统"} default: diff --git a/internal/notify/service.go b/internal/notify/service.go index af3fa19..40069f3 100644 --- a/internal/notify/service.go +++ b/internal/notify/service.go @@ -246,6 +246,12 @@ func (s *Service) enabledFor(cfg RuntimeConfig, event string) bool { return cfg.OnResourceThreshold case EventTunnelDown: return cfg.OnTunnelDown + case EventFrpsServerDown: + return cfg.OnFrpsDown + case EventAgentUnreachable: + return cfg.OnAgentUnreachable + case EventRouteDrift: + return cfg.OnRouteDrift default: return true } diff --git a/internal/notify/store.go b/internal/notify/store.go index 9f0bf0c..1446471 100644 --- a/internal/notify/store.go +++ b/internal/notify/store.go @@ -80,6 +80,9 @@ func (s *Store) load(ctx context.Context, decrypt bool) (Config, runtimeSecrets, cfg.MemoryThreshold = intSetting(s.settings, ctx, settings.KeyNotifyMemoryThreshold, DefaultMemoryThreshold) cfg.DiskThreshold = intSetting(s.settings, ctx, settings.KeyNotifyDiskThreshold, DefaultDiskThreshold) cfg.TunnelFailThreshold = intSetting(s.settings, ctx, settings.KeyNotifyTunnelFailThreshold, DefaultTunnelFailThreshold) + cfg.FrpsFailThreshold = intSetting(s.settings, ctx, settings.KeyNotifyFrpsFailThreshold, DefaultFrpsFailThreshold) + cfg.AgentFailThreshold = intSetting(s.settings, ctx, settings.KeyNotifyAgentFailThreshold, DefaultAgentFailThreshold) + cfg.DriftCooldownHours = intSetting(s.settings, ctx, settings.KeyNotifyDriftCooldownHours, DefaultDriftCooldownHours) if hasNotifyEventSettings(ctx, s.settings) { cfg.OnDDNSIPChange = flag(s.settings, ctx, settings.KeyNotifyOnDDNSIPChange, true) @@ -93,6 +96,9 @@ func (s *Store) load(ctx context.Context, decrypt bool) (Config, runtimeSecrets, // 资源与隧道告警默认关闭：升级后不改变现有用户收到的通知 cfg.OnResourceThreshold = flag(s.settings, ctx, settings.KeyNotifyOnResourceThreshold, false) cfg.OnTunnelDown = flag(s.settings, ctx, settings.KeyNotifyOnTunnelDown, false) + cfg.OnFrpsDown = flag(s.settings, ctx, settings.KeyNotifyOnFrpsDown, false) + cfg.OnAgentUnreachable = flag(s.settings, ctx, settings.KeyNotifyOnAgentUnreachable, false) + cfg.OnRouteDrift = flag(s.settings, ctx, settings.KeyNotifyOnRouteDrift, false) } else { legacyDDNS := flag(s.settings, ctx, settings.KeyNotifyOnDDNSError, true) legacyCert := flag(s.settings, ctx, settings.KeyNotifyOnCertError, true) @@ -193,6 +199,9 @@ func hasNotifyEventSettings(ctx context.Context, store *settings.Store) bool { settings.KeyNotifyOnNginxReloadFailure, settings.KeyNotifyOnResourceThreshold, settings.KeyNotifyOnTunnelDown, + settings.KeyNotifyOnFrpsDown, + settings.KeyNotifyOnAgentUnreachable, + settings.KeyNotifyOnRouteDrift, } for _, key := range keys { raw, err := store.Get(ctx, key) @@ -253,6 +262,24 @@ func (s *Store) Save(ctx context.Context, in SaveInput) error { if err := s.settings.SetInt(ctx, settings.KeyNotifyTunnelFailThreshold, clampThreshold(in.TunnelFailThreshold, DefaultTunnelFailThreshold)); err != nil { return err } + if err := s.settings.SetBool(ctx, settings.KeyNotifyOnFrpsDown, in.OnFrpsDown); err != nil { + return err + } + if err := s.settings.SetBool(ctx, settings.KeyNotifyOnAgentUnreachable, in.OnAgentUnreachable); err != nil { + return err + } + if err := s.settings.SetBool(ctx, settings.KeyNotifyOnRouteDrift, in.OnRouteDrift); err != nil { + return err + } + if err := s.settings.SetInt(ctx, settings.KeyNotifyFrpsFailThreshold, clampThreshold(in.FrpsFailThreshold, DefaultFrpsFailThreshold)); err != nil { + return err + } + if err := s.settings.SetInt(ctx, settings.KeyNotifyAgentFailThreshold, clampThreshold(in.AgentFailThreshold, DefaultAgentFailThreshold)); err != nil { + return err + } + if err := s.settings.SetInt(ctx, settings.KeyNotifyDriftCooldownHours, clampThreshold(in.DriftCooldownHours, DefaultDriftCooldownHours)); err != nil { + return err + } threshold := in.IPFrequentThreshold if threshold <= 0 { threshold = DefaultIPFrequentThreshold @@ -493,6 +520,12 @@ func ParseSaveInputFromMap(values map[string]string) (SaveInput, bool) { in.MemoryThreshold = intSettingFromMap(values, settings.KeyNotifyMemoryThreshold, DefaultMemoryThreshold) in.DiskThreshold = intSettingFromMap(values, settings.KeyNotifyDiskThreshold, DefaultDiskThreshold) in.TunnelFailThreshold = intSettingFromMap(values, settings.KeyNotifyTunnelFailThreshold, DefaultTunnelFailThreshold) + in.OnFrpsDown = values[settings.KeyNotifyOnFrpsDown] == "1" || strings.EqualFold(values[settings.KeyNotifyOnFrpsDown], "true") + in.OnAgentUnreachable = values[settings.KeyNotifyOnAgentUnreachable] == "1" || strings.EqualFold(values[settings.KeyNotifyOnAgentUnreachable], "true") + in.OnRouteDrift = values[settings.KeyNotifyOnRouteDrift] == "1" || strings.EqualFold(values[settings.KeyNotifyOnRouteDrift], "true") + in.FrpsFailThreshold = intSettingFromMap(values, settings.KeyNotifyFrpsFailThreshold, DefaultFrpsFailThreshold) + in.AgentFailThreshold = intSettingFromMap(values, settings.KeyNotifyAgentFailThreshold, DefaultAgentFailThreshold) + in.DriftCooldownHours = intSettingFromMap(values, settings.KeyNotifyDriftCooldownHours, DefaultDriftCooldownHours) return in, true } diff --git a/internal/settings/settings.go b/internal/settings/settings.go index f7e5042..cc94ccd 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -46,6 +46,12 @@ const ( KeyNotifyMemoryThreshold = "notify_memory_threshold" KeyNotifyDiskThreshold = "notify_disk_threshold" KeyNotifyTunnelFailThreshold = "notify_tunnel_fail_threshold" + KeyNotifyOnFrpsDown = "notify_on_frps_down" + KeyNotifyOnAgentUnreachable = "notify_on_agent_unreachable" + KeyNotifyOnRouteDrift = "notify_on_route_drift" + KeyNotifyFrpsFailThreshold = "notify_frps_fail_threshold" + KeyNotifyAgentFailThreshold = "notify_agent_fail_threshold" + KeyNotifyDriftCooldownHours = "notify_drift_cooldown_hours" KeyTrustedProxy = "trusted_proxy_json" KeyGlobalIPBlacklist = "global_ip_blacklist" KeyGlobalIPWhitelist = "global_ip_whitelist" diff --git a/web/package.json b/web/package.json index 3888b72..a0431ce 100644 --- a/web/package.json +++ b/web/package.json @@ -1,7 +1,7 @@ { "name": "fonu-web", "private": true, - "version": "1.2.9", + "version": "1.2.10", "packageManager": "npm@10.9.4", "type": "module", "scripts": { diff --git a/web/src/constants/notifyEvents.ts b/web/src/constants/notifyEvents.ts index 4eb1804..f509190 100644 --- a/web/src/constants/notifyEvents.ts +++ b/web/src/constants/notifyEvents.ts @@ -3,7 +3,7 @@ export interface NotifyEventOption { label: string hint?: string hasThreshold?: boolean - thresholdKey?: 'ip_frequent' | 'login_failure' | 'resource' | 'tunnel_fail' + thresholdKey?: 'ip_frequent' | 'login_failure' | 'resource' | 'tunnel_fail' | 'frps_fail' | 'agent_fail' | 'drift_cooldown' } export interface NotifyEventGroup { @@ -85,6 +85,27 @@ export const NOTIFY_EVENT_GROUPS: NotifyEventGroup[] = [ hasThreshold: true, thresholdKey: 'tunnel_fail', }, + { + key: 'on_frps_down', + label: '公网服务端掉线', + hint: '服务端进程没在跑或隧道连不上 frps（手动停止的不报），恢复时再通知一次', + hasThreshold: true, + thresholdKey: 'frps_fail', + }, + { + key: 'on_agent_unreachable', + label: '公网 agent 不可达', + hint: 'VPS 上的 fonu-agent 连续无响应（重启后没起来最常见），恢复时再通知一次', + hasThreshold: true, + thresholdKey: 'agent_fail', + }, + { + key: 'on_route_drift', + label: '配置漂移', + hint: 'VPS 上的 vhost 与 Fonu 记录不一致（被手工改过或重装后未重部署），同一台按提醒间隔重复提醒', + hasThreshold: true, + thresholdKey: 'drift_cooldown', + }, ], }, ] @@ -97,3 +118,6 @@ export const DEFAULT_CPU_THRESHOLD = 90 export const DEFAULT_MEMORY_THRESHOLD = 90 export const DEFAULT_DISK_THRESHOLD = 85 export const DEFAULT_TUNNEL_FAIL_THRESHOLD = 3 +export const DEFAULT_FRPS_FAIL_THRESHOLD = 2 +export const DEFAULT_AGENT_FAIL_THRESHOLD = 3 +export const DEFAULT_DRIFT_COOLDOWN_HOURS = 24 diff --git a/web/src/constants/notifyProviders.ts b/web/src/constants/notifyProviders.ts index 3cee6a6..ed44999 100644 --- a/web/src/constants/notifyProviders.ts +++ b/web/src/constants/notifyProviders.ts @@ -3,7 +3,8 @@ import type { NotifyType, WebhookProvider } from '../api/types' export const NOTIFY_TYPE_OPTIONS: { label: string; value: NotifyType }[] = [ { label: '关闭', value: '' }, { label: '邮件（SMTP）', value: 'email' }, - { label: 'Webhook', value: 'webhook' }, + // 企业微信 / 钉钉 / 飞书机器人都是「POST 一个 URL + JSON」，归在 Webhook 预设里，标签要说明白 + { label: 'Webhook / IM 机器人', value: 'webhook' }, { label: 'Telegram 机器人', value: 'telegram' }, ] diff --git a/web/src/views/SettingsView.vue b/web/src/views/SettingsView.vue index f784184..81fbf36 100644 --- a/web/src/views/SettingsView.vue +++ b/web/src/views/SettingsView.vue @@ -334,6 +334,24 @@ + + + @@ -716,8 +734,11 @@ import PageHeader from '../components/PageHeader.vue' import StatusBadge from '../components/StatusBadge.vue' import { DEFAULT_PRIVATE_CIDRS, mergeIPList, stripDefaultIPList } from '../constants/ipPolicy' import { + DEFAULT_AGENT_FAIL_THRESHOLD, DEFAULT_CPU_THRESHOLD, DEFAULT_DISK_THRESHOLD, + DEFAULT_DRIFT_COOLDOWN_HOURS, + DEFAULT_FRPS_FAIL_THRESHOLD, DEFAULT_LOGIN_FAILURE_THRESHOLD, DEFAULT_LOGIN_FAILURE_WINDOW_SEC, DEFAULT_MEMORY_THRESHOLD, @@ -767,6 +788,9 @@ const notifyEventFlags = reactive>({ on_nginx_reload_failure: true, on_resource_threshold: false, on_tunnel_down: false, + on_frps_down: false, + on_agent_unreachable: false, + on_route_drift: false, }) const notifyIPFrequentThreshold = ref(100) const notifyIPFrequentWindowSec = ref(60) @@ -776,6 +800,9 @@ const notifyCPUThreshold = ref(DEFAULT_CPU_THRESHOLD) const notifyMemoryThreshold = ref(DEFAULT_MEMORY_THRESHOLD) const notifyDiskThreshold = ref(DEFAULT_DISK_THRESHOLD) const notifyTunnelFailThreshold = ref(DEFAULT_TUNNEL_FAIL_THRESHOLD) +const notifyFrpsFailThreshold = ref(DEFAULT_FRPS_FAIL_THRESHOLD) +const notifyAgentFailThreshold = ref(DEFAULT_AGENT_FAIL_THRESHOLD) +const notifyDriftCooldownHours = ref(DEFAULT_DRIFT_COOLDOWN_HOURS) // API Token：明文只在创建响应里回一次，因此单独用一个 ref 暂存展示 const apiTokens = ref([]) @@ -1278,6 +1305,9 @@ function applyNotifySettings(settings: Record) { notifyEventFlags.on_nginx_reload_failure = flagOn('notify_on_nginx_reload_failure') notifyEventFlags.on_resource_threshold = flagOptIn('notify_on_resource_threshold') notifyEventFlags.on_tunnel_down = flagOptIn('notify_on_tunnel_down') + notifyEventFlags.on_frps_down = flagOptIn('notify_on_frps_down') + notifyEventFlags.on_agent_unreachable = flagOptIn('notify_on_agent_unreachable') + notifyEventFlags.on_route_drift = flagOptIn('notify_on_route_drift') } else { const legacyDDNS = flagOn('notify_on_ddns_error') const legacyCert = flagOn('notify_on_cert_error') @@ -1291,6 +1321,9 @@ function applyNotifySettings(settings: Record) { notifyEventFlags.on_login_failure = false notifyEventFlags.on_resource_threshold = false notifyEventFlags.on_tunnel_down = false + notifyEventFlags.on_frps_down = false + notifyEventFlags.on_agent_unreachable = false + notifyEventFlags.on_route_drift = false } notifyIPFrequentThreshold.value = Number(settings.notify_ip_frequent_threshold || 100) notifyIPFrequentWindowSec.value = Number(settings.notify_ip_frequent_window_sec || 60) @@ -1300,6 +1333,9 @@ function applyNotifySettings(settings: Record) { notifyMemoryThreshold.value = Number(settings.notify_memory_threshold || DEFAULT_MEMORY_THRESHOLD) notifyDiskThreshold.value = Number(settings.notify_disk_threshold || DEFAULT_DISK_THRESHOLD) notifyTunnelFailThreshold.value = Number(settings.notify_tunnel_fail_threshold || DEFAULT_TUNNEL_FAIL_THRESHOLD) + notifyFrpsFailThreshold.value = Number(settings.notify_frps_fail_threshold || DEFAULT_FRPS_FAIL_THRESHOLD) + notifyAgentFailThreshold.value = Number(settings.notify_agent_fail_threshold || DEFAULT_AGENT_FAIL_THRESHOLD) + notifyDriftCooldownHours.value = Number(settings.notify_drift_cooldown_hours || DEFAULT_DRIFT_COOLDOWN_HOURS) } function buildNotifyEmailJSON(): string { @@ -1407,6 +1443,12 @@ function buildSavePayload() { notify_on_nginx_reload_failure: notifyEventFlags.on_nginx_reload_failure ? '1' : '0', notify_on_resource_threshold: notifyEventFlags.on_resource_threshold ? '1' : '0', notify_on_tunnel_down: notifyEventFlags.on_tunnel_down ? '1' : '0', + notify_on_frps_down: notifyEventFlags.on_frps_down ? '1' : '0', + notify_on_agent_unreachable: notifyEventFlags.on_agent_unreachable ? '1' : '0', + notify_on_route_drift: notifyEventFlags.on_route_drift ? '1' : '0', + notify_frps_fail_threshold: String(notifyFrpsFailThreshold.value || DEFAULT_FRPS_FAIL_THRESHOLD), + notify_agent_fail_threshold: String(notifyAgentFailThreshold.value || DEFAULT_AGENT_FAIL_THRESHOLD), + notify_drift_cooldown_hours: String(notifyDriftCooldownHours.value || DEFAULT_DRIFT_COOLDOWN_HOURS), notify_cpu_threshold: String(notifyCPUThreshold.value || DEFAULT_CPU_THRESHOLD), notify_memory_threshold: String(notifyMemoryThreshold.value || DEFAULT_MEMORY_THRESHOLD), notify_disk_threshold: String(notifyDiskThreshold.value || DEFAULT_DISK_THRESHOLD),