备份、隧道生命周期、日志、会话与设置项存在一批「静默失效」问题,本次一并收敛。 正确性: - 备份归档补上 session_secret 与 frp 配置。缺密钥时换机恢复后库里所有凭据都解不开, 界面表现为「配置都在、一执行就报解密失败」;同时支持口令加密(scrypt + AES-256-GCM)。 - 修复「立即备份」必失败:tar 头用的是遍历时的文件尺寸,而 logs、frpc 配置在打包期间 还在增长,读出的字节多于声明值导致 archive/tar 报错、整份备份作废。改为打开后重新 取尺寸并用 LimitReader 兜住,被截短时补零保持条目结构完整。 - 数据库改用 VACUUM INTO 生成一致性快照。直接打包正在写入的 SQLite 文件可能得到 结构完整但内容自相矛盾的副本,恢复后才暴露。 - Cloudflare 启动改用 StartAuto,Shutdown 显式 StopAll,子进程绑定应用 ctx。此前 重启会把用户手动停掉的隧道重新拉起,退出也可能留下孤儿进程。 - Nginx 访问/错误日志加轮转(200MB 阈值、gzip 归档、USR1 重开),读取改为从文件尾 分块倒读,并跨当日归档统计。此前日志只增不减,读接口每次全量扫描。 - 过期会话改由日任务清理;删除反代规则时同步清掉流量累计,避免同名规则重建继承旧读数。 - 启动加就绪门:Nginx bootstrap 完成后再放行 FRP 对账、Agent 隧道与 Cloudflare 自启。 设置项: - 新增设置项注册表(internal/settings/registry.go),白名单与默认值从它派生, 并记录每个键的消费者,防止再出现「界面能改、后端没人读」的黑洞设置。 - 接上三个此前不生效的设置项:timezone(启动与保存时应用到 time.Local)、 theme(后端为权威源,挂载时覆盖本地偏好、顶栏切换回写)、log_retention_days (控制日志归档保留天数)。 工程质量: - gofmt 全量格式化 23 个历史文件,CI 增加格式化门禁。 - 前端引入 ESLint(flat config)与 lint 脚本,CI 增加 lint 步骤;顺带修掉 logDisplay 的 let→const 与 LogsView 模板内直接改 prop 两处问题。 - 删除无引用的 pnpm-lock.yaml,prototype/ 加入忽略列表。 - 打包版本对齐:Docker manifest 同步到 2.0.0,CI 校验 VERSION 与 package.json 一致。 验证:gofmt 0 文件、go vet、go test ./...、npm run lint/typecheck/test/build 全绿。 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
+137
-28
@@ -2,10 +2,14 @@ package logstore
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -298,16 +302,9 @@ func hourlyAccessCountsForDay(path string, day time.Time) ([]int, []string) {
|
||||
counts := make([]int, hourlyBucketCount)
|
||||
labels := calendarHourlyLabels()
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return counts, labels
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
entry, ok := parseAccess(scanner.Text())
|
||||
// 轮转后的历史内容在归档文件里:只读当前文件会让「今日统计」在轮转后突然掉一半。
|
||||
for _, line := range accessLinesForDay(path, day) {
|
||||
entry, ok := parseAccess(line)
|
||||
if !ok || !isDashboardAccess(entry) {
|
||||
continue
|
||||
}
|
||||
@@ -324,6 +321,75 @@ func hourlyAccessCountsForDay(path string, day time.Time) ([]int, []string) {
|
||||
return counts, labels
|
||||
}
|
||||
|
||||
// accessLinesForDay 返回某天的原始日志行:当前文件 + 该日期的归档文件。
|
||||
// 只挑文件名里带目标日期的归档,避免把几个月的归档都解开。
|
||||
func accessLinesForDay(path string, day time.Time) []string {
|
||||
var out []string
|
||||
stamp := day.Format("20060102")
|
||||
for _, archive := range archivesForDay(path, stamp) {
|
||||
out = append(out, readGzipLines(archive)...)
|
||||
}
|
||||
out = append(out, readFileLines(path)...)
|
||||
return out
|
||||
}
|
||||
|
||||
// archivesForDay 列出与目标日期匹配的归档(access-20261024-1305.log.gz):
|
||||
// 文件名里的日期是按轮转时刻取的,所以既可能属于当天,也可能属于前一天跨零点的那一段。
|
||||
func archivesForDay(path, day string) []string {
|
||||
dir := filepath.Dir(path)
|
||||
base := strings.TrimSuffix(filepath.Base(path), ".log")
|
||||
prefix := base + "-" + day
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, 4)
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if entry.IsDir() || !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, ".log.gz") {
|
||||
continue
|
||||
}
|
||||
out = append(out, filepath.Join(dir, name))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func readFileLines(path string) []string {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer file.Close()
|
||||
var lines []string
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Text())
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func readGzipLines(path string) []string {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer file.Close()
|
||||
gz, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer gz.Close()
|
||||
var lines []string
|
||||
scanner := bufio.NewScanner(gz)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Text())
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func calendarHourlyLabels() []string {
|
||||
labels := make([]string, hourlyBucketCount)
|
||||
for i := 0; i < hourlyBucketCount; i++ {
|
||||
@@ -358,17 +424,10 @@ func CountTodayAccess(path string) (total int, errors int, avgMs float64) {
|
||||
}
|
||||
|
||||
func CountAccessForDay(path string, day time.Time) (total int, errors int, avgMs float64) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, 0, 0
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var sum float64
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
entry, ok := parseAccess(scanner.Text())
|
||||
// 与小时曲线一致:轮转后的内容在归档里,只扫当前文件会漏掉当天早些时候的请求
|
||||
for _, line := range accessLinesForDay(path, day) {
|
||||
entry, ok := parseAccess(line)
|
||||
if !ok || !isDashboardAccess(entry) {
|
||||
continue
|
||||
}
|
||||
@@ -388,7 +447,14 @@ func CountAccessForDay(path string, day time.Time) (total int, errors int, avgMs
|
||||
return total, errors, avgMs
|
||||
}
|
||||
|
||||
// tailLines 返回文件末尾的至多 max 行(保持原顺序,旧→新)。
|
||||
//
|
||||
// 之所以从文件尾往前读:日志可能上到几百 MB,而调用方只要最后几十行。
|
||||
// 顺序扫描等于每刷新一次页面就把整个文件读一遍,NAS 这种长期运行的机器上很吃亏。
|
||||
func tailLines(path string, max int) ([]string, error) {
|
||||
if max <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -398,16 +464,59 @@ func tailLines(path string, max int) ([]string, error) {
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
var lines []string
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Text())
|
||||
if len(lines) > max {
|
||||
lines = lines[1:]
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := info.Size()
|
||||
if size == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// 从尾部往前读块,攒够 max 个换行就停。64KB 起步,最多扩到 8MB:
|
||||
// 单行上限 1MB(与 scanner 的上限一致),保证超长行不会被截断成多条。
|
||||
const (
|
||||
startChunk = 64 << 10
|
||||
maxChunk = 8 << 20
|
||||
)
|
||||
chunk := int64(startChunk)
|
||||
var tail []byte
|
||||
offset := size
|
||||
for offset > 0 {
|
||||
readSize := chunk
|
||||
if readSize > offset {
|
||||
readSize = offset
|
||||
}
|
||||
offset -= readSize
|
||||
buf := make([]byte, readSize)
|
||||
if _, err := file.ReadAt(buf, offset); err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
tail = append(buf, tail...)
|
||||
// 已攒到足够的行就不必继续往前读(末尾若没有换行,也算一行)
|
||||
if strings.Count(string(tail), "\n") > max {
|
||||
break
|
||||
}
|
||||
if chunk < maxChunk {
|
||||
chunk *= 2
|
||||
}
|
||||
}
|
||||
return lines, scanner.Err()
|
||||
|
||||
text := string(tail)
|
||||
if offset > 0 {
|
||||
// 从中间开始读时,第一行很可能是半行:丢掉它
|
||||
if i := strings.IndexByte(text, '\n'); i >= 0 {
|
||||
text = text[i+1:]
|
||||
}
|
||||
}
|
||||
lines := strings.Split(strings.TrimRight(text, "\n"), "\n")
|
||||
if len(lines) > max {
|
||||
lines = lines[len(lines)-max:]
|
||||
}
|
||||
if len(lines) == 1 && lines[0] == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
func asString(v any) string {
|
||||
|
||||
Reference in New Issue
Block a user