From 3e4858a624cb6fb2df963129b3b1a44c315cc182 Mon Sep 17 00:00:00 2001 From: Luis Pater Date: Sat, 20 Dec 2025 15:52:59 +0800 Subject: [PATCH] feat(config): add log file size limit configuration #535 This commit introduces a new configuration option `logs-max-total-size-mb` that allows users to set a maximum total size (in MB) for log files in the logs directory. When this limit is exceeded, the oldest log files will be automatically deleted to stay within the specified size. Setting this value to 0 (the default) disables this feature. This change enhances log management by preventing excessive disk space usage. --- cmd/server/main.go | 2 +- config.example.yaml | 4 + internal/api/server.go | 15 +- internal/config/config.go | 9 ++ internal/logging/global_logger.go | 32 +++-- internal/logging/log_dir_cleaner.go | 166 +++++++++++++++++++++++ internal/logging/log_dir_cleaner_test.go | 70 ++++++++++ 7 files changed, 282 insertions(+), 16 deletions(-) create mode 100644 internal/logging/log_dir_cleaner.go create mode 100644 internal/logging/log_dir_cleaner_test.go diff --git a/cmd/server/main.go b/cmd/server/main.go index aec51ab8..2b20bcb5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -405,7 +405,7 @@ func main() { usage.SetStatisticsEnabled(cfg.UsageStatisticsEnabled) coreauth.SetQuotaCooldownDisabled(cfg.DisableCooling) - if err = logging.ConfigureLogOutput(cfg.LoggingToFile); err != nil { + if err = logging.ConfigureLogOutput(cfg.LoggingToFile, cfg.LogsMaxTotalSizeMB); err != nil { log.Errorf("failed to configure log output: %v", err) return } diff --git a/config.example.yaml b/config.example.yaml index 563dd06c..1e084cb4 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -42,6 +42,10 @@ debug: false # When true, write application logs to rotating files instead of stdout logging-to-file: false +# Maximum total size (MB) of log files under the logs directory. When exceeded, the oldest log +# files are deleted until within the limit. Set to 0 to disable. +logs-max-total-size-mb: 0 + # When false, disable in-memory usage statistics aggregation usage-statistics-enabled: false diff --git a/internal/api/server.go b/internal/api/server.go index d6fe91bf..094da118 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -844,11 +844,20 @@ func (s *Server) UpdateClients(cfg *config.Config) { } } - if oldCfg != nil && oldCfg.LoggingToFile != cfg.LoggingToFile { - if err := logging.ConfigureLogOutput(cfg.LoggingToFile); err != nil { + if oldCfg == nil || oldCfg.LoggingToFile != cfg.LoggingToFile || oldCfg.LogsMaxTotalSizeMB != cfg.LogsMaxTotalSizeMB { + if err := logging.ConfigureLogOutput(cfg.LoggingToFile, cfg.LogsMaxTotalSizeMB); err != nil { log.Errorf("failed to reconfigure log output: %v", err) } else { - log.Debugf("logging_to_file updated from %t to %t", oldCfg.LoggingToFile, cfg.LoggingToFile) + if oldCfg == nil { + log.Debug("log output configuration refreshed") + } else { + if oldCfg.LoggingToFile != cfg.LoggingToFile { + log.Debugf("logging_to_file updated from %t to %t", oldCfg.LoggingToFile, cfg.LoggingToFile) + } + if oldCfg.LogsMaxTotalSizeMB != cfg.LogsMaxTotalSizeMB { + log.Debugf("logs_max_total_size_mb updated from %d to %d", oldCfg.LogsMaxTotalSizeMB, cfg.LogsMaxTotalSizeMB) + } + } } } diff --git a/internal/config/config.go b/internal/config/config.go index 2ced3796..cd56bd77 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -42,6 +42,10 @@ type Config struct { // LoggingToFile controls whether application logs are written to rotating files or stdout. LoggingToFile bool `yaml:"logging-to-file" json:"logging-to-file"` + // LogsMaxTotalSizeMB limits the total size (in MB) of log files under the logs directory. + // When exceeded, the oldest log files are deleted until within the limit. Set to 0 to disable. + LogsMaxTotalSizeMB int `yaml:"logs-max-total-size-mb" json:"logs-max-total-size-mb"` + // UsageStatisticsEnabled toggles in-memory usage aggregation; when false, usage data is discarded. UsageStatisticsEnabled bool `yaml:"usage-statistics-enabled" json:"usage-statistics-enabled"` @@ -341,6 +345,7 @@ func LoadConfigOptional(configFile string, optional bool) (*Config, error) { // Set defaults before unmarshal so that absent keys keep defaults. cfg.Host = "" // Default empty: binds to all interfaces (IPv4 + IPv6) cfg.LoggingToFile = false + cfg.LogsMaxTotalSizeMB = 0 cfg.UsageStatisticsEnabled = false cfg.DisableCooling = false cfg.AmpCode.RestrictManagementToLocalhost = false // Default to false: API key auth is sufficient @@ -385,6 +390,10 @@ func LoadConfigOptional(configFile string, optional bool) (*Config, error) { cfg.RemoteManagement.PanelGitHubRepository = DefaultPanelGitHubRepository } + if cfg.LogsMaxTotalSizeMB < 0 { + cfg.LogsMaxTotalSizeMB = 0 + } + // Sync request authentication providers with inline API keys for backwards compatibility. syncInlineAccessProvider(&cfg) diff --git a/internal/logging/global_logger.go b/internal/logging/global_logger.go index 28fde213..e7d795fa 100644 --- a/internal/logging/global_logger.go +++ b/internal/logging/global_logger.go @@ -72,39 +72,45 @@ func SetupBaseLogger() { } // ConfigureLogOutput switches the global log destination between rotating files and stdout. -func ConfigureLogOutput(loggingToFile bool) error { +// When logsMaxTotalSizeMB > 0, a background cleaner removes the oldest log files in the logs directory +// until the total size is within the limit. +func ConfigureLogOutput(loggingToFile bool, logsMaxTotalSizeMB int) error { SetupBaseLogger() writerMu.Lock() defer writerMu.Unlock() + logDir := "logs" + if base := util.WritablePath(); base != "" { + logDir = filepath.Join(base, "logs") + } + + protectedPath := "" if loggingToFile { - logDir := "logs" - if base := util.WritablePath(); base != "" { - logDir = filepath.Join(base, "logs") - } if err := os.MkdirAll(logDir, 0o755); err != nil { return fmt.Errorf("logging: failed to create log directory: %w", err) } if logWriter != nil { _ = logWriter.Close() } + protectedPath = filepath.Join(logDir, "main.log") logWriter = &lumberjack.Logger{ - Filename: filepath.Join(logDir, "main.log"), + Filename: protectedPath, MaxSize: 10, MaxBackups: 0, MaxAge: 0, Compress: false, } log.SetOutput(logWriter) - return nil + } else { + if logWriter != nil { + _ = logWriter.Close() + logWriter = nil + } + log.SetOutput(os.Stdout) } - if logWriter != nil { - _ = logWriter.Close() - logWriter = nil - } - log.SetOutput(os.Stdout) + configureLogDirCleanerLocked(logDir, logsMaxTotalSizeMB, protectedPath) return nil } @@ -112,6 +118,8 @@ func closeLogOutputs() { writerMu.Lock() defer writerMu.Unlock() + stopLogDirCleanerLocked() + if logWriter != nil { _ = logWriter.Close() logWriter = nil diff --git a/internal/logging/log_dir_cleaner.go b/internal/logging/log_dir_cleaner.go new file mode 100644 index 00000000..e563b381 --- /dev/null +++ b/internal/logging/log_dir_cleaner.go @@ -0,0 +1,166 @@ +package logging + +import ( + "context" + "os" + "path/filepath" + "sort" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +const logDirCleanerInterval = time.Minute + +var logDirCleanerCancel context.CancelFunc + +func configureLogDirCleanerLocked(logDir string, maxTotalSizeMB int, protectedPath string) { + stopLogDirCleanerLocked() + + if maxTotalSizeMB <= 0 { + return + } + + maxBytes := int64(maxTotalSizeMB) * 1024 * 1024 + if maxBytes <= 0 { + return + } + + dir := strings.TrimSpace(logDir) + if dir == "" { + return + } + + ctx, cancel := context.WithCancel(context.Background()) + logDirCleanerCancel = cancel + go runLogDirCleaner(ctx, filepath.Clean(dir), maxBytes, strings.TrimSpace(protectedPath)) +} + +func stopLogDirCleanerLocked() { + if logDirCleanerCancel == nil { + return + } + logDirCleanerCancel() + logDirCleanerCancel = nil +} + +func runLogDirCleaner(ctx context.Context, logDir string, maxBytes int64, protectedPath string) { + ticker := time.NewTicker(logDirCleanerInterval) + defer ticker.Stop() + + cleanOnce := func() { + deleted, errClean := enforceLogDirSizeLimit(logDir, maxBytes, protectedPath) + if errClean != nil { + log.WithError(errClean).Warn("logging: failed to enforce log directory size limit") + return + } + if deleted > 0 { + log.Debugf("logging: removed %d old log file(s) to enforce log directory size limit", deleted) + } + } + + cleanOnce() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cleanOnce() + } + } +} + +func enforceLogDirSizeLimit(logDir string, maxBytes int64, protectedPath string) (int, error) { + if maxBytes <= 0 { + return 0, nil + } + + dir := strings.TrimSpace(logDir) + if dir == "" { + return 0, nil + } + dir = filepath.Clean(dir) + + entries, errRead := os.ReadDir(dir) + if errRead != nil { + if os.IsNotExist(errRead) { + return 0, nil + } + return 0, errRead + } + + protected := strings.TrimSpace(protectedPath) + if protected != "" { + protected = filepath.Clean(protected) + } + + type logFile struct { + path string + size int64 + modTime time.Time + } + + var ( + files []logFile + total int64 + ) + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if !isLogFileName(name) { + continue + } + info, errInfo := entry.Info() + if errInfo != nil { + continue + } + if !info.Mode().IsRegular() { + continue + } + path := filepath.Join(dir, name) + files = append(files, logFile{ + path: path, + size: info.Size(), + modTime: info.ModTime(), + }) + total += info.Size() + } + + if total <= maxBytes { + return 0, nil + } + + sort.Slice(files, func(i, j int) bool { + return files[i].modTime.Before(files[j].modTime) + }) + + deleted := 0 + for _, file := range files { + if total <= maxBytes { + break + } + if protected != "" && filepath.Clean(file.path) == protected { + continue + } + if errRemove := os.Remove(file.path); errRemove != nil { + log.WithError(errRemove).Warnf("logging: failed to remove old log file: %s", filepath.Base(file.path)) + continue + } + total -= file.size + deleted++ + } + + return deleted, nil +} + +func isLogFileName(name string) bool { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return false + } + lower := strings.ToLower(trimmed) + return strings.HasSuffix(lower, ".log") || strings.HasSuffix(lower, ".log.gz") +} diff --git a/internal/logging/log_dir_cleaner_test.go b/internal/logging/log_dir_cleaner_test.go new file mode 100644 index 00000000..3670da50 --- /dev/null +++ b/internal/logging/log_dir_cleaner_test.go @@ -0,0 +1,70 @@ +package logging + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestEnforceLogDirSizeLimitDeletesOldest(t *testing.T) { + dir := t.TempDir() + + writeLogFile(t, filepath.Join(dir, "old.log"), 60, time.Unix(1, 0)) + writeLogFile(t, filepath.Join(dir, "mid.log"), 60, time.Unix(2, 0)) + protected := filepath.Join(dir, "main.log") + writeLogFile(t, protected, 60, time.Unix(3, 0)) + + deleted, err := enforceLogDirSizeLimit(dir, 120, protected) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if deleted != 1 { + t.Fatalf("expected 1 deleted file, got %d", deleted) + } + + if _, err := os.Stat(filepath.Join(dir, "old.log")); !os.IsNotExist(err) { + t.Fatalf("expected old.log to be removed, stat error: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "mid.log")); err != nil { + t.Fatalf("expected mid.log to remain, stat error: %v", err) + } + if _, err := os.Stat(protected); err != nil { + t.Fatalf("expected protected main.log to remain, stat error: %v", err) + } +} + +func TestEnforceLogDirSizeLimitSkipsProtected(t *testing.T) { + dir := t.TempDir() + + protected := filepath.Join(dir, "main.log") + writeLogFile(t, protected, 200, time.Unix(1, 0)) + writeLogFile(t, filepath.Join(dir, "other.log"), 50, time.Unix(2, 0)) + + deleted, err := enforceLogDirSizeLimit(dir, 100, protected) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if deleted != 1 { + t.Fatalf("expected 1 deleted file, got %d", deleted) + } + + if _, err := os.Stat(protected); err != nil { + t.Fatalf("expected protected main.log to remain, stat error: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "other.log")); !os.IsNotExist(err) { + t.Fatalf("expected other.log to be removed, stat error: %v", err) + } +} + +func writeLogFile(t *testing.T, path string, size int, modTime time.Time) { + t.Helper() + + data := make([]byte, size) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatalf("set times: %v", err) + } +}