feat (recovery): Introduced panic recovery and monitoring mechanisms.

A new recovery package has been added, containing Recoverer and PanicMetrics, for capturing panics in goroutines.
It records stack trace information and provides statistical metrics. This mechanism is also integrated into the TCP connector and listener.
Enhance service stability and observability.
This commit is contained in:
Gouryella
2025-12-10 16:10:26 +08:00
parent 8e6f9c553a
commit f6f2c6fd5b
4 changed files with 241 additions and 10 deletions

View File

@@ -11,6 +11,7 @@ import (
"drip/internal/server/tunnel"
"drip/internal/shared/pool"
"drip/internal/shared/recovery"
"go.uber.org/zap"
)
@@ -32,6 +33,8 @@ type Listener struct {
connections map[string]*Connection
connMu sync.RWMutex
workerPool *pool.WorkerPool // Worker pool for connection handling
recoverer *recovery.Recoverer
panicMetrics *recovery.PanicMetrics
}
func NewListener(address string, tlsConfig *tls.Config, authToken string, manager *tunnel.Manager, logger *zap.Logger, portAlloc *PortAllocator, domain string, publicPort int, httpHandler http.Handler, responseChans HTTPResponseHandler) *Listener {
@@ -46,6 +49,9 @@ func NewListener(address string, tlsConfig *tls.Config, authToken string, manage
zap.Int("queue_size", queueSize),
)
panicMetrics := recovery.NewPanicMetrics(logger, nil)
recoverer := recovery.NewRecoverer(logger, panicMetrics)
return &Listener{
address: address,
tlsConfig: tlsConfig,
@@ -60,6 +66,8 @@ func NewListener(address string, tlsConfig *tls.Config, authToken string, manage
stopCh: make(chan struct{}),
connections: make(map[string]*Connection),
workerPool: workerPool,
recoverer: recoverer,
panicMetrics: panicMetrics,
}
}
@@ -86,6 +94,7 @@ func (l *Listener) Start() error {
// acceptLoop accepts incoming connections
func (l *Listener) acceptLoop() {
defer l.wg.Done()
defer l.recoverer.Recover("acceptLoop")
for {
select {
@@ -114,12 +123,20 @@ func (l *Listener) acceptLoop() {
}
l.wg.Add(1)
submitted := l.workerPool.Submit(func() {
l.handleConnection(conn)
})
submitted := l.workerPool.Submit(l.recoverer.WrapGoroutine(
fmt.Sprintf("handleConnection-%s", conn.RemoteAddr().String()),
func() {
l.handleConnection(conn)
},
))
if !submitted {
go l.handleConnection(conn)
l.recoverer.SafeGo(
fmt.Sprintf("handleConnection-fallback-%s", conn.RemoteAddr().String()),
func() {
l.handleConnection(conn)
},
)
}
}
}
@@ -128,6 +145,12 @@ func (l *Listener) acceptLoop() {
func (l *Listener) handleConnection(netConn net.Conn) {
defer l.wg.Done()
defer netConn.Close()
defer l.recoverer.RecoverWithCallback("handleConnection", func(p interface{}) {
connID := netConn.RemoteAddr().String()
l.connMu.Lock()
delete(l.connections, connID)
l.connMu.Unlock()
})
tlsConn, ok := netConn.(*tls.Conn)
if !ok {