feat: Add HTTP streaming, compression support, and Docker deployment

enhancements

  - Add adaptive HTTP response handling with automatic streaming for large
  responses (>1MB)
  - Implement zero-copy streaming using buffer pools for better performance
  - Add compression module for reduced bandwidth usage
  - Add GitHub Container Registry workflow for automated Docker builds
  - Add production-optimized Dockerfile and docker-compose configuration
  - Simplify background mode with -d flag and improved daemon management
  - Update documentation with new command syntax and deployment guides
  - Clean up unused code and improve error handling
  - Fix lipgloss style usage (remove unnecessary .Copy() calls)

internal/shared/compression/hpack/decoder.go

@@ -0,0 +1,280 @@
+package hpack
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net/http"
+	"sync"
+)
+
+// Decoder decompresses HPACK-encoded headers
+// Each connection MUST have its own decoder instance to maintain correct state
+type Decoder struct {
+	mu           sync.Mutex
+	dynamicTable *DynamicTable
+	staticTable  *StaticTable
+	maxTableSize uint32
+}
+
+// NewDecoder creates a new HPACK decoder
+func NewDecoder(maxTableSize uint32) *Decoder {
+	if maxTableSize == 0 {
+		maxTableSize = DefaultDynamicTableSize
+	}
+
+	return &Decoder{
+		dynamicTable: NewDynamicTable(maxTableSize),
+		staticTable:  GetStaticTable(),
+		maxTableSize: maxTableSize,
+	}
+}
+
+// Decode decodes HPACK-encoded headers
+func (d *Decoder) Decode(data []byte) (http.Header, error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if len(data) == 0 {
+		return http.Header{}, nil
+	}
+
+	headers := make(http.Header)
+	buf := bytes.NewReader(data)
+
+	for buf.Len() > 0 {
+		b, err := buf.ReadByte()
+		if err != nil {
+			return nil, fmt.Errorf("read header byte: %w", err)
+		}
+
+		// Unread the byte so we can process it properly
+		if err := buf.UnreadByte(); err != nil {
+			return nil, err
+		}
+
+		var name, value string
+
+		if b&0x80 != 0 {
+			// Indexed header field (10xxxxxx)
+			name, value, err = d.decodeIndexedHeader(buf)
+		} else if b&0x40 != 0 {
+			// Literal with incremental indexing (01xxxxxx)
+			name, value, err = d.decodeLiteralWithIndexing(buf)
+		} else {
+			// Literal without indexing (0000xxxx)
+			name, value, err = d.decodeLiteralWithoutIndexing(buf)
+		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		headers.Add(name, value)
+	}
+
+	return headers, nil
+}
+
+// decodeIndexedHeader decodes an indexed header field
+func (d *Decoder) decodeIndexedHeader(buf *bytes.Reader) (string, string, error) {
+	index, err := d.readInteger(buf, 7)
+	if err != nil {
+		return "", "", fmt.Errorf("read index: %w", err)
+	}
+
+	if index == 0 {
+		return "", "", errors.New("invalid index: 0")
+	}
+
+	staticSize := uint32(d.staticTable.Size())
+
+	if index <= staticSize {
+		// Static table
+		return d.staticTable.Get(index - 1)
+	}
+
+	// Dynamic table (indices start after static table)
+	dynamicIndex := index - staticSize - 1
+	return d.dynamicTable.Get(dynamicIndex)
+}
+
+// decodeLiteralWithIndexing decodes a literal header with incremental indexing
+func (d *Decoder) decodeLiteralWithIndexing(buf *bytes.Reader) (string, string, error) {
+	nameIndex, err := d.readInteger(buf, 6)
+	if err != nil {
+		return "", "", err
+	}
+
+	var name string
+	if nameIndex == 0 {
+		// Name is literal
+		name, err = d.readString(buf)
+		if err != nil {
+			return "", "", fmt.Errorf("read name: %w", err)
+		}
+	} else {
+		// Name is indexed
+		staticSize := uint32(d.staticTable.Size())
+		if nameIndex <= staticSize {
+			name, _, err = d.staticTable.Get(nameIndex - 1)
+		} else {
+			dynamicIndex := nameIndex - staticSize - 1
+			name, _, err = d.dynamicTable.Get(dynamicIndex)
+		}
+		if err != nil {
+			return "", "", fmt.Errorf("get indexed name: %w", err)
+		}
+	}
+
+	// Value is always literal
+	value, err := d.readString(buf)
+	if err != nil {
+		return "", "", fmt.Errorf("read value: %w", err)
+	}
+
+	// Add to dynamic table
+	d.dynamicTable.Add(name, value)
+
+	return name, value, nil
+}
+
+// decodeLiteralWithoutIndexing decodes a literal header without indexing
+func (d *Decoder) decodeLiteralWithoutIndexing(buf *bytes.Reader) (string, string, error) {
+	nameIndex, err := d.readInteger(buf, 4)
+	if err != nil {
+		return "", "", err
+	}
+
+	var name string
+	if nameIndex == 0 {
+		// Name is literal
+		name, err = d.readString(buf)
+		if err != nil {
+			return "", "", fmt.Errorf("read name: %w", err)
+		}
+	} else {
+		// Name is indexed
+		staticSize := uint32(d.staticTable.Size())
+		if nameIndex <= staticSize {
+			name, _, err = d.staticTable.Get(nameIndex - 1)
+		} else {
+			dynamicIndex := nameIndex - staticSize - 1
+			name, _, err = d.dynamicTable.Get(dynamicIndex)
+		}
+		if err != nil {
+			return "", "", fmt.Errorf("get indexed name: %w", err)
+		}
+	}
+
+	// Value is always literal
+	value, err := d.readString(buf)
+	if err != nil {
+		return "", "", fmt.Errorf("read value: %w", err)
+	}
+
+	// Do NOT add to dynamic table
+
+	return name, value, nil
+}
+
+// readInteger reads an HPACK integer
+func (d *Decoder) readInteger(buf *bytes.Reader, prefixBits int) (uint32, error) {
+	if prefixBits < 1 || prefixBits > 8 {
+		return 0, fmt.Errorf("invalid prefix bits: %d", prefixBits)
+	}
+
+	b, err := buf.ReadByte()
+	if err != nil {
+		return 0, err
+	}
+
+	maxPrefix := uint32((1 << prefixBits) - 1)
+	mask := byte(maxPrefix)
+
+	value := uint32(b & mask)
+	if value < maxPrefix {
+		return value, nil
+	}
+
+	// Multi-byte integer
+	m := uint32(0)
+	for {
+		b, err := buf.ReadByte()
+		if err != nil {
+			return 0, err
+		}
+
+		value += uint32(b&0x7f) << m
+		m += 7
+
+		if b&0x80 == 0 {
+			break
+		}
+
+		if m > 28 {
+			return 0, errors.New("integer overflow")
+		}
+	}
+
+	return value, nil
+}
+
+// readString reads an HPACK string
+func (d *Decoder) readString(buf *bytes.Reader) (string, error) {
+	b, err := buf.ReadByte()
+	if err != nil {
+		return "", err
+	}
+
+	if err := buf.UnreadByte(); err != nil {
+		return "", err
+	}
+
+	huffmanEncoded := (b & 0x80) != 0
+
+	length, err := d.readInteger(buf, 7)
+	if err != nil {
+		return "", fmt.Errorf("read string length: %w", err)
+	}
+
+	if length == 0 {
+		return "", nil
+	}
+
+	if length > uint32(buf.Len()) {
+		return "", fmt.Errorf("string length %d exceeds buffer size %d", length, buf.Len())
+	}
+
+	data := make([]byte, length)
+	n, err := buf.Read(data)
+	if err != nil {
+		return "", err
+	}
+	if n != int(length) {
+		return "", fmt.Errorf("expected %d bytes, read %d", length, n)
+	}
+
+	if huffmanEncoded {
+		// TODO: Implement Huffman decoding if needed
+		return "", errors.New("huffman decoding not implemented")
+	}
+
+	return string(data), nil
+}
+
+// SetMaxTableSize updates the dynamic table size
+func (d *Decoder) SetMaxTableSize(size uint32) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	d.maxTableSize = size
+	d.dynamicTable.SetMaxSize(size)
+}
+
+// Reset clears the dynamic table
+func (d *Decoder) Reset() {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.dynamicTable = NewDynamicTable(d.maxTableSize)
+}

internal/shared/compression/hpack/dynamic_table.go

@@ -0,0 +1,124 @@
+package hpack
+
+import (
+	"fmt"
+)
+
+// DynamicTable implements the HPACK dynamic table (RFC 7541 Section 2.3.2)
+// The dynamic table is a FIFO queue where new entries are added at the beginning
+// and old entries are evicted when the table size exceeds the maximum
+type DynamicTable struct {
+	entries  []HeaderField
+	size     uint32 // Current size in bytes
+	maxSize  uint32 // Maximum size in bytes
+}
+
+// HeaderField represents a header name-value pair
+type HeaderField struct {
+	Name  string
+	Value string
+}
+
+// Size returns the size of this header field in bytes
+// RFC 7541: size = len(name) + len(value) + 32
+func (h *HeaderField) Size() uint32 {
+	return uint32(len(h.Name) + len(h.Value) + 32)
+}
+
+// NewDynamicTable creates a new dynamic table with the specified maximum size
+func NewDynamicTable(maxSize uint32) *DynamicTable {
+	return &DynamicTable{
+		entries: make([]HeaderField, 0, 32),
+		size:    0,
+		maxSize: maxSize,
+	}
+}
+
+// Add adds a header field to the dynamic table
+// New entries are added at the beginning (index 0)
+func (dt *DynamicTable) Add(name, value string) {
+	field := HeaderField{Name: name, Value: value}
+	fieldSize := field.Size()
+
+	// If the field is larger than maxSize, don't add it
+	if fieldSize > dt.maxSize {
+		dt.evictAll()
+		return
+	}
+
+	// Evict entries if necessary to make room
+	for dt.size+fieldSize > dt.maxSize && len(dt.entries) > 0 {
+		dt.evictOldest()
+	}
+
+	// Add new entry at the beginning
+	dt.entries = append([]HeaderField{field}, dt.entries...)
+	dt.size += fieldSize
+}
+
+// Get retrieves a header field by index (0-based)
+// Index 0 is the most recently added entry
+func (dt *DynamicTable) Get(index uint32) (string, string, error) {
+	if index >= uint32(len(dt.entries)) {
+		return "", "", fmt.Errorf("index %d out of range (table size: %d)", index, len(dt.entries))
+	}
+
+	field := dt.entries[index]
+	return field.Name, field.Value, nil
+}
+
+// FindExact searches for an exact match (name and value)
+// Returns the index (0-based) and true if found
+func (dt *DynamicTable) FindExact(name, value string) (uint32, bool) {
+	for i, field := range dt.entries {
+		if field.Name == name && field.Value == value {
+			return uint32(i), true
+		}
+	}
+	return 0, false
+}
+
+// FindName searches for a name match
+// Returns the index (0-based) and true if found
+func (dt *DynamicTable) FindName(name string) (uint32, bool) {
+	for i, field := range dt.entries {
+		if field.Name == name {
+			return uint32(i), true
+		}
+	}
+	return 0, false
+}
+
+// SetMaxSize updates the maximum table size
+// If the new size is smaller, entries are evicted
+func (dt *DynamicTable) SetMaxSize(maxSize uint32) {
+	dt.maxSize = maxSize
+
+	// Evict entries if current size exceeds new max
+	for dt.size > dt.maxSize && len(dt.entries) > 0 {
+		dt.evictOldest()
+	}
+}
+
+// CurrentSize returns the current size of the table in bytes
+func (dt *DynamicTable) CurrentSize() uint32 {
+	return dt.size
+}
+
+// evictOldest removes the oldest entry (last in the slice)
+func (dt *DynamicTable) evictOldest() {
+	if len(dt.entries) == 0 {
+		return
+	}
+
+	lastIndex := len(dt.entries) - 1
+	evicted := dt.entries[lastIndex]
+	dt.entries = dt.entries[:lastIndex]
+	dt.size -= evicted.Size()
+}
+
+// evictAll removes all entries
+func (dt *DynamicTable) evictAll() {
+	dt.entries = dt.entries[:0]
+	dt.size = 0
+}

internal/shared/compression/hpack/encoder.go

@@ -0,0 +1,200 @@
+package hpack
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+)
+
+const (
+	// DefaultDynamicTableSize is the default size of the dynamic table (4KB)
+	DefaultDynamicTableSize = 4096
+
+	// IndexedHeaderField represents a fully indexed header field
+	indexedHeaderField = 0x80 // 10xxxxxx
+
+	// LiteralHeaderFieldWithIndexing represents a literal with incremental indexing
+	literalHeaderFieldWithIndexing = 0x40 // 01xxxxxx
+)
+
+// Encoder compresses HTTP headers using HPACK
+// Each connection MUST have its own encoder instance to avoid state corruption
+type Encoder struct {
+	mu            sync.Mutex
+	dynamicTable  *DynamicTable
+	staticTable   *StaticTable
+	maxTableSize  uint32
+}
+
+// NewEncoder creates a new HPACK encoder with the specified dynamic table size
+// This encoder is NOT thread-safe and should be used by a single connection
+func NewEncoder(maxTableSize uint32) *Encoder {
+	if maxTableSize == 0 {
+		maxTableSize = DefaultDynamicTableSize
+	}
+
+	return &Encoder{
+		dynamicTable: NewDynamicTable(maxTableSize),
+		staticTable:  GetStaticTable(),
+		maxTableSize: maxTableSize,
+	}
+}
+
+// Encode encodes HTTP headers into HPACK binary format
+// This method is safe to call concurrently within the same encoder instance
+func (e *Encoder) Encode(headers http.Header) ([]byte, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	if headers == nil {
+		return nil, errors.New("headers cannot be nil")
+	}
+
+	buf := &bytes.Buffer{}
+
+	for name, values := range headers {
+		for _, value := range values {
+			if err := e.encodeHeaderField(buf, name, value); err != nil {
+				return nil, fmt.Errorf("encode header %s: %w", name, err)
+			}
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
+// encodeHeaderField encodes a single header field
+func (e *Encoder) encodeHeaderField(buf *bytes.Buffer, name, value string) error {
+	// HTTP/2 requires header names to be lowercase (RFC 7540 Section 8.1.2)
+	// Convert to lowercase for table lookups and storage
+	nameLower := strings.ToLower(name)
+
+	// Try to find in static table first
+	if index, found := e.staticTable.FindExact(nameLower, value); found {
+		return e.writeIndexedHeader(buf, index+1)
+	}
+
+	// Check if name exists in static table (for literal with name reference)
+	if index, found := e.staticTable.FindName(nameLower); found {
+		return e.writeLiteralWithIndexing(buf, index+1, value, true)
+	}
+
+	// Try dynamic table
+	if index, found := e.dynamicTable.FindExact(nameLower, value); found {
+		// Dynamic table indices start after static table
+		dynamicIndex := uint32(e.staticTable.Size()) + index + 1
+		return e.writeIndexedHeader(buf, dynamicIndex)
+	}
+
+	if index, found := e.dynamicTable.FindName(nameLower); found {
+		dynamicIndex := uint32(e.staticTable.Size()) + index + 1
+		return e.writeLiteralWithIndexing(buf, dynamicIndex, value, true)
+	}
+
+	// Not found anywhere - literal with indexing and new name
+	// Write literal flag
+	buf.WriteByte(literalHeaderFieldWithIndexing)
+
+	// Write name as literal string (must come before value)
+	// Use lowercase name for consistency
+	if err := e.writeString(buf, nameLower, false); err != nil {
+		return err
+	}
+
+	// Write value as literal string
+	if err := e.writeString(buf, value, false); err != nil {
+		return err
+	}
+
+	// Add to dynamic table with lowercase name
+	e.dynamicTable.Add(nameLower, value)
+
+	return nil
+}
+
+// writeIndexedHeader writes an indexed header field (10xxxxxx)
+func (e *Encoder) writeIndexedHeader(buf *bytes.Buffer, index uint32) error {
+	return e.writeInteger(buf, index, 7, indexedHeaderField)
+}
+
+// writeLiteralWithIndexing writes a literal header with incremental indexing (01xxxxxx)
+func (e *Encoder) writeLiteralWithIndexing(buf *bytes.Buffer, nameIndex uint32, value string, hasIndex bool) error {
+	if hasIndex {
+		// Write name as index
+		if err := e.writeInteger(buf, nameIndex, 6, literalHeaderFieldWithIndexing); err != nil {
+			return err
+		}
+	} else {
+		// Write literal flag
+		buf.WriteByte(literalHeaderFieldWithIndexing)
+	}
+
+	// Write value as literal string
+	return e.writeString(buf, value, false)
+}
+
+// writeInteger writes an integer using HPACK integer representation
+func (e *Encoder) writeInteger(buf *bytes.Buffer, value uint32, prefixBits int, prefix byte) error {
+	if prefixBits < 1 || prefixBits > 8 {
+		return fmt.Errorf("invalid prefix bits: %d", prefixBits)
+	}
+
+	maxPrefix := uint32((1 << prefixBits) - 1)
+
+	if value < maxPrefix {
+		buf.WriteByte(prefix | byte(value))
+		return nil
+	}
+
+	// Value >= maxPrefix, need multiple bytes
+	buf.WriteByte(prefix | byte(maxPrefix))
+	value -= maxPrefix
+
+	for value >= 128 {
+		buf.WriteByte(byte(value%128) | 0x80)
+		value /= 128
+	}
+	buf.WriteByte(byte(value))
+
+	return nil
+}
+
+// writeString writes a string using HPACK string representation
+func (e *Encoder) writeString(buf *bytes.Buffer, str string, huffmanEncode bool) error {
+	// For simplicity, we don't use Huffman encoding in this implementation
+	// Huffman flag is bit 7, followed by length in remaining 7 bits
+
+	length := uint32(len(str))
+	if huffmanEncode {
+		// TODO: Implement Huffman encoding if needed
+		return errors.New("huffman encoding not implemented")
+	}
+
+	// Write length with H=0 (no Huffman)
+	if err := e.writeInteger(buf, length, 7, 0x00); err != nil {
+		return err
+	}
+
+	// Write string bytes
+	buf.WriteString(str)
+	return nil
+}
+
+// SetMaxTableSize updates the dynamic table size
+func (e *Encoder) SetMaxTableSize(size uint32) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	e.maxTableSize = size
+	e.dynamicTable.SetMaxSize(size)
+}
+
+// Reset clears the dynamic table
+func (e *Encoder) Reset() {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.dynamicTable = NewDynamicTable(e.maxTableSize)
+}

internal/shared/compression/hpack/static_table.go

@@ -0,0 +1,150 @@
+package hpack
+
+import (
+	"fmt"
+	"sync"
+)
+
+// StaticTable implements the HPACK static table (RFC 7541 Appendix A)
+// The static table is predefined and never changes
+type StaticTable struct {
+	entries []HeaderField
+	nameMap map[string][]uint32 // Maps name to list of indices
+}
+
+var (
+	staticTableInstance *StaticTable
+	staticTableOnce     sync.Once
+)
+
+// GetStaticTable returns the singleton static table instance
+func GetStaticTable() *StaticTable {
+	staticTableOnce.Do(func() {
+		staticTableInstance = newStaticTable()
+	})
+	return staticTableInstance
+}
+
+// newStaticTable creates and initializes the static table
+func newStaticTable() *StaticTable {
+	// RFC 7541 Appendix A - Static Table Definition
+	// We include the most common headers for HTTP
+	entries := []HeaderField{
+		{Name: ":authority", Value: ""},
+		{Name: ":method", Value: "GET"},
+		{Name: ":method", Value: "POST"},
+		{Name: ":path", Value: "/"},
+		{Name: ":path", Value: "/index.html"},
+		{Name: ":scheme", Value: "http"},
+		{Name: ":scheme", Value: "https"},
+		{Name: ":status", Value: "200"},
+		{Name: ":status", Value: "204"},
+		{Name: ":status", Value: "206"},
+		{Name: ":status", Value: "304"},
+		{Name: ":status", Value: "400"},
+		{Name: ":status", Value: "404"},
+		{Name: ":status", Value: "500"},
+		{Name: "accept-charset", Value: ""},
+		{Name: "accept-encoding", Value: "gzip, deflate"},
+		{Name: "accept-language", Value: ""},
+		{Name: "accept-ranges", Value: ""},
+		{Name: "accept", Value: ""},
+		{Name: "access-control-allow-origin", Value: ""},
+		{Name: "age", Value: ""},
+		{Name: "allow", Value: ""},
+		{Name: "authorization", Value: ""},
+		{Name: "cache-control", Value: ""},
+		{Name: "content-disposition", Value: ""},
+		{Name: "content-encoding", Value: ""},
+		{Name: "content-language", Value: ""},
+		{Name: "content-length", Value: ""},
+		{Name: "content-location", Value: ""},
+		{Name: "content-range", Value: ""},
+		{Name: "content-type", Value: ""},
+		{Name: "cookie", Value: ""},
+		{Name: "date", Value: ""},
+		{Name: "etag", Value: ""},
+		{Name: "expect", Value: ""},
+		{Name: "expires", Value: ""},
+		{Name: "from", Value: ""},
+		{Name: "host", Value: ""},
+		{Name: "if-match", Value: ""},
+		{Name: "if-modified-since", Value: ""},
+		{Name: "if-none-match", Value: ""},
+		{Name: "if-range", Value: ""},
+		{Name: "if-unmodified-since", Value: ""},
+		{Name: "last-modified", Value: ""},
+		{Name: "link", Value: ""},
+		{Name: "location", Value: ""},
+		{Name: "max-forwards", Value: ""},
+		{Name: "proxy-authenticate", Value: ""},
+		{Name: "proxy-authorization", Value: ""},
+		{Name: "range", Value: ""},
+		{Name: "referer", Value: ""},
+		{Name: "refresh", Value: ""},
+		{Name: "retry-after", Value: ""},
+		{Name: "server", Value: ""},
+		{Name: "set-cookie", Value: ""},
+		{Name: "strict-transport-security", Value: ""},
+		{Name: "transfer-encoding", Value: ""},
+		{Name: "user-agent", Value: ""},
+		{Name: "vary", Value: ""},
+		{Name: "via", Value: ""},
+		{Name: "www-authenticate", Value: ""},
+	}
+
+	// Build name index map
+	nameMap := make(map[string][]uint32)
+	for i, entry := range entries {
+		nameMap[entry.Name] = append(nameMap[entry.Name], uint32(i))
+	}
+
+	return &StaticTable{
+		entries: entries,
+		nameMap: nameMap,
+	}
+}
+
+// Get retrieves a header field by index (0-based)
+func (st *StaticTable) Get(index uint32) (string, string, error) {
+	if index >= uint32(len(st.entries)) {
+		return "", "", fmt.Errorf("index %d out of range (static table size: %d)", index, len(st.entries))
+	}
+
+	field := st.entries[index]
+	return field.Name, field.Value, nil
+}
+
+// FindExact searches for an exact match (name and value)
+// Returns the index (0-based) and true if found
+func (st *StaticTable) FindExact(name, value string) (uint32, bool) {
+	indices, exists := st.nameMap[name]
+	if !exists {
+		return 0, false
+	}
+
+	for _, index := range indices {
+		field := st.entries[index]
+		if field.Value == value {
+			return index, true
+		}
+	}
+
+	return 0, false
+}
+
+// FindName searches for a name match
+// Returns the first matching index (0-based) and true if found
+func (st *StaticTable) FindName(name string) (uint32, bool) {
+	indices, exists := st.nameMap[name]
+	if !exists || len(indices) == 0 {
+		return 0, false
+	}
+
+	return indices[0], true
+}
+
+// Size returns the number of entries in the static table
+func (st *StaticTable) Size() int {
+	return len(st.entries)
+}

internal/shared/constants/constants.go

@@ -18,9 +18,6 @@ const (
 	// RequestTimeout is the maximum time to wait for a response from the client
 	RequestTimeout = 30 * time.Second
 
-	// MaxRequestBodySize is the maximum size of an HTTP request body (10MB)
-	MaxRequestBodySize = 10 * 1024 * 1024
-
 	// ReconnectBaseDelay is the initial delay for reconnection attempts
 	ReconnectBaseDelay = 1 * time.Second
 

internal/shared/protocol/binary_header.go

@@ -18,11 +18,17 @@ type DataHeader struct {
 type DataType uint8
 
 const (
-	DataTypeData         DataType = 0x00 // 000
-	DataTypeResponse     DataType = 0x01 // 001
-	DataTypeClose        DataType = 0x02 // 010
-	DataTypeHTTPRequest  DataType = 0x03 // 011
-	DataTypeHTTPResponse DataType = 0x04 // 100
+	DataTypeData          DataType = 0x00 // 000
+	DataTypeResponse      DataType = 0x01 // 001
+	DataTypeClose         DataType = 0x02 // 010
+	DataTypeHTTPRequest   DataType = 0x03 // 011
+	DataTypeHTTPResponse  DataType = 0x04 // 100
+	DataTypeHTTPHead      DataType = 0x05 // 101 - streaming headers (shared)
+	DataTypeHTTPBodyChunk DataType = 0x06 // 110 - streaming body chunks (shared)
+
+	// Reuse the same type codes for request streaming to stay within 3 bits.
+	DataTypeHTTPRequestHead      DataType = DataTypeHTTPHead
+	DataTypeHTTPRequestBodyChunk DataType = DataTypeHTTPBodyChunk
 )
 
 // String returns the string representation of DataType
@@ -38,6 +44,10 @@ func (t DataType) String() string {
 		return "http_request"
 	case DataTypeHTTPResponse:
 		return "http_response"
+	case DataTypeHTTPHead:
+		return "http_head"
+	case DataTypeHTTPBodyChunk:
+		return "http_body_chunk"
 	default:
 		return "unknown"
 	}
@@ -56,6 +66,10 @@ func DataTypeFromString(s string) DataType {
 		return DataTypeHTTPRequest
 	case "http_response":
 		return DataTypeHTTPResponse
+	case "http_head":
+		return DataTypeHTTPHead
+	case "http_body_chunk":
+		return DataTypeHTTPBodyChunk
 	default:
 		return DataTypeData
 	}
@@ -118,8 +132,8 @@ func (h *DataHeader) UnmarshalBinary(data []byte) error {
 
 	// Decode flags
 	flags := data[0]
-	h.Type = DataType(flags & 0x07)      // Bits 0-2
-	h.IsLast = (flags & 0x08) != 0       // Bit 3
+	h.Type = DataType(flags & 0x07) // Bits 0-2
+	h.IsLast = (flags & 0x08) != 0  // Bit 3
 
 	// Decode lengths
 	streamIDLen := int(binary.BigEndian.Uint16(data[1:3]))

internal/shared/protocol/http_codec.go

@@ -1,9 +1,10 @@
 package protocol
 
 import (
-	json "github.com/goccy/go-json"
 	"errors"
 
+	json "github.com/goccy/go-json"
+
 	"github.com/vmihailenco/msgpack/v5"
 )
 
@@ -37,6 +38,31 @@ func DecodeHTTPRequest(data []byte) (*HTTPRequest, error) {
 	return &req, nil
 }
 
+// EncodeHTTPRequestHead encodes HTTP request headers for streaming
+func EncodeHTTPRequestHead(head *HTTPRequestHead) ([]byte, error) {
+	return msgpack.Marshal(head)
+}
+
+// DecodeHTTPRequestHead decodes HTTP request headers for streaming
+func DecodeHTTPRequestHead(data []byte) (*HTTPRequestHead, error) {
+	if len(data) == 0 {
+		return nil, errors.New("empty data")
+	}
+
+	var head HTTPRequestHead
+	if data[0] == '{' {
+		if err := json.Unmarshal(data, &head); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := msgpack.Unmarshal(data, &head); err != nil {
+			return nil, err
+		}
+	}
+
+	return &head, nil
+}
+
 // EncodeHTTPResponse encodes HTTPResponse using msgpack encoding (optimized)
 func EncodeHTTPResponse(resp *HTTPResponse) ([]byte, error) {
 	return msgpack.Marshal(resp)
@@ -66,3 +92,28 @@ func DecodeHTTPResponse(data []byte) (*HTTPResponse, error) {
 
 	return &resp, nil
 }
+
+// EncodeHTTPResponseHead encodes HTTP response headers for streaming
+func EncodeHTTPResponseHead(head *HTTPResponseHead) ([]byte, error) {
+	return msgpack.Marshal(head)
+}
+
+// DecodeHTTPResponseHead decodes HTTP response headers for streaming
+func DecodeHTTPResponseHead(data []byte) (*HTTPResponseHead, error) {
+	if len(data) == 0 {
+		return nil, errors.New("empty data")
+	}
+
+	var head HTTPResponseHead
+	if data[0] == '{' {
+		if err := json.Unmarshal(data, &head); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := msgpack.Unmarshal(data, &head); err != nil {
+			return nil, err
+		}
+	}
+
+	return &head, nil
+}

internal/shared/protocol/message.go

@@ -33,6 +33,14 @@ type HTTPRequest struct {
 	Body    []byte              `json:"body,omitempty"`
 }
 
+// HTTPRequestHead represents HTTP request headers for streaming (no body)
+type HTTPRequestHead struct {
+	Method        string              `json:"method"`
+	URL           string              `json:"url"`
+	Headers       map[string][]string `json:"headers"`
+	ContentLength int64               `json:"content_length"` // -1 for unknown/chunked
+}
+
 // HTTPResponse represents an HTTP response from the local service
 type HTTPResponse struct {
 	StatusCode int                 `json:"status_code"`
@@ -41,6 +49,14 @@ type HTTPResponse struct {
 	Body       []byte              `json:"body,omitempty"`
 }
 
+// HTTPResponseHead represents HTTP response headers for streaming (no body)
+type HTTPResponseHead struct {
+	StatusCode    int                 `json:"status_code"`
+	Status        string              `json:"status"`
+	Headers       map[string][]string `json:"headers"`
+	ContentLength int64               `json:"content_length"` // -1 for unknown/chunked
+}
+
 // RegisterData contains information sent when a tunnel is registered
 type RegisterData struct {
 	Subdomain string `json:"subdomain"`

internal/shared/protocol/payload.go

@@ -7,9 +7,8 @@ import (
 	"drip/internal/shared/pool"
 )
 
-// EncodeDataPayload encodes a data header and payload into a frame payload.
-// Deprecated: Use EncodeDataPayloadPooled for better performance.
-func EncodeDataPayload(header DataHeader, data []byte) ([]byte, error) {
+// encodeDataPayload encodes a data header and payload into a frame payload.
+func encodeDataPayload(header DataHeader, data []byte) ([]byte, error) {
 	streamIDLen := len(header.StreamID)
 	requestIDLen := len(header.RequestID)
 
@@ -37,11 +36,6 @@ func EncodeDataPayload(header DataHeader, data []byte) ([]byte, error) {
 
 // EncodeDataPayloadPooled encodes with adaptive allocation based on load.
 // Returns payload slice and pool buffer pointer (may be nil).
-//
-// Adaptive strategy:
-// - Mid-load (<150 conn):   256KB threshold, pool disabled → max QPS
-// - High-load (≥300 conn):  32KB threshold, pool enabled → stable latency
-// - Transition (150-300):   Hysteresis to prevent flapping
 func EncodeDataPayloadPooled(header DataHeader, data []byte) (payload []byte, poolBuffer *[]byte, err error) {
 	streamIDLen := len(header.StreamID)
 	requestIDLen := len(header.RequestID)
@@ -50,12 +44,12 @@ func EncodeDataPayloadPooled(header DataHeader, data []byte) (payload []byte, po
 	dynamicThreshold := GetAdaptiveThreshold()
 
 	if totalLen < dynamicThreshold {
-		regularPayload, err := EncodeDataPayload(header, data)
+		regularPayload, err := encodeDataPayload(header, data)
 		return regularPayload, nil, err
 	}
 
 	if totalLen > pool.SizeLarge {
-		regularPayload, err := EncodeDataPayload(header, data)
+		regularPayload, err := encodeDataPayload(header, data)
 		return regularPayload, nil, err
 	}
 
@@ -100,7 +94,3 @@ func DecodeDataPayload(payload []byte) (DataHeader, []byte, error) {
 	data := payload[headerSize:]
 	return header, data, nil
 }
-
-func GetPayloadHeaderSize(header DataHeader) int {
-	return header.Size()
-}

internal/shared/protocol/writer.go

@@ -172,8 +172,27 @@ func (w *FrameWriter) Close() error {
 
 func (w *FrameWriter) Flush() {
 	w.mu.Lock()
-	defer w.mu.Unlock()
+	if w.closed {
+		w.mu.Unlock()
+		return
+	}
+
+	// First, drain the queue into batch
+	for {
+		select {
+		case frame, ok := <-w.queue:
+			if !ok {
+				break
+			}
+			w.batch = append(w.batch, frame)
+		default:
+			goto done
+		}
+	}
+done:
+	// Then flush the batch
 	w.flushBatchLocked()
+	w.mu.Unlock()
 }
 
 func (w *FrameWriter) EnableHeartbeat(interval time.Duration, callback func() *Frame) {