From ab6f1e1329b356089e2c2c0b96ab5306e18c02ea Mon Sep 17 00:00:00 2001 From: penguinehis Date: Wed, 15 Jul 2026 00:11:56 -0300 Subject: [PATCH] Tunning and memory control --- README.md | 24 ++ admin/assets/js/08-server-config.js | 9 + admin/index.html | 9 +- main.go | 20 +- quota.go | 47 ++- quota_hardening_test.go | 506 ++++++++++++++++++++++++++++ xray_clients.go | 38 ++- xray_integration.go | 184 ++++++++-- xray_native.go | 107 ++++-- xray_native_mux.go | 224 +++++++++--- xray_native_safety.go | 240 ++++++++++++- xray_native_tuning.go | 59 +++- xray_native_udp.go | 67 ++-- xray_quota.go | 201 ++++++++--- xray_vmess.go | 11 +- xray_xhttp.go | 340 +++++++++++++++++-- 16 files changed, 1828 insertions(+), 258 deletions(-) create mode 100644 quota_hardening_test.go diff --git a/README.md b/README.md index 6d21e26..8fd978b 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,18 @@ Para configurações XHTTP antigas, carregue a configuração visual e clique em A confirmação dessa migração é exibida dentro do próprio painel. Se a gravação falhar, o inbound SSH temporário é removido do rascunho e o inbound antigo permanece intacto, permitindo tentar novamente após corrigir o erro exibido. +### Cota de tráfego e proteção de recursos + +Contas SSH e clientes VLESS/VMess do modo nativo podem usar `data_quota_bytes` com ação `block` ou `throttle`. O botão **Reset/Zerar tráfego** limpa apenas os contadores; não renova validade, senha ou configuração da conta. O valor `max_conns` é aplicado no momento em que o usuário VLESS/VMess é autenticado e vale em conjunto para TCP, UDP, WebSocket, XHTTP e conexões Mux (uma conexão Mux autenticada conta como uma conexão, independentemente dos streams filhos). + +O runtime nativo também possui limites globais para impedir crescimento sem controle de sockets, goroutines e sessões HTTP: + +- `max_concurrent_connections`: conexões de transporte TCP/TLS/WebSocket/XHTTP; padrão `4096`; +- `max_concurrent_xhttp_requests`: handlers XHTTP simultâneos; padrão `8192`; +- `xhttp_max_sessions`: sessões XHTTP ativas; padrão `4096`. + +Esses campos ficam em **Configurações → Xray → Native Xray scale tuning**. `0` seleciona o padrão seguro e `-1` desativa o respectivo contador de admissão, o que não é recomendado em listeners públicos; conexões HTTP/2 continuam com um limite de 256 streams simultâneos por conexão. Cada transporte Mux aceita no máximo 64 sessões filhas, com limite global padrão de 8192. Sockets WebSocket incompletos têm timeout de handshake, conexões HTTP ociosas têm timeout, e parar/reiniciar o Xray nativo fecha conexões e sessões existentes. Atualizações de tráfego e de conexões ativas são agregadas e persistidas em lote a cada cinco segundos, sem criar uma goroutine ou consulta PostgreSQL por conexão. Entradas pendentes de usuários removidos são descartadas para manter os mapas de retry limitados ao conjunto atual de contas. + ### Requisitos - Servidor Linux com `systemd` @@ -595,6 +607,18 @@ For older XHTTP configurations, load the visual configuration and click **Enable The migration confirmation is rendered inside the panel. If saving fails, the temporary SSH inbound is removed from the draft and the old inbound remains intact, so the operation can be retried after fixing the displayed error. +### Traffic quotas and resource protection + +SSH accounts and native-mode VLESS/VMess clients can use `data_quota_bytes` with either the `block` or `throttle` action. The **Reset traffic** action clears only usage counters; it does not renew expiry, change a password, or alter account settings. `max_conns` is enforced when a native VLESS/VMess user is authenticated and is shared across TCP, UDP, WebSocket, XHTTP, and Mux transports (one authenticated Mux transport counts as one connection, regardless of its child streams). + +The native runtime also has global ceilings that prevent unbounded socket, goroutine, and HTTP-session growth: + +- `max_concurrent_connections`: TCP/TLS/WebSocket/XHTTP transport connections; default `4096`; +- `max_concurrent_xhttp_requests`: simultaneous XHTTP handlers; default `8192`; +- `xhttp_max_sessions`: active XHTTP sessions; default `4096`. + +These fields are available under **Settings → Xray → Native Xray scale tuning**. `0` selects the safe default and `-1` disables the corresponding admission counter, which is not recommended on public listeners; HTTP/2 connections still retain a 256-stream concurrent guard. Each Mux transport accepts at most 64 child sessions, with a default global ceiling of 8192. Incomplete WebSocket handshakes time out, idle HTTP connections time out, and stopping/restarting native Xray closes existing transports and XHTTP sessions. Traffic and active-connection changes are aggregated and written in five-second batches rather than creating a PostgreSQL query or goroutine for every connection. Pending retry entries for deleted clients are removed so retry maps stay bounded by the current account set. + ### Requirements - Linux server with `systemd` diff --git a/admin/assets/js/08-server-config.js b/admin/assets/js/08-server-config.js index 9fbe855..801c6d1 100644 --- a/admin/assets/js/08-server-config.js +++ b/admin/assets/js/08-server-config.js @@ -36,11 +36,17 @@ const XRAY_NATIVE_TUNING_DEFAULTS = { safe: { runtime_gomaxprocs: 0, mux_global_sessions: 8192, + max_concurrent_connections: 4096, + max_concurrent_xhttp_requests: 8192, + xhttp_max_sessions: 4096, trace_packets: false, }, "2k": { runtime_gomaxprocs: 0, mux_global_sessions: 32768, + max_concurrent_connections: 8192, + max_concurrent_xhttp_requests: 16384, + xhttp_max_sessions: 8192, trace_packets: false, }, }; @@ -48,6 +54,9 @@ const XRAY_NATIVE_TUNING_DEFAULTS = { const XRAY_NATIVE_TUNING_FIELDS = { runtime_gomaxprocs: "cfgXrayRuntimeGomaxprocs", mux_global_sessions: "cfgXrayMuxGlobalSessions", + max_concurrent_connections: "cfgXrayMaxConnections", + max_concurrent_xhttp_requests: "cfgXrayMaxXHTTPRequests", + xhttp_max_sessions: "cfgXrayMaxXHTTPSessions", }; function setXrayNativeTuningDefaults(profile = "2k") { diff --git a/admin/index.html b/admin/index.html index 086b2b9..34c6a3e 100644 --- a/admin/index.html +++ b/admin/index.html @@ -1509,13 +1509,16 @@ Native Xray scale tuning
-
+
+
+
+
-
Transport buffers (HTTP/2 flow control, XHTTP reorder buffer, mux/UDP buffers) are fixed to xray-core defaults and no longer tunable, so they can't be misconfigured. Go CPU threads = 0 means all detected cores. Saved in the panel config and applied live on restart/reload.
+
The transport ceiling rejects sockets before native protocol/TLS work starts; the XHTTP ceilings bound concurrent handlers and session state. Keep the safe defaults unless load testing proves the VPS can sustain more; -1 disables an application ceiling and is not recommended on public listeners. HTTP/2 still keeps a 256-stream guard per connection. Transport buffers remain fixed to safe defaults. Saved in the panel config and applied live on restart/reload.
@@ -1562,7 +1565,7 @@ - + diff --git a/main.go b/main.go index 4d19b3e..50b4306 100644 --- a/main.go +++ b/main.go @@ -578,12 +578,15 @@ var copyBufPool = sync.Pool{ // io.Copy, which allocates a fresh 32 KiB buffer per direction per channel and // never pools it — at thousands of channels that churn dominated GC pressure. func copyWithRateLimit(dst io.Writer, src io.Reader, lim *rate.Limiter) (written int64, err error) { + return copyWithRateLimitContext(context.Background(), dst, src, lim) +} + +func copyWithRateLimitContext(ctx context.Context, dst io.Writer, src io.Reader, lim *rate.Limiter) (written int64, err error) { bufp := copyBufPool.Get().(*[]byte) buf := *bufp defer copyBufPool.Put(bufp) - var ctx context.Context - if lim != nil { + if ctx == nil { ctx = context.Background() } @@ -2566,9 +2569,14 @@ func handleDirectTCPIP(newChan ssh.NewChannel, u *UserState, upLimiter, downLimi // half-close that never completes), both sides are force-closed so the // other direction unblocks. Close is idempotent, so calling it from both // directions is safe and no separate waiter goroutine is needed. + ctx, cancel := context.WithCancel(context.Background()) + var closeOnce sync.Once closeAll := func() { - _ = backend.Close() - _ = ch.Close() + closeOnce.Do(func() { + cancel() + _ = backend.Close() + _ = ch.Close() + }) } // Drain channel requests concurrently so the peer isn't left waiting. @@ -2582,7 +2590,7 @@ func handleDirectTCPIP(newChan ssh.NewChannel, u *UserState, upLimiter, downLimi // upstream: SSH channel -> backend, in its own goroutine. go func() { - _, _ = copyWithRateLimit(sshQuotaWriter{w: backend, user: u, uplink: true}, ch, upLimiter) + _, _ = copyWithRateLimitContext(ctx, sshQuotaWriter{w: backend, user: u, uplink: true, ctx: ctx}, ch, upLimiter) // Signal to the backend that we are done writing. if cw, ok := backend.(interface{ CloseWrite() error }); ok { _ = cw.CloseWrite() @@ -2593,7 +2601,7 @@ func handleDirectTCPIP(newChan ssh.NewChannel, u *UserState, upLimiter, downLimi // downstream: backend -> SSH channel, run in this goroutine. // handleDirectTCPIP already runs as its own goroutine (see handleConn), // so reusing it here avoids spawning a third goroutine per channel. - _, _ = copyWithRateLimit(sshQuotaWriter{w: ch, user: u, uplink: false}, backend, downLimiter) + _, _ = copyWithRateLimitContext(ctx, sshQuotaWriter{w: ch, user: u, uplink: false, ctx: ctx}, backend, downLimiter) closeAll() } diff --git a/quota.go b/quota.go index a73e1fa..292bde2 100644 --- a/quota.go +++ b/quota.go @@ -40,7 +40,36 @@ type sshTrafficDelta struct { Downlink int64 } -var sshTrafficPersistenceMu sync.Mutex +var ( + sshTrafficPersistenceMu sync.Mutex + sshTrafficDirtyMu sync.Mutex + sshTrafficDirty = make(map[string]*UserState) +) + +func markSSHUserTrafficDirty(u *UserState) { + if u == nil || strings.TrimSpace(u.Cfg.Username) == "" { + return + } + sshTrafficDirtyMu.Lock() + sshTrafficDirty[u.Cfg.Username] = u + sshTrafficDirtyMu.Unlock() +} + +func takeSSHUserTrafficDirty() map[string]*UserState { + sshTrafficDirtyMu.Lock() + dirty := sshTrafficDirty + sshTrafficDirty = make(map[string]*UserState) + sshTrafficDirtyMu.Unlock() + return dirty +} + +func clearSSHUserTrafficDirty(username string, u *UserState) { + sshTrafficDirtyMu.Lock() + if current := sshTrafficDirty[username]; u == nil || current == u { + delete(sshTrafficDirty, username) + } + sshTrafficDirtyMu.Unlock() +} func (s *Store) AddSSHUserTrafficBatch(ctx context.Context, deltas map[string]sshTrafficDelta) error { if s == nil || len(deltas) == 0 { @@ -118,6 +147,7 @@ func resetSSHRuntimeUsage(username string) { } u.trafficMu.Lock() resetSSHRuntimeUsageLocked(u) + clearSSHUserTrafficDirty(username, u) u.trafficMu.Unlock() } @@ -134,6 +164,7 @@ func resetSSHUserTrafficAccounting(ctx context.Context, store *Store, username s } if u != nil { resetSSHRuntimeUsageLocked(u) + clearSSHUserTrafficDirty(username, u) } return nil } @@ -228,12 +259,14 @@ func finishSSHUserReservation(u *UserState, uplink bool, reserved, written int) atomic.AddInt64(&u.TotalDownlinkBytes, int64(written)) atomic.AddInt64(&u.pendingDownlinkBytes, int64(written)) } + markSSHUserTrafficDirty(u) } type sshQuotaWriter struct { w io.Writer user *UserState uplink bool + ctx context.Context } func (qw sshQuotaWriter) Write(p []byte) (int, error) { @@ -246,7 +279,11 @@ func (qw sshQuotaWriter) Write(p []byte) (int, error) { return 0, errDataQuotaExceeded } if quotaLimiter != nil { - if err := quotaLimiter.WaitN(context.Background(), allowed); err != nil { + ctx := qw.ctx + if ctx == nil { + ctx = context.Background() + } + if err := quotaLimiter.WaitN(ctx, allowed); err != nil { finishSSHUserReservation(qw.user, qw.uplink, allowed, 0) return 0, err } @@ -283,8 +320,8 @@ func flushSSHUserTraffic(store *Store) { defer sshTrafficPersistenceMu.Unlock() deltas := make(map[string]sshTrafficDelta) states := make(map[string]*UserState) - for _, u := range userMgr.List() { - if u == nil || strings.TrimSpace(u.Cfg.Username) == "" { + for username, u := range takeSSHUserTrafficDirty() { + if u == nil || strings.TrimSpace(username) == "" { continue } up := atomic.SwapInt64(&u.pendingUplinkBytes, 0) @@ -292,7 +329,6 @@ func flushSSHUserTraffic(store *Store) { if up == 0 && down == 0 { continue } - username := u.Cfg.Username deltas[username] = sshTrafficDelta{Uplink: up, Downlink: down} states[username] = u } @@ -307,6 +343,7 @@ func flushSSHUserTraffic(store *Store) { if u := states[username]; u != nil { atomic.AddInt64(&u.pendingUplinkBytes, d.Uplink) atomic.AddInt64(&u.pendingDownlinkBytes, d.Downlink) + markSSHUserTrafficDirty(u) } } } diff --git a/quota_hardening_test.go b/quota_hardening_test.go new file mode 100644 index 0000000..51f6124 --- /dev/null +++ b/quota_hardening_test.go @@ -0,0 +1,506 @@ +package main + +import ( + "context" + "errors" + "io" + "net" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/time/rate" +) + +func TestNativeClientMaxConnectionsAndBatchedActiveDelta(t *testing.T) { + oldStore := statsStore + statsStore = &Store{} + defer func() { statsStore = oldStore }() + + const uuid = "11111111-1111-1111-1111-111111111111" + m := &XrayManager{ + nativeQuotaByUUID: map[string]*xrayNativeQuotaState{ + uuid: {maxConns: 1, generation: 1}, + }, + } + state := m.nativeQuotaState(uuid) + + release, acquiredState, ok := m.acquireNativeClientConnection(uuid, "user@example") + if !ok || release == nil { + t.Fatal("first native connection was rejected") + } + if acquiredState != state { + t.Fatal("connection lease did not retain the authenticated policy state") + } + if _, _, ok := m.acquireNativeClientConnection(uuid, "user@example"); ok { + t.Fatal("connection above max_conns was accepted") + } + + m.nativeDBMu.Lock() + pending := m.nativeActivePending[uuid] + m.nativeDBMu.Unlock() + if pending.Delta != 1 || !pending.Connected || pending.State != state { + t.Fatalf("connect was not queued for batch persistence: %+v", pending) + } + + release() + release() // idempotent release must not underflow counters. + + m.nativeDBMu.Lock() + pending = m.nativeActivePending[uuid] + m.nativeDBMu.Unlock() + if pending.Delta != 0 || !pending.Connected { + t.Fatalf("connect/disconnect batch should net to zero and retain last-active: %+v", pending) + } + + state.mu.Lock() + active := state.activeConns + state.mu.Unlock() + if active != 0 { + t.Fatalf("active connection count = %d, want 0", active) + } + + release2, _, ok := m.acquireNativeClientConnection(uuid, "user@example") + if !ok { + t.Fatal("slot was not reusable after release") + } + release2() +} + +func TestRemoveNativeQuotaPolicyPrunesPendingMaps(t *testing.T) { + m := &XrayManager{ + nativeQuotaByUUID: map[string]*xrayNativeQuotaState{ + "gone": {generation: 1}, + }, + nativeTrafficPending: map[string]xrayPendingTraffic{ + "gone": {Uplink: 10}, + }, + nativeActivePending: map[string]xrayPendingActive{ + "gone": {Delta: 1}, + }, + } + m.removeNativeQuotaPolicy("gone") + if m.nativeQuotaState("gone") != nil { + t.Fatal("quota policy was not removed") + } + m.nativeDBMu.Lock() + _, trafficExists := m.nativeTrafficPending["gone"] + _, activeExists := m.nativeActivePending["gone"] + m.nativeDBMu.Unlock() + if trafficExists || activeExists { + t.Fatal("deleted UUID remained in a pending persistence map") + } +} + +func TestNativeCounterIsBoundedAndReleaseIsIdempotent(t *testing.T) { + var active atomicInt64ForTest + release1, ok := acquireNativeCounter(&active.Int64, 2) + if !ok { + t.Fatal("first slot rejected") + } + release2, ok := acquireNativeCounter(&active.Int64, 2) + if !ok { + t.Fatal("second slot rejected") + } + if _, ok := acquireNativeCounter(&active.Int64, 2); ok { + t.Fatal("slot above limit accepted") + } + release1() + release1() + if got := active.Load(); got != 1 { + t.Fatalf("active after double release = %d, want 1", got) + } + release2() + if got := active.Load(); got != 0 { + t.Fatalf("active after releases = %d, want 0", got) + } +} + +// Embedding keeps the test declaration readable while still passing the exact +// atomic.Int64 type required by acquireNativeCounter. +type atomicInt64ForTest struct{ Int64 atomic.Int64 } + +func (a *atomicInt64ForTest) Load() int64 { return a.Int64.Load() } + +func TestTrackedNativeConnectionsAreClosedOnShutdown(t *testing.T) { + oldAccepting := nativeTransportAccepting.Load() + defer nativeTransportAccepting.Store(oldAccepting) + + beginNativeTransportAccepting() + before := nativeTransportConnections.Load() + serverSide, clientSide := net.Pipe() + defer clientSide.Close() + wrapped, ok := wrapTrackedNativeTransportConn(serverSide) + if !ok { + t.Fatal("tracked connection was rejected") + } + if got := nativeTransportConnections.Load(); got != before+1 { + t.Fatalf("transport count = %d, want %d", got, before+1) + } + + stopNativeTransportAccepting() + closeAllNativeTransportConnections() + _ = clientSide.SetReadDeadline(time.Now().Add(time.Second)) + if _, err := clientSide.Read(make([]byte, 1)); err == nil { + t.Fatal("peer remained open after native shutdown") + } + if err := wrapped.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Fatalf("second close returned unexpected error: %v", err) + } + if got := nativeTransportConnections.Load(); got != before { + t.Fatalf("transport count after shutdown = %d, want %d", got, before) + } +} + +func TestCloseAllXHTTPSessionsReleasesGlobalSlots(t *testing.T) { + oldLimit := nativeTuneXHTTPMaxSessions.Load() + nativeTuneXHTTPMaxSessions.Store(8) + defer nativeTuneXHTTPMaxSessions.Store(oldLimit) + + before := nativeXHTTPSessions.Load() + ib := &nativeInbound{xhttpMaxBufferedPosts: 2} + for _, id := range []string{"one", "two"} { + if sess := ib.upsertXHTTPSession(httptest.NewRecorder(), id); sess == nil { + t.Fatalf("session %q was rejected", id) + } + } + if got := nativeXHTTPSessions.Load(); got != before+2 { + t.Fatalf("global XHTTP sessions = %d, want %d", got, before+2) + } + ib.closeAllXHTTPSessions() + if got := nativeXHTTPSessions.Load(); got != before { + t.Fatalf("global XHTTP sessions after close = %d, want %d", got, before) + } + ib.xhttpMu.Lock() + remaining := len(ib.xhttpSessions) + ib.xhttpMu.Unlock() + if remaining != 0 { + t.Fatalf("inbound retained %d XHTTP sessions", remaining) + } +} + +func TestNegativeXHTTPSessionLimitMeansUnlimited(t *testing.T) { + old := nativeTuneXHTTPMaxSessions.Load() + nativeTuneXHTTPMaxSessions.Store(0) + defer nativeTuneXHTTPMaxSessions.Store(old) + if got := (&nativeInbound{}).xhttpMaxActiveSessions(); got != 0 { + t.Fatalf("unlimited XHTTP session limit normalized to %d", got) + } +} + +func TestNativeProtocolGuardsRemainFinite(t *testing.T) { + oldRequests := nativeTuneMaxXHTTPRequests.Load() + defer nativeTuneMaxXHTTPRequests.Store(oldRequests) + + // Zero is the internal representation of an explicitly disabled application + // request counter. HTTP/2 must still retain a finite per-connection guard. + nativeTuneMaxXHTTPRequests.Store(0) + if got := nativeHTTP2MaxConcurrentStreams(); got != defaultNativeHTTP2MaxStreams { + t.Fatalf("HTTP/2 stream guard = %d, want %d", got, defaultNativeHTTP2MaxStreams) + } + + nativeTuneMaxXHTTPRequests.Store(32) + if got := nativeHTTP2MaxConcurrentStreams(); got != 32 { + t.Fatalf("HTTP/2 stream guard did not honor lower request cap: %d", got) + } + if got := nativeMuxMaxSessionLimit(); got != 64 { + t.Fatalf("per-transport Mux session guard = %d, want 64", got) + } +} + +func TestXHTTPMetadataLengthIsBoundedBeforeSessionAllocation(t *testing.T) { + ib := &nativeInbound{transport: "xhttp", path: "/"} + req := httptest.NewRequest("GET", "/"+strings.Repeat("a", nativeXHTTPMaxSessionIDBytes+1), nil) + rec := httptest.NewRecorder() + ib.ServeHTTP(rec, req) + if rec.Code != 400 { + t.Fatalf("oversized XHTTP session id status = %d, want 400", rec.Code) + } + ib.xhttpMu.Lock() + sessions := len(ib.xhttpSessions) + ib.xhttpMu.Unlock() + if sessions != 0 { + t.Fatalf("oversized metadata allocated %d sessions", sessions) + } +} + +func TestXHTTPUploadMemoryIsReleasedOnReadAndClose(t *testing.T) { + before := nativeXHTTPBufferedBytes.Load() + q := newNativeXHTTPUploadQueue(4, 8) + + lease, ok := acquireNativeXHTTPMemory(8) + if !ok { + t.Fatal("failed to reserve XHTTP test memory") + } + lease.shrink(4) + if err := q.push(context.Background(), nativeXHTTPPacket{Payload: []byte("test"), Seq: 0}, lease); err != nil { + lease.release() + t.Fatalf("queue push failed: %v", err) + } + lease.release() // transferred leases are a no-op for the producer. + if got := nativeXHTTPBufferedBytes.Load(); got != before+4 { + t.Fatalf("buffered bytes after push = %d, want %d", got, before+4) + } + + buf := make([]byte, 4) + if n, err := q.Read(buf); err != nil || n != 4 || string(buf) != "test" { + t.Fatalf("queue read = (%d, %v, %q), want (4, nil, test)", n, err, string(buf)) + } + if got := nativeXHTTPBufferedBytes.Load(); got != before { + t.Fatalf("buffered bytes after read = %d, want %d", got, before) + } + + lease, ok = acquireNativeXHTTPMemory(3) + if !ok { + t.Fatal("failed to reserve second XHTTP test memory") + } + if err := q.push(context.Background(), nativeXHTTPPacket{Payload: []byte("xyz"), Seq: 2}, lease); err != nil { + lease.release() + t.Fatalf("second queue push failed: %v", err) + } + lease.release() + q.close() + if got := nativeXHTTPBufferedBytes.Load(); got != before { + t.Fatalf("buffered bytes after close = %d, want %d", got, before) + } +} + +func TestXHTTPUploadQueueEnforcesPerSessionByteBudget(t *testing.T) { + before := nativeXHTTPBufferedBytes.Load() + q := newNativeXHTTPUploadQueue(4, 4) + defer q.close() + + lease, ok := acquireNativeXHTTPMemory(5) + if !ok { + t.Fatal("failed to reserve XHTTP test memory") + } + defer lease.release() + err := q.push(context.Background(), nativeXHTTPPacket{Payload: make([]byte, 5)}, lease) + if !errors.Is(err, errNativeXHTTPUploadBufferFull) { + t.Fatalf("oversized queue push error = %v, want buffer limit", err) + } + lease.release() + if got := nativeXHTTPBufferedBytes.Load(); got != before { + t.Fatalf("rejected payload retained %d bytes, baseline %d", got, before) + } +} + +func TestNativeQuotaResetWaitsForInFlightTraffic(t *testing.T) { + const uuid = "22222222-2222-2222-2222-222222222222" + state := &xrayNativeQuotaState{usedBytes: 123, generation: 1} + m := &XrayManager{nativeQuotaByUUID: map[string]*xrayNativeQuotaState{uuid: state}} + + state.trafficMu.RLock() + done := make(chan struct{}) + go func() { + m.resetNativeQuotaUsage(uuid) + close(done) + }() + select { + case <-done: + state.trafficMu.RUnlock() + t.Fatal("traffic reset crossed an in-flight writer boundary") + case <-time.After(25 * time.Millisecond): + } + state.trafficMu.RUnlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("traffic reset did not complete after writer released") + } + state.mu.Lock() + used, generation := state.usedBytes, state.generation + state.mu.Unlock() + if used != 0 || generation != 2 { + t.Fatalf("reset state = used %d generation %d, want 0/2", used, generation) + } +} + +func TestNativeRateWaitCanBeCanceled(t *testing.T) { + lim := rate.NewLimiter(1, 1) + if !lim.AllowN(time.Now(), 1) { + t.Fatal("failed to consume initial limiter token") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := waitNativeRate(ctx, lim, 1); !errors.Is(err, context.Canceled) { + t.Fatalf("waitNativeRate error = %v, want context.Canceled", err) + } +} + +func TestSSHDirtyQueueDoesNotScanInactiveUsers(t *testing.T) { + sshTrafficDirtyMu.Lock() + old := sshTrafficDirty + sshTrafficDirty = make(map[string]*UserState) + sshTrafficDirtyMu.Unlock() + defer func() { + sshTrafficDirtyMu.Lock() + sshTrafficDirty = old + sshTrafficDirtyMu.Unlock() + }() + + active := &UserState{Cfg: UserConfig{Username: "active"}} + inactive := &UserState{Cfg: UserConfig{Username: "inactive"}} + markSSHUserTrafficDirty(active) + dirty := takeSSHUserTrafficDirty() + if len(dirty) != 1 || dirty["active"] != active { + t.Fatalf("dirty queue = %#v", dirty) + } + if _, found := dirty[inactive.Cfg.Username]; found { + t.Fatal("inactive user appeared in dirty queue") + } + if next := takeSSHUserTrafficDirty(); len(next) != 0 { + t.Fatalf("dirty queue was not drained: %#v", next) + } +} + +func TestOldNativeConnectionCannotDecrementReplacementAccount(t *testing.T) { + oldStore := statsStore + statsStore = &Store{} + defer func() { statsStore = oldStore }() + + const uuid = "replacement-active-user" + oldState := &xrayNativeQuotaState{maxConns: 1, generation: 1} + m := &XrayManager{nativeQuotaByUUID: map[string]*xrayNativeQuotaState{uuid: oldState}} + + release, acquiredState, ok := m.acquireNativeClientConnection(uuid, "old@example") + if !ok || acquiredState != oldState { + t.Fatal("failed to acquire old account connection") + } + // Discard the old account's successful connect delta so this assertion only + // measures what happens when that old connection later disconnects. + m.nativeDBMu.Lock() + m.nativeActivePending = nil + m.nativeDBMu.Unlock() + + newState := &xrayNativeQuotaState{maxConns: 1, generation: 1} + m.nativeQuotaMu.Lock() + m.nativeQuotaByUUID[uuid] = newState + m.nativeQuotaMu.Unlock() + + release() + + m.nativeDBMu.Lock() + pending := m.nativeActivePending[uuid] + m.nativeDBMu.Unlock() + if pending.Delta != 0 || pending.State != nil { + t.Fatalf("old disconnect was queued against replacement account: %+v", pending) + } + newState.mu.Lock() + active := newState.activeConns + newState.mu.Unlock() + if active != 0 { + t.Fatalf("replacement account active count changed to %d", active) + } +} + +func TestOldNativeTrafficCannotAttachToReplacementAccount(t *testing.T) { + oldStore := statsStore + statsStore = &Store{} + defer func() { statsStore = oldStore }() + + const uuid = "replacement-traffic-user" + oldState := &xrayNativeQuotaState{generation: 1} + newState := &xrayNativeQuotaState{generation: 1} + m := &XrayManager{nativeQuotaByUUID: map[string]*xrayNativeQuotaState{uuid: oldState}} + + meter := newTrafficMeter(uuid, "old@example", true, oldState) + meter.n = 1234 + + m.nativeQuotaMu.Lock() + m.nativeQuotaByUUID[uuid] = newState + m.nativeQuotaMu.Unlock() + + oldMgr := xrayMgr + xrayMgr = m + defer func() { xrayMgr = oldMgr }() + meter.flush() + + m.nativeDBMu.Lock() + pending := m.nativeTrafficPending[uuid] + m.nativeDBMu.Unlock() + if pending.Uplink != 0 || pending.Downlink != 0 || pending.State != nil { + t.Fatalf("old traffic was queued against replacement account: %+v", pending) + } + m.statsMu.RLock() + stat := m.statsByEmail["old@example"] + m.statsMu.RUnlock() + if stat.Uplink != 0 || stat.Downlink != 0 { + t.Fatalf("old traffic resurfaced in runtime stats: %+v", stat) + } +} + +func TestNativeFlusherDropsMismatchedPolicyIdentity(t *testing.T) { + oldStore := statsStore + statsStore = &Store{} + defer func() { statsStore = oldStore }() + + const uuid = "identity-prune-user" + oldState := &xrayNativeQuotaState{generation: 1} + newState := &xrayNativeQuotaState{generation: 1} + m := &XrayManager{ + nativeQuotaByUUID: map[string]*xrayNativeQuotaState{uuid: newState}, + nativeTrafficPending: map[string]xrayPendingTraffic{ + uuid: {Email: "old@example", Uplink: 99, State: oldState}, + }, + nativeActivePending: map[string]xrayPendingActive{ + uuid: {Email: "old@example", Delta: -1, State: oldState}, + }, + } + m.flushNativeStatsToDB() + + m.nativeDBMu.Lock() + defer m.nativeDBMu.Unlock() + if len(m.nativeTrafficPending) != 0 || len(m.nativeActivePending) != 0 { + t.Fatalf("mismatched pending deltas survived prune: traffic=%v active=%v", m.nativeTrafficPending, m.nativeActivePending) + } +} + +func TestNativeMuxFinishRunsOnce(t *testing.T) { + var calls atomic.Int32 + s := &nativeMuxSession{ + closed: make(chan struct{}), + uplink: make(chan nativeMuxUplinkItem), + onClose: func(*nativeMuxSession) { + calls.Add(1) + }, + } + + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + s.finish() + }() + } + wg.Wait() + if got := calls.Load(); got != 1 { + t.Fatalf("mux onClose called %d times, want 1", got) + } +} + +type closeTrackingReader struct { + closed atomic.Bool +} + +func (r *closeTrackingReader) Read([]byte) (int, error) { return 0, io.EOF } +func (r *closeTrackingReader) Close() error { + r.closed.Store(true) + return nil +} + +func TestNativeXHTTPQueueCloseClosesQueuedStreamReader(t *testing.T) { + q := newNativeXHTTPUploadQueue(1, 1024) + r := &closeTrackingReader{} + if err := q.push(context.Background(), nativeXHTTPPacket{Reader: r}, nil); err != nil { + t.Fatalf("queue stream reader: %v", err) + } + q.close() + if !r.closed.Load() { + t.Fatal("queued stream reader was not closed during queue shutdown") + } +} diff --git a/xray_clients.go b/xray_clients.go index 98024f0..05a8fd6 100644 --- a/xray_clients.go +++ b/xray_clients.go @@ -112,6 +112,11 @@ func (s *Store) GetXrayClientMeta(ctx context.Context, uuid string) (*XrayClient } func (s *Store) DeleteXrayClientMeta(ctx context.Context, uuid string) error { + // Serialize deletion with the native stats flusher. Otherwise a batch that + // was swapped out just before DELETE could finish afterward and, if the same + // UUID is recreated quickly, apply stale traffic/active deltas to the new row. + xrayMgr.nativeTrafficPersistMu.Lock() + defer xrayMgr.nativeTrafficPersistMu.Unlock() _, err := s.db.ExecContext(ctx, `DELETE FROM xray_clients WHERE uuid = $1`, uuid) if err == nil { xrayMgr.removeNativeQuotaPolicy(uuid) @@ -228,19 +233,38 @@ func (s *Store) AddXrayClientTrafficBatch(ctx context.Context, deltas map[string return tx.Commit() } -// UpdateXrayClientActive adjusts the native online connection counter. -func (s *Store) UpdateXrayClientActive(ctx context.Context, uuid, email string, delta int) error { - if uuid == "" || delta == 0 { +// AddXrayClientActiveBatch persists native online-counter deltas without +// launching a database goroutine/query for every connect and disconnect. +func (s *Store) AddXrayClientActiveBatch(ctx context.Context, deltas map[string]xrayPendingActive) error { + if len(deltas) == 0 { return nil } - _, err := s.db.ExecContext(ctx, ` + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + stmt, err := tx.PrepareContext(ctx, ` UPDATE xray_clients SET email = CASE WHEN email = '' AND $2 <> '' THEN $2 ELSE email END, name = CASE WHEN name = '' AND $2 <> '' THEN $2 ELSE name END, - last_active = CASE WHEN $3::INT > 0 THEN NOW() ELSE last_active END, + last_active = CASE WHEN $4::BOOLEAN THEN NOW() ELSE last_active END, active_connections = GREATEST(active_connections + $3::INT, 0) - WHERE uuid = $1`, uuid, email, delta) - return err + WHERE uuid = $1`) + if err != nil { + _ = tx.Rollback() + return err + } + defer stmt.Close() + for uuid, d := range deltas { + if uuid == "" || (d.Delta == 0 && !d.Connected) { + continue + } + if _, err := stmt.ExecContext(ctx, uuid, d.Email, d.Delta, d.Connected); err != nil { + _ = tx.Rollback() + return err + } + } + return tx.Commit() } func countOwnedXrayClients(ctx context.Context, store *Store, ownerUsername string) int { diff --git a/xray_integration.go b/xray_integration.go index 8c0c153..2342f4d 100644 --- a/xray_integration.go +++ b/xray_integration.go @@ -261,6 +261,7 @@ type XrayManager struct { nativeDBMu sync.Mutex nativeTrafficPersistMu sync.Mutex nativeTrafficPending map[string]xrayPendingTraffic + nativeActivePending map[string]xrayPendingActive nativeStatsFlushStarted bool nativeQuotaMu sync.RWMutex @@ -284,6 +285,14 @@ type xrayPendingTraffic struct { Email string Uplink int64 Downlink int64 + State *xrayNativeQuotaState +} + +type xrayPendingActive struct { + Email string + Delta int + Connected bool + State *xrayNativeQuotaState } var xrayMgr = &XrayManager{} @@ -455,7 +464,7 @@ func (m *XrayManager) Restart() error { // recordNativeConnect marks a native client stream as online immediately. This // is more accurate than external Xray's Stats API polling because it knows when // the decoded VMess/VLESS stream is authenticated and opened. -func (m *XrayManager) recordNativeConnect(uuid, email string) { +func (m *XrayManager) recordNativeConnect(uuid, email string, state *xrayNativeQuotaState) { uuid = strings.TrimSpace(uuid) email = strings.TrimSpace(email) if email == "" { @@ -476,18 +485,10 @@ func (m *XrayManager) recordNativeConnect(uuid, email string) { m.statsByEmail[email] = st m.statsMu.Unlock() - if statsStore != nil && uuid != "" { - xrayGo("native xray stats active increment", func() { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - if err := statsStore.UpdateXrayClientActive(ctx, uuid, email, 1); err != nil { - xrayLogf("xray native stats: active +1 for %s failed: %v", uuid, err) - } - }) - } + m.queueNativeActiveDelta(uuid, email, 1, true, state) } -func (m *XrayManager) recordNativeDisconnect(uuid, email string) { +func (m *XrayManager) recordNativeDisconnect(uuid, email string, state *xrayNativeQuotaState) { uuid = strings.TrimSpace(uuid) email = strings.TrimSpace(email) if email == "" { @@ -506,21 +507,44 @@ func (m *XrayManager) recordNativeDisconnect(uuid, email string) { } m.statsMu.Unlock() - if statsStore != nil && uuid != "" { - xrayGo("native xray stats active decrement", func() { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - if err := statsStore.UpdateXrayClientActive(ctx, uuid, email, -1); err != nil { - xrayLogf("xray native stats: active -1 for %s failed: %v", uuid, err) - } - }) + m.queueNativeActiveDelta(uuid, email, -1, false, state) +} + +func (m *XrayManager) queueNativeActiveDelta(uuid, email string, delta int, connected bool, state *xrayNativeQuotaState) { + if statsStore == nil || uuid == "" || delta == 0 || state == nil { + return } + // Keep the policy identity stable until the delta is queued. A UUID can be + // deleted and later recreated; an old connection must never decrement or add + // traffic to the replacement account merely because the string key matches. + m.nativeQuotaMu.RLock() + if m.nativeQuotaByUUID[uuid] != state { + m.nativeQuotaMu.RUnlock() + return + } + m.nativeDBMu.Lock() + if m.nativeActivePending == nil { + m.nativeActivePending = make(map[string]xrayPendingActive) + } + p := m.nativeActivePending[uuid] + if p.State != nil && p.State != state { + p = xrayPendingActive{} + } + if p.Email == "" { + p.Email = email + } + p.Delta += delta + p.Connected = p.Connected || connected + p.State = state + m.nativeActivePending[uuid] = p + m.nativeDBMu.Unlock() + m.nativeQuotaMu.RUnlock() } // recordNativeTraffic accumulates in-process byte counters for a client and // queues DB persistence. Used by the native emulator instead of external // `xray api statsquery` polling. -func (m *XrayManager) recordNativeTraffic(uuid, email string, up, down int64, generation uint64) { +func (m *XrayManager) recordNativeTraffic(uuid, email string, up, down int64, generation uint64, state *xrayNativeQuotaState) { uuid = strings.TrimSpace(uuid) email = strings.TrimSpace(email) if email == "" { @@ -529,8 +553,10 @@ func (m *XrayManager) recordNativeTraffic(uuid, email string, up, down int64, ge if email == "" || (up == 0 && down == 0) { return } - state := m.nativeQuotaState(uuid) if state != nil { + // Keep generation validation and queuing in the same critical section as + // resetNativeTrafficAccounting. Otherwise an old meter can validate just + // before a reset and enqueue its bytes immediately after the DB was zeroed. state.mu.Lock() defer state.mu.Unlock() if generation != state.generation { @@ -538,17 +564,30 @@ func (m *XrayManager) recordNativeTraffic(uuid, email string, up, down int64, ge } } - if statsStore != nil && uuid != "" { + // Only DB-backed clients have a native policy state. Config-only clients are + // still shown in runtime stats, but queuing UPDATEs for rows that do not exist + // can make the retry map grow during a database outage. + if statsStore != nil && uuid != "" && state != nil { + m.nativeQuotaMu.RLock() + if m.nativeQuotaByUUID[uuid] != state { + m.nativeQuotaMu.RUnlock() + return + } m.nativeDBMu.Lock() if m.nativeTrafficPending == nil { m.nativeTrafficPending = make(map[string]xrayPendingTraffic) } p := m.nativeTrafficPending[uuid] + if p.State != nil && p.State != state { + p = xrayPendingTraffic{} + } p.Email = email p.Uplink += up p.Downlink += down + p.State = state m.nativeTrafficPending[uuid] = p m.nativeDBMu.Unlock() + m.nativeQuotaMu.RUnlock() } now := time.Now() @@ -612,35 +651,106 @@ func (m *XrayManager) flushNativeStatsToDB() { } m.nativeTrafficPersistMu.Lock() defer m.nativeTrafficPersistMu.Unlock() + persistent := m.nativePersistentStates() m.nativeDBMu.Lock() - pending := m.nativeTrafficPending + for uuid, pending := range m.nativeTrafficPending { + if persistent[uuid] != pending.State { + delete(m.nativeTrafficPending, uuid) + } + } + for uuid, pending := range m.nativeActivePending { + if persistent[uuid] != pending.State { + delete(m.nativeActivePending, uuid) + } + } + pendingTraffic := m.nativeTrafficPending + pendingActive := m.nativeActivePending m.nativeTrafficPending = nil + m.nativeActivePending = nil m.nativeDBMu.Unlock() - if len(pending) == 0 { + if len(pendingTraffic) == 0 && len(pendingActive) == 0 { return } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := statsStore.AddXrayClientTrafficBatch(ctx, pending); err != nil { - xrayLogf("xray native stats: db traffic flush failed: %v", err) - // Put deltas back so a transient DB failure does not lose accounting. + + var trafficErr error + if len(pendingTraffic) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + trafficErr = statsStore.AddXrayClientTrafficBatch(ctx, pendingTraffic) + cancel() + } + var activeErr error + if len(pendingActive) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + activeErr = statsStore.AddXrayClientActiveBatch(ctx, pendingActive) + cancel() + } + + if trafficErr != nil { + xrayLogf("xray native stats: db traffic flush failed: %v", trafficErr) + } + if activeErr != nil { + xrayLogf("xray native stats: db active flush failed: %v", activeErr) + } + if trafficErr != nil || activeErr != nil { + // Put only failed batches back so a successful write is never duplicated. + persistent = m.nativePersistentStates() m.nativeDBMu.Lock() - if m.nativeTrafficPending == nil { + if trafficErr != nil && m.nativeTrafficPending == nil { m.nativeTrafficPending = make(map[string]xrayPendingTraffic) } - for uuid, d := range pending { - p := m.nativeTrafficPending[uuid] - if p.Email == "" { - p.Email = d.Email + if trafficErr != nil { + for uuid, d := range pendingTraffic { + if persistent[uuid] != d.State { + continue + } + p := m.nativeTrafficPending[uuid] + if p.State != nil && p.State != d.State { + p = xrayPendingTraffic{} + } + if p.Email == "" { + p.Email = d.Email + } + p.Uplink += d.Uplink + p.Downlink += d.Downlink + p.State = d.State + m.nativeTrafficPending[uuid] = p + } + } + if activeErr != nil && m.nativeActivePending == nil { + m.nativeActivePending = make(map[string]xrayPendingActive) + } + if activeErr != nil { + for uuid, d := range pendingActive { + if persistent[uuid] != d.State { + continue + } + p := m.nativeActivePending[uuid] + if p.State != nil && p.State != d.State { + p = xrayPendingActive{} + } + if p.Email == "" { + p.Email = d.Email + } + p.Delta += d.Delta + p.Connected = p.Connected || d.Connected + p.State = d.State + m.nativeActivePending[uuid] = p } - p.Uplink += d.Uplink - p.Downlink += d.Downlink - m.nativeTrafficPending[uuid] = p } m.nativeDBMu.Unlock() } } +func (m *XrayManager) nativePersistentStates() map[string]*xrayNativeQuotaState { + m.nativeQuotaMu.RLock() + out := make(map[string]*xrayNativeQuotaState, len(m.nativeQuotaByUUID)) + for uuid, state := range m.nativeQuotaByUUID { + out[uuid] = state + } + m.nativeQuotaMu.RUnlock() + return out +} + // XrayStatusDTO is returned by /api/xray/status. type XrayStatusDTO struct { Enabled bool `json:"enabled"` diff --git a/xray_native.go b/xray_native.go index 0dd5cdf..bf1a721 100644 --- a/xray_native.go +++ b/xray_native.go @@ -138,6 +138,14 @@ func (s *nativeXrayServer) start(configFile string) error { if s.running { return fmt.Errorf("native xray already running") } + beginNativeTransportAccepting() + started := false + defer func() { + if !started { + stopNativeTransportAccepting() + closeAllNativeTransportConnections() + } + }() if configFile == "" { return fmt.Errorf("native xray: no config file configured") } @@ -196,9 +204,11 @@ func (s *nativeXrayServer) start(configFile string) error { } return fmt.Errorf("native xray: listen %s (shared XHTTP): %w", addr, err) } - serveLn := net.Listener(ln) + // Apply the global pre-authentication ceiling before net/http can spawn a + // goroutine or begin a TLS handshake for the accepted socket. + serveLn := limitNativeListener(ln) if group.security == "tls" { - serveLn = tls.NewListener(ln, group.tlsConfig) + serveLn = tls.NewListener(serveLn, group.tlsConfig) } opened = append(opened, serveLn) xrayGo(fmt.Sprintf("native xray shared xhttp listener %s", addr), func() { group.serve(serveLn) }) @@ -212,21 +222,34 @@ func (s *nativeXrayServer) start(configFile string) error { s.inboundsByTag = active s.running = true s.startTime = time.Now() + started = true return nil } func (s *nativeXrayServer) stop() { s.mu.Lock() - defer s.mu.Unlock() if !s.running && len(s.listeners) == 0 { + s.mu.Unlock() return } - for _, l := range s.listeners { - _ = l.Close() + stopNativeTransportAccepting() + listeners := append([]net.Listener(nil), s.listeners...) + inbounds := make([]*nativeInbound, 0, len(s.inboundsByTag)) + for _, ib := range s.inboundsByTag { + inbounds = append(inbounds, ib) } s.listeners = nil s.inboundsByTag = nil s.running = false + s.mu.Unlock() + + for _, l := range listeners { + _ = l.Close() + } + closeAllNativeTransportConnections() + for _, ib := range inbounds { + ib.closeAllXHTTPSessions() + } xrayLogf("native xray: stopped") } @@ -241,6 +264,12 @@ func (ib *nativeInbound) acceptLoop(ln net.Listener) { xrayLogf("native xray: accept error on %s: %v", ln.Addr(), err) continue } + counted, ok := wrapTrackedNativeTransportConn(c) + if !ok { + time.Sleep(nativeOverloadBackoff) + continue + } + c = counted xrayGo(fmt.Sprintf("native xray connection remote=%s", c.RemoteAddr()), func() { ib.serve(c) }) } } @@ -262,7 +291,7 @@ func (ib *nativeInbound) serve(raw net.Conn) { tconn := tls.Server(raw, ib.tlsConfig) _ = tconn.SetDeadline(time.Now().Add(tlsHandshakeTimeout)) if err := tconn.Handshake(); err != nil { - xrayLogf("native xray: tls handshake from %s failed: %v", raw.RemoteAddr(), err) + logNativePreAuthRejection("native xray: tls handshake from %s failed: %v", raw.RemoteAddr(), err) return } _ = tconn.SetDeadline(time.Time{}) @@ -275,11 +304,13 @@ func (ib *nativeInbound) serve(raw net.Conn) { case "tcp", "raw", "": // stream is already the protocol stream case "ws", "websocket": + _ = conn.SetDeadline(time.Now().Add(tlsHandshakeTimeout)) ws, err := wsServerHandshake(conn, ib.path) if err != nil { - xrayLogf("native xray: ws handshake from %s failed: %v", raw.RemoteAddr(), err) + logNativePreAuthRejection("native xray: ws handshake from %s failed: %v", raw.RemoteAddr(), err) return } + _ = conn.SetDeadline(time.Time{}) stream = ws case "xhttp", "splithttp": xrayLogf("native xray: inbound %q got raw connection for XHTTP; this transport is served by http.Server", ib.tag) @@ -331,11 +362,7 @@ const ( func (ib *nativeInbound) handleVLESS(stream net.Conn, remote net.Addr) { defer xrayRecover(fmt.Sprintf("native xray VLESS inbound=%q remote=%s", ib.tag, remote)) - if ib.isXHTTP() { - xrayTracef("native xray: vless handshake start inbound=%q transport=%s remote=%s", ib.tag, ib.transport, remote) - } else { - xrayLogf("native xray: vless handshake start inbound=%q transport=%s remote=%s", ib.tag, ib.transport, remote) - } + xrayTracef("native xray: vless handshake start inbound=%q transport=%s remote=%s", ib.tag, ib.transport, remote) _ = stream.SetReadDeadline(time.Now().Add(30 * time.Second)) head := make([]byte, 1+16+1) // version + uuid + addonLen @@ -349,7 +376,7 @@ func (ib *nativeInbound) handleVLESS(stream net.Conn, remote net.Addr) { client := ib.getNativeClient(id) if client == nil { - xrayLogf("native xray: inbound %q rejected unknown VLESS uuid from %s", ib.tag, remote) + logNativePreAuthRejection("native xray: inbound %q rejected unknown VLESS uuid from %s", ib.tag, remote) return } if xrayMgr.nativeQuotaBlocked(client.uuid) { @@ -400,6 +427,18 @@ func (ib *nativeInbound) handleVLESS(stream net.Conn, remote net.Addr) { } _ = stream.SetReadDeadline(time.Time{}) + switch cmd[0] { + case vlessCmdTCP, vlessCmdUDP, vlessCmdMux: + default: + xrayLogf("native xray: inbound %q VLESS command %d not supported yet", ib.tag, cmd[0]) + return + } + releaseConnection, quotaState, ok := xrayMgr.acquireNativeClientConnection(client.uuid, client.email) + if !ok { + return + } + defer releaseConnection() + // VLESS response header must be sent before relaying payload. CommandMux is // special: official Xray does not read a target from the VLESS header for it; // the following bytes are Mux.Cool/XUDP frames. Reading port/address here @@ -417,7 +456,7 @@ func (ib *nativeInbound) handleVLESS(stream net.Conn, remote net.Addr) { return } ib.nativeSuccessLogf("native xray: vless/tcp user=%s src=%s -> %s (inbound %q)", client.email, backend.LocalAddr(), target, ib.tag) - nativeTunnel(stream, backend, client.uuid, client.email, ib.upLimiter(), ib.downLimiter()) + nativeTunnel(stream, backend, client.uuid, client.email, quotaState, ib.upLimiter(), ib.downLimiter()) case vlessCmdUDP: backend, target, err := ib.nativeDialUDP(host, port) if err != nil { @@ -425,12 +464,10 @@ func (ib *nativeInbound) handleVLESS(stream net.Conn, remote net.Addr) { return } ib.nativeSuccessLogf("native xray: vless/udp user=%s src=%s -> %s (inbound %q)", client.email, backend.LocalAddr(), target, ib.tag) - nativeVLESSUDPTunnel(stream, backend, client.uuid, client.email, ib.upLimiter(), ib.downLimiter()) + nativeVLESSUDPTunnel(stream, backend, client.uuid, client.email, quotaState, ib.upLimiter(), ib.downLimiter()) case vlessCmdMux: ib.nativeSuccessLogf("native xray: vless/mux user=%s remote=%s (inbound %q)", client.email, remote, ib.tag) - ib.nativeVLESSMuxTunnel(stream, client.uuid, client.email) - default: - xrayLogf("native xray: inbound %q VLESS command %d not supported yet", ib.tag, cmd[0]) + ib.nativeVLESSMuxTunnel(stream, client.uuid, client.email, quotaState) } } @@ -444,7 +481,7 @@ func (ib *nativeInbound) logVLESSReadFailure(stage string, remote net.Addr, emai return } if email == "" { - xrayLogf("native xray: vless %s failed inbound=%q transport=%s remote=%s: %v", stage, ib.tag, ib.transport, remote, err) + logNativePreAuthRejection("native xray: vless %s failed inbound=%q transport=%s remote=%s: %v", stage, ib.tag, ib.transport, remote, err) } else { xrayLogf("native xray: vless %s failed inbound=%q transport=%s user=%s remote=%s: %v", stage, ib.tag, ib.transport, email, remote, err) } @@ -661,18 +698,18 @@ func normalizeNativeTargetHost(raw string) string { // backend, applying per-direction rate limits and accounting traffic against // the client's email so the panel's online detection keeps working. It mirrors // handleDirectTCPIP in main.go. -func nativeTunnel(client io.ReadWriteCloser, backend net.Conn, uuid, email string, up, down *rate.Limiter) { - xrayMgr.recordNativeConnect(uuid, email) - defer xrayMgr.recordNativeDisconnect(uuid, email) +func nativeTunnel(client io.ReadWriteCloser, backend net.Conn, uuid, email string, quotaState *xrayNativeQuotaState, up, down *rate.Limiter) { defer xrayRecover(fmt.Sprintf("native xray TCP tunnel user=%s", email)) - upMeter := &trafficMeter{uuid: uuid, email: email, uplink: true} - downMeter := &trafficMeter{uuid: uuid, email: email, uplink: false} + upMeter := newTrafficMeter(uuid, email, true, quotaState) + downMeter := newTrafficMeter(uuid, email, false, quotaState) var wg sync.WaitGroup var closeOnce sync.Once + ctx, cancel := context.WithCancel(context.Background()) closeAll := func() { closeOnce.Do(func() { + cancel() _ = backend.Close() _ = client.Close() }) @@ -682,7 +719,7 @@ func nativeTunnel(client io.ReadWriteCloser, backend net.Conn, uuid, email strin xrayGo("native xray TCP uplink", func() { // client -> backend defer wg.Done() defer closeAll() - _, _ = copyWithRateLimit(xrayQuotaMeteredWriter{w: backend, meter: upMeter}, client, up) + _, _ = copyWithRateLimitContext(ctx, xrayQuotaMeteredWriter{w: backend, meter: upMeter, ctx: ctx}, client, up) if cw, ok := backend.(interface{ CloseWrite() error }); ok { _ = cw.CloseWrite() } @@ -692,7 +729,7 @@ func nativeTunnel(client io.ReadWriteCloser, backend net.Conn, uuid, email strin xrayGo("native xray TCP downlink", func() { // backend -> client defer wg.Done() defer closeAll() - _, _ = copyWithRateLimit(xrayQuotaMeteredWriter{w: client, meter: downMeter}, backend, down) + _, _ = copyWithRateLimitContext(ctx, xrayQuotaMeteredWriter{w: client, meter: downMeter, ctx: ctx}, backend, down) }) wg.Wait() @@ -709,10 +746,17 @@ type trafficMeter struct { uplink bool n int64 quotaGeneration uint64 + state *xrayNativeQuotaState } const trafficFlushThreshold = 1024 * 1024 +func newTrafficMeter(uuid, email string, uplink bool, state *xrayNativeQuotaState) *trafficMeter { + t := &trafficMeter{uuid: uuid, email: email, uplink: uplink, state: state} + t.syncQuotaGeneration() + return t +} + func (t *trafficMeter) add(n int) { t.syncQuotaGeneration() t.n += int64(n) @@ -727,15 +771,20 @@ func (t *trafficMeter) flush() { return } if t.uplink { - xrayMgr.recordNativeTraffic(t.uuid, t.email, t.n, 0, t.quotaGeneration) + xrayMgr.recordNativeTraffic(t.uuid, t.email, t.n, 0, t.quotaGeneration, t.state) } else { - xrayMgr.recordNativeTraffic(t.uuid, t.email, 0, t.n, t.quotaGeneration) + xrayMgr.recordNativeTraffic(t.uuid, t.email, 0, t.n, t.quotaGeneration, t.state) } t.n = 0 } func (t *trafficMeter) syncQuotaGeneration() { - generation := xrayMgr.nativeQuotaGeneration(t.uuid) + var generation uint64 + if t.state != nil { + t.state.mu.Lock() + generation = t.state.generation + t.state.mu.Unlock() + } if t.quotaGeneration == 0 { t.quotaGeneration = generation return diff --git a/xray_native_mux.go b/xray_native_mux.go index 2057a99..2952e5e 100644 --- a/xray_native_mux.go +++ b/xray_native_mux.go @@ -54,7 +54,11 @@ type nativeMuxUplinkItem struct { port uint16 } -const nativeMuxUplinkQueue = 64 +const ( + nativeMuxUplinkQueue = 16 + nativeMuxMaxBufferedBytesPerSession = 1 * 1024 * 1024 + nativeMuxMaxBufferedBytesGlobal = 128 * 1024 * 1024 +) var nativeMuxFramePool = sync.Pool{ New: func() any { @@ -91,6 +95,11 @@ type nativeMuxSession struct { uplink chan nativeMuxUplinkItem closed chan struct{} closeOnce sync.Once + finishOnce sync.Once + enqueueMu sync.Mutex + enqueueWG sync.WaitGroup + enqueueDone bool + buffered atomic.Int64 ctx context.Context cancel context.CancelFunc onClose func(*nativeMuxSession) @@ -98,7 +107,11 @@ type nativeMuxSession struct { globalID [8]byte } -var nativeMuxGlobalActive atomic.Int64 +var ( + nativeMuxGlobalActive atomic.Int64 + nativeMuxBufferedBytes atomic.Int64 + nativeMuxBufferRejected atomic.Int64 +) func acquireNativeMuxGlobalSlot() (func(), bool) { limit := int64(nativeMuxGlobalSessionLimit()) @@ -117,15 +130,73 @@ func acquireNativeMuxGlobalSlot() (func(), bool) { } } +func reserveNativeMuxBufferedBytes(s *nativeMuxSession, n int64) bool { + if s == nil || n <= 0 { + return true + } + for { + current := s.buffered.Load() + if current > nativeMuxMaxBufferedBytesPerSession-n { + logNativeLimitRejection("mux session buffered bytes", &nativeMuxBufferRejected, nativeMuxMaxBufferedBytesPerSession) + return false + } + if s.buffered.CompareAndSwap(current, current+n) { + break + } + } + for { + current := nativeMuxBufferedBytes.Load() + if current > nativeMuxMaxBufferedBytesGlobal-n { + s.buffered.Add(-n) + logNativeLimitRejection("mux global buffered bytes", &nativeMuxBufferRejected, nativeMuxMaxBufferedBytesGlobal) + return false + } + if nativeMuxBufferedBytes.CompareAndSwap(current, current+n) { + return true + } + } +} + +func releaseNativeMuxBufferedBytes(s *nativeMuxSession, n int64) { + if s == nil || n <= 0 { + return + } + for { + current := s.buffered.Load() + release := n + if release > current { + release = current + } + if s.buffered.CompareAndSwap(current, current-release) { + releaseNativeAtomicBytes(&nativeMuxBufferedBytes, release) + return + } + } +} + +func releaseNativeAtomicBytes(counter *atomic.Int64, n int64) { + if counter == nil || n <= 0 { + return + } + for { + current := counter.Load() + next := current - n + if next < 0 { + next = 0 + } + if counter.CompareAndSwap(current, next) { + return + } + } +} + // nativeVLESSMuxTunnel implements the server side of Xray's Mux.Cool framing // for VLESS CommandMux. CommandMux does not carry a VLESS target address; every // child TCP/UDP request is described by mux frame metadata. UDP is treated as a // packet protocol, not as a byte stream, and XUDP-style GlobalID/endpoint // metadata is accepted for full-cone friendly clients. -func (ib *nativeInbound) nativeVLESSMuxTunnel(stream io.ReadWriteCloser, uuid, email string) { +func (ib *nativeInbound) nativeVLESSMuxTunnel(stream io.ReadWriteCloser, uuid, email string, quotaState *xrayNativeQuotaState) { defer xrayRecover(fmt.Sprintf("native xray VLESS mux user=%s", email)) - xrayMgr.recordNativeConnect(uuid, email) - defer xrayMgr.recordNativeDisconnect(uuid, email) writeMu := &sync.Mutex{} sessions := make(map[uint16]*nativeMuxSession) @@ -256,7 +327,7 @@ func (ib *nativeInbound) nativeVLESSMuxTunnel(stream io.ReadWriteCloser, uuid, e } } - s, target, err := ib.newNativeMuxSession(meta.sessionID, meta.network, targetHost, targetPort, isXUDP, meta.globalID, stream, writeMu, uuid, email, removeSession) + s, target, err := ib.newNativeMuxSession(meta.sessionID, meta.network, targetHost, targetPort, isXUDP, meta.globalID, stream, writeMu, uuid, email, quotaState, removeSession) if err != nil { xrayLogf("native xray: VLESS mux session %s setup failed: %v", target, err) writeMu.Lock() @@ -281,8 +352,11 @@ func (ib *nativeInbound) nativeVLESSMuxTunnel(stream io.ReadWriteCloser, uuid, e xrayTracef("native xray: vless/mux %s user=%s -> %s session=%d xudp=%v", nativeMuxNetworkName(meta.network), email, target, meta.sessionID, isXUDP) ib2, host2, port2 := ib, targetHost, targetPort xrayGo(fmt.Sprintf("native xray mux session=%d", s.id), func() { s.run(ib2, host2, port2) }) - if len(pkt.payload) > 0 { - s.enqueueUplink(pkt.payload, pkt.host, pkt.port) + if len(pkt.payload) > 0 && !s.enqueueUplink(pkt.payload, pkt.host, pkt.port) { + closeSession(s.id) + writeMu.Lock() + _ = writeNativeMuxEnd(stream, meta.sessionID, true) + writeMu.Unlock() } case nativeMuxStatusKeep: @@ -319,8 +393,11 @@ func (ib *nativeInbound) nativeVLESSMuxTunnel(stream io.ReadWriteCloser, uuid, e pkt.host = meta.host pkt.port = meta.port } - if len(pkt.payload) > 0 { - s.enqueueUplink(pkt.payload, pkt.host, pkt.port) + if len(pkt.payload) > 0 && !s.enqueueUplink(pkt.payload, pkt.host, pkt.port) { + closeSession(s.id) + writeMu.Lock() + _ = writeNativeMuxEnd(stream, meta.sessionID, true) + writeMu.Unlock() } default: @@ -332,7 +409,7 @@ func (ib *nativeInbound) nativeVLESSMuxTunnel(stream io.ReadWriteCloser, uuid, e } } -func (ib *nativeInbound) newNativeMuxSession(id uint16, network byte, host string, port uint16, xudp bool, globalID [8]byte, client io.Writer, writeMu *sync.Mutex, uuid, email string, onClose func(*nativeMuxSession)) (*nativeMuxSession, string, error) { +func (ib *nativeInbound) newNativeMuxSession(id uint16, network byte, host string, port uint16, xudp bool, globalID [8]byte, client io.Writer, writeMu *sync.Mutex, uuid, email string, quotaState *xrayNativeQuotaState, onClose func(*nativeMuxSession)) (*nativeMuxSession, string, error) { target := net.JoinHostPort(normalizeNativeTargetHost(host), strconv.Itoa(int(port))) if invalidNativeDestination(host, port) { return nil, target, fmt.Errorf("invalid destination") @@ -351,8 +428,8 @@ func (ib *nativeInbound) newNativeMuxSession(id uint16, network byte, host strin email: email, upLimiter: ib.upLimiter(), downLimiter: ib.downLimiter(), - upMeter: &trafficMeter{uuid: uuid, email: email, uplink: true}, - downMeter: &trafficMeter{uuid: uuid, email: email, uplink: false}, + upMeter: newTrafficMeter(uuid, email, true, quotaState), + downMeter: newTrafficMeter(uuid, email, false, quotaState), uplink: make(chan nativeMuxUplinkItem, nativeMuxUplinkQueue), closed: make(chan struct{}), onClose: onClose, @@ -365,6 +442,7 @@ func (ib *nativeInbound) newNativeMuxSession(id uint16, network byte, host strin func (s *nativeMuxSession) run(ib *nativeInbound, host string, port uint16) { defer xrayRecover(fmt.Sprintf("native xray mux run session=%d", s.id)) + defer s.finish() select { case <-s.closed: @@ -410,24 +488,61 @@ func (s *nativeMuxSession) failInit(notifyClient bool) { _ = writeNativeMuxEnd(s.client, s.id, true) s.writeMu.Unlock() } - if s.onClose != nil { - s.onClose(s) - } - s.closeBackend() + s.finish() } -func (s *nativeMuxSession) enqueueUplink(payload []byte, host string, port uint16) { +// finish is the single lifecycle exit for a mux child. The backend reader, +// uplink loop, parent mux stream, and initialization path can all detect the +// terminal condition concurrently, so both cleanup and map removal must be +// exactly-once operations. +func (s *nativeMuxSession) finish() { + s.finishOnce.Do(func() { + s.closeBackend() + if s.onClose != nil { + s.onClose(s) + } + }) +} + +func (s *nativeMuxSession) beginEnqueue() bool { + s.enqueueMu.Lock() + defer s.enqueueMu.Unlock() + if s.enqueueDone { + return false + } + s.enqueueWG.Add(1) + return true +} + +func (s *nativeMuxSession) enqueueUplink(payload []byte, host string, port uint16) bool { if len(payload) == 0 { - return + return true + } + if !s.beginEnqueue() { + return false + } + defer s.enqueueWG.Done() + + bytes := int64(len(payload)) + if !reserveNativeMuxBufferedBytes(s, bytes) { + return false } cp := make([]byte, len(payload)) copy(cp, payload) select { case s.uplink <- nativeMuxUplinkItem{payload: cp, host: host, port: port}: + return true case <-s.closed: + releaseNativeMuxBufferedBytes(s, bytes) + return false } } +func (s *nativeMuxSession) processUplinkItem(item nativeMuxUplinkItem) bool { + defer releaseNativeMuxBufferedBytes(s, int64(len(item.payload))) + return s.writeBackendItem(item) +} + func (s *nativeMuxSession) uplinkLoop() { defer s.upMeter.flush() for { @@ -435,7 +550,7 @@ func (s *nativeMuxSession) uplinkLoop() { case <-s.closed: return case item := <-s.uplink: - if !s.writeBackendItem(item) { + if !s.processUplinkItem(item) { s.closeBackend() return } @@ -481,15 +596,12 @@ func (s *nativeMuxSession) writeBackendItem(item nativeMuxUplinkItem) bool { return false } } - quotaLimiter, quotaErr := reserveNativePacketQuota(s.upMeter, len(payload)) + quotaReservation, quotaErr := reserveNativePacketQuota(s.upMeter, len(payload)) if quotaErr != nil { return false } - if quotaLimiter != nil { - if err := quotaLimiter.WaitN(s.ctx, len(payload)); err != nil { - finishNativePacketQuota(s.upMeter, len(payload), 0) - return false - } + if err := quotaReservation.wait(s.ctx); err != nil { + return false } var n int @@ -502,7 +614,7 @@ func (s *nativeMuxSession) writeBackendItem(item nativeMuxUplinkItem) bool { if isNativeDNSSinkTarget(item.host) || invalidNativeDestination(item.host, item.port) { // AdGuard/blocked endpoints must be ignored at the cheapest possible // point. Do not resolve, dial, log loudly, or keep the mux child busy. - finishNativePacketQuota(s.upMeter, len(payload), 0) + quotaReservation.finish(0) xrayTracef("native xray: VLESS mux UDP fast-ignored override sink session=%d target=%s:%d", s.id, item.host, item.port) return true } @@ -514,7 +626,7 @@ func (s *nativeMuxSession) writeBackendItem(item nativeMuxUplinkItem) bool { s.lastUDPPort = item.port s.lastUDPAddr = addr } else { - finishNativePacketQuota(s.upMeter, len(payload), 0) + quotaReservation.finish(0) xrayTracef("native xray: VLESS mux UDP override resolve failed session=%d target=%s:%d: %v", s.id, item.host, item.port, rerr) return true } @@ -524,7 +636,7 @@ func (s *nativeMuxSession) writeBackendItem(item nativeMuxUplinkItem) bool { if s.network == nativeMuxNetworkUDP && err == nil { _ = s.udp.SetReadDeadline(time.Now().Add(nativeMuxUDPIdleTimeout())) } - finishNativePacketQuota(s.upMeter, len(payload), n) + quotaReservation.finish(n) if err != nil { xrayLogf("native xray: VLESS mux backend write failed session=%d: %v", s.id, err) return false @@ -542,10 +654,7 @@ func (s *nativeMuxSession) readBackendLoop() { _ = writeNativeMuxEnd(s.client, s.id, false) s.writeMu.Unlock() } - if s.onClose != nil { - s.onClose(s) - } - s.closeBackend() + s.finish() }() if s.network == nativeMuxNetworkTCP { @@ -581,25 +690,22 @@ func (s *nativeMuxSession) readTCPBackendLoop() { if err := s.waitDownRate(n); err != nil { return } - quotaLimiter, quotaErr := reserveNativePacketQuota(s.downMeter, n) + quotaReservation, quotaErr := reserveNativePacketQuota(s.downMeter, n) if quotaErr != nil { return } - if quotaLimiter != nil { - if err := quotaLimiter.WaitN(s.ctx, n); err != nil { - finishNativePacketQuota(s.downMeter, n, 0) - return - } + if err := quotaReservation.wait(s.ctx); err != nil { + return } s.writeMu.Lock() werr := writeNativeMuxData(s.client, s.id, nativeMuxStatusKeep, buf[:n]) s.writeMu.Unlock() if werr != nil { - finishNativePacketQuota(s.downMeter, n, 0) + quotaReservation.finish(0) xrayLogf("native xray: VLESS mux TCP client write failed session=%d: %v", s.id, werr) return } - finishNativePacketQuota(s.downMeter, n, n) + quotaReservation.finish(n) } } @@ -623,15 +729,12 @@ func (s *nativeMuxSession) readUDPBackendLoop() bool { if err := s.waitDownRate(n); err != nil { return true } - quotaLimiter, quotaErr := reserveNativePacketQuota(s.downMeter, n) + quotaReservation, quotaErr := reserveNativePacketQuota(s.downMeter, n) if quotaErr != nil { return true } - if quotaLimiter != nil { - if err := quotaLimiter.WaitN(s.ctx, n); err != nil { - finishNativePacketQuota(s.downMeter, n, 0) - return true - } + if err := quotaReservation.wait(s.ctx); err != nil { + return true } s.writeMu.Lock() // Include the UDP source endpoint on XUDP responses so clients that rely on @@ -640,29 +743,46 @@ func (s *nativeMuxSession) readUDPBackendLoop() bool { werr := writeNativeMuxPacketData(s.client, s.id, nativeMuxStatusKeep, buf[:n], addr, s.xudp) s.writeMu.Unlock() if werr != nil { - finishNativePacketQuota(s.downMeter, n, 0) + quotaReservation.finish(0) xrayLogf("native xray: VLESS mux UDP client write failed session=%d: %v", s.id, werr) return true } - finishNativePacketQuota(s.downMeter, n, n) + quotaReservation.finish(n) } } func (s *nativeMuxSession) closeBackend() { s.closeOnce.Do(func() { + s.enqueueMu.Lock() + s.enqueueDone = true close(s.closed) + s.enqueueMu.Unlock() if s.cancel != nil { s.cancel() } - if s.releaseSlot != nil { - s.releaseSlot() - } if s.tcp != nil { _ = s.tcp.Close() } if s.udp != nil { _ = s.udp.Close() } + + // Wait for producers that passed beginEnqueue before the close flag, then + // discard any payloads the consumer did not take. This returns every byte + // reservation even when shutdown races a full queue. + s.enqueueWG.Wait() + for { + select { + case item := <-s.uplink: + releaseNativeMuxBufferedBytes(s, int64(len(item.payload))) + item.payload = nil + default: + if s.releaseSlot != nil { + s.releaseSlot() + } + return + } + } }) } diff --git a/xray_native_safety.go b/xray_native_safety.go index f93f391..0f11394 100644 --- a/xray_native_safety.go +++ b/xray_native_safety.go @@ -1,6 +1,14 @@ package main -import "runtime/debug" +import ( + "net" + "runtime/debug" + "sync" + "sync/atomic" + "time" +) + +const nativeOverloadBackoff = 10 * time.Millisecond // xrayRecover prevents a bad client packet, closed HTTP stream, or mux/session // race from taking down the whole sshpanel process. A panic should only kill the @@ -18,3 +26,233 @@ func xrayGo(where string, fn func()) { fn() }() } + +func init() { + // Direct native-inbound tests and embedders may run an accept loop without + // the singleton server start method. Production stop() flips this to false. + nativeTransportAccepting.Store(true) +} + +var ( + nativeTransportConnections atomic.Int64 + nativeTransportRejected atomic.Int64 + nativeXHTTPRequests atomic.Int64 + nativeXHTTPRequestsRejected atomic.Int64 + nativeXHTTPSessions atomic.Int64 + nativeXHTTPSessionsRejected atomic.Int64 + nativeClientConnsRejected atomic.Int64 + nativePreAuthRejected atomic.Int64 + + nativeTransportAccepting atomic.Bool + nativeTransportRegistry = struct { + sync.Mutex + conns map[*nativeCountedConn]struct{} + }{conns: make(map[*nativeCountedConn]struct{})} +) + +// acquireNativeCounter reserves one slot without blocking. Blocking the accept +// loop or an HTTP handler when the process is already at its safety ceiling +// would retain yet more sockets/goroutines, so overload is rejected promptly. +func acquireNativeCounter(active *atomic.Int64, limit int) (func(), bool) { + for { + current := active.Load() + if limit > 0 && current >= int64(limit) { + return nil, false + } + if active.CompareAndSwap(current, current+1) { + var once sync.Once + return func() { + once.Do(func() { + if active.Add(-1) < 0 { + active.Store(0) + } + }) + }, true + } + } +} + +func shouldLogNativeSample(counter *atomic.Int64) (n int64, ok bool) { + n = counter.Add(1) + // Keep attacks visible without allowing logging itself to become a CPU/disk + // amplifier. The first event and one event per 1024 repetitions are logged. + return n, n == 1 || n%1024 == 0 +} + +func logNativeLimitRejection(kind string, rejected *atomic.Int64, limit int) { + n, ok := shouldLogNativeSample(rejected) + if ok { + xrayLogf("native xray: rejected %s at safety limit=%d (rejected=%d)", kind, limit, n) + } +} + +func logNativeClientLimitRejection(email string, limit int) { + n, ok := shouldLogNativeSample(&nativeClientConnsRejected) + if ok { + xrayLogf("native xray: rejected authenticated user %s at max_conns=%d (rejected=%d)", email, limit, n) + } +} + +func logNativePreAuthRejection(format string, args ...interface{}) { + if _, ok := shouldLogNativeSample(&nativePreAuthRejected); ok { + xrayLogf(format, args...) + } +} + +func acquireNativeTransportConnection() (func(), bool) { + limit := nativeMaxConnectionLimit() + release, ok := acquireNativeCounter(&nativeTransportConnections, limit) + if !ok { + logNativeLimitRejection("transport connection", &nativeTransportRejected, limit) + } + return release, ok +} + +func acquireNativeXHTTPRequest() (func(), bool) { + limit := nativeMaxXHTTPRequestLimit() + release, ok := acquireNativeCounter(&nativeXHTTPRequests, limit) + if !ok { + logNativeLimitRejection("XHTTP request", &nativeXHTTPRequestsRejected, limit) + } + return release, ok +} + +func acquireNativeXHTTPSession() (func(), bool) { + limit := nativeXHTTPMaxSessionLimit() + release, ok := acquireNativeCounter(&nativeXHTTPSessions, limit) + if !ok { + logNativeLimitRejection("XHTTP session", &nativeXHTTPSessionsRejected, limit) + } + return release, ok +} + +func configureNativeTransportSocket(c net.Conn) { + if tc, ok := c.(*net.TCPConn); ok { + _ = tc.SetKeepAlive(true) + _ = tc.SetKeepAlivePeriod(30 * time.Second) + _ = tc.SetNoDelay(true) + } +} + +// nativeCountedConn releases its global transport slot and unregisters itself +// exactly once, even when several tunnel paths race to close the same socket. +type nativeCountedConn struct { + net.Conn + release func() + onClose func() + closeOnce sync.Once + closeErr error +} + +func (c *nativeCountedConn) Close() error { + c.closeOnce.Do(func() { + c.closeErr = c.Conn.Close() + if c.release != nil { + c.release() + } + if c.onClose != nil { + c.onClose() + } + }) + return c.closeErr +} + +// wrapNativeTransportConn applies only the global counter. It is useful for +// focused tests and for callers that own connection lifetime themselves. +func wrapNativeTransportConn(c net.Conn) (net.Conn, bool) { + if c == nil { + return nil, false + } + configureNativeTransportSocket(c) + release, ok := acquireNativeTransportConnection() + if !ok { + _ = c.Close() + return nil, false + } + return &nativeCountedConn{Conn: c, release: release}, true +} + +// wrapTrackedNativeTransportConn additionally registers the accepted socket so +// stopping/restarting native Xray closes established raw, WebSocket, TLS, HTTP/1 +// and HTTP/2 transports instead of leaving tunnel goroutines alive. +func wrapTrackedNativeTransportConn(c net.Conn) (net.Conn, bool) { + if c == nil { + return nil, false + } + configureNativeTransportSocket(c) + if !nativeTransportAccepting.Load() { + _ = c.Close() + return nil, false + } + release, ok := acquireNativeTransportConnection() + if !ok { + _ = c.Close() + return nil, false + } + + counted := &nativeCountedConn{Conn: c, release: release} + counted.onClose = func() { + nativeTransportRegistry.Lock() + delete(nativeTransportRegistry.conns, counted) + nativeTransportRegistry.Unlock() + } + + nativeTransportRegistry.Lock() + if !nativeTransportAccepting.Load() { + nativeTransportRegistry.Unlock() + _ = counted.Close() + return nil, false + } + nativeTransportRegistry.conns[counted] = struct{}{} + nativeTransportRegistry.Unlock() + return counted, true +} + +func beginNativeTransportAccepting() { + nativeTransportAccepting.Store(true) +} + +func stopNativeTransportAccepting() { + nativeTransportAccepting.Store(false) +} + +func closeAllNativeTransportConnections() { + nativeTransportRegistry.Lock() + conns := make([]*nativeCountedConn, 0, len(nativeTransportRegistry.conns)) + for c := range nativeTransportRegistry.conns { + conns = append(conns, c) + } + nativeTransportRegistry.Unlock() + for _, c := range conns { + _ = c.Close() + } +} + +// nativeLimitedListener applies the same pre-authentication ceiling to XHTTP +// listeners. net/http receives only sockets that own a slot; rejected sockets +// are closed before it can allocate a per-connection goroutine or perform TLS. +type nativeLimitedListener struct { + net.Listener +} + +func (l nativeLimitedListener) Accept() (net.Conn, error) { + for { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + if counted, ok := wrapTrackedNativeTransportConn(c); ok { + return counted, nil + } + // At the ceiling, a hot accept/close loop can itself consume a CPU core. + // A short fixed backoff also lets the kernel backlog absorb brief spikes. + time.Sleep(nativeOverloadBackoff) + } +} + +func limitNativeListener(ln net.Listener) net.Listener { + if ln == nil { + return nil + } + return nativeLimitedListener{Listener: ln} +} diff --git a/xray_native_tuning.go b/xray_native_tuning.go index c724288..55f1f23 100644 --- a/xray_native_tuning.go +++ b/xray_native_tuning.go @@ -7,22 +7,32 @@ import ( ) type XrayNativeTuning struct { - RuntimeGOMAXPROCS int `json:"runtime_gomaxprocs,omitempty"` - MuxGlobalSessions int `json:"mux_global_sessions,omitempty"` - TracePackets bool `json:"trace_packets,omitempty"` + RuntimeGOMAXPROCS int `json:"runtime_gomaxprocs,omitempty"` + MuxGlobalSessions int `json:"mux_global_sessions,omitempty"` + MaxConcurrentConnections int `json:"max_concurrent_connections,omitempty"` + MaxConcurrentXHTTPRequests int `json:"max_concurrent_xhttp_requests,omitempty"` + XHTTPMaxSessions int `json:"xhttp_max_sessions,omitempty"` + TracePackets bool `json:"trace_packets,omitempty"` } const ( defaultNativeRuntimeGOMAXPROCS = 0 - defaultNativeMuxGlobalSessions = 32768 + defaultNativeMuxGlobalSessions = 8192 + defaultNativeMaxConnections = 4096 + defaultNativeMaxXHTTPRequests = 8192 - fixedNativeMuxMaxSessions = 128 + fixedNativeMuxMaxSessions = 64 fixedNativeMuxUDPIdleMS = 120000 fixedNativeMuxUDPReadBuffer = 256 * 1024 fixedNativeMuxUDPWriteBuffer = 256 * 1024 - defaultNativeXHTTPMaxSessions = 16384 - defaultNativeXHTTPBufferedPosts = 512 + defaultNativeXHTTPMaxSessions = 4096 + defaultNativeHTTP2MaxStreams = 256 + // Packet-up posts are also protected by byte budgets in xray_xhttp.go. Keep + // the default reorder queue modest so thousands of unauthenticated sessions + // cannot consume large amounts of memory merely by allocating empty channel + // buffers. Operators may request more, up to the hard cap enforced there. + defaultNativeXHTTPBufferedPosts = 64 // Do not impose an application-level lifetime on a connected XHTTP VPN // session. The official Xray server keeps a connected session for the @@ -35,6 +45,9 @@ const ( var ( nativeTuneRuntimeGOMAXPROCS atomic.Int64 nativeTuneMuxGlobalSessions atomic.Int64 + nativeTuneMaxConnections atomic.Int64 + nativeTuneMaxXHTTPRequests atomic.Int64 + nativeTuneXHTTPMaxSessions atomic.Int64 nativeTuneTracePackets atomic.Bool ) @@ -53,6 +66,15 @@ func normalizeNativeXrayTuning(t *XrayNativeTuning) XrayNativeTuning { if out.MuxGlobalSessions <= 0 { out.MuxGlobalSessions = defaultNativeMuxGlobalSessions } + if out.MaxConcurrentConnections == 0 { + out.MaxConcurrentConnections = defaultNativeMaxConnections + } + if out.MaxConcurrentXHTTPRequests == 0 { + out.MaxConcurrentXHTTPRequests = defaultNativeMaxXHTTPRequests + } + if out.XHTTPMaxSessions == 0 { + out.XHTTPMaxSessions = defaultNativeXHTTPMaxSessions + } return out } @@ -68,19 +90,40 @@ func applyNativeXrayTuning(t *XrayNativeTuning) XrayNativeTuning { runtime.GOMAXPROCS(gomax) nativeTuneRuntimeGOMAXPROCS.Store(int64(gomax)) nativeTuneMuxGlobalSessions.Store(int64(out.MuxGlobalSessions)) + nativeTuneMaxConnections.Store(nativeLimitValue(out.MaxConcurrentConnections)) + nativeTuneMaxXHTTPRequests.Store(nativeLimitValue(out.MaxConcurrentXHTTPRequests)) + nativeTuneXHTTPMaxSessions.Store(nativeLimitValue(out.XHTTPMaxSessions)) nativeTuneTracePackets.Store(out.TracePackets) return out } +// Native tuning limits use zero internally for unlimited. In configuration, +// zero means "use the safe default" and any negative value disables the cap. +func nativeLimitValue(v int) int64 { + if v < 0 { + return 0 + } + return int64(v) +} + func nativeRuntimeGOMAXPROCS() int { return int(nativeTuneRuntimeGOMAXPROCS.Load()) } func nativeMuxGlobalSessionLimit() int { return int(nativeTuneMuxGlobalSessions.Load()) } +func nativeMaxConnectionLimit() int { return int(nativeTuneMaxConnections.Load()) } +func nativeMaxXHTTPRequestLimit() int { return int(nativeTuneMaxXHTTPRequests.Load()) } +func nativeXHTTPMaxSessionLimit() int { return int(nativeTuneXHTTPMaxSessions.Load()) } func nativeTracePacketsEnabled() bool { return nativeTuneTracePackets.Load() } func nativeMuxMaxSessionLimit() int { return fixedNativeMuxMaxSessions } func nativeMuxUDPReadBufferSize() int { return fixedNativeMuxUDPReadBuffer } func nativeMuxUDPWriteBufferSize() int { return fixedNativeMuxUDPWriteBuffer } -func nativeXHTTPMaxSessionLimit() int { return defaultNativeXHTTPMaxSessions } func nativeXHTTPBufferedPostLimit() int { return defaultNativeXHTTPBufferedPosts } +func nativeHTTP2MaxConcurrentStreams() uint32 { + limit := nativeMaxXHTTPRequestLimit() + if limit <= 0 || limit > defaultNativeHTTP2MaxStreams { + return defaultNativeHTTP2MaxStreams + } + return uint32(limit) +} func nativeMuxUDPIdleTimeout() time.Duration { return fixedNativeMuxUDPIdleMS * time.Millisecond } diff --git a/xray_native_udp.go b/xray_native_udp.go index 23c519e..9f5331e 100644 --- a/xray_native_udp.go +++ b/xray_native_udp.go @@ -25,18 +25,18 @@ const ( // check and caused the server to block waiting for a fake second payload. // XUDP belongs to VLESS CommandMux and is handled separately when Mux support // is implemented. -func nativeVLESSUDPTunnel(client io.ReadWriteCloser, backend net.Conn, uuid, email string, up, down *rate.Limiter) { - xrayMgr.recordNativeConnect(uuid, email) - defer xrayMgr.recordNativeDisconnect(uuid, email) +func nativeVLESSUDPTunnel(client io.ReadWriteCloser, backend net.Conn, uuid, email string, quotaState *xrayNativeQuotaState, up, down *rate.Limiter) { defer xrayRecover(fmt.Sprintf("native xray VLESS UDP tunnel user=%s", email)) - upMeter := &trafficMeter{uuid: uuid, email: email, uplink: true} - downMeter := &trafficMeter{uuid: uuid, email: email, uplink: false} + upMeter := newTrafficMeter(uuid, email, true, quotaState) + downMeter := newTrafficMeter(uuid, email, false, quotaState) var wg sync.WaitGroup var closeOnce sync.Once + ctx, cancel := context.WithCancel(context.Background()) closeAll := func() { closeOnce.Do(func() { + cancel() _ = backend.Close() _ = client.Close() }) @@ -57,19 +57,18 @@ func nativeVLESSUDPTunnel(client io.ReadWriteCloser, backend net.Conn, uuid, ema if len(payload) == 0 { continue } - if err := waitNativeRate(up, len(payload)); err != nil { + if err := waitNativeRate(ctx, up, len(payload)); err != nil { return } - quotaLimiter, err := reserveNativePacketQuota(upMeter, len(payload)) + quotaReservation, err := reserveNativePacketQuota(upMeter, len(payload)) if err != nil { return } - if err := waitNativeRate(quotaLimiter, len(payload)); err != nil { - finishNativePacketQuota(upMeter, len(payload), 0) + if err := quotaReservation.wait(ctx); err != nil { return } n, err := backend.Write(payload) - finishNativePacketQuota(upMeter, len(payload), n) + quotaReservation.finish(n) if err != nil { xrayLogf("native xray: VLESS UDP backend write failed: %v", err) return @@ -97,23 +96,22 @@ func nativeVLESSUDPTunnel(client io.ReadWriteCloser, backend net.Conn, uuid, ema if n <= 0 { continue } - if err := waitNativeRate(down, n); err != nil { + if err := waitNativeRate(ctx, down, n); err != nil { return } - quotaLimiter, err := reserveNativePacketQuota(downMeter, n) + quotaReservation, err := reserveNativePacketQuota(downMeter, n) if err != nil { return } - if err := waitNativeRate(quotaLimiter, n); err != nil { - finishNativePacketQuota(downMeter, n, 0) + if err := quotaReservation.wait(ctx); err != nil { return } if err := writeVLESSLengthPacket(client, buf[:n]); err != nil { - finishNativePacketQuota(downMeter, n, 0) + quotaReservation.finish(0) xrayLogf("native xray: VLESS UDP client write failed: %v", err) return } - finishNativePacketQuota(downMeter, n, n) + quotaReservation.finish(n) } }) @@ -316,18 +314,18 @@ func writeVLESSXUDPPacket(w io.Writer, payload []byte) error { // nativeVMessUDPTunnel maps one VMess body chunk to one UDP datagram. VMess AEAD // chunking already preserves packet boundaries, so no extra VLESS length prefix // is added inside the encrypted body. -func nativeVMessUDPTunnel(client nativeVMessStream, backend net.Conn, uuid, email string, up, down *rate.Limiter) { - xrayMgr.recordNativeConnect(uuid, email) - defer xrayMgr.recordNativeDisconnect(uuid, email) +func nativeVMessUDPTunnel(client nativeVMessStream, backend net.Conn, uuid, email string, quotaState *xrayNativeQuotaState, up, down *rate.Limiter) { defer xrayRecover(fmt.Sprintf("native xray VMess UDP tunnel user=%s", email)) - upMeter := &trafficMeter{uuid: uuid, email: email, uplink: true} - downMeter := &trafficMeter{uuid: uuid, email: email, uplink: false} + upMeter := newTrafficMeter(uuid, email, true, quotaState) + downMeter := newTrafficMeter(uuid, email, false, quotaState) var wg sync.WaitGroup var closeOnce sync.Once + ctx, cancel := context.WithCancel(context.Background()) closeAll := func() { closeOnce.Do(func() { + cancel() _ = backend.Close() _ = client.Close() }) @@ -348,19 +346,18 @@ func nativeVMessUDPTunnel(client nativeVMessStream, backend net.Conn, uuid, emai if len(pkt) == 0 { continue } - if err := waitNativeRate(up, len(pkt)); err != nil { + if err := waitNativeRate(ctx, up, len(pkt)); err != nil { return } - quotaLimiter, err := reserveNativePacketQuota(upMeter, len(pkt)) + quotaReservation, err := reserveNativePacketQuota(upMeter, len(pkt)) if err != nil { return } - if err := waitNativeRate(quotaLimiter, len(pkt)); err != nil { - finishNativePacketQuota(upMeter, len(pkt), 0) + if err := quotaReservation.wait(ctx); err != nil { return } n, err := backend.Write(pkt) - finishNativePacketQuota(upMeter, len(pkt), n) + quotaReservation.finish(n) if err != nil { xrayLogf("native xray: VMess UDP backend write failed: %v", err) return @@ -388,23 +385,22 @@ func nativeVMessUDPTunnel(client nativeVMessStream, backend net.Conn, uuid, emai if n <= 0 { continue } - if err := waitNativeRate(down, n); err != nil { + if err := waitNativeRate(ctx, down, n); err != nil { return } - quotaLimiter, err := reserveNativePacketQuota(downMeter, n) + quotaReservation, err := reserveNativePacketQuota(downMeter, n) if err != nil { return } - if err := waitNativeRate(quotaLimiter, n); err != nil { - finishNativePacketQuota(downMeter, n, 0) + if err := quotaReservation.wait(ctx); err != nil { return } if err := client.WritePacket(buf[:n]); err != nil { - finishNativePacketQuota(downMeter, n, 0) + quotaReservation.finish(0) xrayLogf("native xray: VMess UDP client write failed: %v", err) return } - finishNativePacketQuota(downMeter, n, n) + quotaReservation.finish(n) } }) @@ -414,9 +410,12 @@ func nativeVMessUDPTunnel(client nativeVMessStream, backend net.Conn, uuid, emai closeAll() } -func waitNativeRate(lim *rate.Limiter, n int) error { +func waitNativeRate(ctx context.Context, lim *rate.Limiter, n int) error { if lim == nil || n <= 0 { return nil } - return lim.WaitN(context.Background(), n) + if ctx == nil { + ctx = context.Background() + } + return lim.WaitN(ctx, n) } diff --git a/xray_quota.go b/xray_quota.go index 01baeea..2abf8e5 100644 --- a/xray_quota.go +++ b/xray_quota.go @@ -10,6 +10,12 @@ import ( ) type xrayNativeQuotaState struct { + // trafficMu establishes a clean reset boundary. Native stream and packet + // writers hold a read lock from quota reservation through the actual write + // and metering; traffic resets take the write lock. This prevents an + // in-flight pre-reset reservation from being accounted in the new period or + // subtracting from freshly reset usage. + trafficMu sync.RWMutex mu sync.Mutex usedBytes int64 quotaBytes int64 @@ -17,6 +23,8 @@ type xrayNativeQuotaState struct { throttleMbps int limiter *rate.Limiter generation uint64 + maxConns int + activeConns int } func (m *XrayManager) reloadNativeQuotaPolicies() { @@ -51,9 +59,17 @@ func newXrayNativeQuotaState(meta *XrayClientMeta) *xrayNativeQuotaState { action: normalizeQuotaAction(meta.QuotaAction), throttleMbps: quotaThrottleMbpsOrDefault(meta.QuotaThrottleMbps), generation: 1, + maxConns: normalizeXrayMaxConns(meta.MaxConns), } } +func normalizeXrayMaxConns(v int) int { + if v < 0 { + return 0 + } + return v +} + func (m *XrayManager) setNativeQuotaPolicy(meta *XrayClientMeta) { if meta == nil || strings.TrimSpace(meta.UUID) == "" { return @@ -75,6 +91,7 @@ func (m *XrayManager) setNativeQuotaPolicy(meta *XrayClientMeta) { existing.quotaBytes = meta.DataQuotaBytes existing.action = normalizeQuotaAction(meta.QuotaAction) existing.throttleMbps = quotaThrottleMbpsOrDefault(meta.QuotaThrottleMbps) + existing.maxConns = normalizeXrayMaxConns(meta.MaxConns) existing.limiter = nil existing.mu.Unlock() } @@ -87,6 +104,13 @@ func (m *XrayManager) removeNativeQuotaPolicy(uuid string) { m.nativeQuotaMu.Lock() delete(m.nativeQuotaByUUID, uuid) m.nativeQuotaMu.Unlock() + + // Do not retain failed traffic/active deltas for a client that no longer + // exists. This also bounds the pending maps during a prolonged DB outage. + m.nativeDBMu.Lock() + delete(m.nativeTrafficPending, uuid) + delete(m.nativeActivePending, uuid) + m.nativeDBMu.Unlock() } func (m *XrayManager) resetNativeQuotaUsage(uuid string) { @@ -97,6 +121,8 @@ func (m *XrayManager) resetNativeQuotaUsage(uuid string) { if state == nil { return } + state.trafficMu.Lock() + defer state.trafficMu.Unlock() state.mu.Lock() state.usedBytes = 0 state.limiter = nil @@ -107,26 +133,23 @@ func (m *XrayManager) resetNativeQuotaUsage(uuid string) { state.mu.Unlock() } -func (m *XrayManager) nativeQuotaGeneration(uuid string) uint64 { - state := m.nativeQuotaState(uuid) - if state == nil { - return 0 - } - state.mu.Lock() - generation := state.generation - state.mu.Unlock() - return generation -} - func (m *XrayManager) resetNativeTrafficAccounting(ctx context.Context, store *Store, uuid, email string) error { state := m.nativeQuotaState(uuid) if state != nil { + state.trafficMu.Lock() + defer state.trafficMu.Unlock() state.mu.Lock() defer state.mu.Unlock() } m.nativeTrafficPersistMu.Lock() defer m.nativeTrafficPersistMu.Unlock() + + // Remove this client's queued pre-reset delta while holding only the short + // map mutex. The database call may take seconds during an outage; keeping + // nativeDBMu locked across it would stall traffic/accounting updates for + // every other native user and could amplify a slow database into a goroutine + // pile-up. m.nativeDBMu.Lock() key := strings.TrimSpace(uuid) var pending xrayPendingTraffic @@ -135,14 +158,27 @@ func (m *XrayManager) resetNativeTrafficAccounting(ctx context.Context, store *S pending, hadPending = m.nativeTrafficPending[key] delete(m.nativeTrafficPending, key) } + m.nativeDBMu.Unlock() + err := store.ResetXrayClientTraffic(ctx, uuid) - if err != nil && hadPending { + if err != nil && hadPending && pending.State == state { + m.nativeDBMu.Lock() if m.nativeTrafficPending == nil { m.nativeTrafficPending = make(map[string]xrayPendingTraffic) } - m.nativeTrafficPending[key] = pending + current := m.nativeTrafficPending[key] + if current.State != nil && current.State != state { + current = xrayPendingTraffic{} + } + if current.Email == "" { + current.Email = pending.Email + } + current.Uplink += pending.Uplink + current.Downlink += pending.Downlink + current.State = state + m.nativeTrafficPending[key] = current + m.nativeDBMu.Unlock() } - m.nativeDBMu.Unlock() if err != nil { return err } @@ -177,8 +213,44 @@ func (m *XrayManager) nativeQuotaState(uuid string) *xrayNativeQuotaState { return state } -func (m *XrayManager) nativeQuotaBlocked(uuid string) bool { +// acquireNativeClientConnection enforces the DB-backed max_conns policy across +// every native inbound and transport. The returned release function is safe to +// call more than once and keeps runtime/DB online counters in sync. +func (m *XrayManager) acquireNativeClientConnection(uuid, email string) (func(), *xrayNativeQuotaState, bool) { state := m.nativeQuotaState(uuid) + if state != nil { + state.mu.Lock() + if state.maxConns > 0 && state.activeConns >= state.maxConns { + limit := state.maxConns + state.mu.Unlock() + logNativeClientLimitRejection(email, limit) + return nil, state, false + } + state.activeConns++ + state.mu.Unlock() + } + + m.recordNativeConnect(uuid, email, state) + var once sync.Once + return func() { + once.Do(func() { + if state != nil { + state.mu.Lock() + if state.activeConns > 0 { + state.activeConns-- + } + state.mu.Unlock() + } + m.recordNativeDisconnect(uuid, email, state) + }) + }, state, true +} + +func (m *XrayManager) nativeQuotaBlocked(uuid string) bool { + return nativeQuotaStateBlocked(m.nativeQuotaState(uuid)) +} + +func nativeQuotaStateBlocked(state *xrayNativeQuotaState) bool { if state == nil { return false } @@ -187,11 +259,10 @@ func (m *XrayManager) nativeQuotaBlocked(uuid string) bool { return state.quotaBytes > 0 && normalizeQuotaAction(state.action) == quotaActionBlock && state.usedBytes >= state.quotaBytes } -func (m *XrayManager) reserveNativeQuota(uuid string, requested int) (allowed int, limiter *rate.Limiter, stopAfter bool) { +func (m *XrayManager) reserveNativeQuota(state *xrayNativeQuotaState, requested int) (allowed int, limiter *rate.Limiter, stopAfter bool) { if requested <= 0 { return 0, nil, false } - state := m.nativeQuotaState(uuid) if state == nil { return requested, nil, false } @@ -232,14 +303,13 @@ func (m *XrayManager) reserveNativeQuota(uuid string, requested int) (allowed in return int(take), nil, take < n } -func (m *XrayManager) finishNativeQuotaReservation(uuid string, reserved, written int) { +func (m *XrayManager) finishNativeQuotaReservation(state *xrayNativeQuotaState, reserved, written int) { if reserved <= 0 || written >= reserved { return } if written < 0 { written = 0 } - state := m.nativeQuotaState(uuid) if state == nil { return } @@ -254,56 +324,107 @@ func (m *XrayManager) finishNativeQuotaReservation(uuid string, reserved, writte type xrayQuotaMeteredWriter struct { w io.Writer meter *trafficMeter + ctx context.Context } func (mw xrayQuotaMeteredWriter) Write(p []byte) (int, error) { if mw.meter == nil { return mw.w.Write(p) } - allowed, limiter, stopAfter := xrayMgr.reserveNativeQuota(mw.meter.uuid, len(p)) + state := mw.meter.state + if state != nil { + state.trafficMu.RLock() + defer state.trafficMu.RUnlock() + } + allowed, limiter, stopAfter := xrayMgr.reserveNativeQuota(state, len(p)) if allowed <= 0 { return 0, errDataQuotaExceeded } if limiter != nil { - if err := limiter.WaitN(context.Background(), allowed); err != nil { - xrayMgr.finishNativeQuotaReservation(mw.meter.uuid, allowed, 0) + ctx := mw.ctx + if ctx == nil { + ctx = context.Background() + } + if err := limiter.WaitN(ctx, allowed); err != nil { + xrayMgr.finishNativeQuotaReservation(state, allowed, 0) return 0, err } } n, err := mw.w.Write(p[:allowed]) - xrayMgr.finishNativeQuotaReservation(mw.meter.uuid, allowed, n) + xrayMgr.finishNativeQuotaReservation(state, allowed, n) if n > 0 { mw.meter.add(n) } if err != nil { return n, err } - if stopAfter || allowed < len(p) || xrayMgr.nativeQuotaBlocked(mw.meter.uuid) { + if stopAfter || allowed < len(p) || nativeQuotaStateBlocked(state) { return n, errDataQuotaExceeded } return n, nil } -func reserveNativePacketQuota(meter *trafficMeter, n int) (*rate.Limiter, error) { - if meter == nil || n <= 0 { - return nil, nil - } - allowed, limiter, stopAfter := xrayMgr.reserveNativeQuota(meter.uuid, n) - if allowed != n || stopAfter { - if allowed > 0 { - xrayMgr.finishNativeQuotaReservation(meter.uuid, allowed, 0) - } - return nil, errDataQuotaExceeded - } - return limiter, nil +type nativePacketQuotaReservation struct { + meter *trafficMeter + state *xrayNativeQuotaState + limiter *rate.Limiter + reserved int + finished bool } -func finishNativePacketQuota(meter *trafficMeter, reserved, written int) { - if meter == nil || reserved <= 0 { +func reserveNativePacketQuota(meter *trafficMeter, n int) (nativePacketQuotaReservation, error) { + if meter == nil || n <= 0 { + return nativePacketQuotaReservation{}, nil + } + state := meter.state + if state != nil { + state.trafficMu.RLock() + } + allowed, limiter, stopAfter := xrayMgr.reserveNativeQuota(state, n) + if allowed != n || stopAfter { + if allowed > 0 { + xrayMgr.finishNativeQuotaReservation(state, allowed, 0) + } + if state != nil { + state.trafficMu.RUnlock() + } + return nativePacketQuotaReservation{}, errDataQuotaExceeded + } + return nativePacketQuotaReservation{ + meter: meter, + state: state, + limiter: limiter, + reserved: n, + }, nil +} + +func (r *nativePacketQuotaReservation) wait(ctx context.Context) error { + if r == nil || r.finished || r.limiter == nil || r.reserved <= 0 { + return nil + } + if err := waitNativeRate(ctx, r.limiter, r.reserved); err != nil { + r.finish(0) + return err + } + return nil +} + +func (r *nativePacketQuotaReservation) finish(written int) { + if r == nil || r.finished { return } - xrayMgr.finishNativeQuotaReservation(meter.uuid, reserved, written) + r.finished = true + if r.meter == nil || r.reserved <= 0 { + if r.state != nil { + r.state.trafficMu.RUnlock() + } + return + } + xrayMgr.finishNativeQuotaReservation(r.state, r.reserved, written) if written > 0 { - meter.add(written) + r.meter.add(written) + } + if r.state != nil { + r.state.trafficMu.RUnlock() } } diff --git a/xray_vmess.go b/xray_vmess.go index bce182a..92d1e6e 100644 --- a/xray_vmess.go +++ b/xray_vmess.go @@ -670,7 +670,7 @@ func (ib *nativeInbound) handleVMess(stream net.Conn, remote net.Addr) { } client := ib.matchVMess(authid, time.Now().Unix()) if client == nil { - log.Printf("native xray: inbound %q rejected unknown/expired VMess auth id from %s", ib.tag, remote) + logNativePreAuthRejection("native xray: inbound %q rejected unknown/expired VMess auth id from %s", ib.tag, remote) return } if xrayMgr.nativeQuotaBlocked(client.uuid) { @@ -694,6 +694,11 @@ func (ib *nativeInbound) handleVMess(stream net.Conn, remote net.Addr) { log.Printf("native xray: inbound %q VMess command %d not supported yet", ib.tag, req.command) return } + releaseConnection, quotaState, ok := xrayMgr.acquireNativeClientConnection(client.uuid, client.email) + if !ok { + return + } + defer releaseConnection() respBodyKey := sha256.Sum256(req.bodyKey[:]) respBodyIV := sha256.Sum256(req.bodyIV[:]) @@ -719,7 +724,7 @@ func (ib *nativeInbound) handleVMess(stream net.Conn, remote net.Addr) { return } log.Printf("native xray: vmess/tcp user=%s src=%s -> %s (inbound %q)", client.email, backend.LocalAddr(), target, ib.tag) - nativeTunnel(vc, backend, client.uuid, client.email, ib.upLimiter(), ib.downLimiter()) + nativeTunnel(vc, backend, client.uuid, client.email, quotaState, ib.upLimiter(), ib.downLimiter()) case vmessCmdUDP: backend, target, err := ib.nativeDialUDP(req.host, req.port) if err != nil { @@ -727,6 +732,6 @@ func (ib *nativeInbound) handleVMess(stream net.Conn, remote net.Addr) { return } log.Printf("native xray: vmess/udp user=%s src=%s -> %s (inbound %q)", client.email, backend.LocalAddr(), target, ib.tag) - nativeVMessUDPTunnel(vc, backend, client.uuid, client.email, ib.upLimiter(), ib.downLimiter()) + nativeVMessUDPTunnel(vc, backend, client.uuid, client.email, quotaState, ib.upLimiter(), ib.downLimiter()) } } diff --git a/xray_xhttp.go b/xray_xhttp.go index f816104..861ecf5 100644 --- a/xray_xhttp.go +++ b/xray_xhttp.go @@ -14,12 +14,31 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" ) +const nativeXHTTPServerIdleTimeout = 90 * time.Second + +const ( + nativeXHTTPMaxSessionIDBytes = 256 + nativeXHTTPMaxSequenceBytes = 20 + nativeXHTTPHardMaxHeaderBytes = 256 * 1024 + nativeXHTTPHardMaxPostBytes int64 = 16 * 1024 * 1024 + nativeXHTTPMaxBufferedPosts = 512 + nativeXHTTPMaxBufferedSessionBytes = 16 * 1024 * 1024 + nativeXHTTPMaxBufferedGlobalBytes = 128 * 1024 * 1024 +) + +var ( + nativeXHTTPBufferedBytes atomic.Int64 + nativeXHTTPBufferRejected atomic.Int64 + errNativeXHTTPUploadBufferFull = errors.New("xhttp upload buffer limit reached") +) + const ( xhttpPlacementPath = "path" xhttpPlacementQuery = "query" @@ -157,7 +176,10 @@ func (ib *nativeInbound) serveXHTTPListener(ln net.Listener) { func (g *nativeXHTTPListener) serve(ln net.Listener) { defer xrayRecover(fmt.Sprintf("native xray shared XHTTP listener addr=%s", ln.Addr())) - h2s := &http2.Server{} + h2s := &http2.Server{ + IdleTimeout: nativeXHTTPServerIdleTimeout, + MaxConcurrentStreams: nativeHTTP2MaxConcurrentStreams(), + } handler := http.Handler(g) // Official Xray accepts plaintext HTTP/1.1 and h2c on non-TLS XHTTP // listeners, and negotiates h2/http1 through ALPN on TLS listeners. Without @@ -169,6 +191,7 @@ func (g *nativeXHTTPListener) serve(ln net.Listener) { srv := &http.Server{ Handler: handler, ReadHeaderTimeout: 4 * time.Second, + IdleTimeout: nativeXHTTPServerIdleTimeout, MaxHeaderBytes: g.headerSize, } if g.security == "tls" && g.tlsConfig != nil { @@ -258,6 +281,9 @@ func (ib *nativeInbound) reapStaleXHTTPSessions(idle time.Duration) { func (ib *nativeInbound) xhttpServerMaxHeaderBytes() int { if ib.xhttpMaxHeaderBytes > 0 { + if ib.xhttpMaxHeaderBytes > nativeXHTTPHardMaxHeaderBytes { + return nativeXHTTPHardMaxHeaderBytes + } return ib.xhttpMaxHeaderBytes } // Xray defaults to 8192. Keep a little room for custom headers/cookies used @@ -269,19 +295,26 @@ func (ib *nativeInbound) xhttpServerMaxHeaderBytes() int { // byte stream to the VLESS/VMess handlers as a net.Conn. func (ib *nativeInbound) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer xrayRecover(fmt.Sprintf("native xray XHTTP request inbound=%q method=%s path=%q remote=%s", ib.tag, r.Method, r.URL.RequestURI(), r.RemoteAddr)) + releaseRequest, ok := acquireNativeXHTTPRequest() + if !ok { + w.Header().Set("Retry-After", "1") + http.Error(w, "native XHTTP request limit reached", http.StatusTooManyRequests) + return + } + defer releaseRequest() if !ib.isXHTTP() { - xrayLogf("native xray: xhttp reject inbound=%q reason=not-xhttp method=%s path=%q host=%q remote=%s", ib.tag, r.Method, r.URL.RequestURI(), r.Host, r.RemoteAddr) + logNativePreAuthRejection("native xray: xhttp reject inbound=%q reason=not-xhttp method=%s path=%q host=%q remote=%s", ib.tag, r.Method, r.URL.RequestURI(), r.Host, r.RemoteAddr) xhttpBadRequest(w) return } if !ib.xhttpHostAllowed(r.Host) { - xrayLogf("native xray: xhttp reject inbound=%q reason=host method=%s path=%q host=%q want=%q remote=%s", ib.tag, r.Method, r.URL.RequestURI(), r.Host, ib.xhttpHost, r.RemoteAddr) + logNativePreAuthRejection("native xray: xhttp reject inbound=%q reason=host method=%s path=%q host=%q want=%q remote=%s", ib.tag, r.Method, r.URL.RequestURI(), r.Host, ib.xhttpHost, r.RemoteAddr) w.WriteHeader(http.StatusNotFound) return } base, ok := ib.matchXHTTPPath(r.URL.Path) if !ok { - xrayLogf("native xray: xhttp reject inbound=%q reason=path method=%s path=%q want=%q host=%q remote=%s", ib.tag, r.Method, r.URL.RequestURI(), ib.path, r.Host, r.RemoteAddr) + logNativePreAuthRejection("native xray: xhttp reject inbound=%q reason=path method=%s path=%q want=%q host=%q remote=%s", ib.tag, r.Method, r.URL.RequestURI(), ib.path, r.Host, r.RemoteAddr) w.WriteHeader(http.StatusNotFound) return } @@ -293,6 +326,11 @@ func (ib *nativeInbound) ServeHTTP(w http.ResponseWriter, r *http.Request) { } sessionID, seqStr := ib.extractXHTTPMeta(r, base) + if len(sessionID) > nativeXHTTPMaxSessionIDBytes || len(seqStr) > nativeXHTTPMaxSequenceBytes { + logNativePreAuthRejection("native xray: xhttp reject inbound=%q reason=metadata-size remote=%s", ib.tag, r.RemoteAddr) + xhttpBadRequest(w) + return + } mode := ib.normalizedXHTTPMode() xrayTracef("native xray: xhttp request inbound=%q method=%s proto=%s path=%q host=%q session=%q seq=%q len=%d mode=%s remote=%s", ib.tag, r.Method, r.Proto, r.URL.RequestURI(), r.Host, sessionID, seqStr, r.ContentLength, mode, r.RemoteAddr) @@ -520,17 +558,24 @@ func (ib *nativeInbound) upsertXHTTPSession(w http.ResponseWriter, id string) *n return s } if max := ib.xhttpMaxActiveSessions(); max > 0 && len(ib.xhttpSessions) >= max { - // XHTTP uses many HTTP requests/sessions by design. Returning HTTP 429 - // makes Xray clients tear down active tunnels, which is worse than allowing - // a short soft-limit overflow and relying on stale-session cleanup. - xrayTracef("native xray: xhttp session soft limit exceeded inbound=%q active=%d limit=%d", ib.tag, len(ib.xhttpSessions), max) + w.Header().Set("Retry-After", "1") + http.Error(w, "native XHTTP session limit reached", http.StatusTooManyRequests) + logNativePreAuthRejection("native xray: xhttp session rejected inbound=%q active=%d limit=%d", ib.tag, len(ib.xhttpSessions), max) + return nil + } + releaseSlot, ok := acquireNativeXHTTPSession() + if !ok { + w.Header().Set("Retry-After", "1") + http.Error(w, "native XHTTP global session limit reached", http.StatusTooManyRequests) + return nil } s := &nativeXHTTPSession{ id: id, - queue: newNativeXHTTPUploadQueue(ib.xhttpMaxBufferedPosts), + queue: newNativeXHTTPUploadQueue(ib.xhttpMaxBufferedPosts, nativeXHTTPMaxBufferedSessionBytes), done: make(chan struct{}), connectedCh: make(chan struct{}), lastSeen: time.Now(), + releaseSlot: releaseSlot, } ib.xhttpSessions[id] = s xrayTracef("native xray: xhttp session created inbound=%q session=%q active=%d", ib.tag, id, len(ib.xhttpSessions)) @@ -539,10 +584,9 @@ func (ib *nativeInbound) upsertXHTTPSession(w http.ResponseWriter, id string) *n } func (ib *nativeInbound) xhttpMaxActiveSessions() int { - if nativeXHTTPMaxSessionLimit() > 0 { - return nativeXHTTPMaxSessionLimit() - } - return defaultNativeXHTTPMaxSessions + // normalizeNativeXrayTuning already installs the safe default. A zero value + // here therefore intentionally means the operator configured -1 (unlimited). + return nativeXHTTPMaxSessionLimit() } func (ib *nativeInbound) reapUnconnectedXHTTPSession(id string, s *nativeXHTTPSession) { @@ -570,6 +614,19 @@ func (ib *nativeInbound) deleteXHTTPSession(id string, s *nativeXHTTPSession) { } } +func (ib *nativeInbound) closeAllXHTTPSessions() { + ib.xhttpMu.Lock() + sessions := make([]*nativeXHTTPSession, 0, len(ib.xhttpSessions)) + for id, session := range ib.xhttpSessions { + delete(ib.xhttpSessions, id) + sessions = append(sessions, session) + } + ib.xhttpMu.Unlock() + for _, session := range sessions { + session.close() + } +} + func (ib *nativeInbound) handleXHTTPStreamUpload(w http.ResponseWriter, r *http.Request, sess *nativeXHTTPSession) { sess.touch() xrayTracef("native xray: xhttp stream-up inbound=%q session=%q len=%d remote=%s", ib.tag, sess.id, r.ContentLength, r.RemoteAddr) @@ -577,7 +634,7 @@ func (ib *nativeInbound) handleXHTTPStreamUpload(w http.ResponseWriter, r *http. http.Error(w, "xhttp stream-up mode is not allowed", http.StatusBadRequest) return } - if err := sess.queue.push(r.Context(), nativeXHTTPPacket{Reader: r.Body}); err != nil { + if err := sess.queue.push(r.Context(), nativeXHTTPPacket{Reader: r.Body}, nil); err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return } @@ -605,16 +662,29 @@ func (ib *nativeInbound) handleXHTTPPacketUpload(w http.ResponseWriter, r *http. http.Error(w, "bad xhttp sequence", http.StatusBadRequest) return } + memory, ok := acquireNativeXHTTPMemory(ib.xhttpMaxPostBytes()) + if !ok { + w.Header().Set("Retry-After", "1") + http.Error(w, errNativeXHTTPUploadBufferFull.Error(), http.StatusTooManyRequests) + return + } + defer memory.release() payload, err := ib.readXHTTPPayload(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } + memory.shrink(int64(len(payload))) xrayTracef("native xray: xhttp packet-up inbound=%q session=%q seq=%d payload=%d remote=%s", ib.tag, sess.id, seq, len(payload), r.RemoteAddr) - if err := sess.queue.push(r.Context(), nativeXHTTPPacket{Payload: payload, Seq: seq}); err != nil { + if err := sess.queue.push(r.Context(), nativeXHTTPPacket{Payload: payload, Seq: seq}, memory); err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return } + if errors.Is(err, errNativeXHTTPUploadBufferFull) { + w.Header().Set("Retry-After", "1") + http.Error(w, err.Error(), http.StatusTooManyRequests) + return + } xrayTracef("native xray: xhttp packet-up push failed inbound=%q session=%q seq=%d: %v", ib.tag, sess.id, seq, err) http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -719,6 +789,9 @@ func (ib *nativeInbound) readXHTTPBodyPayload(r *http.Request) ([]byte, error) { func (ib *nativeInbound) xhttpMaxPostBytes() int64 { if ib.xhttpMaxEachPostBytes > 0 { + if ib.xhttpMaxEachPostBytes > nativeXHTTPHardMaxPostBytes { + return nativeXHTTPHardMaxPostBytes + } return ib.xhttpMaxEachPostBytes } return 1_000_000 @@ -794,7 +867,7 @@ func (ib *nativeInbound) handleXHTTPDownload(w http.ResponseWriter, r *http.Requ // The stream-down HTTP request is the lifetime owner of an XHTTP // session. Log the actual transport cancellation so a CDN/proxy // timeout can be distinguished from a server idle policy. - xrayLogf("native xray: xhttp stream-down ended inbound=%q session=%q remote=%s err=%v", + xrayTracef("native xray: xhttp stream-down ended inbound=%q session=%q remote=%s err=%v", ib.tag, sessionID, r.RemoteAddr, r.Context().Err()) _ = xc.Close() case <-sess.done: @@ -859,6 +932,7 @@ type nativeXHTTPSession struct { mu sync.Mutex connected bool lastSeen time.Time + releaseSlot func() } func (s *nativeXHTTPSession) touch() { @@ -879,6 +953,9 @@ func (s *nativeXHTTPSession) close() { s.closeOnce.Do(func() { close(s.done) s.queue.close() + if s.releaseSlot != nil { + s.releaseSlot() + } }) } @@ -980,6 +1057,70 @@ func (w *nativeXHTTPResponseWriter) close() { w.mu.Unlock() } +// nativeXHTTPMemoryLease reserves from a process-wide byte budget before a +// packet-up handler allocates its payload. The same lease is transferred to the +// session queue, so active request bodies and queued reassembly data share one +// hard ceiling instead of each having an independent amplification window. +type nativeXHTTPMemoryLease struct { + bytes int64 +} + +func acquireNativeXHTTPMemory(n int64) (*nativeXHTTPMemoryLease, bool) { + if n <= 0 { + return &nativeXHTTPMemoryLease{}, true + } + for { + current := nativeXHTTPBufferedBytes.Load() + if current > nativeXHTTPMaxBufferedGlobalBytes-n { + logNativeLimitRejection("XHTTP buffered upload bytes", &nativeXHTTPBufferRejected, nativeXHTTPMaxBufferedGlobalBytes) + return nil, false + } + if nativeXHTTPBufferedBytes.CompareAndSwap(current, current+n) { + return &nativeXHTTPMemoryLease{bytes: n}, true + } + } +} + +func releaseNativeXHTTPMemory(n int64) { + if n <= 0 { + return + } + for { + current := nativeXHTTPBufferedBytes.Load() + next := current - n + if next < 0 { + next = 0 + } + if nativeXHTTPBufferedBytes.CompareAndSwap(current, next) { + return + } + } +} + +func (l *nativeXHTTPMemoryLease) shrink(n int64) { + if l == nil { + return + } + if n < 0 { + n = 0 + } + if n >= l.bytes { + return + } + release := l.bytes - n + l.bytes = n + releaseNativeXHTTPMemory(release) +} + +func (l *nativeXHTTPMemoryLease) release() { + if l == nil || l.bytes <= 0 { + return + } + n := l.bytes + l.bytes = 0 + releaseNativeXHTTPMemory(n) +} + type nativeXHTTPPacket struct { Reader io.ReadCloser Payload []byte @@ -989,44 +1130,127 @@ type nativeXHTTPPacket struct { type nativeXHTTPUploadQueue struct { pushedPackets chan nativeXHTTPPacket maxPackets int + maxBytes int64 - mu sync.Mutex - reader io.ReadCloser - heap nativeXHTTPHeap - nextSeq uint64 - readDeadline time.Time + // readMu serializes the single decoded stream reader with close-time queue + // cleanup. pushWG lets close wait until every producer that started before + // closedFlag was set has either transferred or released its memory lease. + readMu sync.Mutex + pushWG sync.WaitGroup + mu sync.Mutex + reader io.ReadCloser + readerQueued bool + heap nativeXHTTPHeap + nextSeq uint64 + readDeadline time.Time + bufferedBytes int64 + closedFlag bool closed chan struct{} closeOnce sync.Once } -func newNativeXHTTPUploadQueue(maxPackets int) *nativeXHTTPUploadQueue { +func newNativeXHTTPUploadQueue(maxPackets int, maxBytes int64) *nativeXHTTPUploadQueue { if maxPackets <= 0 { maxPackets = defaultNativeXHTTPBufferedPosts } + if maxPackets > nativeXHTTPMaxBufferedPosts { + maxPackets = nativeXHTTPMaxBufferedPosts + } + if maxBytes <= 0 || maxBytes > nativeXHTTPMaxBufferedSessionBytes { + maxBytes = nativeXHTTPMaxBufferedSessionBytes + } return &nativeXHTTPUploadQueue{ pushedPackets: make(chan nativeXHTTPPacket, maxPackets), maxPackets: maxPackets, + maxBytes: maxBytes, closed: make(chan struct{}), } } -func (q *nativeXHTTPUploadQueue) push(ctx context.Context, p nativeXHTTPPacket) error { +func (q *nativeXHTTPUploadQueue) beginPush() bool { + q.mu.Lock() + defer q.mu.Unlock() + if q.closedFlag { + return false + } + q.pushWG.Add(1) + return true +} + +func (q *nativeXHTTPUploadQueue) adoptPayloadMemory(memory *nativeXHTTPMemoryLease, n int64) bool { + if n <= 0 { + return true + } + if memory == nil || memory.bytes != n { + return false + } + q.mu.Lock() + defer q.mu.Unlock() + if q.closedFlag || q.bufferedBytes > q.maxBytes-n { + return false + } + q.bufferedBytes += n + memory.bytes = 0 + return true +} + +func (q *nativeXHTTPUploadQueue) releasePayloadMemory(n int64) { + if n <= 0 { + return + } + q.mu.Lock() + release := n + if release > q.bufferedBytes { + release = q.bufferedBytes + } + q.bufferedBytes -= release + q.mu.Unlock() + releaseNativeXHTTPMemory(release) +} + +func (q *nativeXHTTPUploadQueue) push(ctx context.Context, p nativeXHTTPPacket, memory *nativeXHTTPMemoryLease) error { + if !q.beginPush() { + return io.ErrClosedPipe + } + defer q.pushWG.Done() + + readerReserved := false if p.Reader != nil { q.mu.Lock() - if q.reader != nil { + if q.reader != nil || q.readerQueued || q.closedFlag { q.mu.Unlock() return errors.New("xhttp upload reader already exists") } + q.readerQueued = true + readerReserved = true q.mu.Unlock() + defer func() { + if readerReserved { + q.mu.Lock() + q.readerQueued = false + q.mu.Unlock() + } + }() + } + payloadBytes := int64(len(p.Payload)) + if !q.adoptPayloadMemory(memory, payloadBytes) { + return errNativeXHTTPUploadBufferFull + } + transferred := payloadBytes > 0 + if transferred { + defer func() { + if payloadBytes > 0 { + q.releasePayloadMemory(payloadBytes) + } + }() } select { case q.pushedPackets <- p: - select { - case <-q.closed: - return io.ErrClosedPipe - default: - } + // Ownership has moved to the queue. close() waits for this producer and + // then drains/releases anything not consumed by the stream reader. + payloadBytes = 0 + readerReserved = false return nil case <-q.closed: return io.ErrClosedPipe @@ -1037,13 +1261,41 @@ func (q *nativeXHTTPUploadQueue) push(ctx context.Context, p nativeXHTTPPacket) func (q *nativeXHTTPUploadQueue) close() { q.closeOnce.Do(func() { - close(q.closed) q.mu.Lock() + q.closedFlag = true reader := q.reader + close(q.closed) q.mu.Unlock() if reader != nil { _ = reader.Close() } + + q.pushWG.Wait() + q.readMu.Lock() + // No producers or readers can now change the queue. Drop references to + // buffered payloads promptly and return their exact byte reservation. + for { + select { + case p := <-q.pushedPackets: + if p.Reader != nil { + _ = p.Reader.Close() + } + p.Payload = nil + default: + goto drained + } + } + drained: + q.mu.Lock() + remaining := q.bufferedBytes + q.bufferedBytes = 0 + for i := range q.heap { + q.heap[i].Payload = nil + } + q.heap = nil + q.mu.Unlock() + q.readMu.Unlock() + releaseNativeXHTTPMemory(remaining) }) } @@ -1085,6 +1337,9 @@ func (q *nativeXHTTPUploadQueue) recv() (nativeXHTTPPacket, error) { } func (q *nativeXHTTPUploadQueue) Read(b []byte) (int, error) { + q.readMu.Lock() + defer q.readMu.Unlock() + if reader := q.loadReader(); reader != nil { return reader.Read(b) } @@ -1101,9 +1356,18 @@ func (q *nativeXHTTPUploadQueue) Read(b []byte) (int, error) { return 0, err } if p.Reader != nil { - q.setReader(p.Reader) + if !q.setReader(p.Reader) { + _ = p.Reader.Close() + return 0, io.EOF + } return p.Reader.Read(b) } + select { + case <-q.closed: + q.releasePayloadMemory(int64(len(p.Payload))) + return 0, io.EOF + default: + } heap.Push(&q.heap, p) } @@ -1112,6 +1376,7 @@ func (q *nativeXHTTPUploadQueue) Read(b []byte) (int, error) { if packet.Seq == q.nextSeq { n := copy(b, packet.Payload) + q.releasePayloadMemory(int64(n)) if n < len(packet.Payload) { packet.Payload = packet.Payload[n:] heap.Push(&q.heap, packet) @@ -1131,10 +1396,15 @@ func (q *nativeXHTTPUploadQueue) Read(b []byte) (int, error) { return 0, err } if p.Reader != nil { + _ = p.Reader.Close() return 0, errors.New("xhttp mixed stream-up and packet-up upload") } heap.Push(&q.heap, p) + continue } + + // A duplicate/late packet is discarded; release the bytes it owned. + q.releasePayloadMemory(int64(len(packet.Payload))) } return 0, nil @@ -1146,10 +1416,14 @@ func (q *nativeXHTTPUploadQueue) loadReader() io.ReadCloser { return q.reader } -func (q *nativeXHTTPUploadQueue) setReader(r io.ReadCloser) { +func (q *nativeXHTTPUploadQueue) setReader(r io.ReadCloser) bool { q.mu.Lock() + defer q.mu.Unlock() + if q.closedFlag { + return false + } q.reader = r - q.mu.Unlock() + return true } type nativeXHTTPHeap []nativeXHTTPPacket