penguinehis/DragonTCP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mult Port + TCP Calibration (SSH DEAD)

Browse Source
This commit is contained in:
penguinehis
2026-08-17 17:08:57 -03:00
parent 7ea221a99c
commit b997294607
58 changed files with 6033 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/internal/xorchunk/chunk.go
+382 -26
View File
@@ -7,6 +7,7 @@
package xorchunk
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
@@ -29,7 +30,7 @@ var requestCounter atomic.Uint32
// NewOptions builds the transport options from the values the CLI already
// parses, keeping the struct fields unexported as in the original.
func NewOptions(startSize, minSize, maxSize int, adaptive bool, adaptSuccesses int, adaptLog bool,
func NewOptions(startSize, minSize, maxSize int, adaptive bool, adaptSuccesses, shrinkAfter int, adaptLog bool,
pollers, reconnectEvery int, pollDelay, txnTimeout time.Duration, tcpBuffer int) Options {
return Options{
startSize: startSize,
@@ -37,6 +38,7 @@ func NewOptions(startSize, minSize, maxSize int, adaptive bool, adaptSuccesses i
maxSize: maxSize,
adaptive: adaptive,
adaptSuccesses: adaptSuccesses,
shrinkAfter: shrinkAfter,
adaptLog: adaptLog,
pollers: pollers,
reconnectEvery: reconnectEvery,
@@ -47,19 +49,24 @@ func NewOptions(startSize, minSize, maxSize int, adaptive bool, adaptSuccesses i
}
type Options struct {
startSize int
minSize int
maxSize int
adaptive bool
adaptSuccesses int
adaptLog bool
pollers int
reconnectEvery int
pollDelay time.Duration
txnTimeout time.Duration
tcpBuffer int
headerMask byte
coverProfile cover.Profile
startSize int
minSize int
maxSize int
uploadStartSize int
downloadStartSize int
uploadMaxSize int
downloadMaxSize int
adaptive bool
adaptSuccesses int
shrinkAfter int
adaptLog bool
pollers int
reconnectEvery int
pollDelay time.Duration
txnTimeout time.Duration
tcpBuffer int
headerMask byte
coverProfile cover.Profile
}
// WithHeaderMask returns a copy using one fixed frame-magic profile. The mask
@@ -78,6 +85,38 @@ func (o Options) WithCoverProfile(profile cover.Profile) Options {
return o
}
// MinSize and MaxSize expose the configured X carrier calibration bounds.
func (o Options) MinSize() int { return o.minSize }
func (o Options) MaxSize() int { return o.maxSize }
// WithCalibratedChunks locks X to the UP/DW sizes proven by the pre-tunnel
// fake-iperf calibration. Once calibration succeeds, runtime adaptive sizing is
// disabled for X: transport successes cannot grow the chunk and transport
// failures cannot shrink it. Failed physical transactions reconnect/retry using
// the same calibrated size.
func (o Options) WithCalibratedChunks(upload, download int) Options {
if upload < o.minSize {
upload = o.minSize
}
if download < o.minSize {
download = o.minSize
}
if upload > o.maxSize {
upload = o.maxSize
}
if download > o.maxSize {
download = o.maxSize
}
o.uploadStartSize = upload
o.downloadStartSize = download
o.uploadMaxSize = upload
o.downloadMaxSize = download
// Calibration replaces runtime X chunk adaptation. The calibrated values are
// the operating sizes for this session, not merely adaptive ceilings.
o.adaptive = false
return o
}
func wireToken(token string) string {
if token == "" {
return "-"
@@ -93,28 +132,44 @@ type adaptiveSizer struct {
max int
adaptive bool
adaptSuccesses int
shrinkAfter int
failures int
successes int
good int
bad int
logChanges bool
}
func newAdaptiveSizer(name string, opts Options) *adaptiveSizer {
start := opts.startSize
func newAdaptiveSizer(name string, start, ceiling int, opts Options) *adaptiveSizer {
if ceiling <= 0 || ceiling > opts.maxSize {
ceiling = opts.maxSize
}
if ceiling < opts.minSize {
ceiling = opts.minSize
}
if start <= 0 {
start = opts.startSize
}
if start < opts.minSize {
start = opts.minSize
}
if start > opts.maxSize {
start = opts.maxSize
if start > ceiling {
start = ceiling
}
return &adaptiveSizer{
name: name,
current: start,
min: opts.minSize,
max: opts.maxSize,
max: ceiling,
adaptive: opts.adaptive,
adaptSuccesses: opts.adaptSuccesses,
logChanges: opts.adaptLog,
shrinkAfter: func() int {
if opts.shrinkAfter > 0 {
return opts.shrinkAfter
}
return 1
}(),
logChanges: opts.adaptLog,
}
}
@@ -129,7 +184,7 @@ func (s *adaptiveSizer) Success(attempted int) {
s.mu.Lock()
defer s.mu.Unlock()
if !s.adaptive || s.current >= s.max {
if !s.adaptive {
return
}
// Ignore stale successes from records that were already in flight when
@@ -137,6 +192,10 @@ func (s *adaptiveSizer) Success(attempted int) {
if attempted != s.current {
return
}
s.failures = 0
if s.current >= s.max {
return
}
if attempted > s.good {
s.good = attempted
@@ -201,6 +260,14 @@ func (s *adaptiveSizer) Failure(attempted int) (old, next int) {
return old, old
}
s.successes = 0
s.failures++
if s.failures < s.shrinkAfter {
if s.logChanges && s.shrinkAfter > 1 {
fmt.Printf("adaptive %s chunk: holding %d after failure %d/%d\n", s.name, old, s.failures, s.shrinkAfter)
}
return old, old
}
s.failures = 0
if s.bad == 0 || attempted < s.bad {
s.bad = attempted
@@ -373,6 +440,288 @@ func ProbeProfile(serverAddr, token string, opts Options) bool {
return opts.headerMask == 0 && strings.HasPrefix(string(resp), "ERR expected TUNNEL")
}
type calibrationProbeResult struct {
ok bool
bytes int
elapsed time.Duration
err error
}
func (r calibrationProbeResult) mbps() float64 {
if r.bytes <= 0 || r.elapsed <= 0 {
return 0
}
return float64(r.bytes*8) / r.elapsed.Seconds() / 1_000_000
}
func calibrationPattern(n int) []byte {
b := make([]byte, n)
for i := range b {
b[i] = byte((i*31 + 17) & 0xff)
}
return b
}
func calibrationBurstCount(chunk int) int {
// Calibration is strictly single-poller/single-outstanding-request. Runtime
// X traffic may use its normal concurrency after calibration completes.
return 1
}
func calibrationTimeout(opts Options) time.Duration {
t := opts.txnTimeout
if t < 8*time.Second {
t = 8 * time.Second
}
if t > 20*time.Second {
t = 20 * time.Second
}
return t
}
func isCalibrationTimeout(err error) bool {
if err == nil {
return false
}
if ne, ok := err.(net.Error); ok && ne.Timeout() {
return true
}
return strings.Contains(strings.ToLower(err.Error()), "i/o timeout") || strings.Contains(strings.ToLower(err.Error()), "timeout")
}
const calibrationDecisionAttempts = 3
// confirmCalibrationResult uses a 2-of-3 decision near the carrier boundary.
// Each retry gets a fresh X connection. Timeouts are treated as inconclusive
// connection failures, not as evidence that the candidate chunk is too large.
func confirmCalibrationResult(serverAddr, token string, opts Options, download bool, candidate int, first calibrationProbeResult, stage string) calibrationProbeResult {
name := "upload"
if download {
name = "download"
}
successes, failures := 0, 0
var lastSuccess, lastFailure, lastTimeout calibrationProbeResult
observe := func(r calibrationProbeResult) {
if r.ok {
successes++
lastSuccess = r
return
}
if isCalibrationTimeout(r.err) {
lastTimeout = r
return
}
failures++
lastFailure = r
}
observe(first)
for attempt := 2; attempt <= calibrationDecisionAttempts && successes < 2 && failures < 2; attempt++ {
r := probeCalibrationSize(serverAddr, token, opts, download, candidate)
observe(r)
result := "failure"
if r.ok {
result = "success"
} else if isCalibrationTimeout(r.err) {
result = "connection_timeout"
}
fmt.Printf("[D-TCP] phase=CALIBRATION fake_iperf=%s wire=x stage=%s chunk=%d confirmation=%d/%d result=%s\n", name, stage, candidate, attempt, calibrationDecisionAttempts, result)
}
if successes >= 2 {
return lastSuccess
}
if failures >= 2 {
return lastFailure
}
if lastTimeout.err != nil {
return lastTimeout
}
if successes > failures && lastSuccess.ok {
return lastSuccess
}
return lastFailure
}
// probeCalibrationSize runs a short repeated UP or DW transfer over one X
// physical connection. It exercises the same UP/OK framing and XOR payload path
// as real X traffic without opening a destination tunnel.
func probeCalibrationSize(serverAddr, token string, opts Options, download bool, candidate int) calibrationProbeResult {
timeout := calibrationTimeout(opts)
lane := newTxnLane(serverAddr, opts.tcpBuffer, 0, timeout, opts.headerMask, opts.coverProfile)
defer lane.Close()
count := calibrationBurstCount(candidate)
started := time.Now()
total := 0
want := calibrationPattern(candidate)
for i := 0; i < count; i++ {
if download {
resp, err := lane.Do([]byte(fmt.Sprintf("CIPERFDW %s %d", wireToken(token), candidate)))
if err != nil {
return calibrationProbeResult{bytes: total, elapsed: time.Since(started), err: err}
}
if !bytes.Equal(resp, want) {
return calibrationProbeResult{bytes: total, elapsed: time.Since(started), err: fmt.Errorf("X download validation failed len=%d want=%d", len(resp), candidate)}
}
total += len(resp)
continue
}
prefix := []byte(fmt.Sprintf("CIPERFUP %s %d ", wireToken(token), candidate))
payload := make([]byte, len(prefix)+len(want))
copy(payload, prefix)
copy(payload[len(prefix):], want)
resp, err := lane.Do(payload)
if err != nil {
return calibrationProbeResult{bytes: total, elapsed: time.Since(started), err: err}
}
if string(resp) != "IPERFOK" {
return calibrationProbeResult{bytes: total, elapsed: time.Since(started), err: fmt.Errorf("X upload rejected: %s", string(resp))}
}
total += candidate
}
return calibrationProbeResult{ok: true, bytes: total, elapsed: time.Since(started)}
}
func calibrateMaximum(serverAddr, token string, opts Options, download bool, fine int) int {
if fine < 1 {
fine = 32
}
name := "upload"
if download {
name = "download"
}
candidate := opts.minSize
if candidate < 32 {
candidate = 32
}
if candidate > opts.maxSize {
candidate = opts.maxSize
}
good, bad := 0, 0
for {
r := probeCalibrationSize(serverAddr, token, opts, download, candidate)
if r.ok {
fmt.Printf("[D-TCP] phase=CALIBRATION fake_iperf=%s wire=x stage=ascend chunk=%d records=%d bytes=%d mbps=%.2f result=success\n", name, candidate, calibrationBurstCount(candidate), r.bytes, r.mbps())
good = candidate
if candidate >= opts.maxSize {
return opts.maxSize
}
next := candidate * 4
if candidate == opts.minSize && next < 512 && opts.maxSize >= 512 {
next = 512
}
if next > opts.maxSize {
next = opts.maxSize
}
if next <= candidate {
return good
}
fmt.Printf("[D-TCP] phase=CALIBRATION probe=%s wire=x stage=ascend chunk_upgrade=%d->%d\n", name, candidate, next)
candidate = next
continue
}
r = confirmCalibrationResult(serverAddr, token, opts, download, candidate, r, "ascend-confirm")
if r.ok {
fmt.Printf("[D-TCP] phase=CALIBRATION fake_iperf=%s wire=x stage=ascend chunk=%d result=recovered_after_retry\n", name, candidate)
good = candidate
if candidate >= opts.maxSize {
return opts.maxSize
}
next := candidate * 4
if candidate == opts.minSize && next < 512 && opts.maxSize >= 512 {
next = 512
}
if next > opts.maxSize {
next = opts.maxSize
}
if next <= candidate {
return good
}
fmt.Printf("[D-TCP] phase=CALIBRATION probe=%s wire=x stage=ascend chunk_upgrade=%d->%d\n", name, candidate, next)
candidate = next
continue
}
if isCalibrationTimeout(r.err) {
selected := good
if selected == 0 {
selected = opts.minSize
}
fmt.Printf("[D-TCP] phase=CALIBRATION fake_iperf=%s wire=x stage=ascend chunk=%d result=connection_timeout action=keep_known_good known_good=%d err=%v\n", name, candidate, selected, r.err)
return selected
}
fmt.Printf("[D-TCP] phase=CALIBRATION fake_iperf=%s wire=x stage=ascend chunk=%d records=%d bytes=%d mbps=%.2f result=failure err=%v\n", name, candidate, calibrationBurstCount(candidate), r.bytes, r.mbps(), r.err)
bad = candidate
if good == 0 {
return opts.minSize
}
break
}
for bad-good > fine {
next := good + (bad-good)/2
if next <= good || next >= bad {
break
}
r := probeCalibrationSize(serverAddr, token, opts, download, next)
r = confirmCalibrationResult(serverAddr, token, opts, download, next, r, "refine-confirm")
if r.ok {
fmt.Printf("[D-TCP] phase=CALIBRATION fake_iperf=%s wire=x stage=refine chunk=%d records=%d bytes=%d mbps=%.2f result=success\n", name, next, calibrationBurstCount(next), r.bytes, r.mbps())
good = next
continue
}
if isCalibrationTimeout(r.err) {
fmt.Printf("[D-TCP] phase=CALIBRATION fake_iperf=%s wire=x stage=refine chunk=%d result=connection_timeout action=keep_known_good known_good=%d err=%v\n", name, next, good, r.err)
return good
}
fmt.Printf("[D-TCP] phase=CALIBRATION fake_iperf=%s wire=x stage=refine chunk=%d records=%d bytes=%d mbps=%.2f result=failure err=%v\n", name, next, calibrationBurstCount(next), r.bytes, r.mbps(), r.err)
bad = next
}
fmt.Printf("[D-TCP] phase=CALIBRATION probe=%s wire=x stage=refine selected=%d failed_above=%d resolution=%d\n", name, good, bad, fine)
return good
}
func probePersistent(serverAddr, token string, opts Options) bool {
lane := newTxnLane(serverAddr, opts.tcpBuffer, 0, minDurationX(calibrationTimeout(opts), 2500*time.Millisecond), opts.headerMask, opts.coverProfile)
defer lane.Close()
for i := 0; i < 8; i++ {
resp, err := lane.Do([]byte("CPROBE " + wireToken(token)))
if err != nil || string(resp) != "PROBEOK" {
return false
}
}
return true
}
func minDurationX(a, b time.Duration) time.Duration {
if a < b {
return a
}
return b
}
// Calibrate performs the X wire's pre-tunnel UP/DW fake-iperf calibration.
// It ascends by 4x and only spends extra probes around the first failure, where
// it resolves the highest stable boundary to the requested byte precision.
func Calibrate(serverAddr, token string, opts Options, fine int) (upload, download int, persistent bool) {
if opts.minSize < 32 {
opts.minSize = 32
}
if opts.maxSize < opts.minSize {
opts.maxSize = opts.minSize
}
if opts.maxSize > protocol.MaxChunkPayload {
opts.maxSize = protocol.MaxChunkPayload
}
upload = calibrateMaximum(serverAddr, token, opts, false, fine)
download = calibrateMaximum(serverAddr, token, opts, true, fine)
persistent = probePersistent(serverAddr, token, opts)
return
}
type chunkResult struct {
seq uint64
data []byte
@@ -471,8 +820,8 @@ func Open(serverAddr, token, targetHost string, targetPort int, opts Options) (n
pending: make(map[uint64][]byte, opts.pollers*2),
}
c.ack.Store(-1)
c.upSizer = newAdaptiveSizer("upload", opts)
c.downSizer = newAdaptiveSizer("download", opts)
c.upSizer = newAdaptiveSizer("upload", opts.uploadStartSize, opts.uploadMaxSize, opts)
c.downSizer = newAdaptiveSizer("download", opts.downloadStartSize, opts.downloadMaxSize, opts)
c.pushLane = newTxnLane(serverAddr, opts.tcpBuffer, opts.reconnectEvery, opts.txnTimeout, opts.headerMask, opts.coverProfile)
@@ -589,14 +938,18 @@ func (c *chunkConn) pullWorker(lane *txnLane) {
resp, err := lane.Do(payload)
if err != nil {
old, next := c.downSizer.Failure(limit)
if next == old && next == c.opts.minSize {
if !c.opts.adaptive || (next == old && next == c.opts.minSize) {
consecutiveMinFailures++
} else {
consecutiveMinFailures = 0
}
if consecutiveMinFailures >= 8 {
detail := fmt.Sprintf("download failed at minimum chunk %d", next)
if !c.opts.adaptive {
detail = fmt.Sprintf("download failed repeatedly at fixed calibrated chunk %d", next)
}
select {
case c.results <- chunkResult{seq: seq, err: fmt.Errorf("download failed at minimum chunk %d: %w", next, err)}:
case c.results <- chunkResult{seq: seq, err: fmt.Errorf("%s: %w", detail, err)}:
case <-c.ctx.Done():
}
return
@@ -773,12 +1126,15 @@ func (c *chunkConn) Write(p []byte) (int, error) {
resp, err := c.pushLane.Do(payload)
if err != nil {
old, next := c.upSizer.Failure(size)
if next == old && next == c.opts.minSize {
if !c.opts.adaptive || (next == old && next == c.opts.minSize) {
consecutiveMinFailures++
} else {
consecutiveMinFailures = 0
}
if consecutiveMinFailures >= 8 {
if !c.opts.adaptive {
return total, fmt.Errorf("upload failed repeatedly at fixed calibrated chunk %d: %w", next, err)
}
return total, fmt.Errorf("upload failed at minimum chunk %d: %w", next, err)
}
time.Sleep(30 * time.Millisecond)