Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 24 additions & 18 deletions bigtable/internal/accelerator/integration_roundtrip_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,10 +165,14 @@ func TestReadRows_MutateReadRoundTrip_Prod(t *testing.T) {
client := btpb.NewBigtableClient(conn)

// Pre-flight: confirm the accelerator's session/vRPC transport can
// actually establish here before entering the (unbounded) round-trip
// loop. Where the session client can't run, no session is ever handed
// to the caller and a MutateRow on context.Background() would park on
// the pool's waiter queue indefinitely, so skip instead.
// actually establish here before entering the round-trip loop. Where the
// vRPC OpenSession API is not served, the session pool now surfaces the
// failure deterministically: after ConsecutiveSessionFailureThreshold
// never-activated closes the circuit breaker trips and CheckoutSession
// returns a consecutive-failure error carrying the underlying
// Unimplemented code (previously the caller parked on the waiter queue
// forever). Probe once and skip on that signal rather than failing the
// round-trip in an environment that can't run it.
skipIfSessionUnavailable(ctx, t, client, fullTableName, []byte("session-probe"),
[]*btpb.Mutation{{Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
FamilyName: prodColumnFamilies[0],
Expand Down Expand Up @@ -376,16 +380,18 @@ func boundedDo(parent context.Context, fn func(context.Context) error) error {
// skipIfSessionUnavailable issues one bounded MutateRow to confirm the
// accelerator's session/vRPC transport can actually establish in this
// environment. Cloud Bigtable's vRPC OpenSession API is not served in every
// environment; where it isn't, no session is ever handed to the caller and an
// unbounded RPC parks on the pool's waiter queue forever. Rather than let that
// stall the run, probe once under a bounded context and t.Skip when the
// session client can't run here.
// environment; where it isn't, the session pool's consecutive-failure
// breaker trips and the RPC returns a consecutive-failure error carrying the
// underlying Unimplemented code. Probe once and t.Skip on that signal.
//
// A backend-reached error (e.g. NotFound / FailedPrecondition on a table that
// isn't writable yet) is NOT a session-availability failure — it proves a
// session was established — so the probe returns and lets the main loop's
// retryTransient handle table readiness. Only a code that indicates the
// transport itself is not served triggers the skip.
// The skip is deliberately narrow: only Unimplemented — the deterministic
// "this transport is not served here" signal now that the pool surfaces the
// underlying code instead of hanging. Transient codes (DeadlineExceeded,
// Unavailable) are NOT skipped: they indicate a slow or flaky backend, not
// an absent API, and are left to the main loop's retryTransient. A
// backend-reached error (e.g. NotFound / FailedPrecondition on a table that
// isn't writable yet) also proves a session was established, so the probe
// returns and lets the round-trip proceed.
func skipIfSessionUnavailable(ctx context.Context, t *testing.T, client btpb.BigtableClient, fullTableName string, key []byte, muts []*btpb.Mutation) {
t.Helper()
probeCtx, cancel := context.WithTimeout(ctx, sessionProbeTimeout)
Expand All @@ -398,12 +404,12 @@ func skipIfSessionUnavailable(ctx context.Context, t *testing.T, client btpb.Big
if err == nil {
return
}
switch status.Code(err) {
case codes.DeadlineExceeded, codes.Unavailable, codes.Unimplemented:
t.Skipf("accelerator session transport unavailable in this environment (%v); skipping prod round-trip", err)
if status.Code(err) == codes.Unimplemented {
t.Skipf("accelerator vRPC OpenSession API not served in this environment (%v); skipping prod round-trip", err)
}
// Any other code (incl. NotFound / FailedPrecondition on a not-yet-ready
// table) means a session was established — proceed with the round-trip.
// Any other code (incl. transient Unavailable / DeadlineExceeded and
// NotFound / FailedPrecondition on a not-yet-ready table) means the
// transport is served — proceed and let retryTransient handle readiness.
}

// retryTransient retries fn on error codes that are expected while a freshly
Expand Down
19 changes: 19 additions & 0 deletions bigtable/internal/transport/session_pool_lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,25 @@ func (p *SessionPoolImpl) onClose(sh *SessionHandle, err error) {
p.mu.Unlock()
p.bumpStartingClose(sh.session)
p.noteAbnormalCloseIfAny(sh)
// Re-drive creation after a session died before ever reaching
// Ready. This is the load-bearing fix for the deadline-less
// CheckoutSession hang: a session that dies in Starting fires
// neither onClosing's replace-on-death kick (it early-returns for
// starting handles) nor any other of the sizer's event-driven wake
// sites, so without this kick a cold-start cohort that all fails in
// Starting stalls — the consecutive-failure counter freezes below
// ConsecutiveSessionFailureThreshold (a MinSessionCount cohort is
// smaller than the threshold), the breaker never trips, and a caller
// parked on a deadline-less context waits forever. Kicking here lets
// the pool keep working toward its floor: each re-driven cohort
// fails again, the counter climbs to the threshold, and the breaker
// trips and drains waiters with ErrConsecutiveFailures carrying the
// underlying cause (e.g. Unimplemented). Paced by the throttler's
// failure penalty so this is not a tight retry loop. Gate mirrors
// onClosing; spawnTickOnce re-checks p.closed under p.mu.
if p.sl.ReadyCount() < p.sizer.MaxSessions() {
p.spawnTickOnce(p.poolCtx)
}
return
}
p.mu.Unlock()
Expand Down
166 changes: 166 additions & 0 deletions bigtable/internal/transport/session_pool_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"time"

spb "cloud.google.com/go/bigtable/apiv2/bigtablepb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

// newStubStreamFactory returns a streamFactory that hands out fresh
Expand Down Expand Up @@ -417,6 +419,170 @@ func TestCheckoutSession_ParkedWaiter_DeadlineExceeded(t *testing.T) {
}
}

// waitUntilParked blocks until at least one caller is enqueued on the
// pool's waiter queue, or fails the test after a generous timeout. Direct
// evidence of the waiter path so create-failure wake tests don't rely on
// timing to infer that the caller parked.
func waitUntilParked(t testing.TB, p *SessionPoolImpl) {
t.Helper()
deadline := time.NewTimer(2 * time.Second)
defer deadline.Stop()
tick := time.NewTicker(time.Millisecond)
defer tick.Stop()
for {
if p.waitersCount.Load() > 0 {
return
}
select {
case <-deadline.C:
t.Fatal("caller never parked on the waiter queue")
case <-tick.C:
}
}
}

// newDyingStreamFactory returns a streamFactory whose every stream dies in
// Starting: the stream's first Recv yields dieErr, so the session's
// readLoop drives handleClose before an OpenSessionResponse ever arrives —
// exactly what happens when the vRPC OpenSession API is not served and the
// server returns an error on the stream. closeAll closes every stream and
// refuses new ones (mirrors newStubStreamFactory) so the pool's
// replace-on-death re-drive loop stops once teardown begins.
func newDyingStreamFactory(dieErr error) (factory func(context.Context) (Stream, error), closeAll func()) {
var mu sync.Mutex
var streams []*fakeStream
var closed bool
factory = func(_ context.Context) (Stream, error) {
mu.Lock()
defer mu.Unlock()
if closed {
return nil, errors.New("newDyingStreamFactory: closed")
}
s := newFakeStream()
// Pre-load the terminal error so readLoop's first Recv returns it.
// recv is buffered, so this never blocks the factory.
s.recv <- recvOp{err: dieErr}
streams = append(streams, s)
return s, nil
}
closeAll = func() {
mu.Lock()
defer mu.Unlock()
closed = true
for _, s := range streams {
s.Close()
}
}
return factory, closeAll
}

// newDyingPool builds a pool whose every session dies in Starting with
// dieErr. Cleanup mirrors newTestPool: bridge poolCtx cancellation to
// closeStreams and close streams before Close so no readLoop leaks.
func newDyingPool(t testing.TB, min, max int, dieErr error) *SessionPoolImpl {
t.Helper()
factory, closeStreams := newDyingStreamFactory(dieErr)
p := NewSessionPoolImpl(
uint64(1),
"dying-pool",
factory,
&spb.OpenSessionRequest{ProtocolVersion: 1},
nil,
SessionTypeTable, true,
)
p.sizer.UpdateConfig(&spb.SessionClientConfiguration_SessionPoolConfiguration{
MinSessionCount: int32(min), MaxSessionCount: int32(max),
})
go func() {
<-p.poolCtx.Done()
closeStreams()
}()
t.Cleanup(func() {
closeStreams()
_ = p.Close()
})
return p
}

// TestConsecutiveFailures_StartingDeathsReDriveUntilBreakerTrips is the
// regression test for the deadline-less CheckoutSession hang. Every
// session dies in Starting (vRPC OpenSession not served → Unimplemented),
// so no session ever activates. The fix is that each never-activated close
// re-drives creation, letting the consecutive-failure counter climb past
// the MinSessionCount cohort to ConsecutiveSessionFailureThreshold; the
// breaker then trips and wakes the parked caller with ErrConsecutiveFailures
// carrying the underlying Unimplemented code. Before the fix the counter
// froze at the cold-start cohort, the breaker was unreachable, and a caller
// on context.Background() parked forever.
func TestConsecutiveFailures_StartingDeathsReDriveUntilBreakerTrips(t *testing.T) {
dieErr := status.Error(codes.Unimplemented, "OpenSession API not served")
p := newDyingPool(t, 1, 5, dieErr)
// Threshold 3 exceeds the MinSessionCount(1) cold-start cohort, so the
// breaker is reachable ONLY because each starting-death re-drives
// creation. This is the exact gap the fix closes.
p.consecutiveFailureThreshold.Store(3)

done := make(chan error, 1)
go func() {
// context.Background(): no deadline can rescue this caller — only
// the breaker trip does. CheckoutSession parks AND kicks the first
// Tick, seeding the cohort that dies and re-drives.
_, err := p.CheckoutSession(context.Background())
done <- err
}()

select {
case err := <-done:
if !errors.Is(err, ErrConsecutiveFailures) {
t.Fatalf("CheckoutSession err = %v, want ErrConsecutiveFailures", err)
}
if got := status.Code(err); got != codes.Unimplemented {
t.Fatalf("status.Code(err) = %v, want Unimplemented (underlying cause must propagate through the trip)", got)
}
case <-time.After(10 * time.Second):
t.Fatal("CheckoutSession never returned — starting-deaths did not re-drive creation to the breaker (deadline-less hang regressed)")
}
}

// TestConsecutiveFailures_SingleStartingCloseDoesNotWakeWaiters pins the
// corrected semantics against the reverted first-failure behavior: a single
// never-activated close increments the counter but must NOT wake parked
// waiters. Only the breaker trip (X consecutive failures) may fail them.
func TestConsecutiveFailures_SingleStartingCloseDoesNotWakeWaiters(t *testing.T) {
p := newTestPool(t, 0, 1)
// High threshold so one abnormal close can't trip the breaker.
p.consecutiveFailureThreshold.Store(100)
// Freeze scaling so onClose's replace-on-death re-drive can't spawn a
// phantom createSession that muddies the assertion.
p.mu.Lock()
p.scalingInProgress = true
p.mu.Unlock()

w := &waiter{ready: make(chan struct{})}
p.waitersMu.Lock()
w.elem = p.waiters.PushBack(w)
p.waitersMu.Unlock()
p.waitersCount.Add(1)

// A single session dies in Starting.
sh := injectStartingSession(t, p, "s1")
p.onClose(sh, nil)

select {
case <-w.ready:
t.Fatal("waiter woken by a single starting-close; only the breaker trip may wake parked waiters")
case <-time.After(100 * time.Millisecond):
// Still parked — correct.
}
if got := p.consecutiveFailures.Load(); got != 1 {
t.Fatalf("counter = %d, want 1 (single abnormal close increments but does not trip)", got)
}

// Drain the waiter so cleanup's Close doesn't race a live parker.
p.drainWaitersWithErr(ErrPoolClosed)
<-w.ready
}

// TestRecordPickDecision_RingWrap verifies the O(1) circular-buffer
// implementation of pickHistory: pre-wrap events are in insertion order,
// post-wrap events preserve oldest-first ordering, and the ring keeps
Expand Down
Loading
⌂ 换网址