diff --git a/bigtable/internal/transport/debug_tracer.go b/bigtable/internal/transport/debug_tracer.go new file mode 100644 index 000000000000..d0479ec30078 --- /dev/null +++ b/bigtable/internal/transport/debug_tracer.go @@ -0,0 +1,292 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// debug_tracer.go — counter for "this branch shouldn't be reached" +// sites in the session pool, session, and configuration manager. One +// atomic add plus one OTel Int64Counter increment per emission, so it's +// cheap enough to sprinkle on cold paths. The metric name matches +// java-bigtable's ClientDebugTagCount so cross-language dashboards can +// join on the tag column. +// +// # Entry points +// +// - recordDebugTag(name) — Warn-level observation. +// - recordDebugTagAt(level, name) — same, non-Warn level. +// - assertDebugTag(expr, name) — invariant check, no format args. +// - assertDebugTagf(expr, name, fmt…) — invariant check with log context. +// +// The assert forms return `expr` and never panic; use as `if !assert…` +// so the site records + logs + bails in one line. +// +// # Rules +// +// - `name` is always a constant from the catalog below — never a +// format string. Dynamic context belongs in the log message. +// - Emission is additive: keep whatever log/err/return the branch +// already does. +// - Default to recordDebugTag (Warn). Reserve Error for assert +// failures and the rare "this is really wrong" observation. + +package internal + +import ( + "context" + "fmt" + "sort" + "sync" + "sync/atomic" + "time" + + btopt "cloud.google.com/go/bigtable/internal/option" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// debugLevel gates whether an emission is admitted. Values match +// Java's TelemetryConfiguration.Level so future wire-config plumbing +// is a straight cast. +type debugLevel int32 + +// lvl namespaces the two levels so call sites read as `lvl.Warn` / +// `lvl.Error`. Named `lvl` (not `tag`) so it doesn't collide visually +// with the `tag…` catalog constants at call sites. +var lvl = struct { + Warn debugLevel + Error debugLevel +}{ + Warn: 1, + Error: 2, +} + +// debugTagCounterName is the OTel instrument name. Kept short because +// the Cloud Monitoring exporter prepends +// `bigtable.googleapis.com/internal/client/` itself; a fully-qualified +// name here would double-prefix and Cloud Monitoring would reject it. +const debugTagCounterName = "debug_tags" + +// debugTagAttrKey is the sole OTel attribute carrying the tag string. +// Cardinality is bounded by the catalog below. +const debugTagAttrKey = "tag" + +// Debug-tag catalog. Every emission site passes one of these constants; +// inline literals are never used. Wire names are snake_case (matching +// java-bigtable); Go identifiers are camelCase. Grep either form to +// jump between the definition and its emission sites. +const ( + // Session-lifecycle observations. + tagSessionUnknownResponse = "session_unknown_response" + tagSessionOpenWrongState = "session_open_wrong_state" + tagSessionGoawayAfterClose = "session_goaway_after_close" + tagSessionGoawayBeforeStart = "session_goaway_before_start" + tagSessionAbnormalClose = "session_abnormal_close" + tagSessionHeartbeatMissed = "session_heartbeat_missed" + tagSessionForceCloseNeverStarted = "session_force_close_never_started" + tagSessionCloseNoReason = "session_close_no_reason" + + // vRPC dispatch observations. + tagSessionVRPCNil = "session_vrpc_nil" + tagSessionVRPCErrorNil = "session_vrpc_error_nil" + tagSessionVRPCIDMismatch = "session_vrpc_id_mismatch" + tagSessionVRPCResponseWrongState = "session_vrpc_response_wrong_state" + tagSessionVRPCDuplicateResult = "session_vrpc_duplicate_result" + + // Pool-scoped anomalies. + tagSessionPoolStuckSessionSwept = "session_pool_stuck_session_swept" + tagSessionPoolDrainTimeout = "session_pool_drain_timeout" + tagSessionPoolCreateFailed = "session_pool_create_failed" + tagSessionPoolPickLostRace = "session_pool_pick_lost_race" + + // Client configuration polling. + tagClientConfigPollFailed = "client_config_poll_failed" + tagClientConfigPollCtxExpired = "client_config_poll_ctx_expired" +) + +var ( + // debugTagCounter is the OTel Int64Counter registered by + // registerDebugTagCounter. Held in an atomic.Value so the + // register-once write and every-emission reads don't race on the + // two-word interface value. Load returns nil until initialization + // runs, so the tracer is safe to call before InitializeSessionMetrics + // or in tests that don't wire OTel. + debugTagCounter atomic.Value + + // debugTagLevelFloor drops any emission with level < floor before it + // touches the counter or the in-memory map. Defaults to Warn. + debugTagLevelFloor atomic.Int32 + + // debugTagCountsMu guards debugTagStats. Contention is negligible: + // emissions are cold-path and reads (tests, /debugtagsz/) are rare. + debugTagCountsMu sync.RWMutex + // debugTagStats is the in-process view of every tag seen since + // process start. Kept alongside the OTel counter so tests and + // /debugtagsz/ can read state without an exporter wired up. + debugTagStats = map[string]*tagStat{} +) + +// tagStat holds one tag's counters. Fields are atomic so the emission +// path stays lock-free after the map entry exists (RLock the map, then +// bump atomics). +type tagStat struct { + count atomic.Int64 // total emissions since process start + firstSeen atomic.Int64 // unix-nano of first emission; write-once + lastSeen atomic.Int64 // unix-nano of most-recent emission +} + +// DebugTagSnapshot is one row of DebugTags output — a tag's count plus +// its first- and last-seen timestamps. Exported for /debugtagsz/. +type DebugTagSnapshot struct { + Name string + Count int64 + FirstSeen time.Time + LastSeen time.Time +} + +func init() { + debugTagLevelFloor.Store(int32(lvl.Warn)) +} + +// registerDebugTagCounter is called once from InitializeSessionMetrics +// after the meter provider is validated non-nil. +func registerDebugTagCounter(meter metric.Meter) error { + c, err := meter.Int64Counter( + debugTagCounterName, + metric.WithDescription("Count of unexpected events tagged by call site — the Go client's parity with java-bigtable's ClientDebugTagCount."), + ) + if err != nil { + return fmt.Errorf("create debug_tags counter: %w", err) + } + debugTagCounter.Store(c) + return nil +} + +// setDebugTagLevelFloor sets the emission floor. Intended for future +// wiring from TelemetryConfiguration.debug_tag_level. +func setDebugTagLevelFloor(l debugLevel) { + debugTagLevelFloor.Store(int32(l)) +} + +// recordDebugTag increments the debug_tags counter for `name` at Warn. +// Safe to call before InitializeSessionMetrics — only the in-memory +// map increments until the OTel counter is registered. +func recordDebugTag(name string) { + recordDebugTagAt(lvl.Warn, name) +} + +// recordDebugTagAt is the level-explicit form. Prefer recordDebugTag +// for ordinary observations; use this only when a site needs a +// non-Warn level outside an assertDebugTag. +func recordDebugTagAt(level debugLevel, name string) { + if int32(level) < debugTagLevelFloor.Load() { + return + } + bumpDebugTagCount(name) + if c, ok := debugTagCounter.Load().(metric.Int64Counter); ok && c != nil { + c.Add(context.Background(), 1, + metric.WithAttributes(attribute.String(debugTagAttrKey, name))) + } +} + +// assertDebugTag returns `expr`. On false it records an Error tag and +// logs "debug-tag assertion failed [name]". Never panics — the caller +// decides whether to bail, drop, or continue. +func assertDebugTag(expr bool, name string) bool { + if expr { + return true + } + recordDebugTagAt(lvl.Error, name) + btopt.Debugf(nil, "bigtable: debug-tag assertion failed [%s]", name) + return false +} + +// assertDebugTagf is the format-string form of assertDebugTag. Use it +// when the site has diagnostic context (state, ids, timing) worth +// putting in the log line; the counter increment is identical. +func assertDebugTagf(expr bool, name, format string, args ...interface{}) bool { + if expr { + return true + } + recordDebugTagAt(lvl.Error, name) + btopt.Debugf(nil, "bigtable: debug-tag assertion failed [%s]: "+format, append([]interface{}{name}, args...)...) + return false +} + +// bumpDebugTagCount bumps the in-memory count for `name` and stamps +// its emission timestamps. The first emission creates the entry under +// the write lock; subsequent ones take the RLock and touch atomics. +// The function handles its own locking; the name deliberately avoids +// the "…Locked" suffix (which by convention means the caller holds +// the lock). +func bumpDebugTagCount(name string) { + now := time.Now().UnixNano() + debugTagCountsMu.RLock() + s, ok := debugTagStats[name] + debugTagCountsMu.RUnlock() + if ok { + s.count.Add(1) + s.lastSeen.Store(now) + return + } + debugTagCountsMu.Lock() + if s, ok = debugTagStats[name]; !ok { + s = &tagStat{} + s.firstSeen.Store(now) + debugTagStats[name] = s + } + s.count.Add(1) + s.lastSeen.Store(now) + debugTagCountsMu.Unlock() +} + +// DebugTags returns every tag emitted since process start, sorted by +// LastSeen descending. The catalog is small (bounded by the const +// block above) so callers can render the whole slice without paging. +func DebugTags() []DebugTagSnapshot { + debugTagCountsMu.RLock() + out := make([]DebugTagSnapshot, 0, len(debugTagStats)) + for name, s := range debugTagStats { + out = append(out, DebugTagSnapshot{ + Name: name, + Count: s.count.Load(), + FirstSeen: time.Unix(0, s.firstSeen.Load()), + LastSeen: time.Unix(0, s.lastSeen.Load()), + }) + } + debugTagCountsMu.RUnlock() + // Sort after releasing the lock — `out` is a local slice of value + // copies, so the sort touches no shared state. + sort.Slice(out, func(i, j int) bool { + return out[i].LastSeen.After(out[j].LastSeen) + }) + return out +} + +// snapshotDebugTagCounts returns a bare name→count map for tests that +// only care about counts. New callers should prefer DebugTags. +func snapshotDebugTagCounts() map[string]int64 { + debugTagCountsMu.RLock() + defer debugTagCountsMu.RUnlock() + out := make(map[string]int64, len(debugTagStats)) + for name, s := range debugTagStats { + out[name] = s.count.Load() + } + return out +} + +// resetDebugTagCountsForTest wipes the map so a test can assert on a +// specific tag's count without cross-test contamination. Test-only. +func resetDebugTagCountsForTest() { + debugTagCountsMu.Lock() + debugTagStats = map[string]*tagStat{} + debugTagCountsMu.Unlock() +} diff --git a/bigtable/internal/transport/debug_tracer_test.go b/bigtable/internal/transport/debug_tracer_test.go new file mode 100644 index 000000000000..25a1af43499d --- /dev/null +++ b/bigtable/internal/transport/debug_tracer_test.go @@ -0,0 +1,192 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "sync" + "testing" + "time" +) + +// TestDebugTag_RecordCounts verifies the in-memory counter tracks every +// recordDebugTag call, including repeats, and snapshotDebugTagCounts returns an +// independent snapshot (mutating the returned map doesn't affect state). +func TestDebugTag_RecordCounts(t *testing.T) { + resetDebugTagCountsForTest() + recordDebugTag("tag_a") + recordDebugTag("tag_a") + recordDebugTagAt(lvl.Error, "tag_b") + + got := snapshotDebugTagCounts() + if got["tag_a"] != 2 { + t.Errorf("tag_a: got %d, want 2", got["tag_a"]) + } + if got["tag_b"] != 1 { + t.Errorf("tag_b: got %d, want 1", got["tag_b"]) + } + + // Mutation on the returned map must not bleed into the tracer. + got["tag_a"] = 999 + if again := snapshotDebugTagCounts(); again["tag_a"] != 2 { + t.Errorf("snapshot leaked mutation: got %d, want 2", again["tag_a"]) + } +} + +// TestDebugTag_LevelFloor verifies emissions below the runtime floor are +// dropped in-process. Restores the default floor at the end so downstream +// tests aren't polluted. +func TestDebugTag_LevelFloor(t *testing.T) { + resetDebugTagCountsForTest() + t.Cleanup(func() { setDebugTagLevelFloor(lvl.Warn) }) + + setDebugTagLevelFloor(lvl.Error) + recordDebugTag("warn_below_floor") + recordDebugTagAt(lvl.Error, "error_at_floor") + + got := snapshotDebugTagCounts() + if _, present := got["warn_below_floor"]; present { + t.Errorf("warn_below_floor should have been dropped, got %v", got) + } + if got["error_at_floor"] != 1 { + t.Errorf("error_at_floor: got %d, want 1", got["error_at_floor"]) + } +} + +// TestDebugTag_AssertPassAndFail verifies both assert forms +// (formatted and format-free) return the predicate result and +// increment the counter only on the failing branch. +func TestDebugTag_AssertPassAndFail(t *testing.T) { + resetDebugTagCountsForTest() + + // Formatted form — captures diagnostic context in the log message. + if ok := assertDebugTagf(true, "assert_pass_f", "should not fire"); !ok { + t.Errorf("assertDebugTagf(true) returned false") + } + if ok := assertDebugTagf(false, "assert_fail_f", "context=%s", "test"); ok { + t.Errorf("assertDebugTagf(false) returned true") + } + + // Format-free form — the tag name is the whole message. + if ok := assertDebugTag(true, "assert_pass"); !ok { + t.Errorf("assertDebugTag(true) returned false") + } + if ok := assertDebugTag(false, "assert_fail"); ok { + t.Errorf("assertDebugTag(false) returned true") + } + + got := snapshotDebugTagCounts() + for _, name := range []string{"assert_pass_f", "assert_pass"} { + if _, present := got[name]; present { + t.Errorf("%s fired despite predicate holding: %v", name, got) + } + } + for _, name := range []string{"assert_fail_f", "assert_fail"} { + if got[name] != 1 { + t.Errorf("%s: got %d, want 1", name, got[name]) + } + } +} + +// TestDebugTag_FirstAndLastSeen verifies that FirstSeen is stamped +// exactly once at first emission and LastSeen advances with every +// subsequent emission. Uses a short sleep between the two record calls +// so timestamps land in distinct nanoseconds even on fast machines. +func TestDebugTag_FirstAndLastSeen(t *testing.T) { + resetDebugTagCountsForTest() + + recordDebugTag("first_last_tag") + firstPass := DebugTags() + if len(firstPass) != 1 { + t.Fatalf("DebugTags length = %d, want 1", len(firstPass)) + } + first := firstPass[0] + if first.Name != "first_last_tag" { + t.Errorf("Name = %q, want %q", first.Name, "first_last_tag") + } + if first.Count != 1 || first.FirstSeen.IsZero() || first.LastSeen.IsZero() { + t.Errorf("first snapshot missing fields: %+v", first) + } + if !first.FirstSeen.Equal(first.LastSeen) { + t.Errorf("first-emission FirstSeen != LastSeen: %v vs %v", first.FirstSeen, first.LastSeen) + } + + time.Sleep(2 * time.Millisecond) + recordDebugTag("first_last_tag") + second := DebugTags()[0] + if second.Count != 2 { + t.Errorf("Count after 2 emissions = %d, want 2", second.Count) + } + if !second.FirstSeen.Equal(first.FirstSeen) { + t.Errorf("FirstSeen changed after re-emission: was %v, now %v", first.FirstSeen, second.FirstSeen) + } + if !second.LastSeen.After(first.LastSeen) { + t.Errorf("LastSeen did not advance: %v -> %v", first.LastSeen, second.LastSeen) + } +} + +// TestDebugTag_SortedByLastSeen verifies DebugTags returns entries +// sorted by LastSeen descending — the debugview page depends on this +// ordering to surface just-fired tags at the top. +func TestDebugTag_SortedByLastSeen(t *testing.T) { + resetDebugTagCountsForTest() + + recordDebugTag("sorted_older") + time.Sleep(2 * time.Millisecond) + recordDebugTag("sorted_newer") + time.Sleep(2 * time.Millisecond) + recordDebugTag("sorted_middle") + // Emit "sorted_older" again to keep its Count > 1 but leave its + // LastSeen stale — no, actually the point of the test is order; skip. + + snaps := DebugTags() + if len(snaps) < 3 { + t.Fatalf("want 3 snapshots, got %d: %+v", len(snaps), snaps) + } + if snaps[0].Name != "sorted_middle" { + t.Errorf("top row = %q, want %q (most-recently emitted)", snaps[0].Name, "sorted_middle") + } + // The other two entries should follow in reverse-time order. + for i := 1; i < len(snaps); i++ { + if snaps[i-1].LastSeen.Before(snaps[i].LastSeen) { + t.Errorf("out-of-order at index %d: %v then %v", i, snaps[i-1].LastSeen, snaps[i].LastSeen) + } + } +} + +// TestDebugTag_ConcurrentEmission is a smoke test for the RWMutex-guarded +// map: many goroutines hammering the same tag should tally exactly, no +// dropped increments, no data race under -race. +func TestDebugTag_ConcurrentEmission(t *testing.T) { + resetDebugTagCountsForTest() + + const goroutines = 16 + const perGoroutine = 500 + var wg sync.WaitGroup + wg.Add(goroutines) + for i := 0; i < goroutines; i++ { + go func() { + defer wg.Done() + for j := 0; j < perGoroutine; j++ { + recordDebugTag("hot_tag") + } + }() + } + wg.Wait() + + want := int64(goroutines * perGoroutine) + if got := snapshotDebugTagCounts()["hot_tag"]; got != want { + t.Errorf("hot_tag: got %d, want %d", got, want) + } +}