Repository navigation
fix(bigtable): Fix NoopMetricsProvider panic #12709
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -36,8 +36,6 @@ import ( | |
| "cloud.google.com/go/internal/trace" | ||
| gax "github.com/googleapis/gax-go/v2" | ||
| "github.com/googleapis/gax-go/v2/apierror" | ||
| "go.opentelemetry.io/otel/attribute" | ||
| "go.opentelemetry.io/otel/metric" | ||
| "google.golang.org/api/option" | ||
| "google.golang.org/api/option/internaloption" | ||
| gtransport "google.golang.org/api/transport/grpc" | ||
|
|
@@ -137,19 +135,7 @@ func NewClientWithConfig(ctx context.Context, project, instance string, config C | |
| o = append(o, opts...) | ||
|
|
||
| // TODO(b/372244283): Remove after b/358175516 has been fixed | ||
| asyncRefreshMetricAttrs := metricsTracerFactory.clientAttributes | ||
| asyncRefreshMetricAttrs = append(asyncRefreshMetricAttrs, | ||
| attribute.String(metricLabelKeyTag, "async_refresh_dry_run"), | ||
| // Table, cluster and zone are unknown at this point | ||
| // Use default values | ||
| attribute.String(monitoredResLabelKeyTable, defaultTable), | ||
| attribute.String(monitoredResLabelKeyCluster, defaultCluster), | ||
| attribute.String(monitoredResLabelKeyZone, defaultZone), | ||
| ) | ||
| o = append(o, internaloption.EnableAsyncRefreshDryRun(func() { | ||
| metricsTracerFactory.debugTags.Add(context.Background(), 1, | ||
| metric.WithAttributes(asyncRefreshMetricAttrs...)) | ||
| })) | ||
| o = append(o, internaloption.EnableAsyncRefreshDryRun(metricsTracerFactory.newAsyncRefreshErrHandler())) | ||
|
|
||
| connPool, err := gtransport.DialPool(ctx, o...) | ||
| if err != nil { | ||
|
|
@@ -525,11 +511,11 @@ func (c *Client) prepareStatementWithMetadata(ctx context.Context, query string, | |
| defer func() { trace.EndSpan(ctx, err) }() | ||
|
|
||
| mt := c.newBuiltinMetricsTracer(ctx, "", false) | ||
| defer recordOperationCompletion(mt) | ||
| defer mt.recordOperationCompletion() | ||
|
|
||
| preparedStatement, err = c.prepareStatement(ctx, mt, query, paramTypes, opts...) | ||
| statusCode, statusErr := convertToGrpcStatusErr(err) | ||
| mt.currOp.setStatus(statusCode.String()) | ||
| mt.setCurrOpStatus(statusCode.String()) | ||
| return preparedStatement, statusErr | ||
| } | ||
|
|
||
|
|
@@ -704,11 +690,11 @@ func (bs *BoundStatement) Execute(ctx context.Context, f func(ResultRow) bool, o | |
| defer func() { trace.EndSpan(ctx, err) }() | ||
|
|
||
| mt := bs.ps.c.newBuiltinMetricsTracer(ctx, "", true) | ||
| defer recordOperationCompletion(mt) | ||
| defer mt.recordOperationCompletion() | ||
|
|
||
| err = bs.execute(ctx, f, mt) | ||
| statusCode, statusErr := convertToGrpcStatusErr(err) | ||
| mt.currOp.setStatus(statusCode.String()) | ||
| mt.setCurrOpStatus(statusCode.String()) | ||
| return statusErr | ||
| } | ||
|
|
||
|
|
@@ -962,11 +948,11 @@ func (t *Table) ReadRows(ctx context.Context, arg RowSet, f func(Row) bool, opts | |
| defer func() { trace.EndSpan(ctx, err) }() | ||
|
|
||
| mt := t.newBuiltinMetricsTracer(ctx, true) | ||
| defer recordOperationCompletion(mt) | ||
| defer mt.recordOperationCompletion() | ||
|
|
||
| err = t.readRows(ctx, arg, f, mt, opts...) | ||
| statusCode, statusErr := convertToGrpcStatusErr(err) | ||
| mt.currOp.setStatus(statusCode.String()) | ||
| mt.setCurrOpStatus(statusCode.String()) | ||
| return statusErr | ||
| } | ||
|
|
||
|
|
@@ -1079,7 +1065,7 @@ func (t *Table) readRows(ctx context.Context, arg RowSet, f func(Row) bool, mt * | |
| appBlockingLatencyStart := time.Now() | ||
| continueReading := f(row) | ||
| numRowsRead++ | ||
| mt.currOp.incrementAppBlockingLatency(convertToMs(time.Since(appBlockingLatencyStart))) | ||
| mt.incrementAppBlockingLatency(convertToMs(time.Since(appBlockingLatencyStart))) | ||
|
|
||
| if !continueReading { | ||
| // Cancel and drain stream. | ||
|
|
@@ -1637,11 +1623,11 @@ func (t *Table) Apply(ctx context.Context, row string, m *Mutation, opts ...Appl | |
| ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigtable/Apply") | ||
| defer func() { trace.EndSpan(ctx, err) }() | ||
| mt := t.newBuiltinMetricsTracer(ctx, false) | ||
| defer recordOperationCompletion(mt) | ||
| defer mt.recordOperationCompletion() | ||
|
|
||
| err = t.apply(ctx, mt, row, m, opts...) | ||
| statusCode, statusErr := convertToGrpcStatusErr(err) | ||
| mt.currOp.setStatus(statusCode.String()) | ||
| mt.setCurrOpStatus(statusCode.String()) | ||
| return statusErr | ||
| } | ||
|
|
||
|
|
@@ -1915,7 +1901,7 @@ func (t *Table) ApplyBulk(ctx context.Context, rowKeys []string, muts []*Mutatio | |
| func (t *Table) applyGroup(ctx context.Context, group []*entryErr, opts ...ApplyOption) (err error) { | ||
| attrMap := make(map[string]interface{}) | ||
| mt := t.newBuiltinMetricsTracer(ctx, true) | ||
| defer recordOperationCompletion(mt) | ||
| defer mt.recordOperationCompletion() | ||
|
|
||
| err = gaxInvokeWithRecorder(ctx, mt, "MutateRows", func(ctx context.Context, headerMD, trailerMD *metadata.MD, _ gax.CallSettings) error { | ||
| attrMap["rowCount"] = len(group) | ||
|
|
@@ -1936,7 +1922,7 @@ func (t *Table) applyGroup(ctx context.Context, group []*entryErr, opts ...Apply | |
| }, t.c.retryOption) | ||
|
|
||
| statusCode, statusErr := convertToGrpcStatusErr(err) | ||
| mt.currOp.setStatus(statusCode.String()) | ||
| mt.setCurrOpStatus(statusCode.String()) | ||
| return statusErr | ||
| } | ||
|
|
||
|
|
@@ -2077,11 +2063,11 @@ func (t *Table) ApplyReadModifyWrite(ctx context.Context, row string, m *ReadMod | |
| ctx = mergeOutgoingMetadata(ctx, t.md) | ||
|
|
||
| mt := t.newBuiltinMetricsTracer(ctx, false) | ||
| defer recordOperationCompletion(mt) | ||
| defer mt.recordOperationCompletion() | ||
|
|
||
| updatedRow, err := t.applyReadModifyWrite(ctx, mt, row, m) | ||
| statusCode, statusErr := convertToGrpcStatusErr(err) | ||
| mt.currOp.setStatus(statusCode.String()) | ||
| mt.setCurrOpStatus(statusCode.String()) | ||
| return updatedRow, statusErr | ||
| } | ||
|
|
||
|
|
@@ -2158,11 +2144,11 @@ func (t *Table) SampleRowKeys(ctx context.Context) ([]string, error) { | |
| ctx = mergeOutgoingMetadata(ctx, t.md) | ||
|
|
||
| mt := t.newBuiltinMetricsTracer(ctx, true) | ||
| defer recordOperationCompletion(mt) | ||
| defer mt.recordOperationCompletion() | ||
|
|
||
| rowKeys, err := t.sampleRowKeys(ctx, mt) | ||
| statusCode, statusErr := convertToGrpcStatusErr(err) | ||
| mt.currOp.setStatus(statusCode.String()) | ||
| mt.setCurrOpStatus(statusCode.String()) | ||
| return rowKeys, statusErr | ||
| } | ||
|
|
||
|
|
@@ -2224,34 +2210,6 @@ func (c *Client) newBuiltinMetricsTracer(ctx context.Context, table string, isSt | |
| return &mt | ||
| } | ||
|
|
||
| // recordOperationCompletion records as many operation specific metrics as it can | ||
| // Ignores error seen while creating metric attributes since metric can still | ||
| // be recorded with rest of the attributes | ||
| func recordOperationCompletion(mt *builtinMetricsTracer) { | ||
| if !mt.builtInEnabled { | ||
| return | ||
| } | ||
|
|
||
| // Calculate elapsed time | ||
| elapsedTimeMs := convertToMs(time.Since(mt.currOp.startTime)) | ||
|
|
||
| // Record operation_latencies | ||
| opLatAttrs, _ := mt.toOtelMetricAttrs(metricNameOperationLatencies) | ||
| mt.instrumentOperationLatencies.Record(mt.ctx, elapsedTimeMs, metric.WithAttributeSet(opLatAttrs)) | ||
|
|
||
| // Record retry_count | ||
| retryCntAttrs, _ := mt.toOtelMetricAttrs(metricNameRetryCount) | ||
| if mt.currOp.attemptCount > 1 { | ||
| // Only record when retry count is greater than 0 so the retry | ||
| // graph will be less confusing | ||
| mt.instrumentRetryCount.Add(mt.ctx, mt.currOp.attemptCount-1, metric.WithAttributeSet(retryCntAttrs)) | ||
| } | ||
|
|
||
| // Record application_latencies | ||
| appBlockingLatAttrs, _ := mt.toOtelMetricAttrs(metricNameAppBlockingLatencies) | ||
| mt.instrumentAppBlockingLatencies.Record(mt.ctx, mt.currOp.appBlockingLatency, metric.WithAttributeSet(appBlockingLatAttrs)) | ||
| } | ||
|
|
||
|
Comment on lines
-2230
to
-2257
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Moved to metrics.go |
||
| // gaxInvokeWithRecorder: | ||
| // - wraps 'f' in a new function 'callWrapper' that: | ||
| // - updates tracer state and records built in attempt specific metrics | ||
|
|
@@ -2264,82 +2222,16 @@ func gaxInvokeWithRecorder(ctx context.Context, mt *builtinMetricsTracer, method | |
| attempTrailerMD := metadata.New(nil) | ||
| mt.setMethod(method) | ||
|
|
||
| var callWrapper func(context.Context, gax.CallSettings) error | ||
| if !mt.builtInEnabled { | ||
| callWrapper = func(ctx context.Context, callSettings gax.CallSettings) error { | ||
| // f makes calls to CBT service | ||
| return f(ctx, &attemptHeaderMD, &attempTrailerMD, callSettings) | ||
| } | ||
| } else { | ||
| callWrapper = func(ctx context.Context, callSettings gax.CallSettings) error { | ||
| // Increment number of attempts | ||
| mt.currOp.incrementAttemptCount() | ||
|
|
||
| mt.currOp.currAttempt = attemptTracer{} | ||
|
|
||
| // record start time | ||
| mt.currOp.currAttempt.setStartTime(time.Now()) | ||
|
Comment on lines
-2278
to
-2284
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Moved to recordAttemptStart |
||
|
|
||
| // f makes calls to CBT service | ||
| err := f(ctx, &attemptHeaderMD, &attempTrailerMD, callSettings) | ||
|
|
||
| // Set attempt status | ||
| statusCode, _ := convertToGrpcStatusErr(err) | ||
| mt.currOp.currAttempt.setStatus(statusCode.String()) | ||
| callWrapper := func(ctx context.Context, callSettings gax.CallSettings) error { | ||
| mt.recordAttemptStart() | ||
|
|
||
| // Get location attributes from metadata and set it in tracer | ||
| // Ignore get location error since the metric can still be recorded with rest of the attributes | ||
| clusterID, zoneID, _ := extractLocation(attemptHeaderMD, attempTrailerMD) | ||
| mt.currOp.currAttempt.setClusterID(clusterID) | ||
| mt.currOp.currAttempt.setZoneID(zoneID) | ||
| // f makes calls to CBT service | ||
| err := f(ctx, &attemptHeaderMD, &attempTrailerMD, callSettings) | ||
|
|
||
| // Set server latency in tracer | ||
| serverLatency, serverLatencyErr := extractServerLatency(attemptHeaderMD, attempTrailerMD) | ||
| mt.currOp.currAttempt.setServerLatencyErr(serverLatencyErr) | ||
| mt.currOp.currAttempt.setServerLatency(serverLatency) | ||
|
Comment on lines
-2289
to
-2302
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Moved to recordAttemptCompletion |
||
|
|
||
| // Record attempt specific metrics | ||
| recordAttemptCompletion(mt) | ||
| return err | ||
| } | ||
| } | ||
| return gax.Invoke(ctx, callWrapper, opts...) | ||
| } | ||
|
|
||
| // recordAttemptCompletion records as many attempt specific metrics as it can | ||
| // Ignore errors seen while creating metric attributes since metric can still | ||
| // be recorded with rest of the attributes | ||
| func recordAttemptCompletion(mt *builtinMetricsTracer) { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. MOved to metrics.go |
||
| if !mt.builtInEnabled { | ||
| return | ||
| } | ||
|
|
||
| // Calculate elapsed time | ||
| elapsedTime := convertToMs(time.Since(mt.currOp.currAttempt.startTime)) | ||
|
|
||
| // Record attempt_latencies | ||
| attemptLatAttrs, _ := mt.toOtelMetricAttrs(metricNameAttemptLatencies) | ||
| mt.instrumentAttemptLatencies.Record(mt.ctx, elapsedTime, metric.WithAttributeSet(attemptLatAttrs)) | ||
|
|
||
| // Record server_latencies | ||
| serverLatAttrs, _ := mt.toOtelMetricAttrs(metricNameServerLatencies) | ||
| if mt.currOp.currAttempt.serverLatencyErr == nil { | ||
| mt.instrumentServerLatencies.Record(mt.ctx, mt.currOp.currAttempt.serverLatency, metric.WithAttributeSet(serverLatAttrs)) | ||
| // Record attempt specific metrics | ||
| mt.recordAttemptCompletion(attemptHeaderMD, attempTrailerMD, err) | ||
| return err | ||
| } | ||
|
|
||
| // Record connectivity_error_count | ||
| connErrCountAttrs, _ := mt.toOtelMetricAttrs(metricNameConnErrCount) | ||
| // Determine if connection error should be incremented. | ||
| // A true connectivity error occurs only when we receive NO server-side signals. | ||
| // 1. Server latency (from server-timing header) is a signal, but absent in DirectPath. | ||
| // 2. Location (from x-goog-ext header) is a signal present in both paths. | ||
| // Therefore, we only count an error if BOTH signals are missing. | ||
| isServerLatencyEffectivelyEmpty := mt.currOp.currAttempt.serverLatencyErr != nil || mt.currOp.currAttempt.serverLatency == 0 | ||
| isLocationEmpty := mt.currOp.currAttempt.clusterID == defaultCluster | ||
| if isServerLatencyEffectivelyEmpty && isLocationEmpty { | ||
| // This is a connectivity error: the request likely never reached Google's network. | ||
| mt.instrumentConnErrCount.Add(mt.ctx, 1, metric.WithAttributeSet(connErrCountAttrs)) | ||
| } else { | ||
| mt.instrumentConnErrCount.Add(mt.ctx, 0, metric.WithAttributeSet(connErrCountAttrs)) | ||
| } | ||
| return gax.Invoke(ctx, callWrapper, opts...) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Moved to newAsyncRefreshErrHandler