Repository navigation
feat(storage): introduce gRPC client-side metrics #10639
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
7df55c0
73d1742
db28808
4883da3
ddf0d41
210280d
d25adfb
f91c109
8a1cb19
09f37d6
f64fb6a
7d4e3fd
e448378
edf13f9
116afbd
8b5bd88
e422ebc
ae16ca8
976adf6
af0b2f6
b94b03e
0f46a93
0147355
1f76ef6
7a38f41
0a0a5be
686feb1
1c53758
024e6d3
a0a8edc
ccc7bd4
a50c50c
41b469c
be3f5d5
498b6e0
ad9a8c5
2334322
c0610c8
83bb31b
75e8f6b
0348728
5a5d8cb
48c0fdf
1fa2883
dab45eb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -364,10 +364,11 @@ gRPC telemetry discussed in [gRFC/66] and [gRFC/78] to | |
| [Google Cloud Monitoring]. The metrics are accessible to through Cloud Monitoring API and you incur no additional cost for publishing the metrics.Google Cloud Support can use this information to more quickly diagnose | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: line length, spaces. (here and below) |
||
| problems related to GCS and gRPC. | ||
| Sending this data does not incur any billing charges, and requires minimal | ||
| CPU (a single RPC every few minutes) or memory (a few KiB to batch the | ||
| CPU (a single RPC every minute) or memory (a few KiB to batch the | ||
| telemetry). | ||
|
|
||
| To access the metrics you can view them through [Cloud Monitoring Metric explorer] with the prefix `storage.googleapis.com/client`. | ||
| To access the metrics you can view them through [Cloud Monitoring Metric explorer] with the prefix `storage.googleapis.com/client`. Metrics are emitted | ||
| every minute. | ||
|
|
||
| You can disable metrics using the following example | ||
| when creating a new gRPC client. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -32,7 +32,6 @@ import ( | |
| "google.golang.org/api/option" | ||
| "google.golang.org/api/transport" | ||
| htransport "google.golang.org/api/transport/http" | ||
| estats "google.golang.org/grpc/experimental/stats" | ||
| "google.golang.org/grpc/stats/opentelemetry" | ||
| ) | ||
|
|
||
|
|
@@ -79,26 +78,8 @@ func sizeHistogramBoundaries() []float64 { | |
| return boundaries | ||
| } | ||
|
|
||
| func getViewMasks(defaultMetrics map[estats.Metric]bool, additionalMetrics []estats.Metric) []sdkmetric.View { | ||
| views := []sdkmetric.View{} | ||
| for k, include := range defaultMetrics { | ||
| if !include { | ||
| continue | ||
| } | ||
| views = append(views, sdkmetric.NewView(sdkmetric.Instrument{ | ||
| Name: string(k), | ||
| }, sdkmetric.Stream{Name: strings.ReplaceAll(string(k), ".", "/")})) | ||
| } | ||
| for _, m := range additionalMetrics { | ||
| views = append(views, sdkmetric.NewView(sdkmetric.Instrument{ | ||
| Name: string(m), | ||
| }, sdkmetric.Stream{Name: strings.ReplaceAll(string(m), ".", "/")})) | ||
| } | ||
| return views | ||
| } | ||
|
|
||
| func metricFormatter(m metricdata.Metrics) string { | ||
| return "storage.googleapis.com/client/" + m.Name | ||
| return "storage.googleapis.com/client/" + strings.ReplaceAll(string(m.Name), ".", "/") | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should this use the monitoredResourceName const?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No, from tests, monitored resource is capital Client and metric prefix is lower client. Pulling it up into consts.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Monitred resource requires |
||
| } | ||
|
|
||
| func gcpAttributeExpectedDefaults() []attribute.KeyValue { | ||
|
|
@@ -219,31 +200,28 @@ func newGRPCMetricContext(ctx context.Context, config internalMetricsConfig) (*i | |
| if err != nil { | ||
| return nil, err | ||
| } | ||
| metricsToEnable := []estats.Metric{ | ||
| "grpc.lb.wrr.rr_fallback", | ||
| "grpc.lb.wrr.endpoint_weight_not_yet_usable", | ||
| "grpc.lb.wrr.endpoint_weight_stale", | ||
| "grpc.lb.wrr.endpoint_weights", | ||
| "grpc.lb.rls.cache_entries", | ||
| "grpc.lb.rls.cache_size", | ||
| "grpc.lb.rls.default_target_picks", | ||
| "grpc.lb.rls.target_picks", | ||
| "grpc.lb.rls.failed_picks", | ||
| metricViews := []sdkmetric.View{ | ||
| createHistogramView("grpc.client.attempt.duration", "A view of grpc.client.attempt.duration with histogram boundaries more appropriate for Google Cloud Storage RPCs", "s", latencyHistogramBoundaries()), | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What are these description strings used for?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. They're exposed to the user in Monitoring Metric Explorer to describe the metric. That said, it should be cached so clarifying if we really need to have this hardcoded client side. |
||
| createHistogramView("grpc.client.attempt.rcvd_total_compressed_message_size", "A view of grpc.client.attempt.rcvd_total_compressed_message_size with histogram boundaries more appropriate for Google Cloud Storage RPCs", "By", sizeHistogramBoundaries()), | ||
| createHistogramView("grpc.client.attempt.sent_total_compressed_message_size", "A view of grpc.client.attempt.sent_total_compressed_message_size with histogram boundaries more appropriate for Google Cloud Storage RPCs", "By", sizeHistogramBoundaries()), | ||
| } | ||
| defaultMetrics := opentelemetry.DefaultMetrics() | ||
| metricViews := getViewMasks(defaultMetrics.Metrics(), metricsToEnable) | ||
| metricViews = append(metricViews, | ||
| createHistogramView("grpc/client/attempt/duration", "A view of grpc/client/attempt/duration with histogram boundaries more appropriate for Google Cloud Storage RPCs", "s", latencyHistogramBoundaries()), | ||
| createHistogramView("grpc/client/attempt/rcvd_total_compressed_message_size", "A view of grpc/client/attempt/rcvd_total_compressed_message_size with histogram boundaries more appropriate for Google Cloud Storage RPCs", "By", sizeHistogramBoundaries()), | ||
| createHistogramView("grpc/client/attempt/sent_total_compressed_message_size", "A view of grpc/client/attempt/sent_total_compressed_message_size with histogram boundaries more appropriate for Google Cloud Storage RPCs", "By", sizeHistogramBoundaries())) | ||
| provider := sdkmetric.NewMeterProvider( | ||
| sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter, sdkmetric.WithInterval(time.Minute))), | ||
| sdkmetric.WithResource(preparedResource.resource), | ||
| sdkmetric.WithView(metricViews...), | ||
| ) | ||
| mo := opentelemetry.MetricsOptions{ | ||
| MeterProvider: provider, | ||
| Metrics: defaultMetrics.Add(metricsToEnable...), | ||
| MeterProvider: provider, | ||
| Metrics: opentelemetry.DefaultMetrics().Add( | ||
| "grpc.lb.wrr.rr_fallback", | ||
| "grpc.lb.wrr.endpoint_weight_not_yet_usable", | ||
| "grpc.lb.wrr.endpoint_weight_stale", | ||
| "grpc.lb.wrr.endpoint_weights", | ||
| "grpc.lb.rls.cache_entries", | ||
| "grpc.lb.rls.cache_size", | ||
| "grpc.lb.rls.default_target_picks", | ||
| "grpc.lb.rls.target_picks", | ||
| "grpc.lb.rls.failed_picks"), | ||
| OptionalLabels: []string{"grpc.lb.locality"}, | ||
| } | ||
| do := option.WithGRPCDialOption(opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo})) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.