Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
7df55c0
feat: initial prototype testing client-side metrics
Jul 31, 2024
73d1742
added tests and refactored to help test
Aug 5, 2024
db28808
use 1.21 and refactor test
Aug 7, 2024
4883da3
fix go.sum go.work.sum
Aug 7, 2024
ddf0d41
add option to disable client metrics
Aug 12, 2024
210280d
wip
Aug 15, 2024
d25adfb
add endpoint and project detection
Aug 15, 2024
f91c109
revert unnecessary changes
Aug 15, 2024
8a1cb19
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 16, 2024
09f37d6
Merge remote-tracking branch 'origin/main' into grpc-storage-metrics
Aug 20, 2024
f64fb6a
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 20, 2024
7d4e3fd
update documentation
Aug 21, 2024
e448378
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 21, 2024
edf13f9
remove unnecessary code and update docs
Aug 21, 2024
116afbd
address vet workflow feedback
Aug 21, 2024
8b5bd88
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 22, 2024
e422ebc
update missed name changes
Aug 22, 2024
ae16ca8
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 22, 2024
976adf6
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 22, 2024
af0b2f6
revert version downgrades
Aug 22, 2024
b94b03e
add exporterLogSuppressor; remove sdkmetric alias; log when metric fa…
Aug 22, 2024
0f46a93
fix typo after rename
Aug 22, 2024
0147355
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 23, 2024
1f76ef6
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 24, 2024
7a38f41
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 26, 2024
0a0a5be
remove internalMetricsConfig, remove use of internal, and clean up do…
Aug 26, 2024
686feb1
remove smoke test
Aug 26, 2024
1c53758
const metricPrefix
Aug 27, 2024
024e6d3
doc edit
Aug 27, 2024
a0a8edc
Merge remote-tracking branch 'origin' into grpc-storage-metrics
Aug 27, 2024
ccc7bd4
add additional comments and remove metric descriptions; fix failing test
Aug 28, 2024
a50c50c
Merge remote-tracking branch 'origin/main' into grpc-storage-metrics
Aug 28, 2024
41b469c
create ConnPool to access endpoint before GAPIC is initialized
Aug 28, 2024
be3f5d5
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 28, 2024
498b6e0
double up on ConnPool :(
Aug 28, 2024
ad9a8c5
Merge branch 'main' into grpc-storage-metrics
Aug 28, 2024
2334322
go work sync
Aug 28, 2024
c0610c8
clean up DialPool creation for metrics
Aug 28, 2024
83bb31b
add todo to remove workaround
Aug 28, 2024
75e8f6b
revert line removals
Aug 28, 2024
0348728
make sure we emit RPC name and remove initializing second connpool
Aug 29, 2024
5a5d8cb
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 29, 2024
48c0fdf
remove endpoint magic
Aug 29, 2024
1fa2883
edit docs
Aug 29, 2024
dab45eb
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
remove unnecessary code and update docs
  • Loading branch information
Frank Natividad
Frank Natividad committed Aug 21, 2024
commit edf13f98b1e898ec3b3948e7cce71146d5f2e495
5 changes: 3 additions & 2 deletions storage/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -364,10 +364,11 @@ gRPC telemetry discussed in [gRFC/66] and [gRFC/78] to
[Google Cloud Monitoring]. The metrics are accessible to through Cloud Monitoring API and you incur no additional cost for publishing the metrics.Google Cloud Support can use this information to more quickly diagnose

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
[Google Cloud Monitoring]. The metrics are accessible to through Cloud Monitoring API and you incur no additional cost for publishing the metrics.Google Cloud Support can use this information to more quickly diagnose
[Google Cloud Monitoring]. The metrics are accessible through Cloud Monitoring API and you incur no additional cost for publishing the metrics. Google Cloud Support can use this information to more quickly diagnose

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: line length, spaces. (here and below)

problems related to GCS and gRPC.
Sending this data does not incur any billing charges, and requires minimal
CPU (a single RPC every few minutes) or memory (a few KiB to batch the
CPU (a single RPC every minute) or memory (a few KiB to batch the
telemetry).

To access the metrics you can view them through [Cloud Monitoring Metric explorer] with the prefix `storage.googleapis.com/client`.
To access the metrics you can view them through [Cloud Monitoring Metric explorer] with the prefix `storage.googleapis.com/client`. Metrics are emitted
every minute.

You can disable metrics using the following example
when creating a new gRPC client.
Expand Down
54 changes: 16 additions & 38 deletions storage/grpc_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
"google.golang.org/api/option"
"google.golang.org/api/transport"
htransport "google.golang.org/api/transport/http"
estats "google.golang.org/grpc/experimental/stats"
"google.golang.org/grpc/stats/opentelemetry"
)

Expand Down Expand Up @@ -79,26 +78,8 @@ func sizeHistogramBoundaries() []float64 {
return boundaries
}

func getViewMasks(defaultMetrics map[estats.Metric]bool, additionalMetrics []estats.Metric) []sdkmetric.View {
views := []sdkmetric.View{}
for k, include := range defaultMetrics {
if !include {
continue
}
views = append(views, sdkmetric.NewView(sdkmetric.Instrument{
Name: string(k),
}, sdkmetric.Stream{Name: strings.ReplaceAll(string(k), ".", "/")}))
}
for _, m := range additionalMetrics {
views = append(views, sdkmetric.NewView(sdkmetric.Instrument{
Name: string(m),
}, sdkmetric.Stream{Name: strings.ReplaceAll(string(m), ".", "/")}))
}
return views
}

func metricFormatter(m metricdata.Metrics) string {
return "storage.googleapis.com/client/" + m.Name
return "storage.googleapis.com/client/" + strings.ReplaceAll(string(m.Name), ".", "/")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this use the monitoredResourceName const?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, from tests, monitored resource is capital Client and metric prefix is lower client. Pulling it up into consts.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Monitred resource requires storage.googleapis.com/Client while the metric is storage.googleapis.com/client

}

func gcpAttributeExpectedDefaults() []attribute.KeyValue {
Expand Down Expand Up @@ -219,31 +200,28 @@ func newGRPCMetricContext(ctx context.Context, config internalMetricsConfig) (*i
if err != nil {
return nil, err
}
metricsToEnable := []estats.Metric{
"grpc.lb.wrr.rr_fallback",
"grpc.lb.wrr.endpoint_weight_not_yet_usable",
"grpc.lb.wrr.endpoint_weight_stale",
"grpc.lb.wrr.endpoint_weights",
"grpc.lb.rls.cache_entries",
"grpc.lb.rls.cache_size",
"grpc.lb.rls.default_target_picks",
"grpc.lb.rls.target_picks",
"grpc.lb.rls.failed_picks",
metricViews := []sdkmetric.View{
createHistogramView("grpc.client.attempt.duration", "A view of grpc.client.attempt.duration with histogram boundaries more appropriate for Google Cloud Storage RPCs", "s", latencyHistogramBoundaries()),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What are these description strings used for?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They're exposed to the user in Monitoring Metric Explorer to describe the metric. That said, it should be cached so clarifying if we really need to have this hardcoded client side.

createHistogramView("grpc.client.attempt.rcvd_total_compressed_message_size", "A view of grpc.client.attempt.rcvd_total_compressed_message_size with histogram boundaries more appropriate for Google Cloud Storage RPCs", "By", sizeHistogramBoundaries()),
createHistogramView("grpc.client.attempt.sent_total_compressed_message_size", "A view of grpc.client.attempt.sent_total_compressed_message_size with histogram boundaries more appropriate for Google Cloud Storage RPCs", "By", sizeHistogramBoundaries()),
}
defaultMetrics := opentelemetry.DefaultMetrics()
metricViews := getViewMasks(defaultMetrics.Metrics(), metricsToEnable)
metricViews = append(metricViews,
createHistogramView("grpc/client/attempt/duration", "A view of grpc/client/attempt/duration with histogram boundaries more appropriate for Google Cloud Storage RPCs", "s", latencyHistogramBoundaries()),
createHistogramView("grpc/client/attempt/rcvd_total_compressed_message_size", "A view of grpc/client/attempt/rcvd_total_compressed_message_size with histogram boundaries more appropriate for Google Cloud Storage RPCs", "By", sizeHistogramBoundaries()),
createHistogramView("grpc/client/attempt/sent_total_compressed_message_size", "A view of grpc/client/attempt/sent_total_compressed_message_size with histogram boundaries more appropriate for Google Cloud Storage RPCs", "By", sizeHistogramBoundaries()))
provider := sdkmetric.NewMeterProvider(
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter, sdkmetric.WithInterval(time.Minute))),
sdkmetric.WithResource(preparedResource.resource),
sdkmetric.WithView(metricViews...),
)
mo := opentelemetry.MetricsOptions{
MeterProvider: provider,
Metrics: defaultMetrics.Add(metricsToEnable...),
MeterProvider: provider,
Metrics: opentelemetry.DefaultMetrics().Add(
"grpc.lb.wrr.rr_fallback",
"grpc.lb.wrr.endpoint_weight_not_yet_usable",
"grpc.lb.wrr.endpoint_weight_stale",
"grpc.lb.wrr.endpoint_weights",
"grpc.lb.rls.cache_entries",
"grpc.lb.rls.cache_size",
"grpc.lb.rls.default_target_picks",
"grpc.lb.rls.target_picks",
"grpc.lb.rls.failed_picks"),
OptionalLabels: []string{"grpc.lb.locality"},
}
do := option.WithGRPCDialOption(opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo}))
Expand Down
36 changes: 2 additions & 34 deletions storage/grpc_metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,9 @@ import (
"testing"

"go.opentelemetry.io/otel/attribute"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"go.opentelemetry.io/otel/sdk/resource"
"google.golang.org/api/iterator"
estats "google.golang.org/grpc/experimental/stats"
)

func TestMetrics(t *testing.T) {
Expand Down Expand Up @@ -81,39 +79,9 @@ func TestNonDefaultEndpoint(t *testing.T) {
}
}

func TestGetViewMasks(t *testing.T) {
testDefaultMetrics := map[estats.Metric]bool{
estats.Metric("default.metric.name"): true,
}
testAdditionalMetrics := []estats.Metric{"test.metric.name"}
views := getViewMasks(testDefaultMetrics, testAdditionalMetrics)
wantSlice := []struct {
inputFormat string
outputFormat string
}{
{
inputFormat: "default.metric.name",
outputFormat: "default/metric/name",
},
{
inputFormat: "test.metric.name",
outputFormat: "test/metric/name",
},
}
// Order matters for the wantSlice and views slice
for idx, want := range wantSlice {
stream, b := views[idx](sdkmetric.Instrument{
Name: want.inputFormat,
})
if !b || stream.Name != want.outputFormat {
t.Errorf("getViewMasks: For metric: %v got=%v, want=%v", want.inputFormat, stream.Name, want.outputFormat)
}
}
}

func TestMetricFormatter(t *testing.T) {
want := "storage.googleapis.com/client/metric"
s := metricdata.Metrics{Name: "metric", Description: "", Unit: "", Data: nil}
want := "storage.googleapis.com/client/metric/name"
s := metricdata.Metrics{Name: "metric.name", Description: "", Unit: "", Data: nil}
got := metricFormatter(s)
if want != got {
t.Errorf("got: %v, want %v", got, want)
Expand Down
⌂ 换网址