Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
7df55c0
feat: initial prototype testing client-side metrics
Jul 31, 2024
73d1742
added tests and refactored to help test
Aug 5, 2024
db28808
use 1.21 and refactor test
Aug 7, 2024
4883da3
fix go.sum go.work.sum
Aug 7, 2024
ddf0d41
add option to disable client metrics
Aug 12, 2024
210280d
wip
Aug 15, 2024
d25adfb
add endpoint and project detection
Aug 15, 2024
f91c109
revert unnecessary changes
Aug 15, 2024
8a1cb19
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 16, 2024
09f37d6
Merge remote-tracking branch 'origin/main' into grpc-storage-metrics
Aug 20, 2024
f64fb6a
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 20, 2024
7d4e3fd
update documentation
Aug 21, 2024
e448378
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 21, 2024
edf13f9
remove unnecessary code and update docs
Aug 21, 2024
116afbd
address vet workflow feedback
Aug 21, 2024
8b5bd88
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 22, 2024
e422ebc
update missed name changes
Aug 22, 2024
ae16ca8
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 22, 2024
976adf6
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 22, 2024
af0b2f6
revert version downgrades
Aug 22, 2024
b94b03e
add exporterLogSuppressor; remove sdkmetric alias; log when metric fa…
Aug 22, 2024
0f46a93
fix typo after rename
Aug 22, 2024
0147355
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 23, 2024
1f76ef6
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 24, 2024
7a38f41
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 26, 2024
0a0a5be
remove internalMetricsConfig, remove use of internal, and clean up do…
Aug 26, 2024
686feb1
remove smoke test
Aug 26, 2024
1c53758
const metricPrefix
Aug 27, 2024
024e6d3
doc edit
Aug 27, 2024
a0a8edc
Merge remote-tracking branch 'origin' into grpc-storage-metrics
Aug 27, 2024
ccc7bd4
add additional comments and remove metric descriptions; fix failing test
Aug 28, 2024
a50c50c
Merge remote-tracking branch 'origin/main' into grpc-storage-metrics
Aug 28, 2024
41b469c
create ConnPool to access endpoint before GAPIC is initialized
Aug 28, 2024
be3f5d5
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 28, 2024
498b6e0
double up on ConnPool :(
Aug 28, 2024
ad9a8c5
Merge branch 'main' into grpc-storage-metrics
Aug 28, 2024
2334322
go work sync
Aug 28, 2024
c0610c8
clean up DialPool creation for metrics
Aug 28, 2024
83bb31b
add todo to remove workaround
Aug 28, 2024
75e8f6b
revert line removals
Aug 28, 2024
0348728
make sure we emit RPC name and remove initializing second connpool
Aug 29, 2024
5a5d8cb
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 29, 2024
48c0fdf
remove endpoint magic
Aug 29, 2024
1fa2883
edit docs
Aug 29, 2024
dab45eb
Merge branch 'main' into grpc-storage-metrics
frankyn Aug 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
create ConnPool to access endpoint before GAPIC is initialized
  • Loading branch information
Frank Natividad
Frank Natividad committed Aug 28, 2024
commit 41b469c9b9ae3b538797c7ca3180b72397b03b93
24 changes: 22 additions & 2 deletions storage/grpc_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"hash/crc32"
"io"
"log"
"math"
"net/url"
"os"

Expand All @@ -34,6 +35,7 @@ import (
"google.golang.org/api/iterator"
"google.golang.org/api/option"
"google.golang.org/api/option/internaloption"
gtransport "google.golang.org/api/transport/grpc"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/encoding"
Expand Down Expand Up @@ -76,6 +78,16 @@ const (
func defaultGRPCOptions() []option.ClientOption {
defaults := []option.ClientOption{
option.WithGRPCConnectionPool(defaultConnPoolSize),
internaloption.WithDefaultEndpoint("storage.googleapis.com:443"),
internaloption.WithDefaultEndpointTemplate("storage.UNIVERSE_DOMAIN:443"),
internaloption.WithDefaultMTLSEndpoint("storage.mtls.googleapis.com:443"),
internaloption.WithDefaultUniverseDomain("googleapis.com"),
internaloption.WithDefaultAudience("https://storage.googleapis.com/"),
internaloption.WithDefaultScopes(gapic.DefaultAuthScopes()...),
internaloption.EnableJwtWithScope(),
internaloption.EnableNewAuthLibrary(),
option.WithGRPCDialOption(grpc.WithDefaultCallOptions(
grpc.MaxCallRecvMsgSize(math.MaxInt32))),
}

// Set emulator options for gRPC if an emulator was specified. Note that in a
Expand Down Expand Up @@ -124,16 +136,24 @@ func newGRPCStorageClient(ctx context.Context, opts ...storageOption) (storageCl
if config.readAPIWasSet {
return nil, errors.New("storage: GRPC is incompatible with any option that specifies an API for reads")
}
// Metrics requires the endpoint before GAPIC is created which is only available in connPool after GAPIC is created.
// Seperated initialization of pool beforehand and passing the instance to
// the GAPIC client.
pool, err := gtransport.DialPool(ctx, s.clientOption...)
if err != nil {
return nil, err
}
s.clientOption = append(s.clientOption, gtransport.WithConnPool(pool))
if !config.disableClientMetrics {
// Do not fail client creation if enabling metrics fails.
if metricsContext, err := enableClientMetrics(ctx, s); err == nil {
ep := pool.Conn().Target()
if metricsContext, err := enableClientMetrics(ctx, ep, s); err == nil {
s.metricsContext = metricsContext
s.clientOption = append(s.clientOption, metricsContext.clientOpts...)
} else {
log.Printf("Failed to enable client metrics: %v", err)
}
}

g, err := gapic.NewClient(ctx, s.clientOption...)
if err != nil {
return nil, err
Expand Down
7 changes: 1 addition & 6 deletions storage/grpc_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"go.opentelemetry.io/otel/sdk/resource"
"google.golang.org/api/option"
"google.golang.org/api/transport"
htransport "google.golang.org/api/transport/http"
"google.golang.org/grpc/stats/opentelemetry"
)

Expand Down Expand Up @@ -234,11 +233,7 @@ func newGRPCMetricContext(ctx context.Context, endpoint, project string) (*metri
return context, nil
}

func enableClientMetrics(ctx context.Context, s *settings) (*metricsContext, error) {
_, ep, err := htransport.NewClient(ctx, s.clientOption...)
if err != nil {
return nil, fmt.Errorf("gRPC Metrics: %w", err)
}
func enableClientMetrics(ctx context.Context, ep string, s *settings) (*metricsContext, error) {
project := ""
c, err := transport.Creds(ctx, s.clientOption...)
if err == nil {
Expand Down
⌂ 换网址