From bee724f738cf9c4bebe7680ac25588a6b57b20a1 Mon Sep 17 00:00:00 2001 From: Chris Cotter Date: Wed, 31 Jan 2024 20:10:04 +0000 Subject: [PATCH 01/12] [WIP] feat(storage/transfermanager): prototype sketch of an interface for downloads --- storage/tmsample/main.go | 88 +++++++++++++++++ storage/transfermanager/downloader.go | 130 ++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 storage/tmsample/main.go create mode 100644 storage/transfermanager/downloader.go diff --git a/storage/tmsample/main.go b/storage/tmsample/main.go new file mode 100644 index 000000000000..5551ca9f4df4 --- /dev/null +++ b/storage/tmsample/main.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "log" + "os" + "time" + + "cloud.google.com/go/storage" + "cloud.google.com/go/storage/transfermanager" +) + +func main() { + ctx := context.Background() + + // Pass in any client opts or set retry policy here + client, err := storage.NewClient(ctx) // can also use NewGRPCClient + if err != nil { + // handle error + } + + // Create Downloader with desired options, including number of workers, + // part size, per operation timeout, etc. + d, err := transfermanager.NewDownloader(client, transfermanager.WithWorkers(16)) + if err != nil { + // handle error + } + + // Create go routine to wait on & process download results. + // This is maybe too much to expect from users -- would making results + // available at the end of the job and/or a callback be preferable? + go func() { + for { + select { + case out := <- d.Output: + if out.Err != nil { + log.Printf("download of %v failed with error %v", out.Name, out.Err) + } else { + log.Printf("download of %v succeeded", out.Name) + } + case <- d.Done: + log.Printf("all downloads complete") + return + } + }() + } + + // Sharded and/or parallelized download + // Create local file writer for output + f, err := os.Create("/path/to/localfile") + if err != nil { + // handle error + } + + // Create download input + in := &transfermanager.DownloadObjectInput{ + Bucket: "mybucket", + Source: "myblob", + Destination: f, + // Optionally specify params to apply to download. + EncryptionKey: []byte("mykey"), + } + + // Can set timeout on this download using context. + ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + // Add to Downloader + d.DownloadObject(ctx, in) + + // Repeat if desired + + // Download many files to path + // Create query, using any GCS list objects options + dirIn := &transfermanager.DownloadDirectoryInput{ + Bucket: "mybucket", + LocalDirectory: "/path/to/dir", + + // Optional filtering within bucket. + Prefix: "objectprefix/", + MatchGlob: "objectprefix/**abc**", + } + + d.DownloadDirectory(ctx, dirIn) + + // Wait for all downloads to complete. + d.WaitAndClose() +} diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go new file mode 100644 index 000000000000..79d644ab9111 --- /dev/null +++ b/storage/transfermanager/downloader.go @@ -0,0 +1,130 @@ +package transfermanager + +import ( + "context" + "io" + "time" + + "cloud.google.com/go/storage" +) + +type TransferManagerOption interface { + apply(*transferManagerConfig) +} + +func WithWorkers(numWorkers int) TransferManagerOption { + return &withWorkers{numWorkers: numWorkers} +} + +type withWorkers struct { + numWorkers int +} + +func (ww withWorkers) apply(tm *transferManagerConfig) { + tm.numWorkers = ww.numWorkers +} + +func WithPartSize(partSize int) TransferManagerOption { + return &withPartSize{partSize: partSize} +} + +type withPartSize struct { + partSize int +} + +func (wps withPartSize) apply(tm *transferManagerConfig) { + tm.partSize = wps.partSize +} + +// etc for additional options. + +type transferManagerConfig struct { + // Workers in thread pool; default numCPU/2 based on previous benchmarks? + numWorkers int + // Size of shards to transfer; Python found 32 MiB to be good default for + // JSON downloads but gRPC may benefit from larger. + partSize int + // Timeout for a single operation (including all retries). + perOperationTimeout time.Duration + + // others, including opts that apply to uploads. +} + +// etc + +type Downloader struct { + ctx context.Context + client *storage.Client + config *transferManagerConfig + objectInputs []DownloadObjectInput // Will be sent via a channel to workers + directoryInputs []DownloadDirectoryInput // This does obj listing/os.File calls and then converts to object inputs + Output <-chan *DownloadOutput // Application receives on this channel for completed downloads. + Done <-chan bool + // etc +} + +// Create a new Downloader to add operations to. +// Choice of transport, etc is configured on the client that's passed in. +func NewDownloader(c *storage.Client, opts ...TransferManagerOption) (*Downloader, error) { + + return nil, nil +} + +// Input for a single object to download. +type DownloadObjectInput struct { + // Required fields + Bucket string + Source string + Destination io.WriterAt + + // Optional fields + Generation *int64 + Conditions *storage.Conditions + EncryptionKey []byte + Offset, Length int64 // if specified, reads only a range. +} + +// Download a single object. If it's larger than the specified part size, +// the operation will automatically be broken up into multiple range reads. +// This will initiate the download but is non-blocking; wait on Downloader.Output +// to process the results. +func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectInput) { +} + +type DownloadDirectoryInput struct { + // Required fields + Bucket string + LocalDirectory string + + // Optional fields to filter objects in bucket. Selection from storage.Query. + Prefix string + StartOffset string + EndOffset string + IncludeTrailingDelimiter bool // maybe unnecessary? + MatchGlob string + + // Maybe some others about local file naming. +} + +// Download a set of objects to a local path. Downloader will +// resolve the query and created the needed directory structure locally as the +// operations progress. +// This will initiate the download but is non-blocking; wait on Downloader.Output +// to process the result. +func (d *Downloader) DownloadDirectory(ctx context.Context, input *DownloadDirectoryInput) { +} + +// Waits for all outstanding downloads to complete. Then, closes the Output +// channel. +func (d *Downloader) WaitAndClose() error { + return nil +} + +// DownloadOutput provides output for a single object download, including all +// errors received while downloading object parts. If the download was successful, +// Attrs will be populated. +type DownloadOutput struct { + Name string // name of object + Err error // error occurring during download. Can use multi-error in Go 1.20+ if multiple failures. + Attrs *storage.ReaderObjectAttrs // Attributes of downloaded object, if successful. +} From 189f87b801ef8ee0f41a608ba76aad258abecbb2 Mon Sep 17 00:00:00 2001 From: Chris Cotter Date: Tue, 23 Apr 2024 00:09:26 +0000 Subject: [PATCH 02/12] result iterator --- go.work.sum | 58 ++++++++++++++++++++++++++- storage/tmsample/main.go | 39 +++++++++--------- storage/transfermanager/downloader.go | 23 ++++++++++- 3 files changed, 98 insertions(+), 22 deletions(-) diff --git a/go.work.sum b/go.work.sum index 2ebf9dcbf209..5891aebac6ce 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,4 +1,60 @@ +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.46.0/go.mod h1:V28hx+cUCZC9e3qcqszMb+Sbt8cQZtHTiXOmyDzoDOg= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go-v2 v1.16.10/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo= +github.com/aws/aws-sdk-go-v2/config v1.15.9/go.mod h1:rv/l/TbZo67kp99v/3Kb0qV6Fm1KEtKyruEV2GvVfgs= +github.com/aws/aws-sdk-go-v2/credentials v1.12.12/go.mod h1:vFHC2HifIWHebmoVsfpqliKuqbAY2LaVlvy03JzF4c4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.11/go.mod h1:38Asv/UyQbDNpSXCurZRlDMjzIl6J+wUe8vY3TtUuzA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.17/go.mod h1:6qtGip7sJEyvgsLjphRZWF9qPe3xJf1mL/MM01E35Wc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.11/go.mod h1:cYAfnB+9ZkmZWpQWmPDsuIGm4EA+6k2ZVtxKjw/XJBY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.18/go.mod h1:hTHq8hL4bAxJyng364s9d4IUGXZOs7Y5LSqAhIiIQ2A= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.18.3/go.mod h1:BrAJyOMrnwzYVQcP5ziqlCpnEuFfkNppZLzqDyW/YTg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.11/go.mod h1:OEofCUKF7Hri4ShOCokF6k6hGq9PCB2sywt/9rLSXjY= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.15/go.mod h1:dDVD4ElJRTQXx7dOQ59EkqGyNU9tnwy1RKln+oLIOTU= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.12/go.mod h1:b53qpmhHk7mTL2J/tfG6f38neZiyBQSiNXGCuNKq4+4= +github.com/aws/smithy-go v1.12.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/fullstorydev/grpcurl v1.8.7/go.mod h1:pVtM4qe3CMoLaIzYS8uvTuDj2jVYmXqMUkZeijnXp/E= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/hoisie/redis v0.0.0-20160730154456-b5c6e81454e0/go.mod h1:pMYMxVaKJqCDC1JUg/XbPJ4/fSazB25zORpFzqsIGIc= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/itchyny/gojq v0.12.9/go.mod h1:T4Ip7AETUXeGpD+436m+UEl3m3tokRgajd5pRfsR5oE= +github.com/itchyny/timefmt-go v0.1.4/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= +github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/miekg/dns v1.1.33/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -go.opentelemetry.io/contrib/detectors/gcp v1.24.0 h1:1Szzq5d735VbnwbEmwPUJ/FIpY9CPM9KYfCRsSNjPgw= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0/go.mod h1:qcTO4xHAxZLaLxPd60TdE88rxtItPHgHWqOhOGRr0as= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.44.0/go.mod h1:sTt30Evb7hJB/gEk27qLb1+l9n4Tb8HvHkR0Wx3S6CU= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/storage/tmsample/main.go b/storage/tmsample/main.go index 5551ca9f4df4..50e023fac659 100644 --- a/storage/tmsample/main.go +++ b/storage/tmsample/main.go @@ -8,6 +8,7 @@ import ( "cloud.google.com/go/storage" "cloud.google.com/go/storage/transfermanager" + "google.golang.org/api/iterator" ) func main() { @@ -26,25 +27,6 @@ func main() { // handle error } - // Create go routine to wait on & process download results. - // This is maybe too much to expect from users -- would making results - // available at the end of the job and/or a callback be preferable? - go func() { - for { - select { - case out := <- d.Output: - if out.Err != nil { - log.Printf("download of %v failed with error %v", out.Name, out.Err) - } else { - log.Printf("download of %v succeeded", out.Name) - } - case <- d.Done: - log.Printf("all downloads complete") - return - } - }() - } - // Sharded and/or parallelized download // Create local file writer for output f, err := os.Create("/path/to/localfile") @@ -85,4 +67,23 @@ func main() { // Wait for all downloads to complete. d.WaitAndClose() + + // Iterate through completed downloads and process results. This can + // also happen async in a go routine as the downloads run. + it := d.Results() + for { + out, err := it.Next() // if async, blocks until next result is available. + if err == iterator.Done { + break + } + if err != nil { + log.Fatalf("error getting next result: %v") + } + if out.Err != nil { + log.Printf("download of %v failed with error %v", out.Name, out.Err) + } else { + log.Printf("download of %v succeeded", out.Name) + } + } + } diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go index 79d644ab9111..60f393a784f4 100644 --- a/storage/transfermanager/downloader.go +++ b/storage/transfermanager/downloader.go @@ -58,8 +58,8 @@ type Downloader struct { config *transferManagerConfig objectInputs []DownloadObjectInput // Will be sent via a channel to workers directoryInputs []DownloadDirectoryInput // This does obj listing/os.File calls and then converts to object inputs - Output <-chan *DownloadOutput // Application receives on this channel for completed downloads. - Done <-chan bool + output <-chan *DownloadOutput // Channel for completed downloads; used to feed results iterator + done <-chan bool // Used to signal completion of all downloads. // etc } @@ -120,6 +120,11 @@ func (d *Downloader) WaitAndClose() error { return nil } +// Results returns the iterator for download outputs. +func (d *Downloader) Results() *DownloadOutputIterator { + return nil +} + // DownloadOutput provides output for a single object download, including all // errors received while downloading object parts. If the download was successful, // Attrs will be populated. @@ -128,3 +133,17 @@ type DownloadOutput struct { Err error // error occurring during download. Can use multi-error in Go 1.20+ if multiple failures. Attrs *storage.ReaderObjectAttrs // Attributes of downloaded object, if successful. } + +// DownloadOutputIterator allows the end user to iterate through completed +// object downloads. +type DownloadOutputIterator struct { + // unexported fields including buffered DownloadOutputs +} + +// Use this to iterate through results. When complete, will return error +// iterator.Done. +// DownloadOutputs will be available as the downloads complete; they can +// be iterated through asynchronously or at the end of the job. +func (it *DownloadOutputIterator) Next() (*DownloadOutput, error) { + return nil, nil +} From a23db2e35a4ff1ce79173099c0f6eeb0098c224f Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Thu, 25 Apr 2024 10:09:38 -0700 Subject: [PATCH 03/12] mvp, just missing some tests --- storage/transfermanager/downloader.go | 276 +++++++++++------ storage/transfermanager/integration_test.go | 326 ++++++++++++++++++++ storage/transfermanager/option.go | 102 ++++++ storage/transfermanager/option_test.go | 41 +++ 4 files changed, 654 insertions(+), 91 deletions(-) create mode 100644 storage/transfermanager/integration_test.go create mode 100644 storage/transfermanager/option.go create mode 100644 storage/transfermanager/option_test.go diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go index 60f393a784f4..a3686464c068 100644 --- a/storage/transfermanager/downloader.go +++ b/storage/transfermanager/downloader.go @@ -1,94 +1,199 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package transfermanager import ( "context" + "errors" "io" + "sync" "time" "cloud.google.com/go/storage" + "google.golang.org/api/iterator" ) -type TransferManagerOption interface { - apply(*transferManagerConfig) -} - -func WithWorkers(numWorkers int) TransferManagerOption { - return &withWorkers{numWorkers: numWorkers} -} - -type withWorkers struct { - numWorkers int -} - -func (ww withWorkers) apply(tm *transferManagerConfig) { - tm.numWorkers = ww.numWorkers -} - -func WithPartSize(partSize int) TransferManagerOption { - return &withPartSize{partSize: partSize} -} - -type withPartSize struct { - partSize int +// Downloader manages parallel download operations from a Cloud Storage bucket. +type Downloader struct { + client *storage.Client + config *transferManagerConfig + work chan *DownloadObjectInput // Piece of work to be executed. + output chan *DownloadOutput // Channel for completed downloads; used to feed results iterator. + workers *sync.WaitGroup // Keeps track of the workers that are currently running. } -func (wps withPartSize) apply(tm *transferManagerConfig) { - tm.partSize = wps.partSize +// DownloadObject queues the download of a single object. If it's larger than +// the specified part size, the download will automatically be broken up into +// multiple range reads. This will initiate the download but is non-blocking; +// call Downloader.Results to process the result. +func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectInput) { + input.ctx = ctx + d.work <- input } -// etc for additional options. - -type transferManagerConfig struct { - // Workers in thread pool; default numCPU/2 based on previous benchmarks? - numWorkers int - // Size of shards to transfer; Python found 32 MiB to be good default for - // JSON downloads but gRPC may benefit from larger. - partSize int - // Timeout for a single operation (including all retries). - perOperationTimeout time.Duration - - // others, including opts that apply to uploads. +// Download a set of objects to a local path. Downloader will create the needed +// directory structure locally as the operations progress. +// This will initiate the download but is non-blocking; wait on Downloader.Results +// to process the result. Results will be split into individual objects. +// NOTE: Do not use, DownloadDirectory is not implemented. +func (d *Downloader) DownloadDirectory(ctx context.Context, input *DownloadDirectoryInput) { + d.output <- &DownloadOutput{Bucket: input.Bucket, Err: errors.New("DownloadDirectory is not implemented")} + // This does obj listing/os.File calls and then converts to object inputs. } -// etc - -type Downloader struct { - ctx context.Context - client *storage.Client - config *transferManagerConfig - objectInputs []DownloadObjectInput // Will be sent via a channel to workers - directoryInputs []DownloadDirectoryInput // This does obj listing/os.File calls and then converts to object inputs - output <-chan *DownloadOutput // Channel for completed downloads; used to feed results iterator - done <-chan bool // Used to signal completion of all downloads. - // etc +// Waits for all outstanding downloads to complete. The Downloader must not be +// used to download more objects or directories after this has been called. +func (d *Downloader) WaitAndClose() error { + close(d.work) + d.workers.Wait() + close(d.output) + return nil } -// Create a new Downloader to add operations to. +// Results returns the iterator for download outputs. +func (d *Downloader) Results() *DownloadOutputIterator { + return &DownloadOutputIterator{ + output: d.output, + } +} + +// downloadWorker continuously processes downloads until the work channel is closed. +func (d *Downloader) downloadWorker() { + d.workers.Add(1) + for { + input, ok := <-d.work + if !ok { + break // no more work; exit + } + + // TODO: break down the input into smaller pieces if necessary; maybe as follows: + // Only request partSize data to begin with. If no error and we haven't finished + // reading the object, enqueue the remaining pieces of work and mark in the + // out var the amount of shards to wait for. + d.output <- input.downloadShard(d.client, d.config.perOperationTimeout) + } + d.workers.Done() +} + +// NewDownloader creates a new Downloader to add operations to. // Choice of transport, etc is configured on the client that's passed in. func NewDownloader(c *storage.Client, opts ...TransferManagerOption) (*Downloader, error) { - - return nil, nil -} - -// Input for a single object to download. + const ( + chanBufferSize = 1000 // how big is it reasonable to make this? + // We should probably expose this as max concurrent ops, because programs can deadlock if calling d.Waitclose before processing it.Next + ) + + d := &Downloader{ + client: c, + config: initTransferManagerConfig(opts...), + output: make(chan *DownloadOutput, chanBufferSize), + work: make(chan *DownloadObjectInput, chanBufferSize), + workers: &sync.WaitGroup{}, + } + + // Start workers in background. + for i := 0; i < d.config.numWorkers; i++ { + go d.downloadWorker() + } + + return d, nil +} + +// DownloadRange specifies the object range. +type DownloadRange struct { + // Offset is the starting offset (inclusive) from with the object is read. + // If offset is negative, the object is read abs(offset) bytes from the end, + // and length must also be negative to indicate all remaining bytes will be read. + Offset int64 + // Length is the number of bytes to read. + // If length is negative or larger than the object size, the object is read + // until the end. + Length int64 +} + +// DownloadObjectInput is the input for a single object to download. type DownloadObjectInput struct { // Required fields Bucket string - Source string + Object string Destination io.WriterAt // Optional fields - Generation *int64 - Conditions *storage.Conditions - EncryptionKey []byte - Offset, Length int64 // if specified, reads only a range. -} - -// Download a single object. If it's larger than the specified part size, -// the operation will automatically be broken up into multiple range reads. -// This will initiate the download but is non-blocking; wait on Downloader.Output -// to process the results. -func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectInput) { + Generation *int64 + Conditions *storage.Conditions + EncryptionKey []byte + Range *DownloadRange // if specified, reads only a range + + ctx context.Context +} + +// downloadShard will read a specific object into in.Destination. +// TODO: download a single shard instead of the entire object. +func (in *DownloadObjectInput) downloadShard(client *storage.Client, timeout time.Duration) (out *DownloadOutput) { + out = &DownloadOutput{Bucket: in.Bucket, Object: in.Object} + + // Set timeout. + ctx := in.ctx + if timeout > 0 { + c, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + ctx = c + } + + // Set options on the object. + o := client.Bucket(in.Bucket).Object(in.Object) + + if in.Conditions != nil { + o.If(*in.Conditions) + } + if in.Generation != nil { + o = o.Generation(*in.Generation) + } + if len(in.EncryptionKey) > 0 { + o = o.Key(in.EncryptionKey) + } + + var offset, length int64 = 0, -1 // get the entire object by default + + if in.Range != nil { + offset, length = in.Range.Offset, in.Range.Length + } + + // Read. + r, err := o.NewRangeReader(ctx, offset, length) + if err != nil { + out.Err = err + return + } + + // TODO: write at a specific offset. + off := io.NewOffsetWriter(in.Destination, 0) + _, err = io.Copy(off, r) + if err != nil { + out.Err = err + r.Close() + return + } + + if err = r.Close(); err != nil { + out.Err = err + return + } + + out.Attrs = &r.Attrs + return } type DownloadDirectoryInput struct { @@ -106,44 +211,33 @@ type DownloadDirectoryInput struct { // Maybe some others about local file naming. } -// Download a set of objects to a local path. Downloader will -// resolve the query and created the needed directory structure locally as the -// operations progress. -// This will initiate the download but is non-blocking; wait on Downloader.Output -// to process the result. -func (d *Downloader) DownloadDirectory(ctx context.Context, input *DownloadDirectoryInput) { -} - -// Waits for all outstanding downloads to complete. Then, closes the Output -// channel. -func (d *Downloader) WaitAndClose() error { - return nil -} - -// Results returns the iterator for download outputs. -func (d *Downloader) Results() *DownloadOutputIterator { - return nil -} - // DownloadOutput provides output for a single object download, including all // errors received while downloading object parts. If the download was successful, // Attrs will be populated. type DownloadOutput struct { - Name string // name of object - Err error // error occurring during download. Can use multi-error in Go 1.20+ if multiple failures. - Attrs *storage.ReaderObjectAttrs // Attributes of downloaded object, if successful. + Bucket string + Object string + Err error // error occurring during download. Can use multi-error in Go 1.20+ if multiple failures. + Attrs *storage.ReaderObjectAttrs // Attributes of downloaded object, if successful. } // DownloadOutputIterator allows the end user to iterate through completed // object downloads. type DownloadOutputIterator struct { - // unexported fields including buffered DownloadOutputs + output <-chan *DownloadOutput } -// Use this to iterate through results. When complete, will return error -// iterator.Done. +// Next iterates through results. When complete, will return the iterator.Done +// error. It is considered complete once WaitAndClose() has been called on the +// Downloader. // DownloadOutputs will be available as the downloads complete; they can // be iterated through asynchronously or at the end of the job. +// Next will block if there are no more completed downloads (and the Downloader +// is not closed). func (it *DownloadOutputIterator) Next() (*DownloadOutput, error) { - return nil, nil + out, ok := <-it.output + if !ok { + return nil, iterator.Done + } + return out, nil } diff --git a/storage/transfermanager/integration_test.go b/storage/transfermanager/integration_test.go new file mode 100644 index 000000000000..d581eafb96f3 --- /dev/null +++ b/storage/transfermanager/integration_test.go @@ -0,0 +1,326 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transfermanager + +import ( + "bytes" + "context" + "crypto/rand" + "hash/crc32" + "io" + "testing" + + "cloud.google.com/go/internal/testutil" + "cloud.google.com/go/internal/uid" + "cloud.google.com/go/storage" + "google.golang.org/api/iterator" +) + +const ( + testPrefix = "go-integration-test" + grpcTestPrefix = "golang-grpc-test" +) + +var uidSpace = uid.NewSpace("", &uid.Options{Short: true}) + +func TestIntegration_Downloader(t *testing.T) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + + bucketName := testPrefix + uidSpace.New() + b := client.Bucket(bucketName) + if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { + t.Fatalf("bucket.Create: %v", err) + } + t.Cleanup(func() { b.Delete(ctx) }) + + // Populate object names and make a map for their contents. + objects := []string{ + "obj1", + "obj2", + "obj/with/slashes", + "obj/", + "./obj", + "!#$&'()*+,/:;=,?@,[] and spaces", + } + contentHashes := make(map[string]uint32) + + // Write objects. + objectSize := int64(507) + for _, obj := range objects { + crc, err := generateFileInGCS(ctx, b.Object(obj), objectSize) + if err != nil { + t.Fatalf("generateFileInGCS: %v", err) + } + contentHashes[obj] = crc + + t.Cleanup(func() { b.Object(obj).Delete(ctx) }) + } + + // Start a downloader. + d, err := NewDownloader(client) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + + w := &testWriter{} + + // Download a single object. + d.DownloadObject(ctx, &DownloadObjectInput{ + Bucket: bucketName, + Object: objects[0], + Destination: w, + }) + + it := d.Results() + + got, err := it.Next() + if err != nil { + t.Fatalf("it.Next: %v", err) + } + + // Close the writer so we can check the contents. + if err := w.Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + } + + if got.Object != objects[0] || got.Bucket != bucketName { + t.Errorf("expected Bucket(%q).Object(%q), got %q.%q", bucketName, objects[0], got.Bucket, got.Object) + } + + if want, got := contentHashes[objects[0]], w.crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != objectSize { + t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + } + + // Download several objects. + writers := make([]*testWriter, len(objects)) + objToWriter := make(map[string]int) // so we can map the resulting content back to the correct object + for i, obj := range objects { + writers[i] = &testWriter{} + objToWriter[obj] = i + d.DownloadObject(ctx, &DownloadObjectInput{ + Bucket: bucketName, + Object: obj, + Destination: writers[i], + }) + } + + if err := d.WaitAndClose(); err != nil { + t.Fatalf("d.WaitAndClose: %v", err) + } + + // Close the writers so we can check the contents. This should be fine, + // since the downloads should all be complete after WaitAndClose. + for i := range objects { + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + } + + numResults := 0 + for { + got, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + t.Errorf("it.Next: %v", err) + } + + numResults++ + writerIdx := objToWriter[got.Object] + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + continue + } + + if want, got := contentHashes[got.Object], writers[writerIdx].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != objectSize { + t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + } + } + + if numResults != len(objects) { + t.Errorf("expected to receive %d results, got %d results", len(objects), numResults) + } + +} + +// generateRandomFileInGCS uploads a file with random contents to GCS and returns +// the crc32c hash of the contents. +func generateFileInGCS(ctx context.Context, o *storage.ObjectHandle, size int64) (uint32, error) { + w := o.Retryer(storage.WithPolicy(storage.RetryAlways)).NewWriter(ctx) + + crc32cHash := crc32.New(crc32.MakeTable(crc32.Castagnoli)) + mw := io.MultiWriter(w, crc32cHash) + + if _, err := io.CopyN(mw, rand.Reader, size); err != nil { + w.Close() + return 0, err + } + return crc32cHash.Sum32(), w.Close() +} + +type testWriter struct { + b []byte + crc32c uint32 + bufs [][]byte // temp +} + +// Close must be called to finalize the buffer +func (tw *testWriter) Close() error { + tw.b = bytes.Join(tw.bufs, nil) + crc := crc32.New(crc32.MakeTable(crc32.Castagnoli)) + + _, err := io.Copy(crc, bytes.NewReader(tw.b)) + tw.crc32c = crc.Sum32() + return err +} + +func (tw *testWriter) WriteAt(b []byte, offset int64) (n int, err error) { + // TODO: use the offset. This is fine for now since reads are not yet sharded. + copiedB := make([]byte, len(b)) + copy(copiedB, b) + tw.bufs = append(tw.bufs, copiedB) + + return len(b), nil +} + +func TestIntegration_DownloadShard(t *testing.T) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + + bucketName := testPrefix + uidSpace.New() + b := client.Bucket(bucketName) + if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { + t.Fatalf("bucket.Create: %v", err) + } + t.Cleanup(func() { b.Delete(ctx) }) + + // Write object. + objectName := "obj" + uidSpace.New() + contentHashes := make(map[string]uint32) + + objectSize := int64(2 * 1024 * 1024) + crc, err := generateFileInGCS(ctx, b.Object(objectName), objectSize) + if err != nil { + t.Fatalf("generateFileInGCS: %v", err) + } + + t.Cleanup(func() { b.Object(objectName).Delete(ctx) }) + + // Start a downloader. + d, err := NewDownloader(client) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + + // Download the object normally ? + + for _, test := range []struct { + desc string + in *DownloadObjectInput + want *DownloadOutput + }{ + { + desc: "basic input", + in: &DownloadObjectInput{ + Bucket: bucketName, + Object: objectName, + //Destination: w, + }, + want: &DownloadOutput{ + Bucket: bucketName, + Object: objectName, + Attrs: &storage.ReaderObjectAttrs{ + Size: objectSize, + StartOffset: 0, + ContentType: "", + ContentEncoding: "", + CacheControl: "", + //LastModified: time.Time{}, + Generation: 0, + Metageneration: 0, + }, + }, + }, + { + desc: "range", + }, + { + desc: "incorrect generation", + }, + { + desc: "conditions: generationmatch", + }, + } { + t.Run(test.desc, func(t *testing.T) { + w := &testWriter{} + + // Download a single object. + + // in.downloadShard() + + // it := d.Results() + + // got, err := it.Next() + // if err != nil { + // t.Fatalf("it.Next: %v", err) + // } + + // // Close the writer so we can check the contents. + // if err := w.Close(); err != nil { + // t.Fatalf("testWriter.Close: %v", err) + // } + + // if got.Err != nil { + // t.Errorf("result.Err: %v", got.Err) + // } + + // if got.Object != objects[0] || got.Bucket != bucketName { + // t.Errorf("expected Bucket(%q).Object(%q), got %q.%q", bucketName, objects[0], got.Bucket, got.Object) + // } + + // if want, got := contentHashes[objects[0]], w.crc32c; got != want { + // t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + // } + + // if got.Attrs.Size != objectSize { + // t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + // } + }) + } + +} + +// test ctx cancel and per op timeout diff --git a/storage/transfermanager/option.go b/storage/transfermanager/option.go new file mode 100644 index 000000000000..fc77d4ef40eb --- /dev/null +++ b/storage/transfermanager/option.go @@ -0,0 +1,102 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transfermanager + +import ( + "runtime" + "time" +) + +// A TransferManagerOption is an option for a transfermanager Downloader or Uploader. +type TransferManagerOption interface { + apply(*transferManagerConfig) +} + +// WithWorkers returns a TransferManagerOption that specifies the maximum number +// of concurrent goroutines that will be used to download or upload objects. +// Defaults to runtime.NumCPU()/2. +func WithWorkers(numWorkers int) TransferManagerOption { + return &withWorkers{numWorkers: numWorkers} +} + +type withWorkers struct { + numWorkers int +} + +func (ww withWorkers) apply(tm *transferManagerConfig) { + tm.numWorkers = ww.numWorkers +} + +// WithPartSize returns a TransferManagerOption that specifies the size of the +// shards to transfer; that is, if the object is larger than this size, it will +// be uploaded or downloaded in concurrent pieces. +// The default is 32 MiB for downloads. +// NOTE: Sharding is not yet implemented. +func WithPartSize(partSize int) TransferManagerOption { + return &withPartSize{partSize: partSize} +} + +type withPartSize struct { + partSize int +} + +func (wps withPartSize) apply(tm *transferManagerConfig) { + tm.partSize = wps.partSize +} + +// WithPerOpTimeout returns a TransferManagerOption that sets a timeout on each +// operation that is performed to download or upload an object. The timeout is +// set when the operation begins processing, not when it is added. +// By default, no timeout is set. +func WithPerOpTimeout(timeout time.Duration) TransferManagerOption { + return &withPerOpTimeout{timeout: timeout} +} + +type withPerOpTimeout struct { + timeout time.Duration +} + +func (wpt withPerOpTimeout) apply(tm *transferManagerConfig) { + tm.perOperationTimeout = wpt.timeout +} + +type transferManagerConfig struct { + // Workers in thread pool; default numCPU/2 based on previous benchmarks? + numWorkers int + // Size of shards to transfer; Python found 32 MiB to be good default for + // JSON downloads but gRPC may benefit from larger. + partSize int + // Timeout for a single operation (including all retries). Zero value means + // no timeout. + perOperationTimeout time.Duration +} + +func defaultTransferManagerConfig() *transferManagerConfig { + return &transferManagerConfig{ + numWorkers: runtime.NumCPU() / 2, + partSize: 32 * 1024 * 1024, // 32 MiB + perOperationTimeout: 0, // no timeout + } +} + +// initTransferManagerConfig initializes a config with the defaults and applies +// the options passed in. +func initTransferManagerConfig(opts ...TransferManagerOption) *transferManagerConfig { + config := defaultTransferManagerConfig() + for _, o := range opts { + o.apply(config) + } + return config +} diff --git a/storage/transfermanager/option_test.go b/storage/transfermanager/option_test.go new file mode 100644 index 000000000000..8f78eadbb0d9 --- /dev/null +++ b/storage/transfermanager/option_test.go @@ -0,0 +1,41 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transfermanager + +import ( + "testing" + "time" +) + +func TestApply(t *testing.T) { + opts := []TransferManagerOption{ + WithWorkers(3), + WithPartSize(2000), + WithPerOpTimeout(time.Hour), + } + var got transferManagerConfig + for _, opt := range opts { + opt.apply(&got) + } + want := transferManagerConfig{ + numWorkers: 3, + partSize: 2000, + perOperationTimeout: time.Hour, + } + + if got != want { + t.Errorf("got: %+v, want: %+v", got, want) + } +} From bbdd5a642f8704b50d2be244bdf1e3a226d4383b Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Fri, 10 May 2024 14:00:06 -0700 Subject: [PATCH 04/12] wip - callbacks and sliced results --- storage/tmsample/main.go | 89 --- storage/transfermanager/downloader.go | 299 ++++++++--- storage/transfermanager/integration_test.go | 567 ++++++++++++++------ storage/transfermanager/option.go | 41 +- storage/transfermanager/option_test.go | 2 - 5 files changed, 654 insertions(+), 344 deletions(-) delete mode 100644 storage/tmsample/main.go diff --git a/storage/tmsample/main.go b/storage/tmsample/main.go deleted file mode 100644 index 50e023fac659..000000000000 --- a/storage/tmsample/main.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - "context" - "log" - "os" - "time" - - "cloud.google.com/go/storage" - "cloud.google.com/go/storage/transfermanager" - "google.golang.org/api/iterator" -) - -func main() { - ctx := context.Background() - - // Pass in any client opts or set retry policy here - client, err := storage.NewClient(ctx) // can also use NewGRPCClient - if err != nil { - // handle error - } - - // Create Downloader with desired options, including number of workers, - // part size, per operation timeout, etc. - d, err := transfermanager.NewDownloader(client, transfermanager.WithWorkers(16)) - if err != nil { - // handle error - } - - // Sharded and/or parallelized download - // Create local file writer for output - f, err := os.Create("/path/to/localfile") - if err != nil { - // handle error - } - - // Create download input - in := &transfermanager.DownloadObjectInput{ - Bucket: "mybucket", - Source: "myblob", - Destination: f, - // Optionally specify params to apply to download. - EncryptionKey: []byte("mykey"), - } - - // Can set timeout on this download using context. - ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - - // Add to Downloader - d.DownloadObject(ctx, in) - - // Repeat if desired - - // Download many files to path - // Create query, using any GCS list objects options - dirIn := &transfermanager.DownloadDirectoryInput{ - Bucket: "mybucket", - LocalDirectory: "/path/to/dir", - - // Optional filtering within bucket. - Prefix: "objectprefix/", - MatchGlob: "objectprefix/**abc**", - } - - d.DownloadDirectory(ctx, dirIn) - - // Wait for all downloads to complete. - d.WaitAndClose() - - // Iterate through completed downloads and process results. This can - // also happen async in a go routine as the downloads run. - it := d.Results() - for { - out, err := it.Next() // if async, blocks until next result is available. - if err == iterator.Done { - break - } - if err != nil { - log.Fatalf("error getting next result: %v") - } - if out.Err != nil { - log.Printf("download of %v failed with error %v", out.Name, out.Err) - } else { - log.Printf("download of %v succeeded", out.Name) - } - } - -} diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go index a3686464c068..3b09de6a6874 100644 --- a/storage/transfermanager/downloader.go +++ b/storage/transfermanager/downloader.go @@ -22,56 +22,98 @@ import ( "time" "cloud.google.com/go/storage" - "google.golang.org/api/iterator" ) -// Downloader manages parallel download operations from a Cloud Storage bucket. +// Downloader manages a set of parallelized downloads. type Downloader struct { - client *storage.Client - config *transferManagerConfig - work chan *DownloadObjectInput // Piece of work to be executed. - output chan *DownloadOutput // Channel for completed downloads; used to feed results iterator. - workers *sync.WaitGroup // Keeps track of the workers that are currently running. + client *storage.Client + config *transferManagerConfig + inputs []DownloadObjectInput + results []DownloadOutput + errors []error + inputsMu *sync.Mutex + resultsMu *sync.Mutex + errorsMu *sync.Mutex + work chan *DownloadObjectInput // Piece of work to be executed. + done chan bool // Indicates to finish up work; expecting no more inputs. + workers *sync.WaitGroup // Keeps track of the workers that are currently running. } -// DownloadObject queues the download of a single object. If it's larger than -// the specified part size, the download will automatically be broken up into -// multiple range reads. This will initiate the download but is non-blocking; -// call Downloader.Results to process the result. +// DownloadObject queues the download of a single object. This will initiate the +// download but is non-blocking; call Downloader.Results to process the result. func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectInput) { input.ctx = ctx - d.work <- input + d.inputsMu.Lock() + d.inputs = append(d.inputs, *input) + d.inputsMu.Unlock() } -// Download a set of objects to a local path. Downloader will create the needed -// directory structure locally as the operations progress. -// This will initiate the download but is non-blocking; wait on Downloader.Results -// to process the result. Results will be split into individual objects. -// NOTE: Do not use, DownloadDirectory is not implemented. -func (d *Downloader) DownloadDirectory(ctx context.Context, input *DownloadDirectoryInput) { - d.output <- &DownloadOutput{Bucket: input.Bucket, Err: errors.New("DownloadDirectory is not implemented")} - // This does obj listing/os.File calls and then converts to object inputs. +// DownloadObject queues the download of a single object. This will initiate the +// download but is non-blocking. The result will not be added to the results obtained +// from Downloader.Results; use the callback to process the result. +func (d *Downloader) DownloadObjectWithCallback(ctx context.Context, input *DownloadObjectInput, callback func(*DownloadOutput)) { + input.ctx = ctx + input.callback = &callback + d.inputsMu.Lock() + d.inputs = append(d.inputs, *input) + d.inputsMu.Unlock() } -// Waits for all outstanding downloads to complete. The Downloader must not be -// used to download more objects or directories after this has been called. +// WaitAndClose waits for all outstanding downloads to complete. The Downloader +// must not be used for any more downloads after this has been called. +// WaitAndClose returns an error if any of the downloads fail. func (d *Downloader) WaitAndClose() error { - close(d.work) + d.done <- true d.workers.Wait() - close(d.output) + + if len(d.errors) > 0 { + // TODO: return a multierror instead with go 1.20+ Join + return errors.New("transfermanager: at least one error encountered downloading objects") + } return nil } // Results returns the iterator for download outputs. -func (d *Downloader) Results() *DownloadOutputIterator { - return &DownloadOutputIterator{ - output: d.output, +func (d *Downloader) Results() []DownloadOutput { + return d.results +} + +// sendInputsToWorkChan listens continuously to the inputs slice until d.done. +// It will send all items in inputs to the d.work chan. +// Once it receives from d.done, it drains the remaining items in the inputs +// (sending them to d.work) and then closes the d.work chan. +func (d *Downloader) sendInputsToWorkChan() { + for { + select { + case <-d.done: + d.drainInput() + close(d.work) + return + default: + d.drainInput() + } + } +} + +// drainInput consumes everything in the inputs slice and sends it to the work chan. +// It will block if there are not enough workers to consume every input, until all +// inputs are received on the work chan(ie. they're dispatched to an available worker). +func (d *Downloader) drainInput() { + for { + d.inputsMu.Lock() + if len(d.inputs) < 1 { + d.inputsMu.Unlock() + return + } + input := d.inputs[0] + d.inputs = d.inputs[1:] + d.inputsMu.Unlock() + d.work <- &input } } // downloadWorker continuously processes downloads until the work channel is closed. func (d *Downloader) downloadWorker() { - d.workers.Add(1) for { input, ok := <-d.work if !ok { @@ -82,7 +124,23 @@ func (d *Downloader) downloadWorker() { // Only request partSize data to begin with. If no error and we haven't finished // reading the object, enqueue the remaining pieces of work and mark in the // out var the amount of shards to wait for. - d.output <- input.downloadShard(d.client, d.config.perOperationTimeout) + out := input.downloadShard(d.client, d.config.perOperationTimeout) + + // Keep track of any error that occurred. + if out.Err != nil { + d.errorsMu.Lock() + d.errors = append(d.errors, out.Err) + d.errorsMu.Unlock() + } + + // Either execute the callback, or append to results. + if input.callback != nil { + (*input.callback)(out) + } else { + d.resultsMu.Lock() + d.results = append(d.results, *out) + d.resultsMu.Unlock() + } } d.workers.Done() } @@ -90,21 +148,26 @@ func (d *Downloader) downloadWorker() { // NewDownloader creates a new Downloader to add operations to. // Choice of transport, etc is configured on the client that's passed in. func NewDownloader(c *storage.Client, opts ...TransferManagerOption) (*Downloader, error) { - const ( - chanBufferSize = 1000 // how big is it reasonable to make this? - // We should probably expose this as max concurrent ops, because programs can deadlock if calling d.Waitclose before processing it.Next - ) - d := &Downloader{ - client: c, - config: initTransferManagerConfig(opts...), - output: make(chan *DownloadOutput, chanBufferSize), - work: make(chan *DownloadObjectInput, chanBufferSize), - workers: &sync.WaitGroup{}, + client: c, + config: initTransferManagerConfig(opts...), + inputs: []DownloadObjectInput{}, + results: []DownloadOutput{}, + errors: []error{}, + inputsMu: &sync.Mutex{}, + resultsMu: &sync.Mutex{}, + errorsMu: &sync.Mutex{}, + work: make(chan *DownloadObjectInput), + done: make(chan bool), + workers: &sync.WaitGroup{}, } - // Start workers in background. + // Start a listener to send work through. + go d.sendInputsToWorkChan() + + // Start workers. for i := 0; i < d.config.numWorkers; i++ { + d.workers.Add(1) go d.downloadWorker() } @@ -136,7 +199,8 @@ type DownloadObjectInput struct { EncryptionKey []byte Range *DownloadRange // if specified, reads only a range - ctx context.Context + ctx context.Context + callback *func(*DownloadOutput) } // downloadShard will read a specific object into in.Destination. @@ -196,48 +260,127 @@ func (in *DownloadObjectInput) downloadShard(client *storage.Client, timeout tim return } -type DownloadDirectoryInput struct { - // Required fields - Bucket string - LocalDirectory string - - // Optional fields to filter objects in bucket. Selection from storage.Query. - Prefix string - StartOffset string - EndOffset string - IncludeTrailingDelimiter bool // maybe unnecessary? - MatchGlob string - - // Maybe some others about local file naming. -} - // DownloadOutput provides output for a single object download, including all // errors received while downloading object parts. If the download was successful, // Attrs will be populated. type DownloadOutput struct { Bucket string Object string - Err error // error occurring during download. Can use multi-error in Go 1.20+ if multiple failures. - Attrs *storage.ReaderObjectAttrs // Attributes of downloaded object, if successful. -} - -// DownloadOutputIterator allows the end user to iterate through completed -// object downloads. -type DownloadOutputIterator struct { - output <-chan *DownloadOutput + Err error // error occurring during download + Attrs *storage.ReaderObjectAttrs // attributes of downloaded object, if successful } -// Next iterates through results. When complete, will return the iterator.Done -// error. It is considered complete once WaitAndClose() has been called on the -// Downloader. -// DownloadOutputs will be available as the downloads complete; they can -// be iterated through asynchronously or at the end of the job. -// Next will block if there are no more completed downloads (and the Downloader -// is not closed). -func (it *DownloadOutputIterator) Next() (*DownloadOutput, error) { - out, ok := <-it.output - if !ok { - return nil, iterator.Done - } - return out, nil -} +// // DownloadOutputIterator allows the end user to iterate through completed +// // object downloads. +// type DownloadOutputIterator struct { +// output <-chan *DownloadOutput +// } + +// // Next iterates through results. When complete, will return the iterator.Done +// // error. It is considered complete once WaitAndClose() has been called on the +// // Downloader. +// // Note that if there was an error reading an object, it will not be returned +// // by Next - check DownloadOutput.Err instead. +// // DownloadOutputs will be available as the downloads complete; they can +// // be iterated through asynchronously or at the end of the job. +// // Next will block if there are no more completed downloads (and the Downloader +// // is not closed). +// func (it *DownloadOutputIterator) Next() (*DownloadOutput, error) { +// out, ok := <-it.output +// if !ok { +// return nil, iterator.Done +// } +// fmt.Println(out) +// return out, nil +// } + +// // listenForAndDelegateWork will receive from the work chan, and start goroutines +// // to execute that work without blocking. +// // It should be called only once. +// func (d *Downloader) listenForAndDelegateWork1() { +// for { +// // Dequeue the work. Can block. +// input, ok := <-d.work +// if !ok { +// break // no more work; exit +// } + +// // Start a worker. This may block. +// d.workerGroup.Go(func() error { +// // Do the download. +// // TODO: break down the input into smaller pieces if necessary; maybe as follows: +// // Only request partSize data to begin with. If no error and we haven't finished +// // reading the object, enqueue the remaining pieces of work (by sending them to d.work) +// // and mark in the out var the amount of shards to wait for. +// out := input.downloadShard(d.client, d.config.perOperationTimeout) + +// // Send the output to be received by Next. This could block until received +// // (ie. the user calls Next), but it's okay; our worker has already returned. +// // Alternatively, we could feed these to a slice that we grab from in Next. +// // This would not block then, but would require synchronization of the slice. +// go func() { +// out := out +// d.output <- out +// }() + +// // We return the error here, to communicate to d.workergroup.Wait +// // that there has been an error. +// // Since the group does not use a shared context, this should not +// // affect any of the other operations using the group. +// // TO-DO: in addition to this, if we want WaitAndClose to return a +// // multi err, we will need to record these errors somewhere. +// // Where-ever we record those, it will need synchronization since +// // this is concurrent. +// return out.Err +// }) + +// } +// } + +// // drainInput consumes everything in the inputs slice and dispatches workers. +// // It will block if there are not enough workers to consume every input, until +// // all inputs are dispatched to an available worker. +// func (d *Downloader) drainInpu1t() { +// fmt.Println(len(d.inputs)) + +// for len(d.inputs) > 0 { +// d.inputsMu.Lock() +// if len(d.inputs) < 1 { +// return +// } +// input := d.inputs[0] +// d.inputs = d.inputs[1:] +// d.inputsMu.Unlock() + +// // Start a worker. This may block, but only if there aren't enough workers. +// d.workerGroup.Go(func() error { +// // Do the download. +// // TODO: break down the input into smaller pieces if necessary; maybe as follows: +// // Only request partSize data to begin with. If no error and we haven't finished +// // reading the object, enqueue the remaining pieces of work +// // and mark in the out var the amount of shards to wait for. +// out := input.downloadShard(d.client, d.config.perOperationTimeout) + +// // Either return the callback, or append to results. +// if input.callback != nil { +// (*input.callback)(out) +// } else { +// d.resultsMu.Lock() +// d.results = append(d.results, *out) +// fmt.Println(len(d.results)) + +// d.resultsMu.Unlock() +// } + +// // We return the error here, to communicate to d.workergroup.Wait +// // that there has been an error. +// // Since the group does not use a shared context, this should not +// // affect any of the other operations using the group. +// // TO-DO: in addition to this, if we want WaitAndClose to return a +// // multi err, we will need to record these errors somewhere. +// // Where-ever we record those, it will need synchronization since +// // this is concurrent. +// return out.Err +// }) +// } +// } diff --git a/storage/transfermanager/integration_test.go b/storage/transfermanager/integration_test.go index d581eafb96f3..9a6135b2c524 100644 --- a/storage/transfermanager/integration_test.go +++ b/storage/transfermanager/integration_test.go @@ -20,12 +20,12 @@ import ( "crypto/rand" "hash/crc32" "io" + "sync" "testing" "cloud.google.com/go/internal/testutil" "cloud.google.com/go/internal/uid" "cloud.google.com/go/storage" - "google.golang.org/api/iterator" ) const ( @@ -35,7 +35,7 @@ const ( var uidSpace = uid.NewSpace("", &uid.Options{Short: true}) -func TestIntegration_Downloader(t *testing.T) { +func TestIntegration_DownloaderSynchronous(t *testing.T) { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { @@ -72,47 +72,106 @@ func TestIntegration_Downloader(t *testing.T) { t.Cleanup(func() { b.Object(obj).Delete(ctx) }) } - // Start a downloader. - d, err := NewDownloader(client) + // Start a downloader. Give it a smaller amount of workers than objects, to + // make sure we aren't blocking anywhere. + d, err := NewDownloader(client, WithWorkers(2)) if err != nil { t.Fatalf("NewDownloader: %v", err) } + // Download several objects. + writers := make([]*testWriter, len(objects)) + objToWriter := make(map[string]int) // so we can map the resulting content back to the correct object + for i, obj := range objects { + writers[i] = &testWriter{} + objToWriter[obj] = i + d.DownloadObject(ctx, &DownloadObjectInput{ + Bucket: bucketName, + Object: obj, + Destination: writers[i], + }) + } - w := &testWriter{} + if err := d.WaitAndClose(); err != nil { + t.Fatalf("d.WaitAndClose: %v", err) + } - // Download a single object. - d.DownloadObject(ctx, &DownloadObjectInput{ - Bucket: bucketName, - Object: objects[0], - Destination: w, - }) + // Close the writers so we can check the contents. This should be fine, + // since the downloads should all be complete after WaitAndClose. + for i := range objects { + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + } - it := d.Results() + results := d.Results() - got, err := it.Next() - if err != nil { - t.Fatalf("it.Next: %v", err) + for _, got := range results { + writerIdx := objToWriter[got.Object] + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + continue + } + + if want, got := contentHashes[got.Object], writers[writerIdx].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != objectSize { + t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + } } - // Close the writer so we can check the contents. - if err := w.Close(); err != nil { - t.Fatalf("testWriter.Close: %v", err) + if len(results) != len(objects) { + t.Errorf("expected to receive %d results, got %d results", len(objects), len(results)) } +} - if got.Err != nil { - t.Errorf("result.Err: %v", got.Err) +// Tests that a single error does not affect the rest of the downloads. +func TestIntegration_DownloaderErrorSync(t *testing.T) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + + bucketName := testPrefix + uidSpace.New() + b := client.Bucket(bucketName) + if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { + t.Fatalf("bucket.Create: %v", err) } + t.Cleanup(func() { b.Delete(ctx) }) - if got.Object != objects[0] || got.Bucket != bucketName { - t.Errorf("expected Bucket(%q).Object(%q), got %q.%q", bucketName, objects[0], got.Bucket, got.Object) + // Populate object names and make a map for their contents. + objects := []string{ + "obj1", + "obj2", + "obj/with/slashes", } + contentHashes := make(map[string]uint32) - if want, got := contentHashes[objects[0]], w.crc32c; got != want { - t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + // Write objects. + objectSize := int64(507) + for _, obj := range objects { + crc, err := generateFileInGCS(ctx, b.Object(obj), objectSize) + if err != nil { + t.Fatalf("generateFileInGCS: %v", err) + } + contentHashes[obj] = crc + + t.Cleanup(func() { b.Object(obj).Delete(ctx) }) } - if got.Attrs.Size != objectSize { - t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + // Add another object to attempt to download; since it hasn't been written, + // this one will fail. Append to the start so that it will (likely) be + // attempted in the first 2 downloads. + nonexistentObject := "not-written" + objects = append([]string{nonexistentObject}, objects...) + + // Start a downloader. + d, err := NewDownloader(client, WithWorkers(2)) + if err != nil { + t.Fatalf("NewDownloader: %v", err) } // Download several objects. @@ -128,8 +187,9 @@ func TestIntegration_Downloader(t *testing.T) { }) } - if err := d.WaitAndClose(); err != nil { - t.Fatalf("d.WaitAndClose: %v", err) + // WaitAndClose should return an error since one of our downloads should have failed. + if err := d.WaitAndClose(); err == nil { + t.Error("d.WaitAndClose should return an error, instead got nil") } // Close the writers so we can check the contents. This should be fine, @@ -140,19 +200,20 @@ func TestIntegration_Downloader(t *testing.T) { } } - numResults := 0 - for { - got, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - t.Errorf("it.Next: %v", err) - } + results := d.Results() - numResults++ + for _, got := range results { writerIdx := objToWriter[got.Object] + // Check that the nonexistent object returned an error. + if got.Object == nonexistentObject { + if got.Err != storage.ErrObjectNotExist { + t.Errorf("Object(%q) should not exist, err found to be %v", got.Object, got.Err) + } + continue + } + + // All other objects should complete correctly. if got.Err != nil { t.Errorf("result.Err: %v", got.Err) continue @@ -167,12 +228,322 @@ func TestIntegration_Downloader(t *testing.T) { } } - if numResults != len(objects) { - t.Errorf("expected to receive %d results, got %d results", len(objects), numResults) + if len(results) != len(objects) { + t.Errorf("expected to receive %d results, got %d results", len(objects), len(results)) + } +} + +func TestIntegration_DownloaderAsynchronous(t *testing.T) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) } + bucketName := testPrefix + uidSpace.New() + b := client.Bucket(bucketName) + if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { + t.Fatalf("bucket.Create: %v", err) + } + t.Cleanup(func() { b.Delete(ctx) }) + + // Populate object names and make a map for their contents. + objects := []string{ + "obj1", + "obj2", + "obj/with/slashes", + "obj/", + "./obj", + "!#$&'()*+,/:;=,?@,[] and spaces", + } + contentHashes := make(map[string]uint32) + + // Write objects. + objectSize := int64(507) + for _, obj := range objects { + crc, err := generateFileInGCS(ctx, b.Object(obj), objectSize) + if err != nil { + t.Fatalf("generateFileInGCS: %v", err) + } + contentHashes[obj] = crc + + t.Cleanup(func() { b.Object(obj).Delete(ctx) }) + } + + // Start a downloader. Give it a smaller amount of workers than objects, to + // make sure we aren't blocking anywhere. + d, err := NewDownloader(client, WithWorkers(2)) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + + numCallbacks := 0 + callbackMu := sync.Mutex{} + + // Download several objects. + writers := make([]*testWriter, len(objects)) + for i, obj := range objects { + i := i + writers[i] = &testWriter{} + d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + Bucket: bucketName, + Object: obj, + Destination: writers[i], + }, func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + } + + // Close the writer so we can check the contents. + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + + if want, got := contentHashes[got.Object], writers[i].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != objectSize { + t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + } + }) + } + + if err := d.WaitAndClose(); err != nil { + t.Fatalf("d.WaitAndClose: %v", err) + } + + if numCallbacks != len(objects) { + t.Errorf("expected to receive %d results, got %d callbacks", len(objects), numCallbacks) + } } +func TestIntegration_DownloaderErrorAsync(t *testing.T) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + + bucketName := testPrefix + uidSpace.New() + b := client.Bucket(bucketName) + if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { + t.Fatalf("bucket.Create: %v", err) + } + t.Cleanup(func() { b.Delete(ctx) }) + + // Populate object names and make a map for their contents. + objects := []string{ + "obj1", + "obj2", + "obj/with/slashes", + "obj/", + "./obj", + "!#$&'()*+,/:;=,?@,[] and spaces", + } + contentHashes := make(map[string]uint32) + + // Write objects. + objectSize := int64(507) + for _, obj := range objects { + crc, err := generateFileInGCS(ctx, b.Object(obj), objectSize) + if err != nil { + t.Fatalf("generateFileInGCS: %v", err) + } + contentHashes[obj] = crc + + t.Cleanup(func() { b.Object(obj).Delete(ctx) }) + } + + // Start a downloader. Give it a smaller amount of workers than objects, to + // make sure we aren't blocking anywhere. + d, err := NewDownloader(client, WithWorkers(2)) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + + numCallbacks := 0 + callbackMu := sync.Mutex{} + + // Download a non-existent object. + nonexistentObject := "not-written" + w := &testWriter{} + + d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + Bucket: bucketName, + Object: nonexistentObject, + Destination: w, + }, func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() + + // Check that the nonexistent object returned an error. + if got.Err != storage.ErrObjectNotExist { + t.Errorf("Object(%q) should not exist, err found to be %v", got.Object, got.Err) + } + }) + + // Download remaining objects. + writers := make([]*testWriter, len(objects)) + for i, obj := range objects { + i := i + writers[i] = &testWriter{} + d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + Bucket: bucketName, + Object: obj, + Destination: writers[i], + }, func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + } + + // Close the writer so we can check the contents. + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + + if want, got := contentHashes[got.Object], writers[i].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != objectSize { + t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + } + }) + } + + // WaitAndClose should return an error since one of our downloads should have failed. + if err := d.WaitAndClose(); err == nil { + t.Error("d.WaitAndClose should return an error, instead got nil") + } + + // We expect num objects callbacks + 1 for the errored call. + if numCallbacks != len(objects)+1 { + t.Errorf("expected to receive %d results, got %d callbacks", len(objects)+1, numCallbacks) + } +} + +// test ctx cancel and per op timeout + +// func TestIntegration_DownloadShard(t *testing.T) { +// ctx := context.Background() +// client, err := storage.NewClient(ctx) +// if err != nil { +// t.Fatalf("NewClient: %v", err) +// } + +// bucketName := testPrefix + uidSpace.New() +// b := client.Bucket(bucketName) +// if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { +// t.Fatalf("bucket.Create: %v", err) +// } +// t.Cleanup(func() { b.Delete(ctx) }) + +// // Write object. +// objectName := "obj" + uidSpace.New() +// contentHashes := make(map[string]uint32) + +// objectSize := int64(2 * 1024 * 1024) +// crc, err := generateFileInGCS(ctx, b.Object(objectName), objectSize) +// if err != nil { +// t.Fatalf("generateFileInGCS: %v", err) +// } + +// t.Cleanup(func() { b.Object(objectName).Delete(ctx) }) + +// // Start a downloader. +// d, err := NewDownloader(client) +// if err != nil { +// t.Fatalf("NewDownloader: %v", err) +// } + +// // Download the object normally ? + +// for _, test := range []struct { +// desc string +// in *DownloadObjectInput +// want *DownloadOutput +// }{ +// { +// desc: "basic input", +// in: &DownloadObjectInput{ +// Bucket: bucketName, +// Object: objectName, +// //Destination: w, +// }, +// want: &DownloadOutput{ +// Bucket: bucketName, +// Object: objectName, +// Attrs: &storage.ReaderObjectAttrs{ +// Size: objectSize, +// StartOffset: 0, +// ContentType: "", +// ContentEncoding: "", +// CacheControl: "", +// //LastModified: time.Time{}, +// Generation: 0, +// Metageneration: 0, +// }, +// }, +// }, +// { +// desc: "range", +// }, +// { +// desc: "incorrect generation", +// }, +// { +// desc: "conditions: generationmatch", +// }, +// } { +// t.Run(test.desc, func(t *testing.T) { +// w := &testWriter{} + +// // Download a single object. + +// // in.downloadShard() + +// // it := d.Results() + +// // got, err := it.Next() +// // if err != nil { +// // t.Fatalf("it.Next: %v", err) +// // } + +// // // Close the writer so we can check the contents. +// // if err := w.Close(); err != nil { +// // t.Fatalf("testWriter.Close: %v", err) +// // } + +// // if got.Err != nil { +// // t.Errorf("result.Err: %v", got.Err) +// // } + +// // if got.Object != objects[0] || got.Bucket != bucketName { +// // t.Errorf("expected Bucket(%q).Object(%q), got %q.%q", bucketName, objects[0], got.Bucket, got.Object) +// // } + +// // if want, got := contentHashes[objects[0]], w.crc32c; got != want { +// // t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) +// // } + +// // if got.Attrs.Size != objectSize { +// // t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) +// // } +// }) +// } + +// } + // generateRandomFileInGCS uploads a file with random contents to GCS and returns // the crc32c hash of the contents. func generateFileInGCS(ctx context.Context, o *storage.ObjectHandle, size int64) (uint32, error) { @@ -188,10 +559,12 @@ func generateFileInGCS(ctx context.Context, o *storage.ObjectHandle, size int64) return crc32cHash.Sum32(), w.Close() } +// TODO: once we provide a DownloaderBuffer that implements WriterAt in the +// library, we can use that instead. type testWriter struct { b []byte crc32c uint32 - bufs [][]byte // temp + bufs [][]byte // temp bufs that will be joined on Close() } // Close must be called to finalize the buffer @@ -212,115 +585,3 @@ func (tw *testWriter) WriteAt(b []byte, offset int64) (n int, err error) { return len(b), nil } - -func TestIntegration_DownloadShard(t *testing.T) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - t.Fatalf("NewClient: %v", err) - } - - bucketName := testPrefix + uidSpace.New() - b := client.Bucket(bucketName) - if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { - t.Fatalf("bucket.Create: %v", err) - } - t.Cleanup(func() { b.Delete(ctx) }) - - // Write object. - objectName := "obj" + uidSpace.New() - contentHashes := make(map[string]uint32) - - objectSize := int64(2 * 1024 * 1024) - crc, err := generateFileInGCS(ctx, b.Object(objectName), objectSize) - if err != nil { - t.Fatalf("generateFileInGCS: %v", err) - } - - t.Cleanup(func() { b.Object(objectName).Delete(ctx) }) - - // Start a downloader. - d, err := NewDownloader(client) - if err != nil { - t.Fatalf("NewDownloader: %v", err) - } - - // Download the object normally ? - - for _, test := range []struct { - desc string - in *DownloadObjectInput - want *DownloadOutput - }{ - { - desc: "basic input", - in: &DownloadObjectInput{ - Bucket: bucketName, - Object: objectName, - //Destination: w, - }, - want: &DownloadOutput{ - Bucket: bucketName, - Object: objectName, - Attrs: &storage.ReaderObjectAttrs{ - Size: objectSize, - StartOffset: 0, - ContentType: "", - ContentEncoding: "", - CacheControl: "", - //LastModified: time.Time{}, - Generation: 0, - Metageneration: 0, - }, - }, - }, - { - desc: "range", - }, - { - desc: "incorrect generation", - }, - { - desc: "conditions: generationmatch", - }, - } { - t.Run(test.desc, func(t *testing.T) { - w := &testWriter{} - - // Download a single object. - - // in.downloadShard() - - // it := d.Results() - - // got, err := it.Next() - // if err != nil { - // t.Fatalf("it.Next: %v", err) - // } - - // // Close the writer so we can check the contents. - // if err := w.Close(); err != nil { - // t.Fatalf("testWriter.Close: %v", err) - // } - - // if got.Err != nil { - // t.Errorf("result.Err: %v", got.Err) - // } - - // if got.Object != objects[0] || got.Bucket != bucketName { - // t.Errorf("expected Bucket(%q).Object(%q), got %q.%q", bucketName, objects[0], got.Bucket, got.Object) - // } - - // if want, got := contentHashes[objects[0]], w.crc32c; got != want { - // t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) - // } - - // if got.Attrs.Size != objectSize { - // t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) - // } - }) - } - -} - -// test ctx cancel and per op timeout diff --git a/storage/transfermanager/option.go b/storage/transfermanager/option.go index fc77d4ef40eb..589fb7770926 100644 --- a/storage/transfermanager/option.go +++ b/storage/transfermanager/option.go @@ -24,6 +24,19 @@ type TransferManagerOption interface { apply(*transferManagerConfig) } +// WithCallbacks returns a TransferManagerOption that allows the use of callbacks +// to process the results. If this option is set, then the results will not be +// returned at the end. +func WithCallbacks() TransferManagerOption { + return &withCallbacks{} +} + +type withCallbacks struct{} + +func (ww withCallbacks) apply(tm *transferManagerConfig) { + tm.asynchronous = true +} + // WithWorkers returns a TransferManagerOption that specifies the maximum number // of concurrent goroutines that will be used to download or upload objects. // Defaults to runtime.NumCPU()/2. @@ -39,23 +52,6 @@ func (ww withWorkers) apply(tm *transferManagerConfig) { tm.numWorkers = ww.numWorkers } -// WithPartSize returns a TransferManagerOption that specifies the size of the -// shards to transfer; that is, if the object is larger than this size, it will -// be uploaded or downloaded in concurrent pieces. -// The default is 32 MiB for downloads. -// NOTE: Sharding is not yet implemented. -func WithPartSize(partSize int) TransferManagerOption { - return &withPartSize{partSize: partSize} -} - -type withPartSize struct { - partSize int -} - -func (wps withPartSize) apply(tm *transferManagerConfig) { - tm.partSize = wps.partSize -} - // WithPerOpTimeout returns a TransferManagerOption that sets a timeout on each // operation that is performed to download or upload an object. The timeout is // set when the operation begins processing, not when it is added. @@ -75,19 +71,20 @@ func (wpt withPerOpTimeout) apply(tm *transferManagerConfig) { type transferManagerConfig struct { // Workers in thread pool; default numCPU/2 based on previous benchmarks? numWorkers int - // Size of shards to transfer; Python found 32 MiB to be good default for - // JSON downloads but gRPC may benefit from larger. - partSize int + // Timeout for a single operation (including all retries). Zero value means // no timeout. perOperationTimeout time.Duration + + // If true, callbacks are used instead of returning results synchronously + // in a slice at the end. + asynchronous bool } func defaultTransferManagerConfig() *transferManagerConfig { return &transferManagerConfig{ numWorkers: runtime.NumCPU() / 2, - partSize: 32 * 1024 * 1024, // 32 MiB - perOperationTimeout: 0, // no timeout + perOperationTimeout: 0, // no timeout } } diff --git a/storage/transfermanager/option_test.go b/storage/transfermanager/option_test.go index 8f78eadbb0d9..88ae1ddfa2c0 100644 --- a/storage/transfermanager/option_test.go +++ b/storage/transfermanager/option_test.go @@ -22,7 +22,6 @@ import ( func TestApply(t *testing.T) { opts := []TransferManagerOption{ WithWorkers(3), - WithPartSize(2000), WithPerOpTimeout(time.Hour), } var got transferManagerConfig @@ -31,7 +30,6 @@ func TestApply(t *testing.T) { } want := transferManagerConfig{ numWorkers: 3, - partSize: 2000, perOperationTimeout: time.Hour, } From 9fcaca4e567d053fa677a4b36af580f47dd87b5f Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Mon, 13 May 2024 16:46:54 -0700 Subject: [PATCH 05/12] finish tests, some cleanup --- go.work.sum | 1 + storage/transfermanager/doc.go | 83 +++ storage/transfermanager/downloader.go | 165 ++--- storage/transfermanager/integration_test.go | 648 ++++++++++++-------- 4 files changed, 521 insertions(+), 376 deletions(-) create mode 100644 storage/transfermanager/doc.go diff --git a/go.work.sum b/go.work.sum index 5891aebac6ce..b1cb59e2c713 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,3 +1,4 @@ +cloud.google.com/go/dataproc v1.12.0 h1:W47qHL3W4BPkAIbk4SWmIERwsWBaNnWm0P2sdx3YgGU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.46.0/go.mod h1:V28hx+cUCZC9e3qcqszMb+Sbt8cQZtHTiXOmyDzoDOg= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= diff --git a/storage/transfermanager/doc.go b/storage/transfermanager/doc.go new file mode 100644 index 000000000000..f85c91e22c35 --- /dev/null +++ b/storage/transfermanager/doc.go @@ -0,0 +1,83 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package transfermanager provides an easy way to parallelize downloads in Google +Cloud Storage. + +More information about Google Cloud Storage is available at +https://cloud.google.com/storage/docs. + +See https://pkg.go.dev/cloud.google.com/go for authentication, timeouts, +connection pooling and similar aspects of this package. + +NOTE: This package is in alpha. It is not stable, and is likely to change. + +# Example usage + + // Pass in any client opts or set retry policy here + client, err := storage.NewClient(ctx) // can also use NewGRPCClient + if err != nil { + // handle error + } + + // Create Downloader with desired options, including number of workers, + // part size, per operation timeout, etc. + d, err := transfermanager.NewDownloader(client, transfermanager.WithWorkers(16)) + if err != nil { + // handle error + } + + // Create local file writer for output + f, err := os.Create("/path/to/localfile") + if err != nil { + // handle error + } + + // Create download input + in := &transfermanager.DownloadObjectInput{ + Bucket: "mybucket", + Object: "myblob", + Destination: f, + // Optionally specify params to apply to download. + EncryptionKey: []byte("mykey"), + } + + // Can set timeout on this download using context. Note that this download + // may not start immediately if all workers are busy, so this may time out + // before the download even starts. To set a timeout that starts with the + // download, use transfermanager.WithPerOpTimeout(time.Duration). + ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + // Add to Downloader + d.DownloadObject(ctx, in) + + // Repeat if desired + + // Wait for all downloads to complete. + d.WaitAndClose() + + // Iterate through completed downloads and process results. This can + // also happen async in a go routine as the downloads run. + results := d.Results() + for _, out := range results { + if out.Err != nil { + log.Printf("download of %v failed with error %v", out.Name, out.Err) + } else { + log.Printf("download of %v succeeded", out.Object) + } + } +*/ +package transfermanager // import "cloud.google.com/go/storage/transfermanager" diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go index 3b09de6a6874..cc17a5355483 100644 --- a/storage/transfermanager/downloader.go +++ b/storage/transfermanager/downloader.go @@ -43,9 +43,7 @@ type Downloader struct { // download but is non-blocking; call Downloader.Results to process the result. func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectInput) { input.ctx = ctx - d.inputsMu.Lock() - d.inputs = append(d.inputs, *input) - d.inputsMu.Unlock() + d.addInput(input) } // DownloadObject queues the download of a single object. This will initiate the @@ -54,9 +52,7 @@ func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectIn func (d *Downloader) DownloadObjectWithCallback(ctx context.Context, input *DownloadObjectInput, callback func(*DownloadOutput)) { input.ctx = ctx input.callback = &callback - d.inputsMu.Lock() - d.inputs = append(d.inputs, *input) - d.inputsMu.Unlock() + d.addInput(input) } // WaitAndClose waits for all outstanding downloads to complete. The Downloader @@ -73,9 +69,18 @@ func (d *Downloader) WaitAndClose() error { return nil } -// Results returns the iterator for download outputs. +// Results returns all the results of the downloads completed since the last +// time it was called. Call WaitAndClose before calling Results to wait for all +// downloads to complete. +// Results will not return results for downloads initiated with a callback. func (d *Downloader) Results() []DownloadOutput { - return d.results + d.resultsMu.Lock() + r := make([]DownloadOutput, len(d.results)) + copy(r, d.results) + d.results = []DownloadOutput{} + d.resultsMu.Unlock() + + return r } // sendInputsToWorkChan listens continuously to the inputs slice until d.done. @@ -112,6 +117,24 @@ func (d *Downloader) drainInput() { } } +func (d *Downloader) addInput(input *DownloadObjectInput) { + d.inputsMu.Lock() + d.inputs = append(d.inputs, *input) + d.inputsMu.Unlock() +} + +func (d *Downloader) addResult(result *DownloadOutput) { + d.resultsMu.Lock() + d.results = append(d.results, *result) + d.resultsMu.Unlock() +} + +func (d *Downloader) error(err error) { + d.errorsMu.Lock() + d.errors = append(d.errors, err) + d.errorsMu.Unlock() +} + // downloadWorker continuously processes downloads until the work channel is closed. func (d *Downloader) downloadWorker() { for { @@ -128,18 +151,14 @@ func (d *Downloader) downloadWorker() { // Keep track of any error that occurred. if out.Err != nil { - d.errorsMu.Lock() - d.errors = append(d.errors, out.Err) - d.errorsMu.Unlock() + d.error(out.Err) } // Either execute the callback, or append to results. if input.callback != nil { (*input.callback)(out) } else { - d.resultsMu.Lock() - d.results = append(d.results, *out) - d.resultsMu.Unlock() + d.addResult(out) } } d.workers.Done() @@ -204,6 +223,7 @@ type DownloadObjectInput struct { } // downloadShard will read a specific object into in.Destination. +// If timeout is less than 0, no timeout is set. // TODO: download a single shard instead of the entire object. func (in *DownloadObjectInput) downloadShard(client *storage.Client, timeout time.Duration) (out *DownloadOutput) { out = &DownloadOutput{Bucket: in.Bucket, Object: in.Object} @@ -220,7 +240,7 @@ func (in *DownloadObjectInput) downloadShard(client *storage.Client, timeout tim o := client.Bucket(in.Bucket).Object(in.Object) if in.Conditions != nil { - o.If(*in.Conditions) + o = o.If(*in.Conditions) } if in.Generation != nil { o = o.Generation(*in.Generation) @@ -269,118 +289,3 @@ type DownloadOutput struct { Err error // error occurring during download Attrs *storage.ReaderObjectAttrs // attributes of downloaded object, if successful } - -// // DownloadOutputIterator allows the end user to iterate through completed -// // object downloads. -// type DownloadOutputIterator struct { -// output <-chan *DownloadOutput -// } - -// // Next iterates through results. When complete, will return the iterator.Done -// // error. It is considered complete once WaitAndClose() has been called on the -// // Downloader. -// // Note that if there was an error reading an object, it will not be returned -// // by Next - check DownloadOutput.Err instead. -// // DownloadOutputs will be available as the downloads complete; they can -// // be iterated through asynchronously or at the end of the job. -// // Next will block if there are no more completed downloads (and the Downloader -// // is not closed). -// func (it *DownloadOutputIterator) Next() (*DownloadOutput, error) { -// out, ok := <-it.output -// if !ok { -// return nil, iterator.Done -// } -// fmt.Println(out) -// return out, nil -// } - -// // listenForAndDelegateWork will receive from the work chan, and start goroutines -// // to execute that work without blocking. -// // It should be called only once. -// func (d *Downloader) listenForAndDelegateWork1() { -// for { -// // Dequeue the work. Can block. -// input, ok := <-d.work -// if !ok { -// break // no more work; exit -// } - -// // Start a worker. This may block. -// d.workerGroup.Go(func() error { -// // Do the download. -// // TODO: break down the input into smaller pieces if necessary; maybe as follows: -// // Only request partSize data to begin with. If no error and we haven't finished -// // reading the object, enqueue the remaining pieces of work (by sending them to d.work) -// // and mark in the out var the amount of shards to wait for. -// out := input.downloadShard(d.client, d.config.perOperationTimeout) - -// // Send the output to be received by Next. This could block until received -// // (ie. the user calls Next), but it's okay; our worker has already returned. -// // Alternatively, we could feed these to a slice that we grab from in Next. -// // This would not block then, but would require synchronization of the slice. -// go func() { -// out := out -// d.output <- out -// }() - -// // We return the error here, to communicate to d.workergroup.Wait -// // that there has been an error. -// // Since the group does not use a shared context, this should not -// // affect any of the other operations using the group. -// // TO-DO: in addition to this, if we want WaitAndClose to return a -// // multi err, we will need to record these errors somewhere. -// // Where-ever we record those, it will need synchronization since -// // this is concurrent. -// return out.Err -// }) - -// } -// } - -// // drainInput consumes everything in the inputs slice and dispatches workers. -// // It will block if there are not enough workers to consume every input, until -// // all inputs are dispatched to an available worker. -// func (d *Downloader) drainInpu1t() { -// fmt.Println(len(d.inputs)) - -// for len(d.inputs) > 0 { -// d.inputsMu.Lock() -// if len(d.inputs) < 1 { -// return -// } -// input := d.inputs[0] -// d.inputs = d.inputs[1:] -// d.inputsMu.Unlock() - -// // Start a worker. This may block, but only if there aren't enough workers. -// d.workerGroup.Go(func() error { -// // Do the download. -// // TODO: break down the input into smaller pieces if necessary; maybe as follows: -// // Only request partSize data to begin with. If no error and we haven't finished -// // reading the object, enqueue the remaining pieces of work -// // and mark in the out var the amount of shards to wait for. -// out := input.downloadShard(d.client, d.config.perOperationTimeout) - -// // Either return the callback, or append to results. -// if input.callback != nil { -// (*input.callback)(out) -// } else { -// d.resultsMu.Lock() -// d.results = append(d.results, *out) -// fmt.Println(len(d.results)) - -// d.resultsMu.Unlock() -// } - -// // We return the error here, to communicate to d.workergroup.Wait -// // that there has been an error. -// // Since the group does not use a shared context, this should not -// // affect any of the other operations using the group. -// // TO-DO: in addition to this, if we want WaitAndClose to return a -// // multi err, we will need to record these errors somewhere. -// // Where-ever we record those, it will need synchronization since -// // this is concurrent. -// return out.Err -// }) -// } -// } diff --git a/storage/transfermanager/integration_test.go b/storage/transfermanager/integration_test.go index 9a6135b2c524..47b9f66c07be 100644 --- a/storage/transfermanager/integration_test.go +++ b/storage/transfermanager/integration_test.go @@ -18,59 +18,62 @@ import ( "bytes" "context" "crypto/rand" + "errors" + "fmt" "hash/crc32" "io" + "log" + "os" "sync" "testing" + "time" "cloud.google.com/go/internal/testutil" "cloud.google.com/go/internal/uid" "cloud.google.com/go/storage" + "github.com/google/go-cmp/cmp" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" ) const ( - testPrefix = "go-integration-test" - grpcTestPrefix = "golang-grpc-test" + testPrefix = "go-integration-test-tm" + grpcTestPrefix = "golang-grpc-test-tm" + bucketExpiryAge = 24 * time.Hour ) -var uidSpace = uid.NewSpace("", &uid.Options{Short: true}) +var ( + uidSpace = uid.NewSpace("", &uid.Options{Short: true}) + testBucket = downloadTestBucket{} // This bucket is shared amongst download tests. It is created with populated objects and cleaned up in TestMain. +) -func TestIntegration_DownloaderSynchronous(t *testing.T) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - t.Fatalf("NewClient: %v", err) +func TestMain(m *testing.M) { + if err := testBucket.Create(testPrefix); err != nil { + log.Fatalf("test bucket creation failed: %v", err) } - bucketName := testPrefix + uidSpace.New() - b := client.Bucket(bucketName) - if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { - t.Fatalf("bucket.Create: %v", err) - } - t.Cleanup(func() { b.Delete(ctx) }) + exit := m.Run() - // Populate object names and make a map for their contents. - objects := []string{ - "obj1", - "obj2", - "obj/with/slashes", - "obj/", - "./obj", - "!#$&'()*+,/:;=,?@,[] and spaces", + if err := testBucket.Cleanup(); err != nil { + log.Printf("test bucket cleanup failed: %v", err) + } + if err := deleteExpiredBuckets(testPrefix); err != nil { + // Don't fail the test if cleanup fails. + log.Printf("Post-test cleanup failed: %v", err) } - contentHashes := make(map[string]uint32) - // Write objects. - objectSize := int64(507) - for _, obj := range objects { - crc, err := generateFileInGCS(ctx, b.Object(obj), objectSize) - if err != nil { - t.Fatalf("generateFileInGCS: %v", err) - } - contentHashes[obj] = crc + os.Exit(exit) +} - t.Cleanup(func() { b.Object(obj).Delete(ctx) }) +func TestIntegration_DownloaderSynchronous(t *testing.T) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) } + t.Cleanup(func() { client.Close() }) + + objects := testBucket.objects // Start a downloader. Give it a smaller amount of workers than objects, to // make sure we aren't blocking anywhere. @@ -85,7 +88,7 @@ func TestIntegration_DownloaderSynchronous(t *testing.T) { writers[i] = &testWriter{} objToWriter[obj] = i d.DownloadObject(ctx, &DownloadObjectInput{ - Bucket: bucketName, + Bucket: testBucket.bucket, Object: obj, Destination: writers[i], }) @@ -103,8 +106,8 @@ func TestIntegration_DownloaderSynchronous(t *testing.T) { } } + // Check the results. results := d.Results() - for _, got := range results { writerIdx := objToWriter[got.Object] @@ -113,12 +116,12 @@ func TestIntegration_DownloaderSynchronous(t *testing.T) { continue } - if want, got := contentHashes[got.Object], writers[writerIdx].crc32c; got != want { + if want, got := testBucket.contentHashes[got.Object], writers[writerIdx].crc32c; got != want { t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) } - if got.Attrs.Size != objectSize { - t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + if got.Attrs.Size != testBucket.objectSize { + t.Errorf("expected object size %d, got %d", testBucket.objectSize, got.Attrs.Size) } } @@ -134,33 +137,11 @@ func TestIntegration_DownloaderErrorSync(t *testing.T) { if err != nil { t.Fatalf("NewClient: %v", err) } + t.Cleanup(func() { client.Close() }) - bucketName := testPrefix + uidSpace.New() - b := client.Bucket(bucketName) - if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { - t.Fatalf("bucket.Create: %v", err) - } - t.Cleanup(func() { b.Delete(ctx) }) - - // Populate object names and make a map for their contents. - objects := []string{ - "obj1", - "obj2", - "obj/with/slashes", - } - contentHashes := make(map[string]uint32) - - // Write objects. - objectSize := int64(507) - for _, obj := range objects { - crc, err := generateFileInGCS(ctx, b.Object(obj), objectSize) - if err != nil { - t.Fatalf("generateFileInGCS: %v", err) - } - contentHashes[obj] = crc - - t.Cleanup(func() { b.Object(obj).Delete(ctx) }) - } + // Make a copy of the objects slice. + objects := make([]string, len(testBucket.objects)) + copy(objects, testBucket.objects) // Add another object to attempt to download; since it hasn't been written, // this one will fail. Append to the start so that it will (likely) be @@ -174,14 +155,14 @@ func TestIntegration_DownloaderErrorSync(t *testing.T) { t.Fatalf("NewDownloader: %v", err) } - // Download several objects. + // Download objects. writers := make([]*testWriter, len(objects)) objToWriter := make(map[string]int) // so we can map the resulting content back to the correct object for i, obj := range objects { writers[i] = &testWriter{} objToWriter[obj] = i d.DownloadObject(ctx, &DownloadObjectInput{ - Bucket: bucketName, + Bucket: testBucket.bucket, Object: obj, Destination: writers[i], }) @@ -200,8 +181,8 @@ func TestIntegration_DownloaderErrorSync(t *testing.T) { } } + // Check the results. results := d.Results() - for _, got := range results { writerIdx := objToWriter[got.Object] @@ -219,12 +200,12 @@ func TestIntegration_DownloaderErrorSync(t *testing.T) { continue } - if want, got := contentHashes[got.Object], writers[writerIdx].crc32c; got != want { + if want, got := testBucket.contentHashes[got.Object], writers[writerIdx].crc32c; got != want { t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) } - if got.Attrs.Size != objectSize { - t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + if got.Attrs.Size != testBucket.objectSize { + t.Errorf("expected object size %d, got %d", testBucket.objectSize, got.Attrs.Size) } } @@ -239,36 +220,9 @@ func TestIntegration_DownloaderAsynchronous(t *testing.T) { if err != nil { t.Fatalf("NewClient: %v", err) } + t.Cleanup(func() { client.Close() }) - bucketName := testPrefix + uidSpace.New() - b := client.Bucket(bucketName) - if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { - t.Fatalf("bucket.Create: %v", err) - } - t.Cleanup(func() { b.Delete(ctx) }) - - // Populate object names and make a map for their contents. - objects := []string{ - "obj1", - "obj2", - "obj/with/slashes", - "obj/", - "./obj", - "!#$&'()*+,/:;=,?@,[] and spaces", - } - contentHashes := make(map[string]uint32) - - // Write objects. - objectSize := int64(507) - for _, obj := range objects { - crc, err := generateFileInGCS(ctx, b.Object(obj), objectSize) - if err != nil { - t.Fatalf("generateFileInGCS: %v", err) - } - contentHashes[obj] = crc - - t.Cleanup(func() { b.Object(obj).Delete(ctx) }) - } + objects := testBucket.objects // Start a downloader. Give it a smaller amount of workers than objects, to // make sure we aren't blocking anywhere. @@ -280,13 +234,13 @@ func TestIntegration_DownloaderAsynchronous(t *testing.T) { numCallbacks := 0 callbackMu := sync.Mutex{} - // Download several objects. + // Download objects. writers := make([]*testWriter, len(objects)) for i, obj := range objects { i := i writers[i] = &testWriter{} d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ - Bucket: bucketName, + Bucket: testBucket.bucket, Object: obj, Destination: writers[i], }, func(got *DownloadOutput) { @@ -303,12 +257,12 @@ func TestIntegration_DownloaderAsynchronous(t *testing.T) { t.Fatalf("testWriter.Close: %v", err) } - if want, got := contentHashes[got.Object], writers[i].crc32c; got != want { + if want, got := testBucket.contentHashes[got.Object], writers[i].crc32c; got != want { t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) } - if got.Attrs.Size != objectSize { - t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + if got.Attrs.Size != testBucket.objectSize { + t.Errorf("expected object size %d, got %d", testBucket.objectSize, got.Attrs.Size) } }) } @@ -328,36 +282,9 @@ func TestIntegration_DownloaderErrorAsync(t *testing.T) { if err != nil { t.Fatalf("NewClient: %v", err) } + t.Cleanup(func() { client.Close() }) - bucketName := testPrefix + uidSpace.New() - b := client.Bucket(bucketName) - if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { - t.Fatalf("bucket.Create: %v", err) - } - t.Cleanup(func() { b.Delete(ctx) }) - - // Populate object names and make a map for their contents. - objects := []string{ - "obj1", - "obj2", - "obj/with/slashes", - "obj/", - "./obj", - "!#$&'()*+,/:;=,?@,[] and spaces", - } - contentHashes := make(map[string]uint32) - - // Write objects. - objectSize := int64(507) - for _, obj := range objects { - crc, err := generateFileInGCS(ctx, b.Object(obj), objectSize) - if err != nil { - t.Fatalf("generateFileInGCS: %v", err) - } - contentHashes[obj] = crc - - t.Cleanup(func() { b.Object(obj).Delete(ctx) }) - } + objects := testBucket.objects // Start a downloader. Give it a smaller amount of workers than objects, to // make sure we aren't blocking anywhere. @@ -366,6 +293,8 @@ func TestIntegration_DownloaderErrorAsync(t *testing.T) { t.Fatalf("NewDownloader: %v", err) } + // Keep track of the number of callbacks. Since the callbacks may happen + // in parallel, we sync access to this variable. numCallbacks := 0 callbackMu := sync.Mutex{} @@ -374,7 +303,7 @@ func TestIntegration_DownloaderErrorAsync(t *testing.T) { w := &testWriter{} d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ - Bucket: bucketName, + Bucket: testBucket.bucket, Object: nonexistentObject, Destination: w, }, func(got *DownloadOutput) { @@ -394,7 +323,7 @@ func TestIntegration_DownloaderErrorAsync(t *testing.T) { i := i writers[i] = &testWriter{} d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ - Bucket: bucketName, + Bucket: testBucket.bucket, Object: obj, Destination: writers[i], }, func(got *DownloadOutput) { @@ -411,12 +340,12 @@ func TestIntegration_DownloaderErrorAsync(t *testing.T) { t.Fatalf("testWriter.Close: %v", err) } - if want, got := contentHashes[got.Object], writers[i].crc32c; got != want { + if want, got := testBucket.contentHashes[got.Object], writers[i].crc32c; got != want { t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) } - if got.Attrs.Size != objectSize { - t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) + if got.Attrs.Size != testBucket.objectSize { + t.Errorf("expected object size %d, got %d", testBucket.objectSize, got.Attrs.Size) } }) } @@ -432,117 +361,219 @@ func TestIntegration_DownloaderErrorAsync(t *testing.T) { } } +func TestIntegration_DownloaderTimeout(t *testing.T) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + t.Cleanup(func() { client.Close() }) + + // Start a downloader. + d, err := NewDownloader(client, WithPerOpTimeout(time.Nanosecond)) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + + // Download an object. + d.DownloadObject(ctx, &DownloadObjectInput{ + Bucket: testBucket.bucket, + Object: testBucket.objects[0], + Destination: &testWriter{}, + }) + + // WaitAndClose should return an error since the timeout is too short. + if err := d.WaitAndClose(); err == nil { + t.Error("d.WaitAndClose should return an error, instead got nil") + } + + // Check the result. + results := d.Results() + got := results[0] + + // Check that the nonexistent object returned an error. + if got.Err != context.DeadlineExceeded { + t.Errorf("expected deadline exceeded error, got: %v", got.Err) + } +} + +func TestIntegration_DownloadShard(t *testing.T) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + t.Cleanup(func() { client.Close() }) + + // Use an object from the test bucket. + objectName := testBucket.objects[0] + + // Get expected Attrs. + o := client.Bucket(testBucket.bucket).Object(objectName) + r, err := o.NewReader(ctx) + if err != nil { + t.Fatalf("o.Attrs: %v", err) + } + + incorrectGen := r.Attrs.Generation - 1 + + cancelledCtx, cancel := context.WithCancel(ctx) + cancel() + + for _, test := range []struct { + desc string + timeout time.Duration + in *DownloadObjectInput + want *DownloadOutput + }{ + { + desc: "basic input", + in: &DownloadObjectInput{ + Bucket: testBucket.bucket, + Object: objectName, + }, + want: &DownloadOutput{ + Bucket: testBucket.bucket, + Object: objectName, + Attrs: &r.Attrs, + }, + }, + { + desc: "range", + in: &DownloadObjectInput{ + Bucket: testBucket.bucket, + Object: objectName, + Range: &DownloadRange{ + Offset: testBucket.objectSize - 5, + Length: -1, + }, + }, + want: &DownloadOutput{ + Bucket: testBucket.bucket, + Object: objectName, + Attrs: &storage.ReaderObjectAttrs{ + Size: testBucket.objectSize, + StartOffset: testBucket.objectSize - 5, + ContentType: r.Attrs.ContentType, + ContentEncoding: r.Attrs.ContentEncoding, + CacheControl: r.Attrs.CacheControl, + LastModified: r.Attrs.LastModified, + Generation: r.Attrs.Generation, + Metageneration: r.Attrs.Metageneration, + }, + }, + }, + { + desc: "incorrect generation", + in: &DownloadObjectInput{ + Bucket: testBucket.bucket, + Object: objectName, + Generation: &incorrectGen, + }, + want: &DownloadOutput{ + Bucket: testBucket.bucket, + Object: objectName, + Err: storage.ErrObjectNotExist, + }, + }, + { + desc: "conditions: generationmatch", + in: &DownloadObjectInput{ + Bucket: testBucket.bucket, + Object: objectName, + Conditions: &storage.Conditions{ + GenerationMatch: r.Attrs.Generation, + }, + }, + want: &DownloadOutput{ + Bucket: testBucket.bucket, + Object: objectName, + Attrs: &r.Attrs, + }, + }, + { + desc: "conditions do not hold", + in: &DownloadObjectInput{ + Bucket: testBucket.bucket, + Object: objectName, + Conditions: &storage.Conditions{ + GenerationMatch: incorrectGen, + }, + }, + want: &DownloadOutput{ + Bucket: testBucket.bucket, + Object: objectName, + Err: &googleapi.Error{Code: 412}, + }, + }, + { + desc: "timeout", + in: &DownloadObjectInput{ + Bucket: testBucket.bucket, + Object: objectName, + }, + timeout: time.Nanosecond, + want: &DownloadOutput{ + Bucket: testBucket.bucket, + Object: objectName, + Err: context.DeadlineExceeded, + }, + }, + { + desc: "cancelled ctx", + in: &DownloadObjectInput{ + Bucket: testBucket.bucket, + Object: objectName, + ctx: cancelledCtx, + }, + timeout: time.Nanosecond, + want: &DownloadOutput{ + Bucket: testBucket.bucket, + Object: objectName, + Err: context.Canceled, + }, + }, + } { + t.Run(test.desc, func(t *testing.T) { + w := &testWriter{} + + test.in.Destination = w + + if test.in.ctx == nil { + test.in.ctx = ctx + } + + got := test.in.downloadShard(client, test.timeout) + + if got.Bucket != test.want.Bucket || got.Object != test.want.Object { + t.Errorf("wanted bucket %q, object %q, got: %q, %q", test.want.Bucket, test.want.Object, got.Bucket, got.Object) + } + + if diff := cmp.Diff(got.Attrs, test.want.Attrs); diff != "" { + t.Errorf("diff got(-) vs. want(+): %v", diff) + } + + if !errorIs(got.Err, test.want.Err) { + t.Errorf("mismatching errors: got %v, want %v", got.Err, test.want.Err) + } + }) + } + +} + // test ctx cancel and per op timeout -// func TestIntegration_DownloadShard(t *testing.T) { -// ctx := context.Background() -// client, err := storage.NewClient(ctx) -// if err != nil { -// t.Fatalf("NewClient: %v", err) -// } - -// bucketName := testPrefix + uidSpace.New() -// b := client.Bucket(bucketName) -// if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { -// t.Fatalf("bucket.Create: %v", err) -// } -// t.Cleanup(func() { b.Delete(ctx) }) - -// // Write object. -// objectName := "obj" + uidSpace.New() -// contentHashes := make(map[string]uint32) - -// objectSize := int64(2 * 1024 * 1024) -// crc, err := generateFileInGCS(ctx, b.Object(objectName), objectSize) -// if err != nil { -// t.Fatalf("generateFileInGCS: %v", err) -// } - -// t.Cleanup(func() { b.Object(objectName).Delete(ctx) }) - -// // Start a downloader. -// d, err := NewDownloader(client) -// if err != nil { -// t.Fatalf("NewDownloader: %v", err) -// } - -// // Download the object normally ? - -// for _, test := range []struct { -// desc string -// in *DownloadObjectInput -// want *DownloadOutput -// }{ -// { -// desc: "basic input", -// in: &DownloadObjectInput{ -// Bucket: bucketName, -// Object: objectName, -// //Destination: w, -// }, -// want: &DownloadOutput{ -// Bucket: bucketName, -// Object: objectName, -// Attrs: &storage.ReaderObjectAttrs{ -// Size: objectSize, -// StartOffset: 0, -// ContentType: "", -// ContentEncoding: "", -// CacheControl: "", -// //LastModified: time.Time{}, -// Generation: 0, -// Metageneration: 0, -// }, -// }, -// }, -// { -// desc: "range", -// }, -// { -// desc: "incorrect generation", -// }, -// { -// desc: "conditions: generationmatch", -// }, -// } { -// t.Run(test.desc, func(t *testing.T) { -// w := &testWriter{} - -// // Download a single object. - -// // in.downloadShard() - -// // it := d.Results() - -// // got, err := it.Next() -// // if err != nil { -// // t.Fatalf("it.Next: %v", err) -// // } - -// // // Close the writer so we can check the contents. -// // if err := w.Close(); err != nil { -// // t.Fatalf("testWriter.Close: %v", err) -// // } - -// // if got.Err != nil { -// // t.Errorf("result.Err: %v", got.Err) -// // } - -// // if got.Object != objects[0] || got.Bucket != bucketName { -// // t.Errorf("expected Bucket(%q).Object(%q), got %q.%q", bucketName, objects[0], got.Bucket, got.Object) -// // } - -// // if want, got := contentHashes[objects[0]], w.crc32c; got != want { -// // t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) -// // } - -// // if got.Attrs.Size != objectSize { -// // t.Errorf("expected object size %d, got %d", objectSize, got.Attrs.Size) -// // } -// }) -// } - -// } +// errorIs is equivalent to errors.Is, except that it additionally will return +// true if err and targetErr are googleapi.Errors with identical error codes. +func errorIs(err error, targetErr error) bool { + var e, targetE *googleapi.Error + if errors.As(err, &e) && errors.As(targetErr, &targetE) { + return e.Code == targetE.Code + } + + // fallback to regular check + return errors.Is(err, targetErr) +} // generateRandomFileInGCS uploads a file with random contents to GCS and returns // the crc32c hash of the contents. @@ -585,3 +616,128 @@ func (tw *testWriter) WriteAt(b []byte, offset int64) (n int, err error) { return len(b), nil } + +func deleteExpiredBuckets(prefix string) error { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("NewClient: %v", err) + } + + projectID := testutil.ProjID() + it := client.Buckets(ctx, projectID) + it.Prefix = prefix + for { + bktAttrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + if time.Since(bktAttrs.Created) > bucketExpiryAge { + log.Printf("deleting bucket %q, which is more than %s old", bktAttrs.Name, bucketExpiryAge) + if err := killBucket(ctx, client, bktAttrs.Name); err != nil { + return err + } + } + } + return nil +} + +// killBucket deletes a bucket and all its objects. +func killBucket(ctx context.Context, client *storage.Client, bucketName string) error { + bkt := client.Bucket(bucketName) + // Bucket must be empty to delete. + it := bkt.Objects(ctx, nil) + for { + objAttrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + if err := bkt.Object(objAttrs.Name).Delete(ctx); err != nil { + return fmt.Errorf("deleting %q: %v", bucketName+"/"+objAttrs.Name, err) + } + } + // GCS is eventually consistent, so this delete may fail because the + // replica still sees an object in the bucket. We log the error and expect + // a later test run to delete the bucket. + if err := bkt.Delete(ctx); err != nil { + log.Printf("deleting %q: %v", bucketName, err) + } + return nil +} + +// downloadTestBucket provides a bucket that can be reused for tests that only +// download from the bucket. +type downloadTestBucket struct { + bucket string + objects []string + contentHashes map[string]uint32 + objectSize int64 +} + +// Create initializes the downloadTestBucket, creating a bucket and populating +// objects in it. All objects are of the same size but with different contents +// and can be mapped to their respective crc32c hash in contentHashes. +func (tb *downloadTestBucket) Create(prefix string) error { + ctx := context.Background() + + tb.bucket = prefix + uidSpace.New() + tb.objectSize = int64(507) + tb.objects = []string{ + "obj1", + "obj2", + "obj/with/slashes", + "obj/", + "./obj", + "!#$&'()*+,/:;=,?@,[] and spaces", + } + tb.contentHashes = make(map[string]uint32) + + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("NewClient: %v", err) + } + defer client.Close() + + b := client.Bucket(tb.bucket) + if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { + return fmt.Errorf("bucket.Create: %v", err) + } + + // Write objects. + for _, obj := range tb.objects { + crc, err := generateFileInGCS(ctx, b.Object(obj), tb.objectSize) + if err != nil { + return fmt.Errorf("generateFileInGCS: %v", err) + } + tb.contentHashes[obj] = crc + + } + return nil +} + +// Cleanup deletes the objects and bucket created in Create. +func (tb *downloadTestBucket) Cleanup() error { + ctx := context.Background() + + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("NewClient: %v", err) + } + defer client.Close() + + b := client.Bucket(tb.bucket) + + for _, obj := range tb.objects { + if err := b.Object(obj).Delete(ctx); err != nil { + return fmt.Errorf("object.Delete: %v", err) + } + } + + return b.Delete(ctx) +} From ca52f14da2a6cd9eee6e84ff0137fb70c387d376 Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Mon, 13 May 2024 16:58:40 -0700 Subject: [PATCH 06/12] revert go.work.sum --- go.work.sum | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/go.work.sum b/go.work.sum index b3b52921d121..002f0a0872ea 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,3 +1,9 @@ +cloud.google.com/go/auth v0.2.0/go.mod h1:+yb+oy3/P0geX6DLKlqiGHARGR6EX2GRtYCzWOCQSbU= +cloud.google.com/go/auth/oauth2adapt v0.2.0/go.mod h1:AfqujpDAlTfLfeCIl/HJZZlIxD8+nJoZ5e0x1IxGq5k= +cloud.google.com/go/dataproc v1.12.0 h1:W47qHL3W4BPkAIbk4SWmIERwsWBaNnWm0P2sdx3YgGU= +cloud.google.com/go/gaming v1.9.0 h1:7vEhFnZmd931Mo7sZ6pJy7uQPDxF7m7v8xtBheG08tc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.18.0 h1:ugYJK/neZQtQeh2jc5xNoDFiMQojlAkoqJMRb7vTu1U= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.18.0/go.mod h1:Xx0VKh7GJ4si3rmElbh19Mejxz68ibWg/J30ZOMrqzU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.46.0/go.mod h1:V28hx+cUCZC9e3qcqszMb+Sbt8cQZtHTiXOmyDzoDOg= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= @@ -49,11 +55,33 @@ github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFW github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA= +go.opentelemetry.io/otel/bridge/opencensus v0.40.0 h1:pqDiayRhBgoqy1vwnscik+TizcImJ58l053NScJyZso= +go.opentelemetry.io/otel/bridge/opencensus v0.40.0/go.mod h1:1NvVHb6tLTe5A9qCYz+eErW0t8iPn4ZfR6tDKcqlGTM= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0/go.mod h1:qcTO4xHAxZLaLxPd60TdE88rxtItPHgHWqOhOGRr0as= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.44.0/go.mod h1:sTt30Evb7hJB/gEk27qLb1+l9n4Tb8HvHkR0Wx3S6CU= +go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI= +go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +google.golang.org/api v0.174.0/go.mod h1:aC7tB6j0HR1Nl0ni5ghpx6iLasmAX78Zkh/wgxAAjLg= +google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= +google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda/go.mod h1:AHcE/gZH76Bk/ROZhQphlRoWo5xKDEtz3eVEO1LfA8c= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20231120223509-83a465c0220f/go.mod h1:iIgEblxoG4klcXsG0d9cpoxJ4xndv6+1FkDROCHhPRI= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20240102182953-50ed04b92917/go.mod h1:O9TvT7A9NLgdqqF0JJXJ+axpaoYiEb8txGmkvy+AvLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 1d00ad46c78776ea733953c7918ba437a380f7db Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Tue, 14 May 2024 11:31:05 -0700 Subject: [PATCH 07/12] vet fixes --- storage/transfermanager/downloader.go | 12 +++++++----- storage/transfermanager/option.go | 12 ++++++------ storage/transfermanager/option_test.go | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go index cc17a5355483..00805621f82c 100644 --- a/storage/transfermanager/downloader.go +++ b/storage/transfermanager/downloader.go @@ -46,9 +46,11 @@ func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectIn d.addInput(input) } -// DownloadObject queues the download of a single object. This will initiate the -// download but is non-blocking. The result will not be added to the results obtained -// from Downloader.Results; use the callback to process the result. +// DownloadObjectWithCallback queues the download of a single object. This will +// initiate the download but is non-blocking; use the callback to process the +// result. +// The results of downloads initiated with this method will not be provided in +// the results given in Downloader.Results. func (d *Downloader) DownloadObjectWithCallback(ctx context.Context, input *DownloadObjectInput, callback func(*DownloadOutput)) { input.ctx = ctx input.callback = &callback @@ -71,7 +73,7 @@ func (d *Downloader) WaitAndClose() error { // Results returns all the results of the downloads completed since the last // time it was called. Call WaitAndClose before calling Results to wait for all -// downloads to complete. +// downloads to complete. The results are not guaranteed to be in any order. // Results will not return results for downloads initiated with a callback. func (d *Downloader) Results() []DownloadOutput { d.resultsMu.Lock() @@ -166,7 +168,7 @@ func (d *Downloader) downloadWorker() { // NewDownloader creates a new Downloader to add operations to. // Choice of transport, etc is configured on the client that's passed in. -func NewDownloader(c *storage.Client, opts ...TransferManagerOption) (*Downloader, error) { +func NewDownloader(c *storage.Client, opts ...Option) (*Downloader, error) { d := &Downloader{ client: c, config: initTransferManagerConfig(opts...), diff --git a/storage/transfermanager/option.go b/storage/transfermanager/option.go index 589fb7770926..d451ab664000 100644 --- a/storage/transfermanager/option.go +++ b/storage/transfermanager/option.go @@ -19,15 +19,15 @@ import ( "time" ) -// A TransferManagerOption is an option for a transfermanager Downloader or Uploader. -type TransferManagerOption interface { +// A Option is an option for a transfermanager Downloader or Uploader. +type Option interface { apply(*transferManagerConfig) } // WithCallbacks returns a TransferManagerOption that allows the use of callbacks // to process the results. If this option is set, then the results will not be // returned at the end. -func WithCallbacks() TransferManagerOption { +func WithCallbacks() Option { return &withCallbacks{} } @@ -40,7 +40,7 @@ func (ww withCallbacks) apply(tm *transferManagerConfig) { // WithWorkers returns a TransferManagerOption that specifies the maximum number // of concurrent goroutines that will be used to download or upload objects. // Defaults to runtime.NumCPU()/2. -func WithWorkers(numWorkers int) TransferManagerOption { +func WithWorkers(numWorkers int) Option { return &withWorkers{numWorkers: numWorkers} } @@ -56,7 +56,7 @@ func (ww withWorkers) apply(tm *transferManagerConfig) { // operation that is performed to download or upload an object. The timeout is // set when the operation begins processing, not when it is added. // By default, no timeout is set. -func WithPerOpTimeout(timeout time.Duration) TransferManagerOption { +func WithPerOpTimeout(timeout time.Duration) Option { return &withPerOpTimeout{timeout: timeout} } @@ -90,7 +90,7 @@ func defaultTransferManagerConfig() *transferManagerConfig { // initTransferManagerConfig initializes a config with the defaults and applies // the options passed in. -func initTransferManagerConfig(opts ...TransferManagerOption) *transferManagerConfig { +func initTransferManagerConfig(opts ...Option) *transferManagerConfig { config := defaultTransferManagerConfig() for _, o := range opts { o.apply(config) diff --git a/storage/transfermanager/option_test.go b/storage/transfermanager/option_test.go index 88ae1ddfa2c0..515744084914 100644 --- a/storage/transfermanager/option_test.go +++ b/storage/transfermanager/option_test.go @@ -20,7 +20,7 @@ import ( ) func TestApply(t *testing.T) { - opts := []TransferManagerOption{ + opts := []Option{ WithWorkers(3), WithPerOpTimeout(time.Hour), } From f5a6b33f061aa7df4226e8a7194e45f3f3eb1e53 Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Wed, 15 May 2024 14:55:54 -0700 Subject: [PATCH 08/12] add grpc tests and errors.join for waitandclose --- storage/transfermanager/doc.go | 8 +- storage/transfermanager/downloader.go | 9 +- storage/transfermanager/integration_test.go | 873 +++++++++++--------- 3 files changed, 485 insertions(+), 405 deletions(-) diff --git a/storage/transfermanager/doc.go b/storage/transfermanager/doc.go index f85c91e22c35..d7d427e945e7 100644 --- a/storage/transfermanager/doc.go +++ b/storage/transfermanager/doc.go @@ -26,7 +26,7 @@ NOTE: This package is in alpha. It is not stable, and is likely to change. # Example usage - // Pass in any client opts or set retry policy here + // Pass in any client opts or set retry policy here. client, err := storage.NewClient(ctx) // can also use NewGRPCClient if err != nil { // handle error @@ -39,7 +39,7 @@ NOTE: This package is in alpha. It is not stable, and is likely to change. // handle error } - // Create local file writer for output + // Create local file writer for output. f, err := os.Create("/path/to/localfile") if err != nil { // handle error @@ -61,10 +61,10 @@ NOTE: This package is in alpha. It is not stable, and is likely to change. ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) defer cancel() - // Add to Downloader + // Add to Downloader. d.DownloadObject(ctx, in) - // Repeat if desired + // Repeat if desired. // Wait for all downloads to complete. d.WaitAndClose() diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go index 00805621f82c..55ac88991b39 100644 --- a/storage/transfermanager/downloader.go +++ b/storage/transfermanager/downloader.go @@ -17,6 +17,7 @@ package transfermanager import ( "context" "errors" + "fmt" "io" "sync" "time" @@ -59,14 +60,16 @@ func (d *Downloader) DownloadObjectWithCallback(ctx context.Context, input *Down // WaitAndClose waits for all outstanding downloads to complete. The Downloader // must not be used for any more downloads after this has been called. -// WaitAndClose returns an error if any of the downloads fail. +// WaitAndClose returns an error wrapping all errors that were encountered by +// the Downloader when downloading objects. These errors are also returned in +// the DownloadOutput for the failing download. func (d *Downloader) WaitAndClose() error { d.done <- true d.workers.Wait() if len(d.errors) > 0 { - // TODO: return a multierror instead with go 1.20+ Join - return errors.New("transfermanager: at least one error encountered downloading objects") + err := errors.Join(d.errors...) + return fmt.Errorf("transfermanager: at least one error encountered downloading objects:\n%w", err) } return nil } diff --git a/storage/transfermanager/integration_test.go b/storage/transfermanager/integration_test.go index 47b9f66c07be..cb8d7a617b93 100644 --- a/storage/transfermanager/integration_test.go +++ b/storage/transfermanager/integration_test.go @@ -17,13 +17,13 @@ package transfermanager import ( "bytes" "context" - "crypto/rand" + crand "crypto/rand" "errors" "fmt" "hash/crc32" "io" "log" - "os" + "math/rand" "sync" "testing" "time" @@ -34,6 +34,8 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -43,325 +45,342 @@ const ( ) var ( - uidSpace = uid.NewSpace("", &uid.Options{Short: true}) - testBucket = downloadTestBucket{} // This bucket is shared amongst download tests. It is created with populated objects and cleaned up in TestMain. + uidSpace = uid.NewSpace("", &uid.Options{Short: true}) + // These buckets are shared amongst download tests. They are created, + // populated with objects and cleaned up in TestMain. + httpTestBucket = downloadTestBucket{} + grpcTestBucket = downloadTestBucket{} ) func TestMain(m *testing.M) { - if err := testBucket.Create(testPrefix); err != nil { + if err := httpTestBucket.Create(testPrefix); err != nil { log.Fatalf("test bucket creation failed: %v", err) } - exit := m.Run() + if err := grpcTestBucket.Create(grpcTestPrefix); err != nil { + log.Fatalf("test bucket creation failed: %v", err) + } - if err := testBucket.Cleanup(); err != nil { + m.Run() + + if err := httpTestBucket.Cleanup(); err != nil { log.Printf("test bucket cleanup failed: %v", err) } + if err := grpcTestBucket.Cleanup(); err != nil { + log.Printf("grpc test bucket cleanup failed: %v", err) + } if err := deleteExpiredBuckets(testPrefix); err != nil { - // Don't fail the test if cleanup fails. - log.Printf("Post-test cleanup failed: %v", err) + log.Printf("expired http bucket cleanup failed: %v", err) + } + if err := deleteExpiredBuckets(grpcTestPrefix); err != nil { + log.Printf("expired grpc bucket cleanup failed: %v", err) } - - os.Exit(exit) } func TestIntegration_DownloaderSynchronous(t *testing.T) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - t.Fatalf("NewClient: %v", err) - } - t.Cleanup(func() { client.Close() }) - - objects := testBucket.objects + multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, c *storage.Client, tb downloadTestBucket) { + objects := tb.objects - // Start a downloader. Give it a smaller amount of workers than objects, to - // make sure we aren't blocking anywhere. - d, err := NewDownloader(client, WithWorkers(2)) - if err != nil { - t.Fatalf("NewDownloader: %v", err) - } - // Download several objects. - writers := make([]*testWriter, len(objects)) - objToWriter := make(map[string]int) // so we can map the resulting content back to the correct object - for i, obj := range objects { - writers[i] = &testWriter{} - objToWriter[obj] = i - d.DownloadObject(ctx, &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: obj, - Destination: writers[i], - }) - } + // Start a downloader. Give it a smaller amount of workers than objects, + // to make sure we aren't blocking anywhere. + d, err := NewDownloader(c, WithWorkers(2)) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + // Download several objects. + writers := make([]*testWriter, len(objects)) + objToWriter := make(map[string]int) // so we can map the resulting content back to the correct object + for i, obj := range objects { + writers[i] = &testWriter{} + objToWriter[obj] = i + d.DownloadObject(ctx, &DownloadObjectInput{ + Bucket: tb.bucket, + Object: obj, + Destination: writers[i], + }) + } - if err := d.WaitAndClose(); err != nil { - t.Fatalf("d.WaitAndClose: %v", err) - } + if err := d.WaitAndClose(); err != nil { + t.Fatalf("d.WaitAndClose: %v", err) + } - // Close the writers so we can check the contents. This should be fine, - // since the downloads should all be complete after WaitAndClose. - for i := range objects { - if err := writers[i].Close(); err != nil { - t.Fatalf("testWriter.Close: %v", err) + // Close the writers so we can check the contents. This should be fine, + // since the downloads should all be complete after WaitAndClose. + for i := range objects { + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } } - } - // Check the results. - results := d.Results() - for _, got := range results { - writerIdx := objToWriter[got.Object] + // Check the results. + results := d.Results() + for _, got := range results { + writerIdx := objToWriter[got.Object] - if got.Err != nil { - t.Errorf("result.Err: %v", got.Err) - continue - } + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + continue + } - if want, got := testBucket.contentHashes[got.Object], writers[writerIdx].crc32c; got != want { - t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) - } + if want, got := tb.contentHashes[got.Object], writers[writerIdx].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } - if got.Attrs.Size != testBucket.objectSize { - t.Errorf("expected object size %d, got %d", testBucket.objectSize, got.Attrs.Size) + if got.Attrs.Size != tb.objectSize { + t.Errorf("expected object size %d, got %d", tb.objectSize, got.Attrs.Size) + } } - } - if len(results) != len(objects) { - t.Errorf("expected to receive %d results, got %d results", len(objects), len(results)) - } + if len(results) != len(objects) { + t.Errorf("expected to receive %d results, got %d results", len(objects), len(results)) + } + }) } // Tests that a single error does not affect the rest of the downloads. func TestIntegration_DownloaderErrorSync(t *testing.T) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - t.Fatalf("NewClient: %v", err) - } - t.Cleanup(func() { client.Close() }) - - // Make a copy of the objects slice. - objects := make([]string, len(testBucket.objects)) - copy(objects, testBucket.objects) + multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, c *storage.Client, tb downloadTestBucket) { + // Make a copy of the objects slice. + objects := make([]string, len(tb.objects)) + copy(objects, tb.objects) + + // Add another object to attempt to download; since it hasn't been written, + // this one will fail. Append to the start so that it will (likely) be + // attempted in the first 2 downloads. + nonexistentObject := "not-written" + objects = append([]string{nonexistentObject}, objects...) + + // Start a downloader. + d, err := NewDownloader(c, WithWorkers(2)) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } - // Add another object to attempt to download; since it hasn't been written, - // this one will fail. Append to the start so that it will (likely) be - // attempted in the first 2 downloads. - nonexistentObject := "not-written" - objects = append([]string{nonexistentObject}, objects...) + // Download objects. + writers := make([]*testWriter, len(objects)) + objToWriter := make(map[string]int) // so we can map the resulting content back to the correct object + for i, obj := range objects { + writers[i] = &testWriter{} + objToWriter[obj] = i + d.DownloadObject(ctx, &DownloadObjectInput{ + Bucket: tb.bucket, + Object: obj, + Destination: writers[i], + }) + } - // Start a downloader. - d, err := NewDownloader(client, WithWorkers(2)) - if err != nil { - t.Fatalf("NewDownloader: %v", err) - } + // WaitAndClose should return an error since one of our downloads should have failed. + if err := d.WaitAndClose(); err == nil { + t.Error("d.WaitAndClose should return an error, instead got nil") + } - // Download objects. - writers := make([]*testWriter, len(objects)) - objToWriter := make(map[string]int) // so we can map the resulting content back to the correct object - for i, obj := range objects { - writers[i] = &testWriter{} - objToWriter[obj] = i - d.DownloadObject(ctx, &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: obj, - Destination: writers[i], - }) - } + // Close the writers so we can check the contents. This should be fine, + // since the downloads should all be complete after WaitAndClose. + for i := range objects { + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + } - // WaitAndClose should return an error since one of our downloads should have failed. - if err := d.WaitAndClose(); err == nil { - t.Error("d.WaitAndClose should return an error, instead got nil") - } + // Check the results. + results := d.Results() + for _, got := range results { + writerIdx := objToWriter[got.Object] + + // Check that the nonexistent object returned an error. + if got.Object == nonexistentObject { + if got.Err != storage.ErrObjectNotExist { + t.Errorf("Object(%q) should not exist, err found to be %v", got.Object, got.Err) + } + continue + } - // Close the writers so we can check the contents. This should be fine, - // since the downloads should all be complete after WaitAndClose. - for i := range objects { - if err := writers[i].Close(); err != nil { - t.Fatalf("testWriter.Close: %v", err) - } - } + // All other objects should complete correctly. + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + continue + } - // Check the results. - results := d.Results() - for _, got := range results { - writerIdx := objToWriter[got.Object] + if want, got := tb.contentHashes[got.Object], writers[writerIdx].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } - // Check that the nonexistent object returned an error. - if got.Object == nonexistentObject { - if got.Err != storage.ErrObjectNotExist { - t.Errorf("Object(%q) should not exist, err found to be %v", got.Object, got.Err) + if got.Attrs.Size != tb.objectSize { + t.Errorf("expected object size %d, got %d", tb.objectSize, got.Attrs.Size) } - continue } - // All other objects should complete correctly. - if got.Err != nil { - t.Errorf("result.Err: %v", got.Err) - continue + if len(results) != len(objects) { + t.Errorf("expected to receive %d results, got %d results", len(objects), len(results)) } + }) +} - if want, got := testBucket.contentHashes[got.Object], writers[writerIdx].crc32c; got != want { - t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) - } +func TestIntegration_DownloaderAsynchronous(t *testing.T) { + multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, c *storage.Client, tb downloadTestBucket) { + objects := tb.objects - if got.Attrs.Size != testBucket.objectSize { - t.Errorf("expected object size %d, got %d", testBucket.objectSize, got.Attrs.Size) + d, err := NewDownloader(c, WithWorkers(2)) + if err != nil { + t.Fatalf("NewDownloader: %v", err) } - } - if len(results) != len(objects) { - t.Errorf("expected to receive %d results, got %d results", len(objects), len(results)) - } -} + numCallbacks := 0 + callbackMu := sync.Mutex{} + + // Download objects. + writers := make([]*testWriter, len(objects)) + for i, obj := range objects { + i := i + writers[i] = &testWriter{} + d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + Bucket: tb.bucket, + Object: obj, + Destination: writers[i], + }, func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + } + + // Close the writer so we can check the contents. + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + + if want, got := tb.contentHashes[got.Object], writers[i].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != tb.objectSize { + t.Errorf("expected object size %d, got %d", tb.objectSize, got.Attrs.Size) + } + }) + } -func TestIntegration_DownloaderAsynchronous(t *testing.T) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - t.Fatalf("NewClient: %v", err) - } - t.Cleanup(func() { client.Close() }) + if err := d.WaitAndClose(); err != nil { + t.Fatalf("d.WaitAndClose: %v", err) + } - objects := testBucket.objects + if numCallbacks != len(objects) { + t.Errorf("expected to receive %d results, got %d callbacks", len(objects), numCallbacks) + } + }) +} - // Start a downloader. Give it a smaller amount of workers than objects, to - // make sure we aren't blocking anywhere. - d, err := NewDownloader(client, WithWorkers(2)) - if err != nil { - t.Fatalf("NewDownloader: %v", err) - } +func TestIntegration_DownloaderErrorAsync(t *testing.T) { + multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, c *storage.Client, tb downloadTestBucket) { + d, err := NewDownloader(c, WithWorkers(2)) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } - numCallbacks := 0 - callbackMu := sync.Mutex{} + // Keep track of the number of callbacks. Since the callbacks may happen + // in parallel, we sync access to this variable. + numCallbacks := 0 + callbackMu := sync.Mutex{} - // Download objects. - writers := make([]*testWriter, len(objects)) - for i, obj := range objects { - i := i - writers[i] = &testWriter{} + // Download an object with incorrect generation. d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: obj, - Destination: writers[i], + Bucket: tb.bucket, + Object: tb.objects[0], + Destination: &testWriter{}, + Conditions: &storage.Conditions{ + GenerationMatch: -10, + }, }, func(got *DownloadOutput) { callbackMu.Lock() numCallbacks++ callbackMu.Unlock() - if got.Err != nil { - t.Errorf("result.Err: %v", got.Err) - } + // This will match both the expected http and grpc errors. + wantErr := errors.Join(&googleapi.Error{Code: 412}, status.Error(codes.FailedPrecondition, "")) - // Close the writer so we can check the contents. - if err := writers[i].Close(); err != nil { - t.Fatalf("testWriter.Close: %v", err) - } - - if want, got := testBucket.contentHashes[got.Object], writers[i].crc32c; got != want { - t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) - } - - if got.Attrs.Size != testBucket.objectSize { - t.Errorf("expected object size %d, got %d", testBucket.objectSize, got.Attrs.Size) + if !errorIs(got.Err, wantErr) { + t.Errorf("mismatching errors: got %v, want %v", got.Err, wantErr) } }) - } - - if err := d.WaitAndClose(); err != nil { - t.Fatalf("d.WaitAndClose: %v", err) - } - - if numCallbacks != len(objects) { - t.Errorf("expected to receive %d results, got %d callbacks", len(objects), numCallbacks) - } -} - -func TestIntegration_DownloaderErrorAsync(t *testing.T) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - t.Fatalf("NewClient: %v", err) - } - t.Cleanup(func() { client.Close() }) - - objects := testBucket.objects - // Start a downloader. Give it a smaller amount of workers than objects, to - // make sure we aren't blocking anywhere. - d, err := NewDownloader(client, WithWorkers(2)) - if err != nil { - t.Fatalf("NewDownloader: %v", err) - } - - // Keep track of the number of callbacks. Since the callbacks may happen - // in parallel, we sync access to this variable. - numCallbacks := 0 - callbackMu := sync.Mutex{} - - // Download a non-existent object. - nonexistentObject := "not-written" - w := &testWriter{} - - d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: nonexistentObject, - Destination: w, - }, func(got *DownloadOutput) { - callbackMu.Lock() - numCallbacks++ - callbackMu.Unlock() - - // Check that the nonexistent object returned an error. - if got.Err != storage.ErrObjectNotExist { - t.Errorf("Object(%q) should not exist, err found to be %v", got.Object, got.Err) + // Download existing objects. + writers := make([]*testWriter, len(tb.objects)) + for i, obj := range tb.objects { + i := i + writers[i] = &testWriter{} + d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + Bucket: tb.bucket, + Object: obj, + Destination: writers[i], + }, func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + } + + // Close the writer so we can check the contents. + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + + if want, got := tb.contentHashes[got.Object], writers[i].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != tb.objectSize { + t.Errorf("expected object size %d, got %d", tb.objectSize, got.Attrs.Size) + } + }) } - }) - // Download remaining objects. - writers := make([]*testWriter, len(objects)) - for i, obj := range objects { - i := i - writers[i] = &testWriter{} + // Download a nonexistent object. d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: obj, - Destination: writers[i], + Bucket: tb.bucket, + Object: "not-written", + Destination: &testWriter{}, }, func(got *DownloadOutput) { callbackMu.Lock() numCallbacks++ callbackMu.Unlock() - if got.Err != nil { - t.Errorf("result.Err: %v", got.Err) + // Check that the nonexistent object returned an error. + if got.Err != storage.ErrObjectNotExist { + t.Errorf("Object(%q) should not exist, err found to be %v", got.Object, got.Err) } + }) - // Close the writer so we can check the contents. - if err := writers[i].Close(); err != nil { - t.Fatalf("testWriter.Close: %v", err) - } + // WaitAndClose should return an error since 2 of our downloads should have failed. + err = d.WaitAndClose() + if err == nil { + t.Error("d.WaitAndClose should return an error, instead got nil") + } - if want, got := testBucket.contentHashes[got.Object], writers[i].crc32c; got != want { - t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) - } + // Check that both errors were returned. + wantErrs := []error{errors.Join(&googleapi.Error{Code: 412}, status.Error(codes.FailedPrecondition, "")), + storage.ErrObjectNotExist} - if got.Attrs.Size != testBucket.objectSize { - t.Errorf("expected object size %d, got %d", testBucket.objectSize, got.Attrs.Size) + for _, want := range wantErrs { + if !errorIs(err, want) { + t.Errorf("got error does not wrap expected error %q, got:\n%v", want, err) } - }) - } - - // WaitAndClose should return an error since one of our downloads should have failed. - if err := d.WaitAndClose(); err == nil { - t.Error("d.WaitAndClose should return an error, instead got nil") - } + } - // We expect num objects callbacks + 1 for the errored call. - if numCallbacks != len(objects)+1 { - t.Errorf("expected to receive %d results, got %d callbacks", len(objects)+1, numCallbacks) - } + // We expect num objects callbacks + 2 for the errored calls. + if want, got := len(tb.objects)+2, numCallbacks; want != got { + t.Errorf("expected to receive %d results, got %d callbacks", want, got) + } + }) } func TestIntegration_DownloaderTimeout(t *testing.T) { + if testing.Short() { + t.Skip("Integration tests skipped in short mode") + } + ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { @@ -377,8 +396,8 @@ func TestIntegration_DownloaderTimeout(t *testing.T) { // Download an object. d.DownloadObject(ctx, &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: testBucket.objects[0], + Bucket: httpTestBucket.bucket, + Object: httpTestBucket.objects[0], Destination: &testWriter{}, }) @@ -398,181 +417,182 @@ func TestIntegration_DownloaderTimeout(t *testing.T) { } func TestIntegration_DownloadShard(t *testing.T) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - t.Fatalf("NewClient: %v", err) - } - t.Cleanup(func() { client.Close() }) - - // Use an object from the test bucket. - objectName := testBucket.objects[0] + multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, c *storage.Client, tb downloadTestBucket) { + objectName := tb.objects[0] - // Get expected Attrs. - o := client.Bucket(testBucket.bucket).Object(objectName) - r, err := o.NewReader(ctx) - if err != nil { - t.Fatalf("o.Attrs: %v", err) - } + // Get expected Attrs. + o := c.Bucket(tb.bucket).Object(objectName) + r, err := o.NewReader(ctx) + if err != nil { + t.Fatalf("o.Attrs: %v", err) + } - incorrectGen := r.Attrs.Generation - 1 - - cancelledCtx, cancel := context.WithCancel(ctx) - cancel() - - for _, test := range []struct { - desc string - timeout time.Duration - in *DownloadObjectInput - want *DownloadOutput - }{ - { - desc: "basic input", - in: &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: objectName, - }, - want: &DownloadOutput{ - Bucket: testBucket.bucket, - Object: objectName, - Attrs: &r.Attrs, - }, - }, - { - desc: "range", - in: &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: objectName, - Range: &DownloadRange{ - Offset: testBucket.objectSize - 5, - Length: -1, + incorrectGen := r.Attrs.Generation - 1 + + cancelledCtx, cancel := context.WithCancel(ctx) + cancel() + + for _, test := range []struct { + desc string + timeout time.Duration + in *DownloadObjectInput + want *DownloadOutput + }{ + { + desc: "basic input", + in: &DownloadObjectInput{ + Bucket: tb.bucket, + Object: objectName, }, - }, - want: &DownloadOutput{ - Bucket: testBucket.bucket, - Object: objectName, - Attrs: &storage.ReaderObjectAttrs{ - Size: testBucket.objectSize, - StartOffset: testBucket.objectSize - 5, - ContentType: r.Attrs.ContentType, - ContentEncoding: r.Attrs.ContentEncoding, - CacheControl: r.Attrs.CacheControl, - LastModified: r.Attrs.LastModified, - Generation: r.Attrs.Generation, - Metageneration: r.Attrs.Metageneration, + want: &DownloadOutput{ + Bucket: tb.bucket, + Object: objectName, + Attrs: &r.Attrs, }, }, - }, - { - desc: "incorrect generation", - in: &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: objectName, - Generation: &incorrectGen, - }, - want: &DownloadOutput{ - Bucket: testBucket.bucket, - Object: objectName, - Err: storage.ErrObjectNotExist, - }, - }, - { - desc: "conditions: generationmatch", - in: &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: objectName, - Conditions: &storage.Conditions{ - GenerationMatch: r.Attrs.Generation, + { + desc: "range", + in: &DownloadObjectInput{ + Bucket: tb.bucket, + Object: objectName, + Range: &DownloadRange{ + Offset: tb.objectSize - 5, + Length: -1, + }, }, - }, - want: &DownloadOutput{ - Bucket: testBucket.bucket, - Object: objectName, - Attrs: &r.Attrs, - }, - }, - { - desc: "conditions do not hold", - in: &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: objectName, - Conditions: &storage.Conditions{ - GenerationMatch: incorrectGen, + want: &DownloadOutput{ + Bucket: tb.bucket, + Object: objectName, + Attrs: &storage.ReaderObjectAttrs{ + Size: tb.objectSize, + StartOffset: tb.objectSize - 5, + ContentType: r.Attrs.ContentType, + ContentEncoding: r.Attrs.ContentEncoding, + CacheControl: r.Attrs.CacheControl, + LastModified: r.Attrs.LastModified, + Generation: r.Attrs.Generation, + Metageneration: r.Attrs.Metageneration, + }, }, }, - want: &DownloadOutput{ - Bucket: testBucket.bucket, - Object: objectName, - Err: &googleapi.Error{Code: 412}, + { + desc: "incorrect generation", + in: &DownloadObjectInput{ + Bucket: tb.bucket, + Object: objectName, + Generation: &incorrectGen, + }, + want: &DownloadOutput{ + Bucket: tb.bucket, + Object: objectName, + Err: storage.ErrObjectNotExist, + }, }, - }, - { - desc: "timeout", - in: &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: objectName, + { + desc: "conditions: generationmatch", + in: &DownloadObjectInput{ + Bucket: tb.bucket, + Object: objectName, + Conditions: &storage.Conditions{ + GenerationMatch: r.Attrs.Generation, + }, + }, + want: &DownloadOutput{ + Bucket: tb.bucket, + Object: objectName, + Attrs: &r.Attrs, + }, }, - timeout: time.Nanosecond, - want: &DownloadOutput{ - Bucket: testBucket.bucket, - Object: objectName, - Err: context.DeadlineExceeded, + { + desc: "conditions do not hold", + in: &DownloadObjectInput{ + Bucket: tb.bucket, + Object: objectName, + Conditions: &storage.Conditions{ + GenerationMatch: incorrectGen, + }, + }, + want: &DownloadOutput{ + Bucket: tb.bucket, + Object: objectName, + Err: errors.Join(&googleapi.Error{Code: 412}, status.Error(codes.FailedPrecondition, "")), + }, }, - }, - { - desc: "cancelled ctx", - in: &DownloadObjectInput{ - Bucket: testBucket.bucket, - Object: objectName, - ctx: cancelledCtx, + { + desc: "timeout", + in: &DownloadObjectInput{ + Bucket: tb.bucket, + Object: objectName, + }, + timeout: time.Nanosecond, + want: &DownloadOutput{ + Bucket: tb.bucket, + Object: objectName, + Err: context.DeadlineExceeded, + }, }, - timeout: time.Nanosecond, - want: &DownloadOutput{ - Bucket: testBucket.bucket, - Object: objectName, - Err: context.Canceled, + { + desc: "cancelled ctx", + in: &DownloadObjectInput{ + Bucket: tb.bucket, + Object: objectName, + ctx: cancelledCtx, + }, + timeout: time.Nanosecond, + want: &DownloadOutput{ + Bucket: tb.bucket, + Object: objectName, + Err: context.Canceled, + }, }, - }, - } { - t.Run(test.desc, func(t *testing.T) { - w := &testWriter{} + } { + t.Run(test.desc, func(t *testing.T) { + w := &testWriter{} - test.in.Destination = w + test.in.Destination = w - if test.in.ctx == nil { - test.in.ctx = ctx - } + if test.in.ctx == nil { + test.in.ctx = ctx + } - got := test.in.downloadShard(client, test.timeout) + got := test.in.downloadShard(c, test.timeout) - if got.Bucket != test.want.Bucket || got.Object != test.want.Object { - t.Errorf("wanted bucket %q, object %q, got: %q, %q", test.want.Bucket, test.want.Object, got.Bucket, got.Object) - } - - if diff := cmp.Diff(got.Attrs, test.want.Attrs); diff != "" { - t.Errorf("diff got(-) vs. want(+): %v", diff) - } + if got.Bucket != test.want.Bucket || got.Object != test.want.Object { + t.Errorf("wanted bucket %q, object %q, got: %q, %q", test.want.Bucket, test.want.Object, got.Bucket, got.Object) + } - if !errorIs(got.Err, test.want.Err) { - t.Errorf("mismatching errors: got %v, want %v", got.Err, test.want.Err) - } - }) - } + if diff := cmp.Diff(got.Attrs, test.want.Attrs); diff != "" { + t.Errorf("diff got(-) vs. want(+): %v", diff) + } + if !errorIs(got.Err, test.want.Err) { + t.Errorf("mismatching errors: got %v, want %v", got.Err, test.want.Err) + } + }) + } + }) } -// test ctx cancel and per op timeout - // errorIs is equivalent to errors.Is, except that it additionally will return -// true if err and targetErr are googleapi.Errors with identical error codes. +// true if err and targetErr are googleapi.Errors with identical error codes, +// or if both errors have the same gRPC status code. func errorIs(err error, targetErr error) bool { + if errors.Is(err, targetErr) { + return true + } + + // Check http var e, targetE *googleapi.Error if errors.As(err, &e) && errors.As(targetErr, &targetE) { return e.Code == targetE.Code } - // fallback to regular check - return errors.Is(err, targetErr) + // Check grpc errors + if status.Code(err) != codes.Unknown { + return status.Code(err) == status.Code(targetErr) + } + + return false } // generateRandomFileInGCS uploads a file with random contents to GCS and returns @@ -583,13 +603,22 @@ func generateFileInGCS(ctx context.Context, o *storage.ObjectHandle, size int64) crc32cHash := crc32.New(crc32.MakeTable(crc32.Castagnoli)) mw := io.MultiWriter(w, crc32cHash) - if _, err := io.CopyN(mw, rand.Reader, size); err != nil { + if _, err := io.CopyN(mw, crand.Reader, size); err != nil { w.Close() return 0, err } return crc32cHash.Sum32(), w.Close() } +// randomInt64 returns a value in the closed interval [min, max]. +// That is, the endpoints are possible return values. +func randomInt64(min, max int64) int64 { + if min > max { + log.Fatalf("min cannot be larger than max; min: %d max: %d", min, max) + } + return rand.Int63n(max-min+1) + min +} + // TODO: once we provide a DownloaderBuffer that implements WriterAt in the // library, we can use that instead. type testWriter struct { @@ -687,7 +716,7 @@ func (tb *downloadTestBucket) Create(prefix string) error { ctx := context.Background() tb.bucket = prefix + uidSpace.New() - tb.objectSize = int64(507) + tb.objectSize = randomInt64(200, 1024*1024) tb.objects = []string{ "obj1", "obj2", @@ -706,7 +735,7 @@ func (tb *downloadTestBucket) Create(prefix string) error { b := client.Bucket(tb.bucket) if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { - return fmt.Errorf("bucket.Create: %v", err) + return fmt.Errorf("bucket(%q).Create: %v", tb.bucket, err) } // Write objects. @@ -741,3 +770,51 @@ func (tb *downloadTestBucket) Cleanup() error { return b.Delete(ctx) } + +// multiTransportTest initializes fresh clients for each transport, then runs +// given testing function using each transport-specific client, supplying the +// test function with the sub-test instance, the context it was given, a test +// bucket and the client to use. +func multiTransportTest(ctx context.Context, t *testing.T, test func(*testing.T, context.Context, *storage.Client, downloadTestBucket)) { + if testing.Short() { + t.Skip("Integration tests skipped in short mode") + } + + clients, err := initTransportClients(ctx) + if err != nil { + t.Fatalf("init clients: %v", err) + } + + for transport, client := range clients { + t.Run(transport, func(t *testing.T) { + t.Cleanup(func() { + client.Close() + }) + + testBucket := httpTestBucket + + if transport == "grpc" { + testBucket = grpcTestBucket + } + + test(t, ctx, client, testBucket) + }) + } +} + +func initTransportClients(ctx context.Context) (map[string]*storage.Client, error) { + c, err := storage.NewClient(ctx) + if err != nil { + return nil, err + } + + gc, err := storage.NewGRPCClient(ctx) + if err != nil { + return nil, err + } + + return map[string]*storage.Client{ + "http": c, + "grpc": gc, + }, nil +} From 8956b7bd214701c804ab9913db6a29e478f7f7de Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Wed, 15 May 2024 16:41:17 -0700 Subject: [PATCH 09/12] remove bucket creation for short tests --- storage/transfermanager/integration_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/storage/transfermanager/integration_test.go b/storage/transfermanager/integration_test.go index cb8d7a617b93..ca49669be629 100644 --- a/storage/transfermanager/integration_test.go +++ b/storage/transfermanager/integration_test.go @@ -647,6 +647,10 @@ func (tw *testWriter) WriteAt(b []byte, offset int64) (n int, err error) { } func deleteExpiredBuckets(prefix string) error { + if testing.Short() { + return nil + } + ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { @@ -713,6 +717,9 @@ type downloadTestBucket struct { // objects in it. All objects are of the same size but with different contents // and can be mapped to their respective crc32c hash in contentHashes. func (tb *downloadTestBucket) Create(prefix string) error { + if testing.Short() { + return nil + } ctx := context.Background() tb.bucket = prefix + uidSpace.New() @@ -752,6 +759,9 @@ func (tb *downloadTestBucket) Create(prefix string) error { // Cleanup deletes the objects and bucket created in Create. func (tb *downloadTestBucket) Cleanup() error { + if testing.Short() { + return nil + } ctx := context.Background() client, err := storage.NewClient(ctx) From ed19c7f721138517ee1d260af0d352b97a281472 Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Wed, 15 May 2024 16:59:56 -0700 Subject: [PATCH 10/12] parse flags in test main to allow setup to skip in short mode --- storage/transfermanager/integration_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/transfermanager/integration_test.go b/storage/transfermanager/integration_test.go index ca49669be629..04a9053f92c4 100644 --- a/storage/transfermanager/integration_test.go +++ b/storage/transfermanager/integration_test.go @@ -19,6 +19,7 @@ import ( "context" crand "crypto/rand" "errors" + "flag" "fmt" "hash/crc32" "io" @@ -53,6 +54,8 @@ var ( ) func TestMain(m *testing.M) { + flag.Parse() + if err := httpTestBucket.Create(testPrefix); err != nil { log.Fatalf("test bucket creation failed: %v", err) } From 5afae5370efb4038209621ebfbf325268da8cc65 Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Wed, 22 May 2024 11:21:13 -0700 Subject: [PATCH 11/12] suggestions --- storage/transfermanager/doc.go | 58 +----- storage/transfermanager/downloader.go | 76 ++++---- storage/transfermanager/downloader_test.go | 41 +++++ storage/transfermanager/example_test.go | 146 ++++++++++++++++ storage/transfermanager/integration_test.go | 184 +++++++++++--------- storage/transfermanager/option.go | 7 +- storage/transfermanager/option_test.go | 43 +++++ 7 files changed, 373 insertions(+), 182 deletions(-) create mode 100644 storage/transfermanager/downloader_test.go create mode 100644 storage/transfermanager/example_test.go diff --git a/storage/transfermanager/doc.go b/storage/transfermanager/doc.go index d7d427e945e7..c8afb94c2d39 100644 --- a/storage/transfermanager/doc.go +++ b/storage/transfermanager/doc.go @@ -22,62 +22,6 @@ https://cloud.google.com/storage/docs. See https://pkg.go.dev/cloud.google.com/go for authentication, timeouts, connection pooling and similar aspects of this package. -NOTE: This package is in alpha. It is not stable, and is likely to change. - -# Example usage - - // Pass in any client opts or set retry policy here. - client, err := storage.NewClient(ctx) // can also use NewGRPCClient - if err != nil { - // handle error - } - - // Create Downloader with desired options, including number of workers, - // part size, per operation timeout, etc. - d, err := transfermanager.NewDownloader(client, transfermanager.WithWorkers(16)) - if err != nil { - // handle error - } - - // Create local file writer for output. - f, err := os.Create("/path/to/localfile") - if err != nil { - // handle error - } - - // Create download input - in := &transfermanager.DownloadObjectInput{ - Bucket: "mybucket", - Object: "myblob", - Destination: f, - // Optionally specify params to apply to download. - EncryptionKey: []byte("mykey"), - } - - // Can set timeout on this download using context. Note that this download - // may not start immediately if all workers are busy, so this may time out - // before the download even starts. To set a timeout that starts with the - // download, use transfermanager.WithPerOpTimeout(time.Duration). - ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - - // Add to Downloader. - d.DownloadObject(ctx, in) - - // Repeat if desired. - - // Wait for all downloads to complete. - d.WaitAndClose() - - // Iterate through completed downloads and process results. This can - // also happen async in a go routine as the downloads run. - results := d.Results() - for _, out := range results { - if out.Err != nil { - log.Printf("download of %v failed with error %v", out.Name, out.Err) - } else { - log.Printf("download of %v succeeded", out.Object) - } - } +NOTE: This package is in preview. It is not stable, and is likely to change. */ package transfermanager // import "cloud.google.com/go/storage/transfermanager" diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go index 55ac88991b39..207006a6859e 100644 --- a/storage/transfermanager/downloader.go +++ b/storage/transfermanager/downloader.go @@ -41,51 +41,45 @@ type Downloader struct { } // DownloadObject queues the download of a single object. This will initiate the -// download but is non-blocking; call Downloader.Results to process the result. -func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectInput) { - input.ctx = ctx - d.addInput(input) -} +// download but is non-blocking; call Downloader.Results or use the callback to +// process the result. +func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectInput) error { + if d.config.asynchronous && input.Callback == nil { + return errors.New("transfermanager: input.Callback must not be nil when the WithCallbacks option is set") + } + if !d.config.asynchronous && input.Callback != nil { + return errors.New("transfermanager: input.Callback must be nil unless the WithCallbacks option is set") + } + + select { + case <-d.done: + return errors.New("transfermanager: WaitAndClose called before DownloadObject") + default: + } -// DownloadObjectWithCallback queues the download of a single object. This will -// initiate the download but is non-blocking; use the callback to process the -// result. -// The results of downloads initiated with this method will not be provided in -// the results given in Downloader.Results. -func (d *Downloader) DownloadObjectWithCallback(ctx context.Context, input *DownloadObjectInput, callback func(*DownloadOutput)) { input.ctx = ctx - input.callback = &callback d.addInput(input) + return nil } -// WaitAndClose waits for all outstanding downloads to complete. The Downloader -// must not be used for any more downloads after this has been called. -// WaitAndClose returns an error wrapping all errors that were encountered by -// the Downloader when downloading objects. These errors are also returned in -// the DownloadOutput for the failing download. -func (d *Downloader) WaitAndClose() error { +// WaitAndClose waits for all outstanding downloads to complete and closes the +// downloader. Adding new downloads after this has been called will cause an error. +// +// WaitAndClose returns all the results of the downloads and an error wrapping +// all errors that were encountered by the Downloader when downloading objects. +// These errors are also returned in the respective DownloadOutput for the +// failing download. The results are not guaranteed to be in any order. +// Results will be empty if using the [WithCallbacks] option. +func (d *Downloader) WaitAndClose() ([]DownloadOutput, error) { d.done <- true d.workers.Wait() + close(d.done) if len(d.errors) > 0 { err := errors.Join(d.errors...) - return fmt.Errorf("transfermanager: at least one error encountered downloading objects:\n%w", err) + return d.results, fmt.Errorf("transfermanager: at least one error encountered downloading objects:\n%w", err) } - return nil -} - -// Results returns all the results of the downloads completed since the last -// time it was called. Call WaitAndClose before calling Results to wait for all -// downloads to complete. The results are not guaranteed to be in any order. -// Results will not return results for downloads initiated with a callback. -func (d *Downloader) Results() []DownloadOutput { - d.resultsMu.Lock() - r := make([]DownloadOutput, len(d.results)) - copy(r, d.results) - d.results = []DownloadOutput{} - d.resultsMu.Unlock() - - return r + return d.results, nil } // sendInputsToWorkChan listens continuously to the inputs slice until d.done. @@ -156,12 +150,12 @@ func (d *Downloader) downloadWorker() { // Keep track of any error that occurred. if out.Err != nil { - d.error(out.Err) + d.error(fmt.Errorf("downloading %q from bucket %q: %w", input.Object, input.Bucket, out.Err)) } // Either execute the callback, or append to results. - if input.callback != nil { - (*input.callback)(out) + if d.config.asynchronous { + input.Callback(out) } else { d.addResult(out) } @@ -223,8 +217,12 @@ type DownloadObjectInput struct { EncryptionKey []byte Range *DownloadRange // if specified, reads only a range - ctx context.Context - callback *func(*DownloadOutput) + // Callback will be run once the object is finished downloading. It must be + // set if and only if the [WithCallbacks] option is set; otherwise, it must + // not be set. + Callback func(*DownloadOutput) + + ctx context.Context } // downloadShard will read a specific object into in.Destination. diff --git a/storage/transfermanager/downloader_test.go b/storage/transfermanager/downloader_test.go new file mode 100644 index 000000000000..6521ae534249 --- /dev/null +++ b/storage/transfermanager/downloader_test.go @@ -0,0 +1,41 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transfermanager + +import ( + "context" + "strings" + "testing" +) + +func TestWaitAndClose(t *testing.T) { + d, err := NewDownloader(nil) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + + if _, err := d.WaitAndClose(); err != nil { + t.Fatalf("WaitAndClose: %v", err) + } + + expectedErr := "transfermanager: WaitAndClose called before DownloadObject" + err = d.DownloadObject(context.Background(), &DownloadObjectInput{}) + if err == nil { + t.Fatalf("d.DownloadObject err was nil, should be %q", expectedErr) + } + if !strings.Contains(err.Error(), expectedErr) { + t.Errorf("expected err %q, got: %v", expectedErr, err.Error()) + } +} diff --git a/storage/transfermanager/example_test.go b/storage/transfermanager/example_test.go new file mode 100644 index 000000000000..83bb327e9d22 --- /dev/null +++ b/storage/transfermanager/example_test.go @@ -0,0 +1,146 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transfermanager_test + +import ( + "context" + "log" + "os" + "time" + + "cloud.google.com/go/storage" + "cloud.google.com/go/storage/transfermanager" +) + +func ExampleDownloader_synchronous() { + ctx := context.Background() + // Pass in any client opts or set retry policy here. + client, err := storage.NewClient(ctx) // can also use NewGRPCClient + if err != nil { + // handle error + } + + // Create Downloader with desired options, including number of workers, + // part size, per operation timeout, etc. + d, err := transfermanager.NewDownloader(client, transfermanager.WithWorkers(16)) + if err != nil { + // handle error + } + + // Create local file writer for output. + f, err := os.Create("/path/to/localfile") + if err != nil { + // handle error + } + + // Create download input + in := &transfermanager.DownloadObjectInput{ + Bucket: "mybucket", + Object: "myblob", + Destination: f, + // Optionally specify params to apply to download. + EncryptionKey: []byte("mykey"), + } + + // Can set timeout on this download using context. Note that this download + // may not start immediately if all workers are busy, so this may time out + // before the download even starts. To set a timeout that starts with the + // download, use transfermanager.WithPerOpTimeout(time.Duration). + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + // Add to Downloader. + if err := d.DownloadObject(ctx, in); err != nil { + // handle error + } + + // Repeat if desired. + + // Wait for all downloads to complete. + results, err := d.WaitAndClose() + if err != nil { + // handle error + } + + // Iterate through completed downloads and process results. + for _, out := range results { + if out.Err != nil { + log.Printf("download of %v failed with error %v", out.Object, out.Err) + } else { + log.Printf("download of %v succeeded", out.Object) + } + } +} + +func ExampleDownloader_asynchronous() { + ctx := context.Background() + // Pass in any client opts or set retry policy here. + client, err := storage.NewClient(ctx) // can also use NewGRPCClient + if err != nil { + // handle error + } + + // Create Downloader with callbacks plus any desired options, including + // number of workers, part size, per operation timeout, etc. + d, err := transfermanager.NewDownloader(client, transfermanager.WithCallbacks()) + if err != nil { + // handle error + } + defer func() { + if _, err := d.WaitAndClose(); err != nil { + // one or more of the downloads failed + } + }() + + // Create local file writer for output. + f, err := os.Create("/path/to/localfile") + if err != nil { + // handle error + } + + // Create callback function + callback := func(out *transfermanager.DownloadOutput) { + if out.Err != nil { + log.Printf("download of %v failed with error %v", out.Object, out.Err) + } else { + log.Printf("download of %v succeeded", out.Object) + } + } + + // Create download input + in := &transfermanager.DownloadObjectInput{ + Bucket: "mybucket", + Object: "myblob", + Destination: f, + // Optionally specify params to apply to download. + EncryptionKey: []byte("mykey"), + // Specify the callback + Callback: callback, + } + + // Can set timeout on this download using context. Note that this download + // may not start immediately if all workers are busy, so this may time out + // before the download even starts. To set a timeout that starts with the + // download, use transfermanager.WithPerOpTimeout(time.Duration). + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + // Add to Downloader. + if err := d.DownloadObject(ctx, in); err != nil { + // handle error + } + + // Repeat if desired. +} diff --git a/storage/transfermanager/integration_test.go b/storage/transfermanager/integration_test.go index 04a9053f92c4..ae5eb873221c 100644 --- a/storage/transfermanager/integration_test.go +++ b/storage/transfermanager/integration_test.go @@ -96,14 +96,17 @@ func TestIntegration_DownloaderSynchronous(t *testing.T) { for i, obj := range objects { writers[i] = &testWriter{} objToWriter[obj] = i - d.DownloadObject(ctx, &DownloadObjectInput{ + if err := d.DownloadObject(ctx, &DownloadObjectInput{ Bucket: tb.bucket, Object: obj, Destination: writers[i], - }) + }); err != nil { + t.Errorf("d.DownloadObject: %v", err) + } } - if err := d.WaitAndClose(); err != nil { + results, err := d.WaitAndClose() + if err != nil { t.Fatalf("d.WaitAndClose: %v", err) } @@ -116,7 +119,6 @@ func TestIntegration_DownloaderSynchronous(t *testing.T) { } // Check the results. - results := d.Results() for _, got := range results { writerIdx := objToWriter[got.Object] @@ -165,15 +167,18 @@ func TestIntegration_DownloaderErrorSync(t *testing.T) { for i, obj := range objects { writers[i] = &testWriter{} objToWriter[obj] = i - d.DownloadObject(ctx, &DownloadObjectInput{ + if err := d.DownloadObject(ctx, &DownloadObjectInput{ Bucket: tb.bucket, Object: obj, Destination: writers[i], - }) + }); err != nil { + t.Errorf("d.DownloadObject: %v", err) + } } // WaitAndClose should return an error since one of our downloads should have failed. - if err := d.WaitAndClose(); err == nil { + results, err := d.WaitAndClose() + if err == nil { t.Error("d.WaitAndClose should return an error, instead got nil") } @@ -186,7 +191,6 @@ func TestIntegration_DownloaderErrorSync(t *testing.T) { } // Check the results. - results := d.Results() for _, got := range results { writerIdx := objToWriter[got.Object] @@ -223,7 +227,7 @@ func TestIntegration_DownloaderAsynchronous(t *testing.T) { multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, c *storage.Client, tb downloadTestBucket) { objects := tb.objects - d, err := NewDownloader(c, WithWorkers(2)) + d, err := NewDownloader(c, WithWorkers(2), WithCallbacks()) if err != nil { t.Fatalf("NewDownloader: %v", err) } @@ -236,35 +240,38 @@ func TestIntegration_DownloaderAsynchronous(t *testing.T) { for i, obj := range objects { i := i writers[i] = &testWriter{} - d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + if err := d.DownloadObject(ctx, &DownloadObjectInput{ Bucket: tb.bucket, Object: obj, Destination: writers[i], - }, func(got *DownloadOutput) { - callbackMu.Lock() - numCallbacks++ - callbackMu.Unlock() - - if got.Err != nil { - t.Errorf("result.Err: %v", got.Err) - } - - // Close the writer so we can check the contents. - if err := writers[i].Close(); err != nil { - t.Fatalf("testWriter.Close: %v", err) - } - - if want, got := tb.contentHashes[got.Object], writers[i].crc32c; got != want { - t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) - } - - if got.Attrs.Size != tb.objectSize { - t.Errorf("expected object size %d, got %d", tb.objectSize, got.Attrs.Size) - } - }) + Callback: func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + } + + // Close the writer so we can check the contents. + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + + if want, got := tb.contentHashes[got.Object], writers[i].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != tb.objectSize { + t.Errorf("expected object size %d, got %d", tb.objectSize, got.Attrs.Size) + } + }, + }); err != nil { + t.Errorf("d.DownloadObject: %v", err) + } } - if err := d.WaitAndClose(); err != nil { + if _, err := d.WaitAndClose(); err != nil { t.Fatalf("d.WaitAndClose: %v", err) } @@ -276,7 +283,7 @@ func TestIntegration_DownloaderAsynchronous(t *testing.T) { func TestIntegration_DownloaderErrorAsync(t *testing.T) { multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, c *storage.Client, tb downloadTestBucket) { - d, err := NewDownloader(c, WithWorkers(2)) + d, err := NewDownloader(c, WithWorkers(2), WithCallbacks()) if err != nil { t.Fatalf("NewDownloader: %v", err) } @@ -287,77 +294,86 @@ func TestIntegration_DownloaderErrorAsync(t *testing.T) { callbackMu := sync.Mutex{} // Download an object with incorrect generation. - d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + if err := d.DownloadObject(ctx, &DownloadObjectInput{ Bucket: tb.bucket, Object: tb.objects[0], Destination: &testWriter{}, Conditions: &storage.Conditions{ GenerationMatch: -10, }, - }, func(got *DownloadOutput) { - callbackMu.Lock() - numCallbacks++ - callbackMu.Unlock() + Callback: func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() - // This will match both the expected http and grpc errors. - wantErr := errors.Join(&googleapi.Error{Code: 412}, status.Error(codes.FailedPrecondition, "")) + // This will match both the expected http and grpc errors. + wantErr := errors.Join(&googleapi.Error{Code: 412}, status.Error(codes.FailedPrecondition, "")) - if !errorIs(got.Err, wantErr) { - t.Errorf("mismatching errors: got %v, want %v", got.Err, wantErr) - } - }) + if !errorIs(got.Err, wantErr) { + t.Errorf("mismatching errors: got %v, want %v", got.Err, wantErr) + } + }, + }); err != nil { + t.Errorf("d.DownloadObject: %v", err) + } // Download existing objects. writers := make([]*testWriter, len(tb.objects)) for i, obj := range tb.objects { i := i writers[i] = &testWriter{} - d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + if err := d.DownloadObject(ctx, &DownloadObjectInput{ Bucket: tb.bucket, Object: obj, Destination: writers[i], - }, func(got *DownloadOutput) { - callbackMu.Lock() - numCallbacks++ - callbackMu.Unlock() - - if got.Err != nil { - t.Errorf("result.Err: %v", got.Err) - } - - // Close the writer so we can check the contents. - if err := writers[i].Close(); err != nil { - t.Fatalf("testWriter.Close: %v", err) - } - - if want, got := tb.contentHashes[got.Object], writers[i].crc32c; got != want { - t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) - } - - if got.Attrs.Size != tb.objectSize { - t.Errorf("expected object size %d, got %d", tb.objectSize, got.Attrs.Size) - } - }) + Callback: func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() + + if got.Err != nil { + t.Errorf("result.Err: %v", got.Err) + } + + // Close the writer so we can check the contents. + if err := writers[i].Close(); err != nil { + t.Fatalf("testWriter.Close: %v", err) + } + + if want, got := tb.contentHashes[got.Object], writers[i].crc32c; got != want { + t.Fatalf("content crc32c does not match; got: %v, expected: %v", got, want) + } + + if got.Attrs.Size != tb.objectSize { + t.Errorf("expected object size %d, got %d", tb.objectSize, got.Attrs.Size) + } + }, + }); err != nil { + t.Errorf("d.DownloadObject: %v", err) + } } // Download a nonexistent object. - d.DownloadObjectWithCallback(ctx, &DownloadObjectInput{ + if err := d.DownloadObject(ctx, &DownloadObjectInput{ Bucket: tb.bucket, Object: "not-written", Destination: &testWriter{}, - }, func(got *DownloadOutput) { - callbackMu.Lock() - numCallbacks++ - callbackMu.Unlock() + Callback: func(got *DownloadOutput) { + callbackMu.Lock() + numCallbacks++ + callbackMu.Unlock() - // Check that the nonexistent object returned an error. - if got.Err != storage.ErrObjectNotExist { - t.Errorf("Object(%q) should not exist, err found to be %v", got.Object, got.Err) - } - }) + // Check that the nonexistent object returned an error. + if got.Err != storage.ErrObjectNotExist { + t.Errorf("Object(%q) should not exist, err found to be %v", got.Object, got.Err) + } + }, + }); err != nil { + t.Errorf("d.DownloadObject: %v", err) + } // WaitAndClose should return an error since 2 of our downloads should have failed. - err = d.WaitAndClose() + _, err = d.WaitAndClose() if err == nil { t.Error("d.WaitAndClose should return an error, instead got nil") } @@ -398,19 +414,21 @@ func TestIntegration_DownloaderTimeout(t *testing.T) { } // Download an object. - d.DownloadObject(ctx, &DownloadObjectInput{ + if err := d.DownloadObject(ctx, &DownloadObjectInput{ Bucket: httpTestBucket.bucket, Object: httpTestBucket.objects[0], Destination: &testWriter{}, - }) + }); err != nil { + t.Errorf("d.DownloadObject: %v", err) + } // WaitAndClose should return an error since the timeout is too short. - if err := d.WaitAndClose(); err == nil { + results, err := d.WaitAndClose() + if err == nil { t.Error("d.WaitAndClose should return an error, instead got nil") } // Check the result. - results := d.Results() got := results[0] // Check that the nonexistent object returned an error. diff --git a/storage/transfermanager/option.go b/storage/transfermanager/option.go index d451ab664000..50d0611547d9 100644 --- a/storage/transfermanager/option.go +++ b/storage/transfermanager/option.go @@ -25,8 +25,8 @@ type Option interface { } // WithCallbacks returns a TransferManagerOption that allows the use of callbacks -// to process the results. If this option is set, then the results will not be -// returned at the end. +// to process the results. If this option is set, then results will not be returned +// by [Downloader.WaitAndClose] and must be processed through the callback. func WithCallbacks() Option { return &withCallbacks{} } @@ -55,7 +55,8 @@ func (ww withWorkers) apply(tm *transferManagerConfig) { // WithPerOpTimeout returns a TransferManagerOption that sets a timeout on each // operation that is performed to download or upload an object. The timeout is // set when the operation begins processing, not when it is added. -// By default, no timeout is set. +// By default, no timeout is set other than an overall timeout as set on the +// provided context. func WithPerOpTimeout(timeout time.Duration) Option { return &withPerOpTimeout{timeout: timeout} } diff --git a/storage/transfermanager/option_test.go b/storage/transfermanager/option_test.go index 515744084914..1f524069f60d 100644 --- a/storage/transfermanager/option_test.go +++ b/storage/transfermanager/option_test.go @@ -15,6 +15,8 @@ package transfermanager import ( + "context" + "strings" "testing" "time" ) @@ -23,6 +25,7 @@ func TestApply(t *testing.T) { opts := []Option{ WithWorkers(3), WithPerOpTimeout(time.Hour), + WithCallbacks(), } var got transferManagerConfig for _, opt := range opts { @@ -31,9 +34,49 @@ func TestApply(t *testing.T) { want := transferManagerConfig{ numWorkers: 3, perOperationTimeout: time.Hour, + asynchronous: true, } if got != want { t.Errorf("got: %+v, want: %+v", got, want) } } + +func TestWithCallbacks(t *testing.T) { + for _, test := range []struct { + desc string + withCallbacks bool + callback func(*DownloadOutput) + expectedErr string + }{ + { + desc: "cannot use callbacks without the option", + withCallbacks: false, + callback: func(*DownloadOutput) {}, + expectedErr: "transfermanager: input.Callback must be nil unless the WithCallbacks option is set", + }, + { + desc: "must provide callback when option is set", + withCallbacks: true, + expectedErr: "transfermanager: input.Callback must not be nil when the WithCallbacks option is set", + }, + } { + t.Run(test.desc, func(t *testing.T) { + var opts []Option + if test.withCallbacks { + opts = append(opts, WithCallbacks()) + } + d, err := NewDownloader(nil, opts...) + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + + err = d.DownloadObject(context.Background(), &DownloadObjectInput{ + Callback: test.callback, + }) + if !strings.Contains(err.Error(), test.expectedErr) { + t.Errorf("expected err %q, got: %v", test.expectedErr, err.Error()) + } + }) + } +} From 5e42c62b900b5a4875f3ef3032883349f405c1fc Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Thu, 23 May 2024 22:11:44 -0700 Subject: [PATCH 12/12] docs + allow waitandclose to be called more than once --- storage/transfermanager/downloader.go | 32 ++++++++++++++++++------- storage/transfermanager/example_test.go | 15 ------------ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/storage/transfermanager/downloader.go b/storage/transfermanager/downloader.go index 207006a6859e..88ecc2c6e769 100644 --- a/storage/transfermanager/downloader.go +++ b/storage/transfermanager/downloader.go @@ -42,7 +42,11 @@ type Downloader struct { // DownloadObject queues the download of a single object. This will initiate the // download but is non-blocking; call Downloader.Results or use the callback to -// process the result. +// process the result. DownloadObject is thread-safe and can be called +// simultaneously from different goroutines. +// The download may not start immediately if all workers are busy, so a deadline +// set on the ctx may time out before the download even starts. To set a timeout +// that starts with the download, use the [WithPerOpTimeout()] option. func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectInput) error { if d.config.asynchronous && input.Callback == nil { return errors.New("transfermanager: input.Callback must not be nil when the WithCallbacks option is set") @@ -63,7 +67,7 @@ func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectIn } // WaitAndClose waits for all outstanding downloads to complete and closes the -// downloader. Adding new downloads after this has been called will cause an error. +// Downloader. Adding new downloads after this has been called will cause an error. // // WaitAndClose returns all the results of the downloads and an error wrapping // all errors that were encountered by the Downloader when downloading objects. @@ -71,15 +75,24 @@ func (d *Downloader) DownloadObject(ctx context.Context, input *DownloadObjectIn // failing download. The results are not guaranteed to be in any order. // Results will be empty if using the [WithCallbacks] option. func (d *Downloader) WaitAndClose() ([]DownloadOutput, error) { - d.done <- true - d.workers.Wait() - close(d.done) + errMsg := "transfermanager: at least one error encountered downloading objects:" + select { + case <-d.done: // this allows users to call WaitAndClose various times + var err error + if len(d.errors) > 0 { + err = fmt.Errorf("%s\n%w", errMsg, errors.Join(d.errors...)) + } + return d.results, err + default: + d.done <- true + d.workers.Wait() + close(d.done) - if len(d.errors) > 0 { - err := errors.Join(d.errors...) - return d.results, fmt.Errorf("transfermanager: at least one error encountered downloading objects:\n%w", err) + if len(d.errors) > 0 { + return d.results, fmt.Errorf("%s\n%w", errMsg, errors.Join(d.errors...)) + } + return d.results, nil } - return d.results, nil } // sendInputsToWorkChan listens continuously to the inputs slice until d.done. @@ -165,6 +178,7 @@ func (d *Downloader) downloadWorker() { // NewDownloader creates a new Downloader to add operations to. // Choice of transport, etc is configured on the client that's passed in. +// The returned Downloader can be shared across goroutines to initiate downloads. func NewDownloader(c *storage.Client, opts ...Option) (*Downloader, error) { d := &Downloader{ client: c, diff --git a/storage/transfermanager/example_test.go b/storage/transfermanager/example_test.go index 83bb327e9d22..c0e50504d4cb 100644 --- a/storage/transfermanager/example_test.go +++ b/storage/transfermanager/example_test.go @@ -18,7 +18,6 @@ import ( "context" "log" "os" - "time" "cloud.google.com/go/storage" "cloud.google.com/go/storage/transfermanager" @@ -54,13 +53,6 @@ func ExampleDownloader_synchronous() { EncryptionKey: []byte("mykey"), } - // Can set timeout on this download using context. Note that this download - // may not start immediately if all workers are busy, so this may time out - // before the download even starts. To set a timeout that starts with the - // download, use transfermanager.WithPerOpTimeout(time.Duration). - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - // Add to Downloader. if err := d.DownloadObject(ctx, in); err != nil { // handle error @@ -130,13 +122,6 @@ func ExampleDownloader_asynchronous() { Callback: callback, } - // Can set timeout on this download using context. Note that this download - // may not start immediately if all workers are busy, so this may time out - // before the download even starts. To set a timeout that starts with the - // download, use transfermanager.WithPerOpTimeout(time.Duration). - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - // Add to Downloader. if err := d.DownloadObject(ctx, in); err != nil { // handle error