Skip to content

Commit c2d6259

Browse files
committed
Check a download against the checksum taken before it, not after
A download is of the build live when it starts, so a checksum fetched after a long transfer could name a build published meanwhile, and the command called a good file bad and told you to delete it. The checksum is now taken first, and asked once more on a mismatch, for a build that went live between the two requests.
1 parent 78231e9 commit c2d6259

2 files changed

Lines changed: 96 additions & 7 deletions

File tree

‎cmd_database.go‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -263,8 +263,15 @@ func dbDownloads(ctx context.Context, db *internetdata.DatabaseAPI, limit int, a
263263
return w.Flush()
264264
}
265265

266+
// The database calls a download makes, which *internetdata.DatabaseAPI answers and a test can stand in for.
267+
type databaseFiles interface {
268+
Download(ctx context.Context, id string, format internetdata.Format, dst io.Writer) (int64, error)
269+
DownloadFile(ctx context.Context, id string, format internetdata.Format, path string) (int64, error)
270+
Checksums(ctx context.Context, id string, format internetdata.Format) (*internetdata.Checksums, error)
271+
}
272+
266273
func dbDownload(
267-
ctx context.Context, db *internetdata.DatabaseAPI,
274+
ctx context.Context, db databaseFiles,
268275
id, format, out string, toStdout, noVerify bool,
269276
) error {
270277
f := internetdata.Format(format)
@@ -284,6 +291,14 @@ func dbDownload(
284291
out = id + "." + extensionFor(f)
285292
}
286293

294+
// The checksum before the download, not after: a download is of the build live when it starts, so a
295+
// checksum taken after a long transfer could name a build published meanwhile and call a good file bad.
296+
var sums *internetdata.Checksums
297+
var sumsErr error
298+
if !noVerify {
299+
sums, sumsErr = db.Checksums(ctx, id, f)
300+
}
301+
287302
fmt.Fprintf(os.Stderr, "downloading %s (%s) to %s...\n", id, format, out)
288303
n, err := db.DownloadFile(ctx, id, f, out)
289304
if err != nil {
@@ -294,11 +309,10 @@ func dbDownload(
294309
if noVerify {
295310
return nil
296311
}
297-
sums, err := db.Checksums(ctx, id, f)
298-
if err != nil {
312+
if sumsErr != nil {
299313
// The file is on disk and intact as far as we know; failing the command
300314
// over an unavailable checksum would throw away a good download.
301-
fmt.Fprintf(os.Stderr, "warn: could not fetch checksums to verify: %v\n", explain(err))
315+
fmt.Fprintf(os.Stderr, "warn: could not fetch checksums to verify: %v\n", explain(sumsErr))
302316
return nil
303317
}
304318
if sums.SHA256 == "" {
@@ -310,9 +324,13 @@ func dbDownload(
310324
return err
311325
}
312326
if got != sums.SHA256 {
313-
return fmt.Errorf(
314-
"checksum mismatch: got %s, expected %s\n%s is not the published file and should be deleted",
315-
got, sums.SHA256, out)
327+
// A build published between the checksum and the download's start: the checksum now names it.
328+
again, err := db.Checksums(ctx, id, f)
329+
if err != nil || again.SHA256 != got {
330+
return fmt.Errorf(
331+
"checksum mismatch: got %s, expected %s\n%s is not the published file and should be deleted",
332+
got, sums.SHA256, out)
333+
}
316334
}
317335
fmt.Fprintln(os.Stderr, "sha256 verified")
318336
return nil

‎cmd_database_test.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
package main
22

33
import (
4+
"context"
5+
"crypto/sha256"
6+
"encoding/hex"
7+
"io"
8+
"os"
9+
"path/filepath"
410
"testing"
511
"time"
612

@@ -39,3 +45,68 @@ func TestLicenseTerm(t *testing.T) {
3945
}
4046
}
4147
}
48+
49+
// A database API whose live build moves on cue: Checksums answers sums[0] until published, then sums[1], and
50+
// DownloadFile writes file, then runs onDownload, as a build published during the transfer would.
51+
type fakeFiles struct {
52+
file []byte
53+
sums []string
54+
published bool
55+
onDownload func(f *fakeFiles)
56+
}
57+
58+
func (f *fakeFiles) Download(context.Context, string, internetdata.Format, io.Writer) (int64, error) {
59+
return 0, nil
60+
}
61+
62+
func (f *fakeFiles) DownloadFile(_ context.Context, _ string, _ internetdata.Format, path string) (int64, error) {
63+
if err := os.WriteFile(path, f.file, 0o644); err != nil {
64+
return 0, err
65+
}
66+
if f.onDownload != nil {
67+
f.onDownload(f)
68+
}
69+
return int64(len(f.file)), nil
70+
}
71+
72+
func (f *fakeFiles) Checksums(context.Context, string, internetdata.Format) (*internetdata.Checksums, error) {
73+
if f.published {
74+
return &internetdata.Checksums{SHA256: f.sums[1]}, nil
75+
}
76+
return &internetdata.Checksums{SHA256: f.sums[0]}, nil
77+
}
78+
79+
func sum256(b []byte) string {
80+
h := sha256.Sum256(b)
81+
return hex.EncodeToString(h[:])
82+
}
83+
84+
// The download is of the build live when it starts; a build published during it moves the checksum on.
85+
func TestDownloadVerifiesAgainstTheChecksumTakenBeforeIt(t *testing.T) {
86+
a, b := []byte("build a"), []byte("build b")
87+
f := &fakeFiles{file: a, sums: []string{sum256(a), sum256(b)}, onDownload: func(f *fakeFiles) { f.published = true }}
88+
out := filepath.Join(t.TempDir(), "x.csv.gz")
89+
if err := dbDownload(context.Background(), f, "x_v1", "csvgz", out, false, false); err != nil {
90+
t.Fatalf("a good file read as bad: %v", err)
91+
}
92+
}
93+
94+
// A build published between the checksum and the download's start: the download is of it, and the checksum
95+
// asked once more names it.
96+
func TestDownloadAsksAgainWhenABuildWentLiveBeforeItStarted(t *testing.T) {
97+
a, b := []byte("build a"), []byte("build b")
98+
f := &fakeFiles{file: b, sums: []string{sum256(a), sum256(b)}, onDownload: func(f *fakeFiles) { f.published = true }}
99+
out := filepath.Join(t.TempDir(), "x.csv.gz")
100+
if err := dbDownload(context.Background(), f, "x_v1", "csvgz", out, false, false); err != nil {
101+
t.Fatalf("the newer build's file read as bad: %v", err)
102+
}
103+
}
104+
105+
func TestDownloadRefusesAFileNoChecksumDescribes(t *testing.T) {
106+
a := []byte("build a")
107+
f := &fakeFiles{file: []byte("corrupt"), sums: []string{sum256(a), sum256(a)}}
108+
out := filepath.Join(t.TempDir(), "x.csv.gz")
109+
if err := dbDownload(context.Background(), f, "x_v1", "csvgz", out, false, false); err == nil {
110+
t.Fatal("a file no checksum describes passed")
111+
}
112+
}

0 commit comments

Comments
 (0)
⌂ 换网址