Repository navigation
refactor(cli): migrate BigQuery integration tests from KMS-encrypted test key to ADC #2362
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
1ae69b3
f12d708
e2e55fa
4548f9c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -78,7 +78,3 @@ use_repo( | |
| "org_golang_x_net", | ||
| "org_golang_x_text", | ||
| ) | ||
|
|
||
| # Custom gcloud SDK extension | ||
| gcloud_sdk_ext = use_extension("//tools/gcloud:extensions.bzl", "gcloud_sdk_extension") | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The description says
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This PR does delete |
||
| use_repo(gcloud_sdk_ext, "gcloud_sdk") | ||
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,79 @@ | ||
| import { Dataset, Table } from "@google-cloud/bigquery"; | ||
| import { expect } from "chai"; | ||
| import { anything, instance, mock, verify, when } from "ts-mockito"; | ||
| import { anything, instance, mock, when } from "ts-mockito"; | ||
|
|
||
| import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; | ||
| import { | ||
| BigQueryDbAdapter, | ||
| createBigQueryClientProvider, | ||
| getBigQueryClientOptions, | ||
| } from "df/cli/api/dbadapters/bigquery"; | ||
| import { dataform } from "df/protos/ts"; | ||
| import { suite, test } from "df/testing"; | ||
|
|
||
| suite("BigQueryDbAdapter", () => { | ||
| suite("getBigQueryClientOptions", () => { | ||
| test("passes the parsed service account key to the client", () => { | ||
| const serviceAccountKey = { | ||
| type: "service_account", | ||
| project_id: "key-project-id", | ||
| private_key_id: "key-id-123", | ||
| private_key: | ||
| "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC\n-----END PRIVATE KEY-----\n", | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: the PEM-looking
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done, it's |
||
| client_email: "test-sa@key-project-id.iam.gserviceaccount.com", | ||
| client_id: "1234567890", | ||
| }; | ||
| const options = getBigQueryClientOptions( | ||
| dataform.BigQuery.create({ | ||
| projectId: "target-project-id", | ||
| location: "EU", | ||
| credentials: JSON.stringify(serviceAccountKey), | ||
| }), | ||
| ); | ||
| expect(options).to.deep.include({ | ||
| projectId: "target-project-id", | ||
| location: "EU", | ||
| credentials: serviceAccountKey, | ||
| }); | ||
| }); | ||
|
|
||
| test("leaves out credentials when no key is given, so ADC is used", () => { | ||
| const options = getBigQueryClientOptions( | ||
| dataform.BigQuery.create({ projectId: "adc-project-id", location: "US" }), | ||
| ); | ||
| expect(options).to.deep.include({ projectId: "adc-project-id", location: "US" }); | ||
| expect(Boolean(options.credentials)).to.equal(false); | ||
| }); | ||
|
|
||
| test("uses the requested project instead of the default one", () => { | ||
| const options = getBigQueryClientOptions( | ||
| dataform.BigQuery.create({ projectId: "default-project", location: "US" }), | ||
| "other-project", | ||
| ); | ||
| expect(options.projectId).to.equal("other-project"); | ||
| }); | ||
|
|
||
| test("throws on malformed service account key JSON", () => { | ||
| expect(() => | ||
| getBigQueryClientOptions({ | ||
| projectId: "target-project-id", | ||
| location: "US", | ||
| credentials: "{not-valid-json", | ||
| }), | ||
| ).to.throw(SyntaxError); | ||
| }); | ||
| }); | ||
|
|
||
| test("createBigQueryClientProvider creates and caches one client per project", () => { | ||
| const clientProvider = createBigQueryClientProvider( | ||
| dataform.BigQuery.create({ projectId: "default-project", location: "EU" }), | ||
| ); | ||
| const defaultClient = clientProvider(); | ||
| expect(defaultClient.projectId).to.equal("default-project"); | ||
| expect(defaultClient.location).to.equal("EU"); | ||
| expect(clientProvider()).to.equal(defaultClient); | ||
| expect(clientProvider("other-project").projectId).to.equal("other-project"); | ||
| }); | ||
|
|
||
| test("tables() with schema filters correctly", async () => { | ||
| const mockBigQuery = mock<any>(); | ||
| const mockDataset = mock<Dataset>(); | ||
|
|
||
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,8 +4,10 @@ set -e | |
| # Run eslint. | ||
| bazel run //:eslint -- '**/*.ts' | ||
|
|
||
| # Run integration tests | ||
| # Run integration tests (--config=integration forwards the ADC environment, see .bazelrc). | ||
| bazel test ... \ | ||
| --config=integration \ | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How does CI authenticate now? Before, the Bazel
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Since your review I've added service account impersonation, so in CI it works like this:
Permissions:
Green integration run on the current head (e2e55fa): Cloud Build 403983ac. It includes the new |
||
| --test_tag_filters=integration \ | ||
| --build_tests_only \ | ||
| --test_env=USE_CLOUD_BUILD_NETWORK=true | ||
| --test_env=USE_CLOUD_BUILD_NETWORK=true \ | ||
| "$@" | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit:
ensureAdcAvailablehonoursGCE_METADATA_HOST/GCE_METADATA_IP/METADATA_SERVER_DETECTION, but they aren't forwarded here, so in the sandbox they'd only work through an explicit--test_env. Worth forwarding them too, for people using metadata emulators?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done.
--config=integrationnow also forwardsGCE_METADATA_HOST,GCE_METADATA_IPandMETADATA_SERVER_DETECTION, which google-auth-library reads too. Bazel doesn't pass variables that aren't set on the client, so CI is unaffected.