Skip to content

Commit 9d10dd0

Browse files
authored
fix(bigquery): use first page row count for Arrow query pagination offset (#14466)
This PR fixes Arrow query pagination by deriving the initial Storage Read API row offset from the number of rows returned in the first page rather than attempting to parse the REST API's opaque page token as a number. `getPageToken()` returns a `string`, see https://docs.cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#response-body.
1 parent 2775eb1 commit 9d10dd0

2 files changed

Lines changed: 33 additions & 21 deletions

File tree

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@
6767
import com.google.common.collect.Iterables;
6868
import com.google.common.collect.Lists;
6969
import com.google.common.collect.Maps;
70-
import com.google.common.primitives.Longs;
7170
import io.opentelemetry.api.common.Attributes;
7271
import io.opentelemetry.api.trace.Span;
7372
import io.opentelemetry.context.Scope;
@@ -2620,20 +2619,9 @@ && getOptions().getOpenTelemetryTracer() != null) {
26202619

26212620
// Calculate row offset and determine if subsequent pages exist.
26222621
boolean hasMorePages = results.getPageToken() != null;
2623-
long initialRowOffset = 0L;
2622+
long initialRowOffset = (long) firstPageRows.size();
26242623
if (hasMorePages) {
2625-
Long parsedOffset = Longs.tryParse(results.getPageToken());
2626-
if (parsedOffset == null) {
2627-
throw new BigQueryException(
2628-
0,
2629-
String.format(
2630-
"Unable to parse page token '%s' as a numeric row offset for Arrow query pagination",
2631-
results.getPageToken()));
2632-
}
2633-
initialRowOffset = parsedOffset;
2634-
if (content.getMaxResults() != null
2635-
&& (initialRowOffset >= content.getMaxResults()
2636-
|| firstPageRows.size() >= content.getMaxResults())) {
2624+
if (content.getMaxResults() != null && initialRowOffset >= content.getMaxResults()) {
26372625
hasMorePages = false;
26382626
}
26392627
}

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3284,6 +3284,24 @@ void testQueryWithArrowFormatMultiplePagesWithMaxResults()
32843284
schemaBytes = out.toByteArray();
32853285
}
32863286

3287+
// Prepare page 1 Arrow batch for initial response with 1 row
3288+
byte[] page1BatchBytes;
3289+
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
3290+
BigIntVector idVector = new BigIntVector("id", allocator)) {
3291+
idVector.allocateNew(1);
3292+
idVector.set(0, 1L);
3293+
idVector.setValueCount(1);
3294+
try (VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(idVector))) {
3295+
VectorUnloader unloader = new VectorUnloader(root);
3296+
try (ArrowRecordBatch recordBatch = unloader.getRecordBatch();
3297+
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
3298+
WriteChannel channel = new WriteChannel(Channels.newChannel(out));
3299+
MessageSerializer.serialize(channel, recordBatch);
3300+
page1BatchBytes = out.toByteArray();
3301+
}
3302+
}
3303+
}
3304+
32873305
// Prepare page 2 Arrow batch for streaming with 2 rows
32883306
byte[] page2BatchBytes;
32893307
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
@@ -3310,10 +3328,13 @@ void testQueryWithArrowFormatMultiplePagesWithMaxResults()
33103328
.setJobComplete(true)
33113329
.setJobReference(queryJob.toPb())
33123330
.setTotalRows(BigInteger.valueOf(3L))
3313-
.setPageToken("1")
3331+
.setPageToken("BG4QSSWFUAAQAAASA4EAAEEAQCAAKGQIBAABB7X7777QOIFQVYKQ====")
33143332
.setArrowSchema(
33153333
new com.google.api.services.bigquery.model.ArrowSchema()
3316-
.setSerializedSchema(BaseEncoding.base64().encode(schemaBytes)));
3334+
.setSerializedSchema(BaseEncoding.base64().encode(schemaBytes)))
3335+
.setArrowRecordBatch(
3336+
new com.google.api.services.bigquery.model.ArrowRecordBatch()
3337+
.setSerializedRecordBatch(BaseEncoding.base64().encode(page1BatchBytes)));
33173338

33183339
when(bigqueryRpcMock.queryRpcSkipExceptionTranslation(eq(PROJECT), any(QueryRequest.class)))
33193340
.thenReturn(queryResponsePb);
@@ -3632,7 +3653,7 @@ void testQueryWithArrowFormatIncompleteJobJobNotFound() throws Exception {
36323653
}
36333654

36343655
@Test
3635-
void testQueryWithArrowFormatInvalidPageToken() throws Exception {
3656+
void testQueryWithArrowFormatOpaquePageToken() throws Exception {
36363657
org.apache.arrow.vector.types.pojo.Schema arrowSchema =
36373658
new org.apache.arrow.vector.types.pojo.Schema(
36383659
ImmutableList.of(
@@ -3648,11 +3669,11 @@ void testQueryWithArrowFormatInvalidPageToken() throws Exception {
36483669
JobId queryJob = JobId.of(PROJECT, JOB).toBuilder().setLocation(LOCATION).build();
36493670
com.google.api.services.bigquery.model.QueryResponse queryResponsePb =
36503671
new com.google.api.services.bigquery.model.QueryResponse()
3651-
.setQueryId("q-arrow-invalid-token")
3672+
.setQueryId("q-arrow-opaque-token")
36523673
.setJobComplete(true)
36533674
.setJobReference(queryJob.toPb())
36543675
.setTotalRows(BigInteger.valueOf(2L))
3655-
.setPageToken("invalid-non-numeric-token")
3676+
.setPageToken("BG4QSSWFUAAQAAASA4EAAEEAQCAAKGQIBAABB7X7777QOIFQVYKQ====")
36563677
.setArrowSchema(
36573678
new com.google.api.services.bigquery.model.ArrowSchema()
36583679
.setSerializedSchema(BaseEncoding.base64().encode(schemaBytes)));
@@ -3666,8 +3687,11 @@ void testQueryWithArrowFormatInvalidPageToken() throws Exception {
36663687
QueryJobConfiguration.newBuilder("SELECT id FROM test")
36673688
.setQueryResultsFormat(QueryResultsFormat.ARROW)
36683689
.build();
3669-
BigQueryException e = assertThrows(BigQueryException.class, () -> bigquery.query(config));
3670-
assertTrue(e.getMessage().contains("Unable to parse page token 'invalid-non-numeric-token'"));
3690+
TableResult result = bigquery.query(config);
3691+
assertNotNull(result);
3692+
assertTrue(result.hasNextPage());
3693+
assertEquals(
3694+
"BG4QSSWFUAAQAAASA4EAAEEAQCAAKGQIBAABB7X7777QOIFQVYKQ====", result.getNextPageToken());
36713695
}
36723696

36733697
@Test

0 commit comments

Comments
 (0)
⌂ 换网址