Skip to content

Commit e42caea

Browse files
odeke-emolavloite
odeke-em
and
olavloite
authored
feat(debugging): implement x-goog-spanner-request-id propagation per request (#2205)
* feat(debugging): implement x-goog-spanner-request-id propagation per request Implements propagation of the x-goog-spanner-request-id that'll be propagated for every call. Once an error has been encountered, that error will have `.requestId` set. Fixes #2200 * All: update tests * Address review feedback * Remove a redundant prefix check for values to skip * Put TODO in correct place for retries from GAX * test: add test to verify that request-id is received * Revert protos/* spurious updates * Correctly check for ExecuteStreamingSql headers asynchronously in callback not assuming synchronous/procedural completion --------- Co-authored-by: Knut Olav Løite <[email protected]>
1 parent b60a683 commit e42caea

11 files changed

+555
-160
lines changed

‎src/batch-transaction.ts

+5-3
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
addLeaderAwareRoutingHeader,
2727
} from '../src/common';
2828
import {startTrace, setSpanError, traceConfig} from './instrument';
29+
import {injectRequestIDIntoHeaders} from './request_id_header';
2930

3031
export interface TransactionIdentifier {
3132
session: string | Session;
@@ -157,7 +158,7 @@ class BatchTransaction extends Snapshot {
157158
method: 'partitionQuery',
158159
reqOpts,
159160
gaxOpts: query.gaxOptions,
160-
headers: headers,
161+
headers: injectRequestIDIntoHeaders(headers, this.session),
161162
},
162163
(err, partitions, resp) => {
163164
if (err) {
@@ -201,10 +202,11 @@ class BatchTransaction extends Snapshot {
201202
transaction: {id: this.id},
202203
});
203204
config.reqOpts = extend({}, query);
204-
config.headers = {
205+
const headers = {
205206
[CLOUD_RESOURCE_HEADER]: (this.session.parent as Database)
206207
.formattedName_,
207208
};
209+
config.headers = injectRequestIDIntoHeaders(headers, this.session);
208210
delete query.partitionOptions;
209211
this.session.request(config, (err, resp) => {
210212
if (err) {
@@ -293,7 +295,7 @@ class BatchTransaction extends Snapshot {
293295
method: 'partitionRead',
294296
reqOpts,
295297
gaxOpts: options.gaxOptions,
296-
headers: headers,
298+
headers: injectRequestIDIntoHeaders(headers, this.session),
297299
},
298300
(err, partitions, resp) => {
299301
if (err) {

‎src/database.ts

+59-3
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,13 @@ import {
113113
setSpanErrorAndException,
114114
traceConfig,
115115
} from './instrument';
116+
import {
117+
AtomicCounter,
118+
X_GOOG_SPANNER_REQUEST_ID_HEADER,
119+
craftRequestId,
120+
newAtomicCounter,
121+
} from './request_id_header';
122+
116123
export type GetDatabaseRolesCallback = RequestCallback<
117124
IDatabaseRole,
118125
databaseAdmin.spanner.admin.database.v1.IListDatabaseRolesResponse
@@ -355,6 +362,8 @@ class Database extends common.GrpcServiceObject {
355362
> | null;
356363
_observabilityOptions?: ObservabilityOptions; // TODO: exmaine if we can remove it
357364
private _traceConfig: traceConfig;
365+
private _nthRequest: AtomicCounter;
366+
public _clientId: number;
358367
constructor(
359368
instance: Instance,
360369
name: string,
@@ -469,6 +478,12 @@ class Database extends common.GrpcServiceObject {
469478
};
470479

471480
this.request = instance.request;
481+
this._nthRequest = newAtomicCounter(0);
482+
if (this.parent && this.parent.parent) {
483+
this._clientId = (this.parent.parent as Spanner)._nthClientId;
484+
} else {
485+
this._clientId = instance._nthClientId;
486+
}
472487
this._observabilityOptions = instance._observabilityOptions;
473488
this.commonHeaders_ = getCommonHeaders(
474489
this.formattedName_,
@@ -489,6 +504,11 @@ class Database extends common.GrpcServiceObject {
489504
Database.getEnvironmentQueryOptions()
490505
);
491506
}
507+
508+
_nextNthRequest(): number {
509+
return this._nthRequest.increment();
510+
}
511+
492512
/**
493513
* @typedef {array} SetDatabaseMetadataResponse
494514
* @property {object} 0 The {@link Database} metadata.
@@ -697,14 +717,20 @@ class Database extends common.GrpcServiceObject {
697717
addLeaderAwareRoutingHeader(headers);
698718
}
699719

720+
const allHeaders = this._metadataWithRequestId(
721+
this._nextNthRequest(),
722+
1,
723+
headers
724+
);
725+
700726
startTrace('Database.batchCreateSessions', this._traceConfig, span => {
701727
this.request<google.spanner.v1.IBatchCreateSessionsResponse>(
702728
{
703729
client: 'SpannerClient',
704730
method: 'batchCreateSessions',
705731
reqOpts,
706732
gaxOpts: options.gaxOptions,
707-
headers: headers,
733+
headers: allHeaders,
708734
},
709735
(err, resp) => {
710736
if (err) {
@@ -728,6 +754,26 @@ class Database extends common.GrpcServiceObject {
728754
});
729755
}
730756

757+
public _metadataWithRequestId(
758+
nthRequest: number,
759+
attempt: number,
760+
priorMetadata?: {[k: string]: string}
761+
): {[k: string]: string} {
762+
if (!priorMetadata) {
763+
priorMetadata = {};
764+
}
765+
const withReqId = {
766+
...priorMetadata,
767+
};
768+
withReqId[X_GOOG_SPANNER_REQUEST_ID_HEADER] = craftRequestId(
769+
this._clientId || 1,
770+
1, // TODO: Properly infer the channelId
771+
nthRequest,
772+
attempt
773+
);
774+
return withReqId;
775+
}
776+
731777
/**
732778
* Get a reference to a {@link BatchTransaction} object.
733779
*
@@ -993,7 +1039,11 @@ class Database extends common.GrpcServiceObject {
9931039
reqOpts.session.creatorRole =
9941040
options.databaseRole || this.databaseRole || null;
9951041

996-
const headers = this.commonHeaders_;
1042+
const headers = this._metadataWithRequestId(
1043+
this._nextNthRequest(),
1044+
1,
1045+
this.commonHeaders_
1046+
);
9971047
if (this._getSpanner().routeToLeaderEnabled) {
9981048
addLeaderAwareRoutingHeader(headers);
9991049
}
@@ -1915,6 +1965,12 @@ class Database extends common.GrpcServiceObject {
19151965
delete (gaxOpts as GetSessionsOptions).pageToken;
19161966
}
19171967

1968+
const headers = this._metadataWithRequestId(
1969+
this._nextNthRequest(),
1970+
1,
1971+
this.commonHeaders_
1972+
);
1973+
19181974
return startTrace('Database.getSessions', this._traceConfig, span => {
19191975
this.request<
19201976
google.spanner.v1.ISession,
@@ -1925,7 +1981,7 @@ class Database extends common.GrpcServiceObject {
19251981
method: 'listSessions',
19261982
reqOpts,
19271983
gaxOpts,
1928-
headers: this.commonHeaders_,
1984+
headers: headers,
19291985
},
19301986
(err, sessions, nextPageRequest, ...args) => {
19311987
if (err) {

‎src/index.ts

+59-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import {GoogleAuth, GoogleAuthOptions} from 'google-auth-library';
2323
import * as path from 'path';
2424
import {common as p} from 'protobufjs';
2525
import * as streamEvents from 'stream-events';
26+
import {EventEmitter} from 'events';
2627
import * as through from 'through2';
2728
import {
2829
codec,
@@ -87,6 +88,10 @@ import {
8788
ObservabilityOptions,
8889
ensureInitialContextManagerSet,
8990
} from './instrument';
91+
import {
92+
injectRequestIDIntoError,
93+
nextSpannerClientId,
94+
} from './request_id_header';
9095

9196
// eslint-disable-next-line @typescript-eslint/no-var-requires
9297
const gcpApiConfig = require('./spanner_grpc_config.json');
@@ -148,6 +153,7 @@ export interface SpannerOptions extends GrpcClientOptions {
148153
directedReadOptions?: google.spanner.v1.IDirectedReadOptions | null;
149154
defaultTransactionOptions?: Pick<RunTransactionOptions, 'isolationLevel'>;
150155
observabilityOptions?: ObservabilityOptions;
156+
interceptors?: any[];
151157
}
152158
export interface RequestConfig {
153159
client: string;
@@ -251,6 +257,7 @@ class Spanner extends GrpcService {
251257
directedReadOptions: google.spanner.v1.IDirectedReadOptions | null;
252258
defaultTransactionOptions: RunTransactionOptions;
253259
_observabilityOptions: ObservabilityOptions | undefined;
260+
readonly _nthClientId: number;
254261

255262
/**
256263
* Placeholder used to auto populate a column with the commit timestamp.
@@ -312,6 +319,7 @@ class Spanner extends GrpcService {
312319
}
313320
}
314321
}
322+
315323
options = Object.assign(
316324
{
317325
libName: 'gccl',
@@ -388,6 +396,7 @@ class Spanner extends GrpcService {
388396
this._observabilityOptions?.enableEndToEndTracing
389397
);
390398
ensureInitialContextManagerSet();
399+
this._nthClientId = nextSpannerClientId();
391400
}
392401

393402
/**
@@ -1562,7 +1571,56 @@ class Spanner extends GrpcService {
15621571
},
15631572
})
15641573
);
1565-
callback(null, requestFn);
1574+
1575+
// Wrap requestFn to inject the spanner request id into every returned error.
1576+
const wrappedRequestFn = (...args) => {
1577+
const hasCallback =
1578+
args &&
1579+
args.length > 0 &&
1580+
typeof args[args.length - 1] === 'function';
1581+
1582+
switch (hasCallback) {
1583+
case true: {
1584+
const cb = args[args.length - 1];
1585+
const priorArgs = args.slice(0, args.length - 1);
1586+
requestFn(...priorArgs, (...results) => {
1587+
if (results && results.length > 0) {
1588+
const err = results[0] as Error;
1589+
injectRequestIDIntoError(config, err);
1590+
}
1591+
1592+
cb(...results);
1593+
});
1594+
return;
1595+
}
1596+
1597+
case false: {
1598+
const res = requestFn(...args);
1599+
const stream = res as EventEmitter;
1600+
if (stream) {
1601+
stream.on('error', err => {
1602+
injectRequestIDIntoError(config, err as Error);
1603+
});
1604+
}
1605+
1606+
const originallyPromise = res instanceof Promise;
1607+
if (!originallyPromise) {
1608+
return res;
1609+
}
1610+
1611+
return new Promise((resolve, reject) => {
1612+
requestFn(...args)
1613+
.then(resolve)
1614+
.catch(err => {
1615+
injectRequestIDIntoError(config, err as Error);
1616+
reject(err);
1617+
});
1618+
});
1619+
}
1620+
}
1621+
};
1622+
1623+
callback(null, wrappedRequestFn);
15661624
});
15671625
}
15681626

‎src/instance.ts

+4
Original file line numberDiff line numberDiff line change
@@ -980,6 +980,10 @@ class Instance extends common.GrpcServiceObject {
980980
if (!this.databases_.has(key!)) {
981981
const db = new Database(this, name, poolOptions, queryOptions);
982982
db._observabilityOptions = this._observabilityOptions;
983+
const parent = this.parent as Spanner;
984+
if (parent && parent._nthClientId) {
985+
db._clientId = parent._nthClientId;
986+
}
983987
this.databases_.set(key!, db);
984988
}
985989
return this.databases_.get(key!)!;

‎src/request_id_header.ts

+3
Original file line numberDiff line numberDiff line change
@@ -187,8 +187,11 @@ export interface RequestIDError extends grpc.ServiceError {
187187
requestID: string;
188188
}
189189

190+
const X_GOOG_REQ_ID_REGEX = /^1\.[0-9A-Fa-f]{8}(\.\d+){3}\.\d+/;
191+
190192
export {
191193
AtomicCounter,
194+
X_GOOG_REQ_ID_REGEX,
192195
X_GOOG_SPANNER_REQUEST_ID_HEADER,
193196
craftRequestId,
194197
injectRequestIDIntoError,

‎src/session.ts

+29-4
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ import {
4444
import {grpc, CallOptions} from 'google-gax';
4545
import IRequestOptions = google.spanner.v1.IRequestOptions;
4646
import {Spanner} from '.';
47-
47+
import {injectRequestIDIntoHeaders, nextNthRequest} from './request_id_header';
4848
export type GetSessionResponse = [Session, r.Response];
4949

5050
/**
@@ -317,13 +317,19 @@ export class Session extends common.GrpcServiceObject {
317317
const reqOpts = {
318318
name: this.formattedName_,
319319
};
320+
const database = this.parent as Database;
320321
return this.request(
321322
{
322323
client: 'SpannerClient',
323324
method: 'deleteSession',
324325
reqOpts,
325326
gaxOpts,
326-
headers: this.commonHeaders_,
327+
headers: injectRequestIDIntoHeaders(
328+
this.commonHeaders_,
329+
this,
330+
nextNthRequest(database),
331+
1
332+
),
327333
},
328334
callback!
329335
);
@@ -389,13 +395,19 @@ export class Session extends common.GrpcServiceObject {
389395
if (this._getSpanner().routeToLeaderEnabled) {
390396
addLeaderAwareRoutingHeader(headers);
391397
}
398+
const database = this.parent as Database;
392399
return this.request(
393400
{
394401
client: 'SpannerClient',
395402
method: 'getSession',
396403
reqOpts,
397404
gaxOpts,
398-
headers: headers,
405+
headers: injectRequestIDIntoHeaders(
406+
headers,
407+
this.session,
408+
nextNthRequest(database),
409+
1
410+
),
399411
},
400412
(err, resp) => {
401413
if (resp) {
@@ -441,17 +453,25 @@ export class Session extends common.GrpcServiceObject {
441453
session: this.formattedName_,
442454
sql: 'SELECT 1',
443455
};
456+
457+
const database = this.parent as Database;
444458
return this.request(
445459
{
446460
client: 'SpannerClient',
447461
method: 'executeSql',
448462
reqOpts,
449463
gaxOpts,
450-
headers: this.commonHeaders_,
464+
headers: injectRequestIDIntoHeaders(
465+
this.commonHeaders_,
466+
this,
467+
nextNthRequest(database),
468+
1
469+
),
451470
},
452471
callback!
453472
);
454473
}
474+
455475
/**
456476
* Create a PartitionedDml transaction.
457477
*
@@ -534,6 +554,11 @@ export class Session extends common.GrpcServiceObject {
534554
private _getSpanner(): Spanner {
535555
return this.parent.parent.parent as Spanner;
536556
}
557+
558+
private channelId(): number {
559+
// The Node.js client does not use a gRPC channel pool, so this always returns 1.
560+
return 1;
561+
}
537562
}
538563

539564
/*! Developer Documentation

0 commit comments

Comments
 (0)