Skip to content

Commit fc51cc2

Browse files
authored
feat(spanner): add OpenTelemetry implementation (#9254)
* feat(spanner): add opentelemetry instrumentation * feat(spanner): add header * feat(spanner): go mod tidy * feat(spanner): code refactoring * feat(spanner): pass context * feat(spanner): fix vet * feat(spanner): code refactoring * feat(spanner): add lock when seeting ot config to avoid data race * feat(spanner): code refactoring * feat(spanner): add new package for testing open telemetery * feat(spanner): aadd header to new files * feat(spanner): testing * feat(spanner): mark go version as 1.19 * feat(spanner): update metrics test cases * feat(spanner): metrics code refactoring * feat(spanner): add OT traces * feat(spanner): comment OT metric sdk * feat(spanner): hardcode context * feat(spanner): odce refactoring * feat(spanner): add header * feat(spanner): code refactoring * feat(spanner): comment refactoring * feat(spanner): code refactoring * feat(spanner): remove disable from integration_test * feat(spanner): rename metrics with prefix 1 * feat(spanner): should revert: add benchmark code * feat(spanner): should revert: add benchmark code * feat(spanner): should revert: add benchmark code * feat(spanner): should revert: add benchmark code * feat(spanner): should revert: add benchmark code * feat(spanner): should revert: add benchmark code * feat(spanner): should revert: add benchmark code * feat(spanner): should revert: add benchmark code * feat(spanner): revert all benchmark codes * feat(spanner): upgrade metrics SDK to latest 1.22.0 that has fix for context done * feat(spanner): rename metrics back to original name * feat(spanner): remove debugging logs * feat(spanner): rename method * feat(spanner): pass context that is done, OT team has fixed this issue * feat(spanner): skip OT tests for go version 1.19 * feat(spanner): revert * feat(spanner): revert * feat(spanner): avoid skipping tests for Go1.19 * feat(spanner): add build constrints - compile OT test files only for Go1.20 and above * feat(spanner): add deprecation warning to OpenCensus code * feat(spanner): go mod tidy
1 parent a8078f0 commit fc51cc2

17 files changed

+1355
-5
lines changed

go.work

+1
Original file line numberDiff line numberDiff line change
@@ -166,4 +166,5 @@ use (
166166
./websecurityscanner
167167
./workflows
168168
./workstations
169+
./spanner/test/opentelemetry/test
169170
)

spanner/batch.go

+15
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,9 @@ func (t *BatchReadOnlyTransaction) PartitionReadUsingIndexWithOptions(ctx contex
149149
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency. Try disabling and rerunning. Error: %v", err)
150150
}
151151
}
152+
if metricErr := recordGFELatencyMetricsOT(ctx, md, "PartitionReadUsingIndexWithOptions", t.otConfig); metricErr != nil {
153+
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency through OpenTelemetry. Error: %v", metricErr)
154+
}
152155
// Prepare ReadRequest.
153156
req := &sppb.ReadRequest{
154157
Session: sid,
@@ -213,6 +216,9 @@ func (t *BatchReadOnlyTransaction) partitionQuery(ctx context.Context, statement
213216
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency. Try disabling and rerunning. Error: %v", err)
214217
}
215218
}
219+
if metricErr := recordGFELatencyMetricsOT(ctx, md, "partitionQuery", t.otConfig); metricErr != nil {
220+
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency through OpenTelemetry. Error: %v", metricErr)
221+
}
216222

217223
// prepare ExecuteSqlRequest
218224
r := &sppb.ExecuteSqlRequest{
@@ -284,6 +290,9 @@ func (t *BatchReadOnlyTransaction) Cleanup(ctx context.Context) {
284290
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency. Try disabling and rerunning. Error: %v", err)
285291
}
286292
}
293+
if metricErr := recordGFELatencyMetricsOT(ctx, md, "Cleanup", t.otConfig); metricErr != nil {
294+
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency through OpenTelemetry. Error: %v", metricErr)
295+
}
287296

288297
if err != nil {
289298
var logger *log.Logger
@@ -336,6 +345,9 @@ func (t *BatchReadOnlyTransaction) Execute(ctx context.Context, p *Partition) *R
336345
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency. Try disabling and rerunning. Error: %v", err)
337346
}
338347
}
348+
if metricErr := recordGFELatencyMetricsOT(ctx, md, "Execute", t.otConfig); metricErr != nil {
349+
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency through OpenTelemetry. Error: %v", metricErr)
350+
}
339351
return client, err
340352
}
341353
} else {
@@ -363,6 +375,9 @@ func (t *BatchReadOnlyTransaction) Execute(ctx context.Context, p *Partition) *R
363375
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency. Try disabling and rerunning. Error: %v", err)
364376
}
365377
}
378+
if metricErr := recordGFELatencyMetricsOT(ctx, md, "Execute", t.otConfig); metricErr != nil {
379+
trace.TracePrintf(ctx, nil, "Error in recording GFE Latency through OpenTelemetry. Error: %v", metricErr)
380+
}
366381
return client, err
367382
}
368383
}

spanner/client.go

+48
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ import (
2828
"cloud.google.com/go/internal/trace"
2929
sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
3030
"github.com/googleapis/gax-go/v2"
31+
"go.opentelemetry.io/otel/attribute"
32+
"go.opentelemetry.io/otel/metric"
3133
"google.golang.org/api/iterator"
3234
"google.golang.org/api/option"
3335
"google.golang.org/api/option/internaloption"
@@ -104,6 +106,7 @@ type Client struct {
104106
ct *commonTags
105107
disableRouteToLeader bool
106108
dro *sppb.DirectedReadOptions
109+
otConfig *openTelemetryConfig
107110
}
108111

109112
// DatabaseName returns the full name of a database, e.g.,
@@ -112,6 +115,11 @@ func (c *Client) DatabaseName() string {
112115
return c.sc.database
113116
}
114117

118+
// ClientID returns the id of the Client. This is not recommended for customer applications and used internally for testing.
119+
func (c *Client) ClientID() string {
120+
return c.sc.id
121+
}
122+
115123
// ClientConfig has configurations for the client.
116124
type ClientConfig struct {
117125
// NumChannels is the number of gRPC channels.
@@ -192,6 +200,23 @@ type ClientConfig struct {
192200
// and ExecuteSqlRequests for the Client which indicate which replicas or regions
193201
// should be used for non-transactional reads or queries.
194202
DirectedReadOptions *sppb.DirectedReadOptions
203+
204+
OpenTelemetryMeterProvider metric.MeterProvider
205+
}
206+
207+
type openTelemetryConfig struct {
208+
meterProvider metric.MeterProvider
209+
attributeMap []attribute.KeyValue
210+
otMetricRegistration metric.Registration
211+
openSessionCount metric.Int64ObservableGauge
212+
maxAllowedSessionsCount metric.Int64ObservableGauge
213+
sessionsCount metric.Int64ObservableGauge
214+
maxInUseSessionsCount metric.Int64ObservableGauge
215+
getSessionTimeoutsCount metric.Int64Counter
216+
acquiredSessionsCount metric.Int64Counter
217+
releasedSessionsCount metric.Int64Counter
218+
gfeLatency metric.Int64Histogram
219+
gfeHeaderMissingCount metric.Int64Counter
195220
}
196221

197222
func contextWithOutgoingMetadata(ctx context.Context, md metadata.MD, disableRouteToLeader bool) context.Context {
@@ -245,6 +270,7 @@ func NewClientWithConfig(ctx context.Context, database string, config ClientConf
245270
if err != nil {
246271
return nil, err
247272
}
273+
248274
if hasNumChannelsConfig && pool.Num() != config.NumChannels {
249275
pool.Close()
250276
return nil, spannerErrorf(codes.InvalidArgument, "Connection pool mismatch: NumChannels=%v, WithGRPCConnectionPool=%v. Only set one of these options, or set both to the same value.", config.NumChannels, pool.Num())
@@ -276,16 +302,29 @@ func NewClientWithConfig(ctx context.Context, database string, config ClientConf
276302
if config.Compression == gzip.Name {
277303
md.Append(requestsCompressionHeader, gzip.Name)
278304
}
305+
279306
// Create a session client.
280307
sc := newSessionClient(pool, database, config.UserAgent, sessionLabels, config.DatabaseRole, config.DisableRouteToLeader, md, config.BatchTimeout, config.Logger, config.CallOptions)
281308

309+
// Create a OpenTelemetry configuration
310+
otConfig, err := createOpenTelemetryConfig(config.OpenTelemetryMeterProvider, config.Logger, sc.id, database)
311+
if err != nil {
312+
// The error returned here will be due to database name parsing
313+
return nil, err
314+
}
315+
// To prevent data race in unit tests (ex: TestClient_SessionNotFound)
316+
sc.mu.Lock()
317+
sc.otConfig = otConfig
318+
sc.mu.Unlock()
319+
282320
// Create a session pool.
283321
config.SessionPoolConfig.sessionLabels = sessionLabels
284322
sp, err := newSessionPool(sc, config.SessionPoolConfig)
285323
if err != nil {
286324
sc.close()
287325
return nil, err
288326
}
327+
289328
c = &Client{
290329
sc: sc,
291330
idleSessions: sp,
@@ -298,6 +337,7 @@ func NewClientWithConfig(ctx context.Context, database string, config ClientConf
298337
ct: getCommonTags(sc),
299338
disableRouteToLeader: config.DisableRouteToLeader,
300339
dro: config.DirectedReadOptions,
340+
otConfig: otConfig,
301341
}
302342
return c, nil
303343
}
@@ -384,6 +424,7 @@ func (c *Client) Single() *ReadOnlyTransaction {
384424
t.txReadOnly.qo.DirectedReadOptions = c.dro
385425
t.txReadOnly.ro.DirectedReadOptions = c.dro
386426
t.ct = c.ct
427+
t.otConfig = c.otConfig
387428
return t
388429
}
389430

@@ -409,6 +450,7 @@ func (c *Client) ReadOnlyTransaction() *ReadOnlyTransaction {
409450
t.txReadOnly.qo.DirectedReadOptions = c.dro
410451
t.txReadOnly.ro.DirectedReadOptions = c.dro
411452
t.ct = c.ct
453+
t.otConfig = c.otConfig
412454
return t
413455
}
414456

@@ -479,6 +521,7 @@ func (c *Client) BatchReadOnlyTransaction(ctx context.Context, tb TimestampBound
479521
t.txReadOnly.qo.DirectedReadOptions = c.dro
480522
t.txReadOnly.ro.DirectedReadOptions = c.dro
481523
t.ct = c.ct
524+
t.otConfig = c.otConfig
482525
return t, nil
483526
}
484527

@@ -512,6 +555,7 @@ func (c *Client) BatchReadOnlyTransactionFromID(tid BatchReadOnlyTransactionID)
512555
t.txReadOnly.qo.DirectedReadOptions = c.dro
513556
t.txReadOnly.ro.DirectedReadOptions = c.dro
514557
t.ct = c.ct
558+
t.otConfig = c.otConfig
515559
return t
516560
}
517561

@@ -616,6 +660,7 @@ func (c *Client) rwTransaction(ctx context.Context, f func(context.Context, *Rea
616660
t.wb = []*Mutation{}
617661
t.txOpts = c.txo.merge(options)
618662
t.ct = c.ct
663+
t.otConfig = c.otConfig
619664

620665
trace.TracePrintf(ctx, map[string]interface{}{"transactionSelector": t.getTransactionSelector().String()},
621666
"Starting transaction attempt")
@@ -877,6 +922,9 @@ func (c *Client) BatchWriteWithOptions(ctx context.Context, mgs []*MutationGroup
877922
trace.TracePrintf(ct, nil, "Error in recording GFE Latency. Try disabling and rerunning. Error: %v", err)
878923
}
879924
}
925+
if metricErr := recordGFELatencyMetricsOT(ct, md, "BatchWrite", c.otConfig); metricErr != nil {
926+
trace.TracePrintf(ct, nil, "Error in recording GFE Latency through OpenTelemetry. Error: %v", err)
927+
}
880928
return stream, rpcErr
881929
}
882930

spanner/go.mod

+5-3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ require (
1212
github.com/json-iterator/go v1.1.12
1313
github.com/stretchr/testify v1.8.4
1414
go.opencensus.io v0.24.0
15+
go.opentelemetry.io/otel v1.22.0
16+
go.opentelemetry.io/otel/metric v1.22.0
1517
golang.org/x/oauth2 v0.16.0
1618
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028
1719
google.golang.org/api v0.162.0
@@ -38,14 +40,13 @@ require (
3840
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
3941
github.com/google/s2a-go v0.1.7 // indirect
4042
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
41-
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
43+
github.com/kr/text v0.2.0 // indirect
44+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
4245
github.com/modern-go/reflect2 v1.0.2 // indirect
4346
github.com/pmezard/go-difflib v1.0.0 // indirect
4447
github.com/stretchr/objx v0.5.0 // indirect
4548
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
4649
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
47-
go.opentelemetry.io/otel v1.22.0 // indirect
48-
go.opentelemetry.io/otel/metric v1.22.0 // indirect
4950
go.opentelemetry.io/otel/sdk v1.21.0 // indirect
5051
go.opentelemetry.io/otel/trace v1.22.0 // indirect
5152
golang.org/x/crypto v0.18.0 // indirect
@@ -55,5 +56,6 @@ require (
5556
golang.org/x/text v0.14.0 // indirect
5657
golang.org/x/time v0.5.0 // indirect
5758
google.golang.org/appengine v1.6.8 // indirect
59+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
5860
gopkg.in/yaml.v3 v3.0.1 // indirect
5961
)

spanner/go.sum

+11-2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XP
2222
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
2323
github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY=
2424
github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
25+
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
2526
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2627
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
2728
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -78,8 +79,15 @@ github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56
7879
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
7980
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
8081
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
81-
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
82+
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
83+
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
84+
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
85+
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
86+
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
87+
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
8288
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
89+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
90+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
8391
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
8492
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
8593
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -206,8 +214,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
206214
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
207215
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
208216
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
209-
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
210217
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
218+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
219+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
211220
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
212221
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
213222
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

0 commit comments

Comments
 (0)