Skip to content

Commit 0ec6c7c

Browse files
olavloiteolavloite
authored
feat: JSONB support (#328)
* feat: JSONB support Add support for JSONB. * deps: bump Spanner client library * fix: binary jsonb contains a version number * test: add integration tests for JSONB * test: fix one of the integration tests * fix: copy jsonb values * fix: remove strange timezones in test data * fix: remove more strange time zones * fix: binary copy missed jsonb column * test: fix integration tests + add jsonb to pgx tests * fix: simple mode mock server test missed jsonb results * test: add jsonb parser test
1 parent 9c60173 commit 0ec6c7c

33 files changed

+10597
-10243
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
com.google.cloud.spanner.pgadapter.nodejs.NodeJSTest
3535
</excludedTests>
3636

37-
<spanner.version>6.28.0</spanner.version>
37+
<spanner.version>6.30.0</spanner.version>
3838
<junixsocket.version>2.5.1</junixsocket.version>
3939
</properties>
4040

‎src/main/java/com/google/cloud/spanner/pgadapter/parsers/ArrayParser.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ private List<?> toList(Value value, Code arrayElementType) {
6262
return value.getBoolArray();
6363
case DATE:
6464
return value.getDateArray();
65-
case JSON:
66-
return value.getJsonArray();
65+
case PG_JSONB:
66+
return value.getPgJsonbArray();
6767
case BYTES:
6868
return value.getBytesArray();
6969
case INT64:

‎src/main/java/com/google/cloud/spanner/pgadapter/parsers/JsonbParser.java

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.google.cloud.spanner.pgadapter.parsers;
16+
17+
import com.google.api.core.InternalApi;
18+
import com.google.cloud.spanner.ResultSet;
19+
import com.google.cloud.spanner.Statement;
20+
import com.google.cloud.spanner.pgadapter.error.PGException;
21+
import com.google.cloud.spanner.pgadapter.error.SQLState;
22+
import com.google.cloud.spanner.pgadapter.error.Severity;
23+
import java.nio.charset.StandardCharsets;
24+
import java.util.Arrays;
25+
import javax.annotation.Nonnull;
26+
27+
/** Translate from wire protocol to jsonb. */
28+
@InternalApi
29+
public class JsonbParser extends Parser<String> {
30+
31+
JsonbParser(ResultSet item, int position) {
32+
this.item = item.getPgJsonb(position);
33+
}
34+
35+
JsonbParser(Object item) {
36+
this.item = (String) item;
37+
}
38+
39+
JsonbParser(byte[] item, FormatCode formatCode) {
40+
if (item != null) {
41+
switch (formatCode) {
42+
case TEXT:
43+
this.item = toString(item);
44+
break;
45+
case BINARY:
46+
if (item.length > 0) {
47+
if (item[0] == 1) {
48+
this.item = toString(Arrays.copyOfRange(item, 1, item.length));
49+
} else {
50+
throw PGException.newBuilder()
51+
.setSQLState(SQLState.RaiseException)
52+
.setSeverity(Severity.ERROR)
53+
.setMessage("Unknown version in binary jsonb value: " + item[0])
54+
.build();
55+
}
56+
} else {
57+
this.item = "";
58+
}
59+
break;
60+
default:
61+
}
62+
}
63+
}
64+
65+
/** Converts the binary data to an UTF8 string. */
66+
public static String toString(@Nonnull byte[] data) {
67+
return new String(data, UTF8);
68+
}
69+
70+
@Override
71+
public String stringParse() {
72+
return this.item;
73+
}
74+
75+
@Override
76+
protected byte[] binaryParse() {
77+
if (this.item == null) {
78+
return null;
79+
}
80+
byte[] value = this.item.getBytes(StandardCharsets.UTF_8);
81+
byte[] result = new byte[value.length + 1];
82+
// Set version = 1
83+
result[0] = 1;
84+
System.arraycopy(value, 0, result, 1, value.length);
85+
86+
return result;
87+
}
88+
89+
@Override
90+
public void bind(Statement.Builder statementBuilder, String name) {
91+
statementBuilder.bind(name).to(this.item);
92+
}
93+
}

‎src/main/java/com/google/cloud/spanner/pgadapter/parsers/Parser.java

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ public static Parser<?> create(byte[] item, int oidType, FormatCode formatCode)
109109
case Oid.TIMESTAMP:
110110
case Oid.TIMESTAMPTZ:
111111
return new TimestampParser(item, formatCode);
112+
case Oid.JSONB:
113+
return new JsonbParser(item, formatCode);
112114
case Oid.UNSPECIFIED:
113115
// Try to guess the type based on the value. Use an unspecified parser if no type could be
114116
// determined.
@@ -148,9 +150,12 @@ public static Parser<?> create(ResultSet result, Type type, int columnarPosition
148150
return new StringParser(result, columnarPosition);
149151
case TIMESTAMP:
150152
return new TimestampParser(result, columnarPosition);
153+
case PG_JSONB:
154+
return new JsonbParser(result, columnarPosition);
151155
case ARRAY:
152156
return new ArrayParser(result, columnarPosition);
153157
case NUMERIC:
158+
case JSON:
154159
case STRUCT:
155160
default:
156161
throw new IllegalArgumentException("Illegal or unknown element type: " + type);
@@ -182,6 +187,8 @@ protected static Parser<?> create(Object result, Code typeCode) {
182187
return new StringParser(result);
183188
case TIMESTAMP:
184189
return new TimestampParser(result);
190+
case PG_JSONB:
191+
return new JsonbParser(result);
185192
case NUMERIC:
186193
case ARRAY:
187194
case STRUCT:
@@ -208,7 +215,7 @@ public static int toOid(Type type) {
208215
return Oid.FLOAT8;
209216
case STRING:
210217
return Oid.VARCHAR;
211-
case JSON:
218+
case PG_JSONB:
212219
return Oid.JSONB;
213220
case BYTES:
214221
return Oid.BYTEA;
@@ -228,7 +235,7 @@ public static int toOid(Type type) {
228235
return Oid.FLOAT8_ARRAY;
229236
case STRING:
230237
return Oid.VARCHAR_ARRAY;
231-
case JSON:
238+
case PG_JSONB:
232239
return Oid.JSONB_ARRAY;
233240
case BYTES:
234241
return Oid.BYTEA_ARRAY;
@@ -237,13 +244,15 @@ public static int toOid(Type type) {
237244
case DATE:
238245
return Oid.DATE_ARRAY;
239246
case NUMERIC:
247+
case JSON:
240248
case ARRAY:
241249
case STRUCT:
242250
default:
243251
throw SpannerExceptionFactory.newSpannerException(
244252
ErrorCode.INVALID_ARGUMENT, "Unsupported or unknown array type: " + type);
245253
}
246254
case NUMERIC:
255+
case JSON:
247256
case STRUCT:
248257
default:
249258
throw SpannerExceptionFactory.newSpannerException(

‎src/main/java/com/google/cloud/spanner/pgadapter/statements/CopyStatement.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ private static Type parsePostgreSQLDataType(String columnType) {
238238
case "timestamp with time zone":
239239
return Type.timestamp();
240240
case "jsonb":
241-
return Type.json();
241+
return Type.pgJsonb();
242242
default:
243243
throw new IllegalArgumentException(
244244
"Unrecognized or unsupported column data type: " + columnType);

‎src/main/java/com/google/cloud/spanner/pgadapter/statements/JdbcMetadataStatementHelper.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,9 @@ static String replaceJdbcMetadataStatement(String sql) {
131131
if (sql.startsWith(PgJdbcCatalog.PG_JDBC_GET_SQL_KEYWORDS_PREFIX)) {
132132
return PgJdbcCatalog.PG_JDBC_GET_SQL_KEYWORDS_REPLACEMENT;
133133
}
134+
if (sql.equals(PgJdbcCatalog.PG_JDBC_GET_TYPE_NAME)) {
135+
return PgJdbcCatalog.PG_JDBC_GET_TYPE_NAME_REPLACEMENT;
136+
}
134137
if (sql.startsWith(PgJdbcCatalog.PG_JDBC_GET_TYPE_INFO_PREFIX)) {
135138
return PgJdbcCatalog.PG_JDBC_GET_TYPE_INFO_REPLACEMENT;
136139
}

‎src/main/java/com/google/cloud/spanner/pgadapter/utils/BinaryCopyParser.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -260,8 +260,8 @@ public Value getValue(Type type, int columnIndex) {
260260
return Value.float64(field.data == null ? null : DoubleParser.toDouble(field.data));
261261
case STRING:
262262
return Value.string(field.data == null ? null : StringParser.toString(field.data));
263-
case JSON:
264-
return Value.json(field.data == null ? null : StringParser.toString(field.data));
263+
case PG_JSONB:
264+
return Value.pgJsonb(field.data == null ? null : StringParser.toString(field.data));
265265
case BYTES:
266266
return Value.bytes(field.data == null ? null : BinaryParser.toByteArray(field.data));
267267
case TIMESTAMP:

‎src/main/java/com/google/cloud/spanner/pgadapter/utils/CsvCopyParser.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,8 @@ static Value getSpannerValue(Type type, String recordValue) throws SpannerExcept
124124
switch (type.getCode()) {
125125
case STRING:
126126
return Value.string(recordValue);
127-
case JSON:
128-
return Value.json(recordValue);
127+
case PG_JSONB:
128+
return Value.pgJsonb(recordValue);
129129
case BOOL:
130130
return Value.bool(recordValue == null ? null : BooleanParser.toBoolean(recordValue));
131131
case INT64:

‎src/main/java/com/google/cloud/spanner/pgadapter/utils/MutationWriter.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@ static int calculateSize(Mutation mutation) {
407407
size += value.getString().length();
408408
break;
409409
case STRING:
410-
case JSON:
410+
case PG_JSONB:
411411
// Assume four bytes per character to be on the safe side.
412412
size += value.getString().length() * 4;
413413
break;
@@ -436,7 +436,7 @@ static int calculateSize(Mutation mutation) {
436436
size += s.length();
437437
}
438438
break;
439-
case JSON:
439+
case PG_JSONB:
440440
case STRING:
441441
for (String s : value.getStringArray()) {
442442
size += s.length() * 4;
@@ -455,11 +455,13 @@ static int calculateSize(Mutation mutation) {
455455
break;
456456
case ARRAY:
457457
case NUMERIC:
458+
case JSON:
458459
case STRUCT:
459460
break;
460461
}
461462
break;
462463
case NUMERIC:
464+
case JSON:
463465
case STRUCT:
464466
break;
465467
}

‎src/main/java/com/google/cloud/spanner/pgadapter/utils/PgJdbcCatalog.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@ public class PgJdbcCatalog {
3939
+ "statistics,stdin,stdout,storage,strict,sysid,tablespace,temp,template,truncate,trusted,"
4040
+ "unencrypted,unlisten,until,vacuum,valid,validator,verbose,volatile'";
4141

42+
public static final String PG_JDBC_GET_TYPE_NAME =
43+
"SELECT n.nspname = ANY(current_schemas(true)), n.nspname, t.typname "
44+
+ "FROM pg_catalog.pg_type t "
45+
+ "JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid WHERE t.oid = $1";
46+
public static final String PG_JDBC_GET_TYPE_NAME_REPLACEMENT =
47+
"SELECT true, n.nspname, t.typname "
48+
+ "FROM pg_catalog.pg_type t "
49+
+ "JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid WHERE t.oid = $1";
4250
public static final String PG_JDBC_GET_TYPE_INFO_PREFIX_42_2_22 =
4351
"SELECT typinput='array_in'::regproc as is_array, typtype, typname "
4452
+ " FROM pg_catalog.pg_type "

‎src/main/java/com/google/cloud/spanner/pgadapter/wireoutput/RowDescriptionResponse.java

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -146,20 +146,12 @@ int getOidTypeSize(int oid_type) {
146146
return 4;
147147
case Oid.INT8:
148148
return 8;
149-
case Oid.NUMERIC:
150-
return -1;
151149
case Oid.FLOAT4:
152150
return 4;
153151
case Oid.FLOAT8:
154152
return 8;
155153
case Oid.CHAR:
156154
return 1;
157-
case Oid.TEXT:
158-
return -1;
159-
case Oid.VARCHAR:
160-
return -1;
161-
case Oid.BYTEA:
162-
return -1;
163155
case Oid.BOOL:
164156
return 1;
165157
case Oid.DATE:
@@ -168,6 +160,11 @@ int getOidTypeSize(int oid_type) {
168160
return 8;
169161
case Oid.TIMESTAMPTZ:
170162
return 12;
163+
case Oid.NUMERIC:
164+
case Oid.TEXT:
165+
case Oid.VARCHAR:
166+
case Oid.BYTEA:
167+
case Oid.JSONB:
171168
default:
172169
return -1;
173170
}

‎src/test/golang/pgadapter_pgx_tests/go.mod

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@ module cloud.google.com/pgadapter_pgx_tests
33
go 1.17
44

55
require (
6-
github.com/jackc/pgconn v1.11.0
7-
github.com/jackc/pgtype v1.10.0
8-
github.com/jackc/pgx/v4 v4.15.0
6+
github.com/jackc/pgconn v1.13.0
7+
github.com/jackc/pgtype v1.12.0
8+
github.com/jackc/pgx/v4 v4.17.2
99
)
1010

1111
require (
1212
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
1313
github.com/jackc/pgio v1.0.0 // indirect
1414
github.com/jackc/pgpassfile v1.0.0 // indirect
15-
github.com/jackc/pgproto3/v2 v2.2.0 // indirect
15+
github.com/jackc/pgproto3/v2 v2.3.1 // indirect
1616
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
17-
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
18-
golang.org/x/text v0.3.6 // indirect
17+
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
18+
golang.org/x/text v0.3.7 // indirect
1919
)

0 commit comments

Comments
 (0)