[oceanbase] Add new deserialization schema with runtime converter (#1356)
This closes #980.pull/1569/head
He Wang
2 years ago
committed by
GitHub
parent
d3435381a6
commit
b1b092c97e
@ -0,0 +1,47 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Ververica Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.ververica.cdc.connectors.oceanbase.source;
|
||||||
|
|
||||||
|
import com.oceanbase.oms.logmessage.ByteString;
|
||||||
|
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runtime converter that converts objects of OceanBase into objects of Flink Table & SQL internal
|
||||||
|
* data structures.
|
||||||
|
*/
|
||||||
|
public interface OceanBaseDeserializationRuntimeConverter extends Serializable {
|
||||||
|
|
||||||
|
default Object convert(Object object) throws Exception {
|
||||||
|
if (object instanceof ByteString) {
|
||||||
|
return convertChangeEvent(
|
||||||
|
((ByteString) object).toString(StandardCharsets.UTF_8.name()));
|
||||||
|
} else {
|
||||||
|
return convertSnapshotEvent(object);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
default Object convertSnapshotEvent(Object object) throws Exception {
|
||||||
|
throw new NotImplementedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
default Object convertChangeEvent(String string) throws Exception {
|
||||||
|
throw new NotImplementedException();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,220 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2022 Ververica Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package com.ververica.cdc.connectors.oceanbase.source;
|
|
||||||
|
|
||||||
import com.oceanbase.oms.logmessage.ByteString;
|
|
||||||
import com.oceanbase.oms.logmessage.DataMessage;
|
|
||||||
import io.debezium.config.CommonConnectorConfig;
|
|
||||||
import io.debezium.jdbc.JdbcValueConverters;
|
|
||||||
import io.debezium.jdbc.TemporalPrecisionMode;
|
|
||||||
import io.debezium.relational.ValueConverterProvider;
|
|
||||||
import io.debezium.util.NumberConversions;
|
|
||||||
import org.apache.kafka.connect.data.Schema;
|
|
||||||
|
|
||||||
import java.math.BigDecimal;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.nio.ByteBuffer;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.sql.Date;
|
|
||||||
import java.sql.Time;
|
|
||||||
import java.sql.Timestamp;
|
|
||||||
import java.sql.Types;
|
|
||||||
import java.time.ZoneOffset;
|
|
||||||
import java.util.Arrays;
|
|
||||||
|
|
||||||
/** Utils to convert jdbc type and value of a field. */
|
|
||||||
public class OceanBaseJdbcConverter {
|
|
||||||
|
|
||||||
public static ValueConverterProvider valueConverterProvider(ZoneOffset zoneOffset) {
|
|
||||||
return new JdbcValueConverters(
|
|
||||||
JdbcValueConverters.DecimalMode.STRING,
|
|
||||||
TemporalPrecisionMode.ADAPTIVE_TIME_MICROSECONDS,
|
|
||||||
zoneOffset,
|
|
||||||
null,
|
|
||||||
JdbcValueConverters.BigIntUnsignedMode.PRECISE,
|
|
||||||
CommonConnectorConfig.BinaryHandlingMode.BYTES);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Object getField(int jdbcType, Object value) {
|
|
||||||
if (value == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
jdbcType = getType(jdbcType, null);
|
|
||||||
switch (jdbcType) {
|
|
||||||
case Types.BIT:
|
|
||||||
if (value instanceof Boolean) {
|
|
||||||
return new byte[] {NumberConversions.getByte((Boolean) value)};
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
case Types.INTEGER:
|
|
||||||
if (value instanceof Boolean) {
|
|
||||||
return NumberConversions.getInteger((Boolean) value);
|
|
||||||
}
|
|
||||||
if (value instanceof Date) {
|
|
||||||
return ((Date) value).getYear() + 1900;
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
case Types.FLOAT:
|
|
||||||
Float f = (Float) value;
|
|
||||||
return f.doubleValue();
|
|
||||||
case Types.DECIMAL:
|
|
||||||
if (value instanceof BigInteger) {
|
|
||||||
return value.toString();
|
|
||||||
}
|
|
||||||
BigDecimal decimal = (BigDecimal) value;
|
|
||||||
return decimal.toString();
|
|
||||||
case Types.DATE:
|
|
||||||
Date date = (Date) value;
|
|
||||||
return io.debezium.time.Date.toEpochDay(date, null);
|
|
||||||
case Types.TIME:
|
|
||||||
Time time = (Time) value;
|
|
||||||
return io.debezium.time.MicroTime.toMicroOfDay(time, true);
|
|
||||||
case Types.TIMESTAMP:
|
|
||||||
Timestamp timestamp = (Timestamp) value;
|
|
||||||
return io.debezium.time.MicroTimestamp.toEpochMicros(timestamp, null);
|
|
||||||
default:
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Object getField(
|
|
||||||
Schema.Type schemaType, DataMessage.Record.Field.Type fieldType, ByteString value) {
|
|
||||||
if (value == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
int jdbcType = getType(fieldType);
|
|
||||||
switch (jdbcType) {
|
|
||||||
case Types.NULL:
|
|
||||||
return null;
|
|
||||||
case Types.INTEGER:
|
|
||||||
if (schemaType.equals(Schema.Type.INT64)) {
|
|
||||||
return Long.parseLong(value.toString());
|
|
||||||
}
|
|
||||||
return Integer.parseInt(value.toString());
|
|
||||||
case Types.BIGINT:
|
|
||||||
if (schemaType.equals(Schema.Type.STRING)) {
|
|
||||||
return value.toString();
|
|
||||||
}
|
|
||||||
return Long.parseLong(value.toString());
|
|
||||||
case Types.DOUBLE:
|
|
||||||
return Double.parseDouble(value.toString());
|
|
||||||
case Types.DATE:
|
|
||||||
Date date = Date.valueOf(value.toString());
|
|
||||||
return io.debezium.time.Date.toEpochDay(date, null);
|
|
||||||
case Types.TIME:
|
|
||||||
Time time = Time.valueOf(value.toString());
|
|
||||||
return io.debezium.time.MicroTime.toMicroOfDay(time, true);
|
|
||||||
case Types.TIMESTAMP:
|
|
||||||
Timestamp timestamp = Timestamp.valueOf(value.toString());
|
|
||||||
return io.debezium.time.MicroTimestamp.toEpochMicros(timestamp, null);
|
|
||||||
case Types.BIT:
|
|
||||||
long v = Long.parseLong(value.toString());
|
|
||||||
byte[] bytes = ByteBuffer.allocate(8).putLong(v).array();
|
|
||||||
int i = 0;
|
|
||||||
while (bytes[i] == 0 && i < Long.BYTES - 1) {
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
return Arrays.copyOfRange(bytes, i, Long.BYTES);
|
|
||||||
case Types.BINARY:
|
|
||||||
return ByteBuffer.wrap(value.toString().getBytes(StandardCharsets.UTF_8));
|
|
||||||
default:
|
|
||||||
return value.toString(StandardCharsets.UTF_8.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static boolean isBoolean(int jdbcType, String typeName) {
|
|
||||||
return jdbcType == Types.BOOLEAN || (jdbcType == Types.BIT && "TINYINT".equals(typeName));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int getType(int jdbcType, String typeName) {
|
|
||||||
// treat boolean as tinyint type
|
|
||||||
if (isBoolean(jdbcType, typeName)) {
|
|
||||||
jdbcType = Types.TINYINT;
|
|
||||||
}
|
|
||||||
// treat year as int type
|
|
||||||
if ("YEAR".equals(typeName)) {
|
|
||||||
jdbcType = Types.INTEGER;
|
|
||||||
}
|
|
||||||
|
|
||||||
// upcasting
|
|
||||||
if ("INT UNSIGNED".equals(typeName)) {
|
|
||||||
jdbcType = Types.BIGINT;
|
|
||||||
}
|
|
||||||
if ("BIGINT UNSIGNED".equals(typeName)) {
|
|
||||||
jdbcType = Types.DECIMAL;
|
|
||||||
}
|
|
||||||
|
|
||||||
// widening conversion according to com.mysql.jdbc.ResultSetImpl#getObject
|
|
||||||
switch (jdbcType) {
|
|
||||||
case Types.TINYINT:
|
|
||||||
case Types.SMALLINT:
|
|
||||||
return Types.INTEGER;
|
|
||||||
case Types.REAL:
|
|
||||||
return Types.FLOAT;
|
|
||||||
default:
|
|
||||||
return jdbcType;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int getType(DataMessage.Record.Field.Type fieldType) {
|
|
||||||
switch (fieldType) {
|
|
||||||
case NULL:
|
|
||||||
return Types.NULL;
|
|
||||||
case INT8:
|
|
||||||
case INT16:
|
|
||||||
case INT24:
|
|
||||||
case INT32:
|
|
||||||
case YEAR:
|
|
||||||
return Types.INTEGER;
|
|
||||||
case INT64:
|
|
||||||
return Types.BIGINT;
|
|
||||||
case FLOAT:
|
|
||||||
case DOUBLE:
|
|
||||||
return Types.DOUBLE;
|
|
||||||
case DECIMAL:
|
|
||||||
return Types.DECIMAL;
|
|
||||||
case ENUM:
|
|
||||||
case SET:
|
|
||||||
case STRING:
|
|
||||||
case JSON:
|
|
||||||
return Types.CHAR;
|
|
||||||
case TIMESTAMP:
|
|
||||||
case DATETIME:
|
|
||||||
case TIMESTAMP_WITH_TIME_ZONE:
|
|
||||||
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
|
|
||||||
case TIMESTAMP_NANO:
|
|
||||||
return Types.TIMESTAMP;
|
|
||||||
case DATE:
|
|
||||||
return Types.DATE;
|
|
||||||
case TIME:
|
|
||||||
return Types.TIME;
|
|
||||||
case BIT:
|
|
||||||
return Types.BIT;
|
|
||||||
case BLOB:
|
|
||||||
case BINARY:
|
|
||||||
return Types.BINARY;
|
|
||||||
case INTERVAL_YEAR_TO_MONTH:
|
|
||||||
case INTERVAL_DAY_TO_SECOND:
|
|
||||||
case GEOMETRY:
|
|
||||||
case RAW:
|
|
||||||
// it's weird to get wrong type from TEXT column, temporarily treat it as a string
|
|
||||||
case UNKOWN:
|
|
||||||
default:
|
|
||||||
return Types.VARCHAR;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,51 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2022 Ververica Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package com.ververica.cdc.connectors.oceanbase.source;
|
|
||||||
|
|
||||||
import org.apache.kafka.connect.data.Schema;
|
|
||||||
import org.apache.kafka.connect.data.SchemaBuilder;
|
|
||||||
import org.apache.kafka.connect.data.Struct;
|
|
||||||
|
|
||||||
/** Utils to deal with OceanBase SourceRecord schema. */
|
|
||||||
public class OceanBaseSchemaUtils {
|
|
||||||
|
|
||||||
public static Schema sourceSchema() {
|
|
||||||
return SchemaBuilder.struct()
|
|
||||||
.field("tenant", Schema.STRING_SCHEMA)
|
|
||||||
.field("database", Schema.STRING_SCHEMA)
|
|
||||||
.field("table", Schema.STRING_SCHEMA)
|
|
||||||
.field("timestamp", Schema.OPTIONAL_STRING_SCHEMA)
|
|
||||||
.field("unique_id", Schema.OPTIONAL_STRING_SCHEMA)
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Struct sourceStruct(
|
|
||||||
String tenant, String database, String table, String timestamp, String uniqueId) {
|
|
||||||
Struct struct =
|
|
||||||
new Struct(sourceSchema())
|
|
||||||
.put("tenant", tenant)
|
|
||||||
.put("database", database)
|
|
||||||
.put("table", table);
|
|
||||||
if (timestamp != null) {
|
|
||||||
struct.put("timestamp", timestamp);
|
|
||||||
}
|
|
||||||
if (uniqueId != null) {
|
|
||||||
struct.put("unique_id", uniqueId);
|
|
||||||
}
|
|
||||||
return struct;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,128 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2022 Ververica Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package com.ververica.cdc.connectors.oceanbase.source;
|
|
||||||
|
|
||||||
import io.debezium.data.Envelope;
|
|
||||||
import io.debezium.relational.Column;
|
|
||||||
import io.debezium.relational.ColumnEditor;
|
|
||||||
import io.debezium.relational.CustomConverterRegistry;
|
|
||||||
import io.debezium.relational.Table;
|
|
||||||
import io.debezium.relational.TableEditor;
|
|
||||||
import io.debezium.relational.TableId;
|
|
||||||
import io.debezium.relational.TableSchema;
|
|
||||||
import io.debezium.relational.TableSchemaBuilder;
|
|
||||||
import io.debezium.util.SchemaNameAdjuster;
|
|
||||||
import org.apache.kafka.connect.data.Schema;
|
|
||||||
import org.apache.kafka.connect.data.SchemaBuilder;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.sql.Types;
|
|
||||||
import java.time.ZoneOffset;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/** Utils to deal with table schema of OceanBase. */
|
|
||||||
public class OceanBaseTableSchema {
|
|
||||||
|
|
||||||
public static TableSchemaBuilder tableSchemaBuilder(ZoneOffset zoneOffset) {
|
|
||||||
return new TableSchemaBuilder(
|
|
||||||
OceanBaseJdbcConverter.valueConverterProvider(zoneOffset),
|
|
||||||
SchemaNameAdjuster.create(),
|
|
||||||
new CustomConverterRegistry(null),
|
|
||||||
OceanBaseSchemaUtils.sourceSchema(),
|
|
||||||
false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static TableId tableId(String databaseName, String tableName) {
|
|
||||||
return new TableId(databaseName, null, tableName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Column getColumn(String name, int jdbcType) {
|
|
||||||
// we can't get the scale and length of decimal, timestamp and bit columns from log,
|
|
||||||
// so here we set a constant value to these fields to be compatible with the logic of
|
|
||||||
// JdbcValueConverters#schemaBuilder
|
|
||||||
ColumnEditor columnEditor =
|
|
||||||
Column.editor().name(name).jdbcType(jdbcType).optional(true).scale(0);
|
|
||||||
if (columnEditor.jdbcType() == Types.TIMESTAMP || columnEditor.jdbcType() == Types.BIT) {
|
|
||||||
columnEditor.length(6);
|
|
||||||
}
|
|
||||||
return columnEditor.create();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static TableSchema getTableSchema(
|
|
||||||
String topicName,
|
|
||||||
String databaseName,
|
|
||||||
String tableName,
|
|
||||||
String[] columnNames,
|
|
||||||
int[] jdbcTypes,
|
|
||||||
ZoneOffset zoneOffset) {
|
|
||||||
TableEditor tableEditor = Table.editor().tableId(tableId(databaseName, tableName));
|
|
||||||
for (int i = 0; i < columnNames.length; i++) {
|
|
||||||
tableEditor.addColumn(getColumn(columnNames[i], jdbcTypes[i]));
|
|
||||||
}
|
|
||||||
return tableSchemaBuilder(zoneOffset)
|
|
||||||
.create(
|
|
||||||
null,
|
|
||||||
Envelope.schemaName(topicName),
|
|
||||||
tableEditor.create(),
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Schema upcastingSchemaType(Schema schema, String value) {
|
|
||||||
if (schema.type().equals(Schema.Type.INT32) && Long.parseLong(value) > Integer.MAX_VALUE) {
|
|
||||||
return Schema.INT64_SCHEMA;
|
|
||||||
}
|
|
||||||
if (schema.type().equals(Schema.Type.INT64)) {
|
|
||||||
BigInteger bigInt = new BigInteger(value);
|
|
||||||
if (bigInt.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) {
|
|
||||||
return Schema.STRING_SCHEMA;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Schema upcastingValueSchema(Schema valueSchema, Map<String, String> fields) {
|
|
||||||
SchemaBuilder schemaBuilder = SchemaBuilder.struct().optional();
|
|
||||||
for (Map.Entry<String, String> entry : fields.entrySet()) {
|
|
||||||
Schema fieldSchema = valueSchema.field(entry.getKey()).schema();
|
|
||||||
fieldSchema = upcastingSchemaType(fieldSchema, entry.getValue());
|
|
||||||
schemaBuilder.field(entry.getKey(), fieldSchema);
|
|
||||||
}
|
|
||||||
return schemaBuilder.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Envelope getEnvelope(String name, Schema valueSchema) {
|
|
||||||
return Envelope.defineSchema()
|
|
||||||
.withName(name)
|
|
||||||
.withRecord(valueSchema)
|
|
||||||
.withSource(OceanBaseSchemaUtils.sourceSchema())
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static TableSchema upcastingTableSchema(
|
|
||||||
String topicName, TableSchema tableSchema, Map<String, String> fields) {
|
|
||||||
Schema valueSchema = upcastingValueSchema(tableSchema.valueSchema(), fields);
|
|
||||||
return new TableSchema(
|
|
||||||
tableSchema.id(),
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
getEnvelope(Envelope.schemaName(topicName), valueSchema),
|
|
||||||
valueSchema,
|
|
||||||
null);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -0,0 +1,679 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Ververica Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.ververica.cdc.connectors.oceanbase.source;
|
||||||
|
|
||||||
|
import org.apache.flink.api.common.typeinfo.TypeInformation;
|
||||||
|
import org.apache.flink.table.data.DecimalData;
|
||||||
|
import org.apache.flink.table.data.GenericArrayData;
|
||||||
|
import org.apache.flink.table.data.GenericRowData;
|
||||||
|
import org.apache.flink.table.data.RowData;
|
||||||
|
import org.apache.flink.table.data.StringData;
|
||||||
|
import org.apache.flink.table.data.TimestampData;
|
||||||
|
import org.apache.flink.table.types.logical.DecimalType;
|
||||||
|
import org.apache.flink.table.types.logical.LogicalType;
|
||||||
|
import org.apache.flink.table.types.logical.RowType;
|
||||||
|
import org.apache.flink.types.RowKind;
|
||||||
|
import org.apache.flink.util.Collector;
|
||||||
|
|
||||||
|
import com.oceanbase.oms.logmessage.ByteString;
|
||||||
|
import com.ververica.cdc.connectors.oceanbase.table.OceanBaseAppendMetadataCollector;
|
||||||
|
import com.ververica.cdc.connectors.oceanbase.table.OceanBaseDeserializationSchema;
|
||||||
|
import com.ververica.cdc.connectors.oceanbase.table.OceanBaseMetadataConverter;
|
||||||
|
import com.ververica.cdc.connectors.oceanbase.table.OceanBaseRecord;
|
||||||
|
import com.ververica.cdc.debezium.utils.TemporalConversions;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.sql.Date;
|
||||||
|
import java.sql.Time;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.ZoneId;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.apache.flink.util.Preconditions.checkNotNull;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deserialization schema from OceanBase object to Flink Table/SQL internal data structure {@link
|
||||||
|
* RowData}.
|
||||||
|
*/
|
||||||
|
public class RowDataOceanBaseDeserializationSchema
|
||||||
|
implements OceanBaseDeserializationSchema<RowData> {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
/** TypeInformation of the produced {@link RowData}. * */
|
||||||
|
private final TypeInformation<RowData> resultTypeInfo;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runtime converter that OceanBase record data into {@link RowData} consisted of physical
|
||||||
|
* column values.
|
||||||
|
*/
|
||||||
|
private final OceanBaseDeserializationRuntimeConverter physicalConverter;
|
||||||
|
|
||||||
|
/** Whether the deserializer needs to handle metadata columns. */
|
||||||
|
private final boolean hasMetadata;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A wrapped output collector which is used to append metadata columns after physical columns.
|
||||||
|
*/
|
||||||
|
private final OceanBaseAppendMetadataCollector appendMetadataCollector;
|
||||||
|
|
||||||
|
/** Returns a builder to build {@link RowDataOceanBaseDeserializationSchema}. */
|
||||||
|
public static RowDataOceanBaseDeserializationSchema.Builder newBuilder() {
|
||||||
|
return new RowDataOceanBaseDeserializationSchema.Builder();
|
||||||
|
}
|
||||||
|
|
||||||
|
RowDataOceanBaseDeserializationSchema(
|
||||||
|
RowType physicalDataType,
|
||||||
|
OceanBaseMetadataConverter[] metadataConverters,
|
||||||
|
TypeInformation<RowData> resultTypeInfo,
|
||||||
|
ZoneId serverTimeZone) {
|
||||||
|
this.hasMetadata = checkNotNull(metadataConverters).length > 0;
|
||||||
|
this.appendMetadataCollector = new OceanBaseAppendMetadataCollector(metadataConverters);
|
||||||
|
this.physicalConverter = createConverter(checkNotNull(physicalDataType), serverTimeZone);
|
||||||
|
this.resultTypeInfo = checkNotNull(resultTypeInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deserialize(OceanBaseRecord record, Collector<RowData> out) throws Exception {
|
||||||
|
RowData physicalRow;
|
||||||
|
if (record.isSnapshotRecord()) {
|
||||||
|
physicalRow = (GenericRowData) physicalConverter.convert(record.getJdbcFields());
|
||||||
|
physicalRow.setRowKind(RowKind.INSERT);
|
||||||
|
emit(record, physicalRow, out);
|
||||||
|
} else {
|
||||||
|
switch (record.getOpt()) {
|
||||||
|
case INSERT:
|
||||||
|
physicalRow =
|
||||||
|
(GenericRowData)
|
||||||
|
physicalConverter.convert(record.getLogMessageFieldsAfter());
|
||||||
|
physicalRow.setRowKind(RowKind.INSERT);
|
||||||
|
emit(record, physicalRow, out);
|
||||||
|
break;
|
||||||
|
case DELETE:
|
||||||
|
physicalRow =
|
||||||
|
(GenericRowData)
|
||||||
|
physicalConverter.convert(record.getLogMessageFieldsBefore());
|
||||||
|
physicalRow.setRowKind(RowKind.DELETE);
|
||||||
|
emit(record, physicalRow, out);
|
||||||
|
break;
|
||||||
|
case UPDATE:
|
||||||
|
physicalRow =
|
||||||
|
(GenericRowData)
|
||||||
|
physicalConverter.convert(record.getLogMessageFieldsBefore());
|
||||||
|
physicalRow.setRowKind(RowKind.UPDATE_BEFORE);
|
||||||
|
emit(record, physicalRow, out);
|
||||||
|
physicalRow =
|
||||||
|
(GenericRowData)
|
||||||
|
physicalConverter.convert(record.getLogMessageFieldsAfter());
|
||||||
|
physicalRow.setRowKind(RowKind.UPDATE_AFTER);
|
||||||
|
emit(record, physicalRow, out);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Unsupported log message record type: " + record.getOpt());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void emit(OceanBaseRecord row, RowData physicalRow, Collector<RowData> collector) {
|
||||||
|
if (!hasMetadata) {
|
||||||
|
collector.collect(physicalRow);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
appendMetadataCollector.inputRecord = row;
|
||||||
|
appendMetadataCollector.outputCollector = collector;
|
||||||
|
appendMetadataCollector.collect(physicalRow);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TypeInformation<RowData> getProducedType() {
|
||||||
|
return resultTypeInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Builder class of {@link RowDataOceanBaseDeserializationSchema}. */
|
||||||
|
public static class Builder {
|
||||||
|
private RowType physicalRowType;
|
||||||
|
private TypeInformation<RowData> resultTypeInfo;
|
||||||
|
private OceanBaseMetadataConverter[] metadataConverters = new OceanBaseMetadataConverter[0];
|
||||||
|
private ZoneId serverTimeZone = ZoneId.of("UTC");
|
||||||
|
|
||||||
|
public RowDataOceanBaseDeserializationSchema.Builder setPhysicalRowType(
|
||||||
|
RowType physicalRowType) {
|
||||||
|
this.physicalRowType = physicalRowType;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RowDataOceanBaseDeserializationSchema.Builder setMetadataConverters(
|
||||||
|
OceanBaseMetadataConverter[] metadataConverters) {
|
||||||
|
this.metadataConverters = metadataConverters;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RowDataOceanBaseDeserializationSchema.Builder setResultTypeInfo(
|
||||||
|
TypeInformation<RowData> resultTypeInfo) {
|
||||||
|
this.resultTypeInfo = resultTypeInfo;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RowDataOceanBaseDeserializationSchema.Builder setServerTimeZone(
|
||||||
|
ZoneId serverTimeZone) {
|
||||||
|
this.serverTimeZone = serverTimeZone;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RowDataOceanBaseDeserializationSchema build() {
|
||||||
|
return new RowDataOceanBaseDeserializationSchema(
|
||||||
|
physicalRowType, metadataConverters, resultTypeInfo, serverTimeZone);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter createConverter(
|
||||||
|
LogicalType type, ZoneId serverTimeZone) {
|
||||||
|
return wrapIntoNullableConverter(createNotNullConverter(type, serverTimeZone));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter wrapIntoNullableConverter(
|
||||||
|
OceanBaseDeserializationRuntimeConverter converter) {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convert(Object object) throws Exception {
|
||||||
|
if (object == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return converter.convert(object);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public static OceanBaseDeserializationRuntimeConverter createNotNullConverter(
|
||||||
|
LogicalType type, ZoneId serverTimeZone) {
|
||||||
|
switch (type.getTypeRoot()) {
|
||||||
|
case ROW:
|
||||||
|
return createRowConverter((RowType) type, serverTimeZone);
|
||||||
|
case NULL:
|
||||||
|
return convertToNull();
|
||||||
|
case BOOLEAN:
|
||||||
|
return convertToBoolean();
|
||||||
|
case TINYINT:
|
||||||
|
return convertToTinyInt();
|
||||||
|
case SMALLINT:
|
||||||
|
return convertToSmallInt();
|
||||||
|
case INTEGER:
|
||||||
|
case INTERVAL_YEAR_MONTH:
|
||||||
|
return convertToInt();
|
||||||
|
case BIGINT:
|
||||||
|
case INTERVAL_DAY_TIME:
|
||||||
|
return convertToLong();
|
||||||
|
case DATE:
|
||||||
|
return convertToDate();
|
||||||
|
case TIME_WITHOUT_TIME_ZONE:
|
||||||
|
return convertToTime();
|
||||||
|
case TIMESTAMP_WITHOUT_TIME_ZONE:
|
||||||
|
return convertToTimestamp();
|
||||||
|
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
|
||||||
|
return convertToLocalTimeZoneTimestamp(serverTimeZone);
|
||||||
|
case FLOAT:
|
||||||
|
return convertToFloat();
|
||||||
|
case DOUBLE:
|
||||||
|
return convertToDouble();
|
||||||
|
case CHAR:
|
||||||
|
case VARCHAR:
|
||||||
|
return convertToString();
|
||||||
|
case BINARY:
|
||||||
|
return convertToBinary();
|
||||||
|
case VARBINARY:
|
||||||
|
return convertToBytes();
|
||||||
|
case DECIMAL:
|
||||||
|
return createDecimalConverter((DecimalType) type);
|
||||||
|
case ARRAY:
|
||||||
|
return createArrayConverter();
|
||||||
|
default:
|
||||||
|
throw new UnsupportedOperationException("Unsupported type: " + type);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter createRowConverter(
|
||||||
|
RowType rowType, ZoneId serverTimeZone) {
|
||||||
|
final OceanBaseDeserializationRuntimeConverter[] fieldConverters =
|
||||||
|
rowType.getFields().stream()
|
||||||
|
.map(RowType.RowField::getType)
|
||||||
|
.map(logicType -> createConverter(logicType, serverTimeZone))
|
||||||
|
.toArray(OceanBaseDeserializationRuntimeConverter[]::new);
|
||||||
|
final String[] fieldNames = rowType.getFieldNames().toArray(new String[0]);
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convert(Object object) throws Exception {
|
||||||
|
int arity = fieldNames.length;
|
||||||
|
GenericRowData row = new GenericRowData(arity);
|
||||||
|
Map<String, Object> fieldMap = (Map<String, Object>) object;
|
||||||
|
for (int i = 0; i < arity; i++) {
|
||||||
|
String fieldName = fieldNames[i];
|
||||||
|
row.setField(i, fieldConverters[i].convert(fieldMap.get(fieldName)));
|
||||||
|
}
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToNull() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convert(Object object) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToBoolean() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof byte[]) {
|
||||||
|
return "1".equals(new String((byte[]) object, StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
|
return Boolean.parseBoolean(object.toString()) || "1".equals(object.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return "1".equals(string);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToTinyInt() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
return Byte.parseByte(object.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return Byte.parseByte(string);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToSmallInt() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
return Short.parseShort(object.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return Short.parseShort(string);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToInt() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof Integer) {
|
||||||
|
return object;
|
||||||
|
} else if (object instanceof Long) {
|
||||||
|
return ((Long) object).intValue();
|
||||||
|
} else if (object instanceof Date) {
|
||||||
|
return ((Date) object).toLocalDate().getYear();
|
||||||
|
} else {
|
||||||
|
return Integer.parseInt(object.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return Integer.parseInt(string);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToLong() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof Integer) {
|
||||||
|
return ((Integer) object).longValue();
|
||||||
|
} else if (object instanceof Long) {
|
||||||
|
return object;
|
||||||
|
} else {
|
||||||
|
return Long.parseLong(object.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return Long.parseLong(string);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToDouble() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof Float) {
|
||||||
|
return ((Float) object).doubleValue();
|
||||||
|
} else if (object instanceof Double) {
|
||||||
|
return object;
|
||||||
|
} else {
|
||||||
|
return Double.parseDouble(object.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return Double.parseDouble(string);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToFloat() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof Float) {
|
||||||
|
return object;
|
||||||
|
} else if (object instanceof Double) {
|
||||||
|
return ((Double) object).floatValue();
|
||||||
|
} else {
|
||||||
|
return Float.parseFloat(object.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return Float.parseFloat(string);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToDate() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
return (int) TemporalConversions.toLocalDate(object).toEpochDay();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return (int) Date.valueOf(string).toLocalDate().toEpochDay();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToTime() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof Long) {
|
||||||
|
return (int) ((Long) object / 1000_000);
|
||||||
|
}
|
||||||
|
return TemporalConversions.toLocalTime(object).toSecondOfDay() * 1000;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return TemporalConversions.toLocalTime(Time.valueOf(string)).toSecondOfDay() * 1000;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToTimestamp() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof Timestamp) {
|
||||||
|
return TimestampData.fromLocalDateTime(((Timestamp) object).toLocalDateTime());
|
||||||
|
}
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Unable to convert to TimestampData from unexpected value '"
|
||||||
|
+ object
|
||||||
|
+ "' of type "
|
||||||
|
+ object.getClass().getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return TimestampData.fromLocalDateTime(Timestamp.valueOf(string).toLocalDateTime());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToLocalTimeZoneTimestamp(
|
||||||
|
ZoneId serverTimeZone) {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof Timestamp) {
|
||||||
|
return TimestampData.fromInstant(
|
||||||
|
((Timestamp) object)
|
||||||
|
.toLocalDateTime()
|
||||||
|
.atZone(serverTimeZone)
|
||||||
|
.toInstant());
|
||||||
|
}
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Unable to convert to TimestampData from unexpected value '"
|
||||||
|
+ object
|
||||||
|
+ "' of type "
|
||||||
|
+ object.getClass().getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return TimestampData.fromInstant(
|
||||||
|
Timestamp.valueOf(string)
|
||||||
|
.toLocalDateTime()
|
||||||
|
.atZone(serverTimeZone)
|
||||||
|
.toInstant());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToString() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
return StringData.fromString(object.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return StringData.fromString(string);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToBinary() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof byte[]) {
|
||||||
|
String str = new String((byte[]) object, StandardCharsets.US_ASCII);
|
||||||
|
return str.getBytes(StandardCharsets.UTF_8);
|
||||||
|
} else if (object instanceof ByteBuffer) {
|
||||||
|
ByteBuffer byteBuffer = (ByteBuffer) object;
|
||||||
|
byte[] bytes = new byte[byteBuffer.remaining()];
|
||||||
|
byteBuffer.get(bytes);
|
||||||
|
return bytes;
|
||||||
|
} else {
|
||||||
|
throw new UnsupportedOperationException(
|
||||||
|
"Unsupported BINARY value type: " + object.getClass().getSimpleName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
try {
|
||||||
|
long v = Long.parseLong(string);
|
||||||
|
byte[] bytes = ByteBuffer.allocate(8).putLong(v).array();
|
||||||
|
int i = 0;
|
||||||
|
while (i < Long.BYTES - 1 && bytes[i] == 0) {
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
return Arrays.copyOfRange(bytes, i, Long.BYTES);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
return string.getBytes(StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter convertToBytes() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
if (object instanceof byte[]) {
|
||||||
|
return object;
|
||||||
|
} else if (object instanceof ByteBuffer) {
|
||||||
|
ByteBuffer byteBuffer = (ByteBuffer) object;
|
||||||
|
byte[] bytes = new byte[byteBuffer.remaining()];
|
||||||
|
byteBuffer.get(bytes);
|
||||||
|
return bytes;
|
||||||
|
} else {
|
||||||
|
throw new UnsupportedOperationException(
|
||||||
|
"Unsupported BYTES value type: " + object.getClass().getSimpleName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return string.getBytes(StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter createDecimalConverter(
|
||||||
|
DecimalType decimalType) {
|
||||||
|
final int precision = decimalType.getPrecision();
|
||||||
|
final int scale = decimalType.getScale();
|
||||||
|
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertSnapshotEvent(Object object) {
|
||||||
|
BigDecimal bigDecimal;
|
||||||
|
if (object instanceof String) {
|
||||||
|
bigDecimal = new BigDecimal((String) object);
|
||||||
|
} else if (object instanceof Long) {
|
||||||
|
bigDecimal = new BigDecimal((Long) object);
|
||||||
|
} else if (object instanceof BigInteger) {
|
||||||
|
bigDecimal = new BigDecimal((BigInteger) object);
|
||||||
|
} else if (object instanceof Double) {
|
||||||
|
bigDecimal = BigDecimal.valueOf((Double) object);
|
||||||
|
} else if (object instanceof BigDecimal) {
|
||||||
|
bigDecimal = (BigDecimal) object;
|
||||||
|
} else {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Unable to convert to decimal from unexpected value '"
|
||||||
|
+ object
|
||||||
|
+ "' of type "
|
||||||
|
+ object.getClass());
|
||||||
|
}
|
||||||
|
return DecimalData.fromBigDecimal(bigDecimal, precision, scale);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convertChangeEvent(String string) {
|
||||||
|
return DecimalData.fromBigDecimal(new BigDecimal(string), precision, scale);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static OceanBaseDeserializationRuntimeConverter createArrayConverter() {
|
||||||
|
return new OceanBaseDeserializationRuntimeConverter() {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object convert(Object object) {
|
||||||
|
String s;
|
||||||
|
if (object instanceof ByteString) {
|
||||||
|
s = ((ByteString) object).toString(StandardCharsets.UTF_8.name());
|
||||||
|
} else {
|
||||||
|
s = object.toString();
|
||||||
|
}
|
||||||
|
String[] strArray = s.split(",");
|
||||||
|
StringData[] stringDataArray = new StringData[strArray.length];
|
||||||
|
for (int i = 0; i < strArray.length; i++) {
|
||||||
|
stringDataArray[i] = StringData.fromString(strArray[i]);
|
||||||
|
}
|
||||||
|
return new GenericArrayData(stringDataArray);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Ververica Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.ververica.cdc.connectors.oceanbase.table;
|
||||||
|
|
||||||
|
import org.apache.flink.annotation.Internal;
|
||||||
|
import org.apache.flink.table.data.GenericRowData;
|
||||||
|
import org.apache.flink.table.data.RowData;
|
||||||
|
import org.apache.flink.table.data.utils.JoinedRowData;
|
||||||
|
import org.apache.flink.util.Collector;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
/** Emits a row with physical fields and metadata fields. */
|
||||||
|
@Internal
|
||||||
|
public class OceanBaseAppendMetadataCollector implements Collector<RowData>, Serializable {
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
private final OceanBaseMetadataConverter[] metadataConverters;
|
||||||
|
|
||||||
|
public transient OceanBaseRecord inputRecord;
|
||||||
|
public transient Collector<RowData> outputCollector;
|
||||||
|
|
||||||
|
public OceanBaseAppendMetadataCollector(OceanBaseMetadataConverter[] metadataConverters) {
|
||||||
|
this.metadataConverters = metadataConverters;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void collect(RowData physicalRow) {
|
||||||
|
GenericRowData metaRow = new GenericRowData(metadataConverters.length);
|
||||||
|
for (int i = 0; i < metadataConverters.length; i++) {
|
||||||
|
Object meta = metadataConverters[i].read(inputRecord);
|
||||||
|
metaRow.setField(i, meta);
|
||||||
|
}
|
||||||
|
RowData outRow = new JoinedRowData(physicalRow.getRowKind(), physicalRow, metaRow);
|
||||||
|
outputCollector.collect(outRow);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Ververica Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.ververica.cdc.connectors.oceanbase.table;
|
||||||
|
|
||||||
|
import org.apache.flink.annotation.PublicEvolving;
|
||||||
|
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
|
||||||
|
import org.apache.flink.util.Collector;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The deserialization schema describes how to turn the OceanBase record into data types (Java/Scala
|
||||||
|
* objects) that are processed by Flink.
|
||||||
|
*
|
||||||
|
* @param <T> The type created by the deserialization schema.
|
||||||
|
*/
|
||||||
|
@PublicEvolving
|
||||||
|
public interface OceanBaseDeserializationSchema<T> extends Serializable, ResultTypeQueryable<T> {
|
||||||
|
|
||||||
|
/** Deserialize the OceanBase record, it is represented in {@link OceanBaseRecord}. */
|
||||||
|
void deserialize(OceanBaseRecord record, Collector<T> out) throws Exception;
|
||||||
|
}
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Ververica Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.ververica.cdc.connectors.oceanbase.table;
|
||||||
|
|
||||||
|
import org.apache.flink.annotation.Internal;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
/** A converter converts OceanBase record metadata into Flink internal data structures. */
|
||||||
|
@FunctionalInterface
|
||||||
|
@Internal
|
||||||
|
public interface OceanBaseMetadataConverter extends Serializable {
|
||||||
|
Object read(OceanBaseRecord record);
|
||||||
|
}
|
||||||
@ -0,0 +1,118 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Ververica Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.ververica.cdc.connectors.oceanbase.table;
|
||||||
|
|
||||||
|
import com.oceanbase.oms.logmessage.ByteString;
|
||||||
|
import com.oceanbase.oms.logmessage.DataMessage;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/** An internal data structure representing record of OceanBase. */
|
||||||
|
public class OceanBaseRecord implements Serializable {
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
private final SourceInfo sourceInfo;
|
||||||
|
private final boolean isSnapshotRecord;
|
||||||
|
private Map<String, Object> jdbcFields;
|
||||||
|
private DataMessage.Record.Type opt;
|
||||||
|
private Map<String, ByteString> logMessageFieldsBefore;
|
||||||
|
private Map<String, ByteString> logMessageFieldsAfter;
|
||||||
|
|
||||||
|
public OceanBaseRecord(SourceInfo sourceInfo, Map<String, Object> jdbcFields) {
|
||||||
|
this.sourceInfo = sourceInfo;
|
||||||
|
this.isSnapshotRecord = true;
|
||||||
|
this.jdbcFields = jdbcFields;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OceanBaseRecord(
|
||||||
|
SourceInfo sourceInfo,
|
||||||
|
DataMessage.Record.Type opt,
|
||||||
|
List<DataMessage.Record.Field> logMessageFieldList) {
|
||||||
|
this.sourceInfo = sourceInfo;
|
||||||
|
this.isSnapshotRecord = false;
|
||||||
|
this.opt = opt;
|
||||||
|
this.logMessageFieldsBefore = new HashMap<>();
|
||||||
|
this.logMessageFieldsAfter = new HashMap<>();
|
||||||
|
for (DataMessage.Record.Field field : logMessageFieldList) {
|
||||||
|
if (field.isPrev()) {
|
||||||
|
logMessageFieldsBefore.put(field.getFieldname(), field.getValue());
|
||||||
|
} else {
|
||||||
|
logMessageFieldsAfter.put(field.getFieldname(), field.getValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public SourceInfo getSourceInfo() {
|
||||||
|
return sourceInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isSnapshotRecord() {
|
||||||
|
return isSnapshotRecord;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, Object> getJdbcFields() {
|
||||||
|
return jdbcFields;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataMessage.Record.Type getOpt() {
|
||||||
|
return opt;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, ByteString> getLogMessageFieldsBefore() {
|
||||||
|
return logMessageFieldsBefore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, ByteString> getLogMessageFieldsAfter() {
|
||||||
|
return logMessageFieldsAfter;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Information about the source of record. */
|
||||||
|
public static class SourceInfo implements Serializable {
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
private final String tenant;
|
||||||
|
private final String database;
|
||||||
|
private final String table;
|
||||||
|
private final long timestampS;
|
||||||
|
|
||||||
|
public SourceInfo(String tenant, String database, String table, long timestampS) {
|
||||||
|
this.tenant = tenant;
|
||||||
|
this.database = database;
|
||||||
|
this.table = table;
|
||||||
|
this.timestampS = timestampS;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTenant() {
|
||||||
|
return tenant;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDatabase() {
|
||||||
|
return database;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTable() {
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getTimestampS() {
|
||||||
|
return timestampS;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue