diff --git a/flink-connector-mysql-cdc/src/main/java/com/ververica/cdc/connectors/mysql/schema/MySqlTypeUtils.java b/flink-connector-mysql-cdc/src/main/java/com/ververica/cdc/connectors/mysql/schema/MySqlTypeUtils.java index dc8be1120..8a0b70702 100644 --- a/flink-connector-mysql-cdc/src/main/java/com/ververica/cdc/connectors/mysql/schema/MySqlTypeUtils.java +++ b/flink-connector-mysql-cdc/src/main/java/com/ververica/cdc/connectors/mysql/schema/MySqlTypeUtils.java @@ -27,23 +27,46 @@ import io.debezium.relational.Column; public class MySqlTypeUtils { // ------ MySQL Type ------ + // https://dev.mysql.com/doc/refman/8.0/en/data-types.html private static final String BIT = "BIT"; private static final String TINYINT = "TINYINT"; private static final String TINYINT_UNSIGNED = "TINYINT UNSIGNED"; + private static final String TINYINT_UNSIGNED_ZEROFILL = "TINYINT UNSIGNED ZEROFILL"; private static final String SMALLINT = "SMALLINT"; private static final String SMALLINT_UNSIGNED = "SMALLINT UNSIGNED"; + private static final String SMALLINT_UNSIGNED_ZEROFILL = "SMALLINT UNSIGNED ZEROFILL"; private static final String MEDIUMINT = "MEDIUMINT"; private static final String MEDIUMINT_UNSIGNED = "MEDIUMINT UNSIGNED"; + private static final String MEDIUMINT_UNSIGNED_ZEROFILL = "MEDIUMINT UNSIGNED ZEROFILL"; private static final String INT = "INT"; private static final String INT_UNSIGNED = "INT UNSIGNED"; + private static final String INT_UNSIGNED_ZEROFILL = "INT UNSIGNED ZEROFILL"; private static final String BIGINT = "BIGINT"; + private static final String SERIAL = "SERIAL"; private static final String BIGINT_UNSIGNED = "BIGINT UNSIGNED"; + private static final String BIGINT_UNSIGNED_ZEROFILL = "BIGINT UNSIGNED ZEROFILL"; + private static final String REAL = "REAL"; + private static final String REAL_UNSIGNED = "REAL UNSIGNED"; + private static final String REAL_UNSIGNED_ZEROFILL = "REAL UNSIGNED ZEROFILL"; private static final String FLOAT = "FLOAT"; private static final String FLOAT_UNSIGNED = "FLOAT UNSIGNED"; + private static final String FLOAT_UNSIGNED_ZEROFILL = "FLOAT UNSIGNED ZEROFILL"; private static final String DOUBLE = "DOUBLE"; private static final String DOUBLE_UNSIGNED = "DOUBLE UNSIGNED"; + private static final String DOUBLE_UNSIGNED_ZEROFILL = "DOUBLE UNSIGNED ZEROFILL"; + private static final String DOUBLE_PRECISION = "DOUBLE PRECISION"; + private static final String DOUBLE_PRECISION_UNSIGNED = "DOUBLE PRECISION UNSIGNED"; + private static final String DOUBLE_PRECISION_UNSIGNED_ZEROFILL = + "DOUBLE PRECISION UNSIGNED ZEROFILL"; + private static final String NUMERIC = "NUMERIC"; + private static final String NUMERIC_UNSIGNED = "NUMERIC UNSIGNED"; + private static final String NUMERIC_UNSIGNED_ZEROFILL = "NUMERIC UNSIGNED ZEROFILL"; + private static final String FIXED = "FIXED"; + private static final String FIXED_UNSIGNED = "FIXED UNSIGNED"; + private static final String FIXED_UNSIGNED_ZEROFILL = "FIXED UNSIGNED ZEROFILL"; private static final String DECIMAL = "DECIMAL"; private static final String DECIMAL_UNSIGNED = "DECIMAL UNSIGNED"; + private static final String DECIMAL_UNSIGNED_ZEROFILL = "DECIMAL UNSIGNED ZEROFILL"; private static final String CHAR = "CHAR"; private static final String VARCHAR = "VARCHAR"; private static final String TINYTEXT = "TINYTEXT"; @@ -87,24 +110,50 @@ public class MySqlTypeUtils { case TINYINT: return column.length() == 1 ? DataTypes.BOOLEAN() : DataTypes.TINYINT(); case TINYINT_UNSIGNED: + case TINYINT_UNSIGNED_ZEROFILL: case SMALLINT: return DataTypes.SMALLINT(); case SMALLINT_UNSIGNED: + case SMALLINT_UNSIGNED_ZEROFILL: case INT: case MEDIUMINT: return DataTypes.INT(); case INT_UNSIGNED: + case INT_UNSIGNED_ZEROFILL: case MEDIUMINT_UNSIGNED: + case MEDIUMINT_UNSIGNED_ZEROFILL: case BIGINT: return DataTypes.BIGINT(); case BIGINT_UNSIGNED: + case BIGINT_UNSIGNED_ZEROFILL: + case SERIAL: return DataTypes.DECIMAL(20, 0); case FLOAT: + case FLOAT_UNSIGNED: + case FLOAT_UNSIGNED_ZEROFILL: return DataTypes.FLOAT(); + case REAL: + case REAL_UNSIGNED: + case REAL_UNSIGNED_ZEROFILL: case DOUBLE: + case DOUBLE_UNSIGNED: + case DOUBLE_UNSIGNED_ZEROFILL: + case DOUBLE_PRECISION: + case DOUBLE_PRECISION_UNSIGNED: + case DOUBLE_PRECISION_UNSIGNED_ZEROFILL: return DataTypes.DOUBLE(); + case NUMERIC: + case NUMERIC_UNSIGNED: + case NUMERIC_UNSIGNED_ZEROFILL: + case FIXED: + case FIXED_UNSIGNED: + case FIXED_UNSIGNED_ZEROFILL: case DECIMAL: - return DataTypes.DECIMAL(column.length(), column.scale().orElse(0)); + case DECIMAL_UNSIGNED: + case DECIMAL_UNSIGNED_ZEROFILL: + return column.length() <= 38 + ? DataTypes.DECIMAL(column.length(), column.scale().orElse(0)) + : DataTypes.STRING(); case TIME: return column.length() >= 0 ? DataTypes.TIME(column.length()) : DataTypes.TIME(); case DATE: diff --git a/flink-connector-mysql-cdc/src/test/java/com/ververica/cdc/connectors/mysql/table/MySqlConnectorITCase.java b/flink-connector-mysql-cdc/src/test/java/com/ververica/cdc/connectors/mysql/table/MySqlConnectorITCase.java index 5ec59108d..e8e4f61d3 100644 --- a/flink-connector-mysql-cdc/src/test/java/com/ververica/cdc/connectors/mysql/table/MySqlConnectorITCase.java +++ b/flink-connector-mysql-cdc/src/test/java/com/ververica/cdc/connectors/mysql/table/MySqlConnectorITCase.java @@ -351,21 +351,32 @@ public class MySqlConnectorITCase extends MySqlSourceTestBase { + " `id` INT NOT NULL,\n" + " tiny_c TINYINT,\n" + " tiny_un_c SMALLINT ,\n" + + " tiny_un_z_c SMALLINT ,\n" + " small_c SMALLINT,\n" + " small_un_c INT,\n" + + " small_un_z_c INT,\n" + " medium_c INT,\n" + " medium_un_c INT,\n" + + " medium_un_z_c BIGINT,\n" + " int_c INT ,\n" + " int_un_c BIGINT,\n" + + " int_un_z_c BIGINT,\n" + " int11_c BIGINT,\n" + " big_c BIGINT,\n" + " big_un_c DECIMAL(20, 0),\n" + + " big_un_z_c DECIMAL(20, 0),\n" + " varchar_c VARCHAR(255),\n" + " char_c CHAR(3),\n" + " real_c FLOAT,\n" + " float_c FLOAT,\n" + + " float_un_c FLOAT,\n" + + " float_un_z_c FLOAT,\n" + " double_c DOUBLE,\n" + + " double_un_c DOUBLE,\n" + + " double_un_z_c DOUBLE,\n" + " decimal_c DECIMAL(8, 4),\n" + + " decimal_un_c DECIMAL(8, 4),\n" + + " decimal_un_z_c DECIMAL(8, 4),\n" + " numeric_c DECIMAL(6, 0),\n" + " big_decimal_c STRING,\n" + " bit1_c BOOLEAN,\n" @@ -427,21 +438,32 @@ public class MySqlConnectorITCase extends MySqlSourceTestBase { "SELECT id,\n" + "tiny_c,\n" + "tiny_un_c,\n" + + "tiny_un_z_c,\n" + "small_c,\n" + "small_un_c,\n" + + "small_un_z_c,\n" + "medium_c, \n" + "medium_un_c, \n" + + "medium_un_z_c, \n" + "int_c,\n" + "int_un_c,\n" + + "int_un_z_c,\n" + "int11_c,\n" + "big_c,\n" + "big_un_c, \n" + + "big_un_z_c, \n" + "varchar_c,\n" + "char_c,\n" + "real_c, \n" + "float_c,\n" + + "float_un_c,\n" + + "float_un_z_c,\n" + "double_c,\n" + + "double_un_c,\n" + + "double_un_z_c,\n" + "decimal_c,\n" + + "decimal_un_c,\n" + + "decimal_un_z_c,\n" + "numeric_c,\n" + "big_decimal_c,\n" + "bit1_c,\n" @@ -498,7 +520,9 @@ public class MySqlConnectorITCase extends MySqlSourceTestBase { "{\"geometries\":[{\"type\":\"Point\",\"coordinates\":[10,10]},{\"type\":\"Point\",\"coordinates\":[30,30]},{\"type\":\"LineString\",\"coordinates\":[[15,15],[20,20]]}],\"type\":\"GeometryCollection\",\"srid\":0}"; String[] expected = new String[] { - "+I[1, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 9223372036854775807, 18446744073709551615, Hello World, abc, 123.102, 123.102, 404.4443, 123.4567, 346, 34567892.1, false, true, true," + "+I[1, 127, 255, 255, 32767, 65535, 65535, 8388607, 16777215, 16777215, 2147483647, 4294967295, 4294967295," + + " 2147483647, 9223372036854775807, 18446744073709551615, 18446744073709551615, Hello World, abc, 123.102, 123.102, 123.103, 123.104, " + + "404.4443, 404.4444, 404.4445, 123.4567, 123.4568, 123.4569, 346, 34567892.1, false, true, true," + " 2020-07-17, 18:00:22, 2020-07-17T18:00:22.123, 2020-07-17T18:00:22.123456, 2020-07-17T18:00:22, " + "ZRrvv70IOQ9I77+977+977+9Nu+/vT57dAA=, [4, 4, 4, 4, 4, 4, 4, 4], text, [16], [16], [16], [16], 2021, red, [a, b], " + "{\"key1\": \"value1\"}, " @@ -518,7 +542,10 @@ public class MySqlConnectorITCase extends MySqlSourceTestBase { + ", " + expectGeometryCollectionJsonText + "]", - "-U[1, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 9223372036854775807, 18446744073709551615, Hello World, abc, 123.102, 123.102, 404.4443, 123.4567, 346, 34567892.1, false, true, true," + "-U[1, 127, 255, 255, 32767, 65535, 65535, 8388607, 16777215, 16777215, 2147483647, 4294967295, 4294967295," + + " 2147483647, 9223372036854775807, 18446744073709551615, 18446744073709551615, Hello World, abc, 123.102," + + " 123.102, 123.103, 123.104, 404.4443, 404.4444, 404.4445, 123.4567, 123.4568, 123.4569, 346, 34567892.1," + + " false, true, true," + " 2020-07-17, 18:00:22, 2020-07-17T18:00:22.123, 2020-07-17T18:00:22.123456, 2020-07-17T18:00:22, " + "ZRrvv70IOQ9I77+977+977+9Nu+/vT57dAA=, [4, 4, 4, 4, 4, 4, 4, 4], text, [16], [16], [16], [16], 2021, red, [a, b], " + "{\"key1\":\"value1\"}, " @@ -538,7 +565,10 @@ public class MySqlConnectorITCase extends MySqlSourceTestBase { + ", " + expectGeometryCollectionJsonText + "]", - "+U[1, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 9223372036854775807, 18446744073709551615, Hello World, abc, 123.102, 123.102, 404.4443, 123.4567, 346, 34567892.1, false, true, true," + "+U[1, 127, 255, 255, 32767, 65535, 65535, 8388607, 16777215, 16777215, 2147483647, 4294967295, 4294967295," + + " 2147483647, 9223372036854775807, 18446744073709551615, 18446744073709551615, Hello World, abc, 123.102," + + " 123.102, 123.103, 123.104, 404.4443, 404.4444, 404.4445, 123.4567, 123.4568, 123.4569, 346, 34567892.1," + + " false, true, true," + " 2020-07-17, 18:00:22, 2020-07-17T18:00:22.123, 2020-07-17T18:00:22.123456, 2020-07-17T18:33:22, " + "ZRrvv70IOQ9I77+977+977+9Nu+/vT57dAA=, [4, 4, 4, 4, 4, 4, 4, 4], text, [16], [16], [16], [16], 2021, red, [a, b], " + "{\"key1\":\"value1\"}, " diff --git a/flink-connector-mysql-cdc/src/test/resources/ddl/column_type_test.sql b/flink-connector-mysql-cdc/src/test/resources/ddl/column_type_test.sql index 61ac5e490..efba82ad8 100644 --- a/flink-connector-mysql-cdc/src/test/resources/ddl/column_type_test.sql +++ b/flink-connector-mysql-cdc/src/test/resources/ddl/column_type_test.sql @@ -17,71 +17,98 @@ -- DATABASE: column_type_test -- ---------------------------------------------------------------------------------------------------------------- -CREATE TABLE full_types ( - id INT AUTO_INCREMENT NOT NULL, - tiny_c TINYINT, - tiny_un_c TINYINT UNSIGNED, - small_c SMALLINT, - small_un_c SMALLINT UNSIGNED, - medium_c MEDIUMINT, - medium_un_c MEDIUMINT UNSIGNED, - int_c INTEGER , - int_un_c INTEGER UNSIGNED, - int11_c INT(11) , - big_c BIGINT, - big_un_c BIGINT UNSIGNED, - varchar_c VARCHAR(255), - char_c CHAR(3), - real_c REAL, - float_c FLOAT, - double_c DOUBLE, - decimal_c DECIMAL(8, 4), - numeric_c NUMERIC(6, 0), - big_decimal_c DECIMAL(65, 1), - bit1_c BIT, - tiny1_c TINYINT(1), - boolean_c BOOLEAN, - date_c DATE, - time_c TIME(0), - datetime3_c DATETIME(3), - datetime6_c DATETIME(6), - timestamp_c TIMESTAMP, - file_uuid BINARY(16), - bit_c BIT(64), - text_c TEXT, - tiny_blob_c TINYBLOB, - blob_c BLOB, - medium_blob_c MEDIUMBLOB, - long_blob_c LONGBLOB, +CREATE TABLE full_types +( + id SERIAL, + tiny_c TINYINT, + tiny_un_c TINYINT UNSIGNED, + tiny_un_z_c TINYINT UNSIGNED ZEROFILL, + small_c SMALLINT, + small_un_c SMALLINT UNSIGNED, + small_un_z_c SMALLINT UNSIGNED ZEROFILL, + medium_c MEDIUMINT, + medium_un_c MEDIUMINT UNSIGNED, + medium_un_z_c MEDIUMINT UNSIGNED ZEROFILL, + int_c INTEGER, + int_un_c INTEGER UNSIGNED, + int_un_z_c INTEGER UNSIGNED ZEROFILL, + int11_c INT(11), + big_c BIGINT, + big_un_c BIGINT UNSIGNED, + big_un_z_c BIGINT UNSIGNED ZEROFILL, + varchar_c VARCHAR(255), + char_c CHAR(3), + real_c REAL, + float_c FLOAT, + float_un_c FLOAT UNSIGNED, + float_un_z_c FLOAT UNSIGNED ZEROFILL, + double_c DOUBLE, + double_un_c DOUBLE UNSIGNED, + double_un_z_c DOUBLE UNSIGNED ZEROFILL, + decimal_c DECIMAL(8, 4), + decimal_un_c DECIMAL(8, 4) UNSIGNED, + decimal_un_z_c DECIMAL(8, 4) UNSIGNED ZEROFILL, + numeric_c NUMERIC(6, 0), + big_decimal_c DECIMAL(65, 1), + bit1_c BIT, + tiny1_c TINYINT(1), + boolean_c BOOLEAN, + date_c DATE, + time_c TIME(0), + datetime3_c DATETIME(3), + datetime6_c DATETIME(6), + timestamp_c TIMESTAMP, + file_uuid BINARY(16), + bit_c BIT(64), + text_c TEXT, + tiny_blob_c TINYBLOB, + blob_c BLOB, + medium_blob_c MEDIUMBLOB, + long_blob_c LONGBLOB, year_c YEAR, - enum_c enum('red', 'white') default 'red', + enum_c enum('red', 'white') default 'red', set_c SET('a', 'b'), - json_c JSON, - point_c POINT, - geometry_c GEOMETRY, - linestring_c LINESTRING, - polygon_c POLYGON, - multipoint_c MULTIPOINT, - multiline_c MULTILINESTRING, - multipolygon_c MULTIPOLYGON, + json_c JSON, + point_c POINT, + geometry_c GEOMETRY, + linestring_c LINESTRING, + polygon_c POLYGON, + multipoint_c MULTIPOINT, + multiline_c MULTILINESTRING, + multipolygon_c MULTIPOLYGON, geometrycollection_c GEOMETRYCOLLECTION, PRIMARY KEY (id) ) DEFAULT CHARSET=utf8; -INSERT INTO full_types VALUES ( - DEFAULT, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 9223372036854775807, - 18446744073709551615, - 'Hello World', 'abc', 123.102, 123.102, 404.4443, 123.4567, 345.6, 34567892.1, 0, 1, true, - '2020-07-17', '18:00:22', '2020-07-17 18:00:22.123', '2020-07-17 18:00:22.123456', '2020-07-17 18:00:22', - unhex(replace('651aed08-390f-4893-b2f1-36923e7b7400','-','')), b'0000010000000100000001000000010000000100000001000000010000000100', - 'text',UNHEX(HEX(16)),UNHEX(HEX(16)),UNHEX(HEX(16)),UNHEX(HEX(16)), 2021, - 'red', 'a,b,a', '{"key1": "value1"}', - ST_GeomFromText('POINT(1 1)'), - ST_GeomFromText('POLYGON((1 1, 2 1, 2 2, 1 2, 1 1))'), - ST_GeomFromText('LINESTRING(3 0, 3 3, 3 5)'), - ST_GeomFromText('POLYGON((1 1, 2 1, 2 2, 1 2, 1 1))'), - ST_GeomFromText('MULTIPOINT((1 1),(2 2))'), - ST_GeomFromText('MultiLineString((1 1,2 2,3 3),(4 4,5 5))'), - ST_GeomFromText('MULTIPOLYGON(((0 0, 10 0, 10 10, 0 10, 0 0)), ((5 5, 7 5, 7 7, 5 7, 5 5)))'), - ST_GeomFromText('GEOMETRYCOLLECTION(POINT(10 10), POINT(30 30), LINESTRING(15 15, 20 20))') -); \ No newline at end of file +INSERT INTO full_types +VALUES (DEFAULT, 127, 255, 255, 32767, 65535, 65535, 8388607, 16777215, 16777215, 2147483647, + 4294967295, 4294967295, 2147483647, 9223372036854775807, + 18446744073709551615, 18446744073709551615, + 'Hello World', 'abc', 123.102, 123.102, 123.103, 123.104, 404.4443, 404.4444, 404.4445, + 123.4567, 123.4568, 123.4569, 345.6, 34567892.1, 0, 1, true, + '2020-07-17', '18:00:22', '2020-07-17 18:00:22.123', '2020-07-17 18:00:22.123456', + '2020-07-17 18:00:22', + unhex(replace('651aed08-390f-4893-b2f1-36923e7b7400', '-', '')), + b'0000010000000100000001000000010000000100000001000000010000000100', + 'text', UNHEX(HEX(16)), UNHEX(HEX(16)), UNHEX(HEX(16)), UNHEX(HEX(16)), 2021, + 'red', 'a,b,a', '{ + "key1": "value1" + }', + ST_GeomFromText('POINT(1 1)'), + ST_GeomFromText('POLYGON((1 1, 2 1, 2 2, 1 2, 1 1))'), + ST_GeomFromText('LINESTRING(3 0, 3 3, 3 5)'), + ST_GeomFromText('POLYGON((1 1, 2 1, 2 2, 1 2, 1 1))'), + ST_GeomFromText('MULTIPOINT((1 1),(2 2))'), + ST_GeomFromText('MultiLineString((1 1,2 2,3 3),(4 4,5 5))'), + ST_GeomFromText('MULTIPOLYGON(((0 0, 10 0, 10 10, 0 10, 0 0)), ((5 5, 7 5, 7 7, 5 7, 5 5)))'), + ST_GeomFromText('GEOMETRYCOLLECTION(POINT(10 10), POINT(30 30), LINESTRING(15 15, 20 20))')); + +CREATE TABLE user_info +( + user_id VARCHAR(255) NOT NULL, + user_name VARCHAR(255), + email VARCHAR(255), + balance DECIMAL(18, 2), + balance2 DECIMAL(18, 2), + PRIMARY KEY (user_id) +) DEFAULT CHARSET=utf8; \ No newline at end of file diff --git a/flink-connector-mysql-cdc/src/test/resources/ddl/column_type_test_mysql8.sql b/flink-connector-mysql-cdc/src/test/resources/ddl/column_type_test_mysql8.sql index 4a8219528..4264f0338 100644 --- a/flink-connector-mysql-cdc/src/test/resources/ddl/column_type_test_mysql8.sql +++ b/flink-connector-mysql-cdc/src/test/resources/ddl/column_type_test_mysql8.sql @@ -17,71 +17,87 @@ -- DATABASE: column_type_test -- ---------------------------------------------------------------------------------------------------------------- -CREATE TABLE full_types ( - id INT AUTO_INCREMENT NOT NULL, - tiny_c TINYINT, - tiny_un_c TINYINT UNSIGNED, - small_c SMALLINT, - small_un_c SMALLINT UNSIGNED, - medium_c MEDIUMINT, - medium_un_c MEDIUMINT UNSIGNED, - int_c INTEGER , - int_un_c INTEGER UNSIGNED, - int11_c INT(11) , - big_c BIGINT, - big_un_c BIGINT UNSIGNED, - varchar_c VARCHAR(255), - char_c CHAR(3), - real_c REAL, - float_c FLOAT, - double_c DOUBLE, - decimal_c DECIMAL(8, 4), - numeric_c NUMERIC(6, 0), - big_decimal_c DECIMAL(65, 1), - bit1_c BIT, - tiny1_c TINYINT(1), - boolean_c BOOLEAN, - date_c DATE, - time_c TIME(0), - datetime3_c DATETIME(3), - datetime6_c DATETIME(6), - timestamp_c TIMESTAMP, - file_uuid BINARY(16), - bit_c BIT(64), - text_c TEXT, - tiny_blob_c TINYBLOB, - blob_c BLOB, - medium_blob_c MEDIUMBLOB, - long_blob_c LONGBLOB, +CREATE TABLE full_types +( + id SERIAL, + tiny_c TINYINT, + tiny_un_c TINYINT UNSIGNED, + tiny_un_z_c TINYINT UNSIGNED ZEROFILL, + small_c SMALLINT, + small_un_c SMALLINT UNSIGNED, + small_un_z_c SMALLINT UNSIGNED ZEROFILL, + medium_c MEDIUMINT, + medium_un_c MEDIUMINT UNSIGNED, + medium_un_z_c MEDIUMINT UNSIGNED ZEROFILL, + int_c INTEGER, + int_un_c INTEGER UNSIGNED, + int_un_z_c INTEGER UNSIGNED ZEROFILL, + int11_c INT(11), + big_c BIGINT, + big_un_c BIGINT UNSIGNED, + big_un_z_c BIGINT UNSIGNED ZEROFILL, + varchar_c VARCHAR(255), + char_c CHAR(3), + real_c REAL, + float_c FLOAT, + float_un_c FLOAT UNSIGNED, + float_un_z_c FLOAT UNSIGNED ZEROFILL, + double_c DOUBLE, + double_un_c DOUBLE UNSIGNED, + double_un_z_c DOUBLE UNSIGNED ZEROFILL, + decimal_c DECIMAL(8, 4), + decimal_un_c DECIMAL(8, 4) UNSIGNED, + decimal_un_z_c DECIMAL(8, 4) UNSIGNED ZEROFILL, + numeric_c NUMERIC(6, 0), + big_decimal_c DECIMAL(65, 1), + bit1_c BIT, + tiny1_c TINYINT(1), + boolean_c BOOLEAN, + date_c DATE, + time_c TIME(0), + datetime3_c DATETIME(3), + datetime6_c DATETIME(6), + timestamp_c TIMESTAMP, + file_uuid BINARY(16), + bit_c BIT(64), + text_c TEXT, + tiny_blob_c TINYBLOB, + blob_c BLOB, + medium_blob_c MEDIUMBLOB, + long_blob_c LONGBLOB, year_c YEAR, - enum_c enum('red', 'white') default 'red', + enum_c enum('red', 'white') default 'red', set_c SET('a', 'b'), - json_c JSON, - point_c POINT, - geometry_c GEOMETRY, - linestring_c LINESTRING, - polygon_c POLYGON, - multipoint_c MULTIPOINT, - multiline_c MULTILINESTRING, - multipolygon_c MULTIPOLYGON, + json_c JSON, + point_c POINT, + geometry_c GEOMETRY, + linestring_c LINESTRING, + polygon_c POLYGON, + multipoint_c MULTIPOINT, + multiline_c MULTILINESTRING, + multipolygon_c MULTIPOLYGON, geometrycollection_c GEOMCOLLECTION, PRIMARY KEY (id) ) DEFAULT CHARSET=utf8; -INSERT INTO full_types VALUES ( - DEFAULT, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 9223372036854775807, - 18446744073709551615, - 'Hello World', 'abc', 123.102, 123.102, 404.4443, 123.4567, 345.6, 34567892.1, 0, 1, true, - '2020-07-17', '18:00:22', '2020-07-17 18:00:22.123', '2020-07-17 18:00:22.123456', '2020-07-17 18:00:22', - unhex(replace('651aed08-390f-4893-b2f1-36923e7b7400','-','')), b'0000010000000100000001000000010000000100000001000000010000000100', - 'text',UNHEX(HEX(16)),UNHEX(HEX(16)),UNHEX(HEX(16)),UNHEX(HEX(16)), 2021, - 'red', 'a,b,a', '{"key1": "value1"}', - ST_GeomFromText('POINT(1 1)'), - ST_GeomFromText('POLYGON((1 1, 2 1, 2 2, 1 2, 1 1))'), - ST_GeomFromText('LINESTRING(3 0, 3 3, 3 5)'), - ST_GeomFromText('POLYGON((1 1, 2 1, 2 2, 1 2, 1 1))'), - ST_GeomFromText('MULTIPOINT((1 1),(2 2))'), - ST_GeomFromText('MultiLineString((1 1,2 2,3 3),(4 4,5 5))'), - ST_GeomFromText('MULTIPOLYGON(((0 0, 10 0, 10 10, 0 10, 0 0)), ((5 5, 7 5, 7 7, 5 7, 5 5)))'), - ST_GeomFromText('GEOMETRYCOLLECTION(POINT(10 10), POINT(30 30), LINESTRING(15 15, 20 20))') -); \ No newline at end of file +INSERT INTO full_types +VALUES (DEFAULT, 127, 255, 255, 32767, 65535, 65535, 8388607, 16777215, 16777215, 2147483647, + 4294967295, 4294967295, 2147483647, 9223372036854775807, + 18446744073709551615, 18446744073709551615, + 'Hello World', 'abc', 123.102, 123.102, 123.103, 123.104, 404.4443, 404.4444, 404.4445, + 123.4567, 123.4568, 123.4569, 345.6, 34567892.1, 0, 1, true, + '2020-07-17', '18:00:22', '2020-07-17 18:00:22.123', '2020-07-17 18:00:22.123456', '2020-07-17 18:00:22', + unhex(replace('651aed08-390f-4893-b2f1-36923e7b7400', '-', '')), + b'0000010000000100000001000000010000000100000001000000010000000100', + 'text', UNHEX(HEX(16)), UNHEX(HEX(16)), UNHEX(HEX(16)), UNHEX(HEX(16)), 2021, + 'red', 'a,b,a', '{ + "key1": "value1" + }', + ST_GeomFromText('POINT(1 1)'), + ST_GeomFromText('POLYGON((1 1, 2 1, 2 2, 1 2, 1 1))'), + ST_GeomFromText('LINESTRING(3 0, 3 3, 3 5)'), + ST_GeomFromText('POLYGON((1 1, 2 1, 2 2, 1 2, 1 1))'), + ST_GeomFromText('MULTIPOINT((1 1),(2 2))'), + ST_GeomFromText('MultiLineString((1 1,2 2,3 3),(4 4,5 5))'), + ST_GeomFromText('MULTIPOLYGON(((0 0, 10 0, 10 10, 0 10, 0 0)), ((5 5, 7 5, 7 7, 5 7, 5 5)))'), + ST_GeomFromText('GEOMETRYCOLLECTION(POINT(10 10), POINT(30 30), LINESTRING(15 15, 20 20))')); \ No newline at end of file diff --git a/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude-with-numeric-decimal.json b/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude-with-numeric-decimal.json index 6985f9f23..c2d255fb6 100644 --- a/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude-with-numeric-decimal.json +++ b/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude-with-numeric-decimal.json @@ -5,21 +5,32 @@ "id": 1, "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": 18446744073709551615, + "big_un_z_c": 18446744073709551615, "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": 123.4567, + "decimal_un_c": 123.4568, + "decimal_un_z_c": 123.4569, "numeric_c": 346, "big_decimal_c": 34567892.1, "bit1_c": false, @@ -58,21 +69,32 @@ "id": 1, "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": 18446744073709551615, + "big_un_z_c": 18446744073709551615, "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": 123.4567, + "decimal_un_c": 123.4568, + "decimal_un_z_c": 123.4569, "numeric_c": 346, "big_decimal_c": 34567892.1, "bit1_c": false, @@ -107,21 +129,32 @@ "id": 1, "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": 18446744073709551615, + "big_un_z_c": 18446744073709551615, "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": 123.4567, + "decimal_un_c": 123.4568, + "decimal_un_z_c": 123.4569, "numeric_c": 346, "big_decimal_c": 34567892.1, "bit1_c": false, diff --git a/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude.json b/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude.json index 7ae4643df..6b62c0bed 100644 --- a/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude.json +++ b/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude.json @@ -2,24 +2,35 @@ "expected_snapshot": { "before": null, "after": { - "id": 1, + "id": "AQ==", "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": "AP//////////", + "big_un_z_c": "AP//////////", "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": "EtaH", + "decimal_un_c": "EtaI", + "decimal_un_z_c": "EtaJ", "numeric_c": "AVo=", "big_decimal_c": "FJqkSQ==", "bit1_c": false, @@ -55,24 +66,35 @@ }, "expected_binlog": { "before": { - "id": 1, + "id": "AQ==", "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": "AP//////////", + "big_un_z_c": "AP//////////", "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": "EtaH", + "decimal_un_c": "EtaI", + "decimal_un_z_c": "EtaJ", "numeric_c": "AVo=", "big_decimal_c": "FJqkSQ==", "bit1_c": false, @@ -104,24 +126,35 @@ "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} }, "after": { - "id": 1, + "id": "AQ==", "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": "AP//////////", + "big_un_z_c": "AP//////////", "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": "EtaH", + "decimal_un_c": "EtaI", + "decimal_un_z_c": "EtaJ", "numeric_c": "AVo=", "big_decimal_c": "FJqkSQ==", "bit1_c": false, diff --git a/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-include.json b/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-include.json index c192b381a..7b3dd4ef1 100644 --- a/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-include.json +++ b/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-include.json @@ -675,24 +675,35 @@ "payload": { "before": null, "after": { - "id": 1, + "id": "AQ==", "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": "AP//////////", + "big_un_z_c": "AP//////////", "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": "EtaH", + "decimal_un_c": "EtaI", + "decimal_un_z_c": "EtaJ", "numeric_c": "AVo=", "big_decimal_c": "FJqkSQ==", "bit1_c": false, @@ -1019,24 +1030,35 @@ }, "payload": { "before": { - "id": 1, + "id": "AQ==", "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": "AP//////////", + "big_un_z_c": "AP//////////", "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": "EtaH", + "decimal_un_c": "EtaI", + "decimal_un_z_c": "EtaJ", "numeric_c": "AVo=", "big_decimal_c": "FJqkSQ==", "bit1_c": false, @@ -1068,24 +1090,35 @@ "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} }, "after": { - "id": 1, + "id": "AQ==", "tiny_c": 127, "tiny_un_c": 255, + "tiny_un_z_c": 255, "small_c": 32767, "small_un_c": 65535, + "small_un_z_c": 65535, "medium_c": 8388607, "medium_un_c": 16777215, + "medium_un_z_c": 16777215, "int_c": 2147483647, "int_un_c": 4294967295, + "int_un_z_c": 4294967295, "int11_c": 2147483647, "big_c": 9223372036854775807, "big_un_c": "AP//////////", + "big_un_z_c": "AP//////////", "varchar_c": "Hello World", "char_c": "abc", "real_c": 123.102, "float_c": 123.10199737548828, + "float_un_c": 123.10299682617188, + "float_un_z_c": 123.10399627685547, "double_c": 404.4443, + "double_un_c": 404.4444, + "double_un_z_c": 404.4445, "decimal_c": "EtaH", + "decimal_un_c": "EtaI", + "decimal_un_z_c": "EtaJ", "numeric_c": "AVo=", "big_decimal_c": "FJqkSQ==", "bit1_c": false,