[mongodb] Support metadata columns for mongodb-cdc connector (#476)
Jiabao Sun
3 years ago
committed by
GitHub
parent
51f5a90eb3
commit
65665ebb6a
@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mongodb.internal;
|
||||
|
||||
/**
|
||||
* An immutable descriptor for the structure of {@link
|
||||
* com.mongodb.client.model.changestream.ChangeStreamDocument} envelopes.
|
||||
*/
|
||||
public class MongoDBEnvelope {
|
||||
|
||||
public static final String CLUSTER_TIME_FIELD = "clusterTime";
|
||||
|
||||
public static final String FULL_DOCUMENT_FIELD = "fullDocument";
|
||||
|
||||
public static final String DOCUMENT_KEY_FIELD = "documentKey";
|
||||
|
||||
public static final String OPERATION_TYPE_FIELD = "operationType";
|
||||
|
||||
public static final String NAMESPACE_FIELD = "ns";
|
||||
|
||||
public static final String NAMESPACE_DATABASE_FIELD = "db";
|
||||
|
||||
public static final String NAMESPACE_COLLECTION_FIELD = "coll";
|
||||
}
|
||||
@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mongodb.table;
|
||||
|
||||
import org.apache.flink.table.api.DataTypes;
|
||||
import org.apache.flink.table.data.StringData;
|
||||
import org.apache.flink.table.data.TimestampData;
|
||||
import org.apache.flink.table.types.DataType;
|
||||
|
||||
import com.ververica.cdc.connectors.mongodb.internal.MongoDBEnvelope;
|
||||
import com.ververica.cdc.debezium.table.MetadataConverter;
|
||||
import io.debezium.connector.AbstractSourceInfo;
|
||||
import io.debezium.data.Envelope;
|
||||
import org.apache.kafka.connect.data.Struct;
|
||||
import org.apache.kafka.connect.source.SourceRecord;
|
||||
|
||||
/** Defines the supported metadata columns for {@link MongoDBTableSource}. */
|
||||
public enum MongoDBReadableMetadata {
|
||||
|
||||
/** Name of the collection that contain the row. */
|
||||
COLLECTION(
|
||||
"collection_name",
|
||||
DataTypes.STRING().notNull(),
|
||||
new MetadataConverter() {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Override
|
||||
public Object read(SourceRecord record) {
|
||||
Struct value = (Struct) record.value();
|
||||
Struct to = value.getStruct(MongoDBEnvelope.NAMESPACE_FIELD);
|
||||
return StringData.fromString(
|
||||
to.getString(MongoDBEnvelope.NAMESPACE_COLLECTION_FIELD));
|
||||
}
|
||||
}),
|
||||
|
||||
/** Name of the database that contain the row. */
|
||||
DATABASE(
|
||||
"database_name",
|
||||
DataTypes.STRING().notNull(),
|
||||
new MetadataConverter() {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Override
|
||||
public Object read(SourceRecord record) {
|
||||
Struct value = (Struct) record.value();
|
||||
Struct to = value.getStruct(MongoDBEnvelope.NAMESPACE_FIELD);
|
||||
return StringData.fromString(
|
||||
to.getString(MongoDBEnvelope.NAMESPACE_DATABASE_FIELD));
|
||||
}
|
||||
}),
|
||||
|
||||
/**
|
||||
* It indicates the time that the change was made in the database. If the record is read from
|
||||
* snapshot of the table instead of the change stream, the value is always 0.
|
||||
*/
|
||||
OP_TS(
|
||||
"op_ts",
|
||||
DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).notNull(),
|
||||
new MetadataConverter() {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Override
|
||||
public Object read(SourceRecord record) {
|
||||
Struct value = (Struct) record.value();
|
||||
Struct source = value.getStruct(Envelope.FieldName.SOURCE);
|
||||
return TimestampData.fromEpochMillis(
|
||||
(Long) source.get(AbstractSourceInfo.TIMESTAMP_KEY));
|
||||
}
|
||||
});
|
||||
|
||||
private final String key;
|
||||
|
||||
private final DataType dataType;
|
||||
|
||||
private final MetadataConverter converter;
|
||||
|
||||
MongoDBReadableMetadata(String key, DataType dataType, MetadataConverter converter) {
|
||||
this.key = key;
|
||||
this.dataType = dataType;
|
||||
this.converter = converter;
|
||||
}
|
||||
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public DataType getDataType() {
|
||||
return dataType;
|
||||
}
|
||||
|
||||
public MetadataConverter getConverter() {
|
||||
return converter;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue