|
|
|
@ -151,6 +151,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
@ -184,6 +185,90 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, ], after=[2, x], op=UPDATE, meta=()}");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ParameterizedTest
|
|
|
|
|
@EnumSource
|
|
|
|
|
void testSingleSplitMultipleSources(ValuesDataSink.SinkApi sinkApi) throws Exception {
|
|
|
|
|
FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
|
|
|
|
|
|
|
|
|
|
// Setup value source
|
|
|
|
|
Configuration sourceConfig1 = new Configuration();
|
|
|
|
|
sourceConfig1.set(
|
|
|
|
|
ValuesDataSourceOptions.EVENT_SET_ID,
|
|
|
|
|
ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_SINGLE_TABLE);
|
|
|
|
|
Configuration sourceConfig2 = new Configuration();
|
|
|
|
|
sourceConfig2.set(
|
|
|
|
|
ValuesDataSourceOptions.EVENT_SET_ID,
|
|
|
|
|
ValuesDataSourceHelper.EventSetId.SINGLE_SPLIT_MULTI_TABLES);
|
|
|
|
|
SourceDef sourceDef1 =
|
|
|
|
|
new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source1", sourceConfig1);
|
|
|
|
|
SourceDef sourceDef2 =
|
|
|
|
|
new SourceDef(ValuesDataFactory.IDENTIFIER, "Value Source2", sourceConfig2);
|
|
|
|
|
|
|
|
|
|
// Setup value sink
|
|
|
|
|
Configuration sinkConfig = new Configuration();
|
|
|
|
|
sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
|
|
|
|
|
sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
|
|
|
|
|
SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
|
|
|
|
|
|
|
|
|
|
// Setup pipeline
|
|
|
|
|
Configuration pipelineConfig = new Configuration();
|
|
|
|
|
pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
|
|
|
|
|
pipelineConfig.set(
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
List<SourceDef> sourceDefs = new ArrayList<>();
|
|
|
|
|
sourceDefs.add(sourceDef1);
|
|
|
|
|
sourceDefs.add(sourceDef2);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
sourceDefs,
|
|
|
|
|
null,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
|
pipelineConfig);
|
|
|
|
|
|
|
|
|
|
// Execute the pipeline
|
|
|
|
|
PipelineExecution execution = composer.compose(pipelineDef);
|
|
|
|
|
execution.execute();
|
|
|
|
|
|
|
|
|
|
// Check result in ValuesDatabase
|
|
|
|
|
List<String> table1Results = ValuesDatabase.getResults(TABLE_1);
|
|
|
|
|
assertThat(table1Results)
|
|
|
|
|
.containsExactly(
|
|
|
|
|
"default_namespace.default_schema.table1:col1=2;newCol3=x",
|
|
|
|
|
"default_namespace.default_schema.table1:col1=3;newCol3=");
|
|
|
|
|
List<String> table2Results = ValuesDatabase.getResults(TABLE_2);
|
|
|
|
|
assertThat(table2Results)
|
|
|
|
|
.contains(
|
|
|
|
|
"default_namespace.default_schema.table2:col1=1;col2=1",
|
|
|
|
|
"default_namespace.default_schema.table2:col1=2;col2=2",
|
|
|
|
|
"default_namespace.default_schema.table2:col1=3;col2=3");
|
|
|
|
|
|
|
|
|
|
// Check the order and content of all received events
|
|
|
|
|
String[] outputEvents = outCaptor.toString().trim().split("\n");
|
|
|
|
|
assertThat(outputEvents)
|
|
|
|
|
.containsExactlyInAnyOrder(
|
|
|
|
|
"CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
|
|
|
|
|
"CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
|
|
|
|
|
"AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}",
|
|
|
|
|
"RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
|
|
|
|
|
"DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1], after=[], op=DELETE, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, 2], after=[2, x], op=UPDATE, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, null], op=INSERT, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, null], op=INSERT, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, null], op=INSERT, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1, 1], after=[], op=DELETE, meta=()}",
|
|
|
|
|
"DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, 2, 2], after=[2, x, x], op=UPDATE, meta=()}");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ParameterizedTest
|
|
|
|
|
@EnumSource
|
|
|
|
|
void testSingleSplitMultipleTables(ValuesDataSink.SinkApi sinkApi) throws Exception {
|
|
|
|
@ -210,6 +295,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
@ -277,6 +363,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, MAX_PARALLELISM);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
@ -335,6 +422,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
@ -398,6 +486,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
@ -470,6 +559,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.emptyList(),
|
|
|
|
@ -527,6 +617,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
routeDef,
|
|
|
|
@ -602,6 +693,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
routeDef,
|
|
|
|
@ -801,6 +893,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
routeDef,
|
|
|
|
@ -1010,6 +1103,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
routeDef,
|
|
|
|
@ -1075,6 +1169,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.singletonList(
|
|
|
|
@ -1141,6 +1236,7 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
pipelineConfig.set(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE, "America/New_York");
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Arrays.asList(
|
|
|
|
@ -1251,8 +1347,11 @@ class FlinkPipelineComposerITCase {
|
|
|
|
|
pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, 1);
|
|
|
|
|
pipelineConfig.set(
|
|
|
|
|
PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
|
|
|
|
|
List<SourceDef> sourceDefs = new ArrayList<>();
|
|
|
|
|
sourceDefs.add(sourceDef);
|
|
|
|
|
PipelineDef pipelineDef =
|
|
|
|
|
new PipelineDef(
|
|
|
|
|
null,
|
|
|
|
|
sourceDef,
|
|
|
|
|
sinkDef,
|
|
|
|
|
Collections.singletonList(
|
|
|
|
|