[mysql] Support capture newly added tables for existed pipeline
元组
3 years ago
committed by
Leonard Xu
parent
df295d80b5
commit
c21aa91e5d
@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mysql.source.assigners;
|
||||
|
||||
import org.apache.flink.util.Preconditions;
|
||||
|
||||
/** The snapshot assigner state machine. */
|
||||
public enum SnapshotAssignerStatus {
|
||||
/**
|
||||
* The assigner state machine goes this way.
|
||||
*
|
||||
* <pre>
|
||||
* INIT -> INIT_FINISH -> SUSPENDED -> RESUMED -> RESUMED_FINISH
|
||||
* ^ |
|
||||
* |_________________________|
|
||||
* </pre>
|
||||
*/
|
||||
INIT {
|
||||
@Override
|
||||
public int getValue() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SnapshotAssignerStatus nextState() {
|
||||
return INIT_FINISH;
|
||||
}
|
||||
},
|
||||
INIT_FINISH {
|
||||
@Override
|
||||
public int getValue() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SnapshotAssignerStatus nextState() {
|
||||
return SUSPENDED;
|
||||
}
|
||||
},
|
||||
SUSPENDED {
|
||||
@Override
|
||||
public int getValue() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SnapshotAssignerStatus nextState() {
|
||||
return RESUMED;
|
||||
}
|
||||
},
|
||||
RESUMED {
|
||||
@Override
|
||||
public int getValue() {
|
||||
return 3;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SnapshotAssignerStatus nextState() {
|
||||
return RESUMED_FINISH;
|
||||
}
|
||||
},
|
||||
RESUMED_FINISH {
|
||||
@Override
|
||||
public int getValue() {
|
||||
return 4;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SnapshotAssignerStatus nextState() {
|
||||
return SUSPENDED;
|
||||
}
|
||||
};
|
||||
|
||||
public abstract int getValue();
|
||||
|
||||
public abstract SnapshotAssignerStatus nextState();
|
||||
|
||||
public static SnapshotAssignerStatus fromInteger(int x) {
|
||||
Preconditions.checkState(x >= 0 && x < 5);
|
||||
switch (x) {
|
||||
case 0:
|
||||
return INIT;
|
||||
case 1:
|
||||
return INIT_FINISH;
|
||||
case 2:
|
||||
return SUSPENDED;
|
||||
case 3:
|
||||
return RESUMED;
|
||||
case 4:
|
||||
return RESUMED_FINISH;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mysql.source.events;
|
||||
|
||||
import org.apache.flink.api.connector.source.SourceEvent;
|
||||
|
||||
import com.ververica.cdc.connectors.mysql.source.enumerator.MySqlSourceEnumerator;
|
||||
import com.ververica.cdc.connectors.mysql.source.reader.MySqlSourceReader;
|
||||
|
||||
/**
|
||||
* The {@link SourceEvent} that {@link MySqlSourceReader} sends to {@link MySqlSourceEnumerator} to
|
||||
* notify the binlog split reader has been suspended.
|
||||
*/
|
||||
public class BinlogSplitReaderSuspendedReportEvent implements SourceEvent {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
public BinlogSplitReaderSuspendedReportEvent() {}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mysql.source.events;
|
||||
|
||||
import org.apache.flink.api.connector.source.SourceEvent;
|
||||
|
||||
import com.ververica.cdc.connectors.mysql.source.enumerator.MySqlSourceEnumerator;
|
||||
import com.ververica.cdc.connectors.mysql.source.reader.MySqlSourceReader;
|
||||
|
||||
/**
|
||||
* The {@link SourceEvent} that {@link MySqlSourceEnumerator} sends to {@link MySqlSourceReader} to
|
||||
* tell the source reader to suspend.
|
||||
*/
|
||||
public class SuspendBinlogReaderEvent implements SourceEvent {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
public SuspendBinlogReaderEvent() {}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mysql.source.events;
|
||||
|
||||
import org.apache.flink.api.connector.source.SourceEvent;
|
||||
|
||||
import com.ververica.cdc.connectors.mysql.source.enumerator.MySqlSourceEnumerator;
|
||||
import com.ververica.cdc.connectors.mysql.source.reader.MySqlSourceReader;
|
||||
|
||||
/**
|
||||
* The {@link SourceEvent} that {@link MySqlSourceEnumerator} sends to {@link MySqlSourceReader} to
|
||||
* notify the source reader to the offset has been propagated to Enumerator.
|
||||
*/
|
||||
public class TotalFinishedSplitSizeRequestEvent implements SourceEvent {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
public TotalFinishedSplitSizeRequestEvent() {}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mysql.source.events;
|
||||
|
||||
import org.apache.flink.api.connector.source.SourceEvent;
|
||||
|
||||
import com.ververica.cdc.connectors.mysql.source.enumerator.MySqlSourceEnumerator;
|
||||
import com.ververica.cdc.connectors.mysql.source.reader.MySqlSourceReader;
|
||||
|
||||
/**
|
||||
* The {@link SourceEvent} that {@link MySqlSourceReader} sends to {@link MySqlSourceEnumerator} to
|
||||
* ask if the offset has been propagated to Enumerator.
|
||||
*/
|
||||
public class TotalFinishedSplitSizeResponseEvent implements SourceEvent {
|
||||
private static final long serialVersionUID = 1L;
|
||||
private final int totalFinishedSplitSize;
|
||||
|
||||
public TotalFinishedSplitSizeResponseEvent(final int totalFinishedSplitSize) {
|
||||
this.totalFinishedSplitSize = totalFinishedSplitSize;
|
||||
}
|
||||
|
||||
public int getTotalFinishedSplitSize() {
|
||||
return totalFinishedSplitSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TotalFinishedSplitSizeResponseEvent{"
|
||||
+ "totalFinishedSplitSize="
|
||||
+ totalFinishedSplitSize
|
||||
+ '}';
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mysql.source.events;
|
||||
|
||||
import org.apache.flink.api.connector.source.SourceEvent;
|
||||
|
||||
import com.ververica.cdc.connectors.mysql.source.enumerator.MySqlSourceEnumerator;
|
||||
import com.ververica.cdc.connectors.mysql.source.reader.MySqlSourceReader;
|
||||
|
||||
/**
|
||||
* The {@link SourceEvent} that {@link MySqlSourceEnumerator} sends to {@link MySqlSourceReader} to
|
||||
* wake up source reader to request split again.
|
||||
*/
|
||||
public class WakeupReaderEvent implements SourceEvent {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/** Wake up type. */
|
||||
public enum WakeUpType {
|
||||
SNAPSHOT,
|
||||
BINLOG
|
||||
}
|
||||
|
||||
private WakeUpType type;
|
||||
|
||||
public WakeupReaderEvent(WakeUpType type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public WakeUpType getType() {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.ververica.cdc.connectors.mysql.source.reader;
|
||||
|
||||
import org.apache.flink.api.connector.source.SourceReaderContext;
|
||||
|
||||
/**
|
||||
* A wrapper class that wraps SourceReaderContext and some data shared by {@link MySqlSourceReader}
|
||||
* and {@link MySqlSplitReader}.
|
||||
*/
|
||||
public class MySqlSourceReaderContext {
|
||||
SourceReaderContext sourceReaderContext;
|
||||
boolean shouldBinlogSplitReaderStopped;
|
||||
|
||||
public MySqlSourceReaderContext(final SourceReaderContext sourceReaderContext) {
|
||||
this.sourceReaderContext = sourceReaderContext;
|
||||
this.shouldBinlogSplitReaderStopped = false;
|
||||
}
|
||||
|
||||
public SourceReaderContext getSourceReaderContext() {
|
||||
return sourceReaderContext;
|
||||
}
|
||||
|
||||
public void setSourceReaderContext(final SourceReaderContext sourceReaderContext) {
|
||||
this.sourceReaderContext = sourceReaderContext;
|
||||
}
|
||||
|
||||
public boolean isShouldBinlogSplitReaderStopped() {
|
||||
return shouldBinlogSplitReaderStopped;
|
||||
}
|
||||
|
||||
public void setShouldBinlogSplitReaderStopped(final boolean shouldBinlogSplitReaderStopped) {
|
||||
this.shouldBinlogSplitReaderStopped = shouldBinlogSplitReaderStopped;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue