[FLINK-35129][postgres] Introduce scan.lsn-commit.checkpoints-num-delay option to control LSN offset commit lazily
This close #3349.pull/3394/head
Muhammet Orazov
8 months ago
committed by
GitHub
parent
33891869a9
commit
5b28d1a579
@ -0,0 +1,94 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.flink.cdc.connectors.postgres.source.reader;
|
||||||
|
|
||||||
|
import org.apache.flink.cdc.connectors.postgres.source.MockPostgresDialect;
|
||||||
|
import org.apache.flink.cdc.connectors.postgres.source.PostgresSourceBuilder;
|
||||||
|
import org.apache.flink.cdc.connectors.postgres.source.config.PostgresSourceConfigFactory;
|
||||||
|
import org.apache.flink.cdc.connectors.postgres.source.offset.PostgresOffsetFactory;
|
||||||
|
import org.apache.flink.cdc.connectors.postgres.testutils.TestTable;
|
||||||
|
import org.apache.flink.cdc.debezium.DebeziumDeserializationSchema;
|
||||||
|
import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
|
||||||
|
import org.apache.flink.table.catalog.Column;
|
||||||
|
import org.apache.flink.table.catalog.ResolvedSchema;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.apache.flink.table.api.DataTypes.BIGINT;
|
||||||
|
import static org.apache.flink.util.Preconditions.checkNotNull;
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|
||||||
|
/** Tests for {@link PostgresSourceReader}. */
|
||||||
|
class PostgresSourceReaderTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testNotifyCheckpointWindowSizeOne() throws Exception {
|
||||||
|
final PostgresSourceReader reader = createReader(1);
|
||||||
|
final List<Long> completedCheckpointIds = new ArrayList<>();
|
||||||
|
MockPostgresDialect.setNotifyCheckpointCompleteCallback(
|
||||||
|
id -> completedCheckpointIds.add(id));
|
||||||
|
reader.notifyCheckpointComplete(11L);
|
||||||
|
assertThat(completedCheckpointIds).isEmpty();
|
||||||
|
reader.notifyCheckpointComplete(12L);
|
||||||
|
assertThat(completedCheckpointIds).containsExactly(11L);
|
||||||
|
reader.notifyCheckpointComplete(13L);
|
||||||
|
assertThat(completedCheckpointIds).containsExactly(11L, 12L);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testNotifyCheckpointWindowSizeDefault() throws Exception {
|
||||||
|
final PostgresSourceReader reader = createReader(3);
|
||||||
|
final List<Long> completedCheckpointIds = new ArrayList<>();
|
||||||
|
MockPostgresDialect.setNotifyCheckpointCompleteCallback(
|
||||||
|
id -> completedCheckpointIds.add(id));
|
||||||
|
reader.notifyCheckpointComplete(103L);
|
||||||
|
assertThat(completedCheckpointIds).isEmpty();
|
||||||
|
reader.notifyCheckpointComplete(102L);
|
||||||
|
assertThat(completedCheckpointIds).isEmpty();
|
||||||
|
reader.notifyCheckpointComplete(101L);
|
||||||
|
assertThat(completedCheckpointIds).isEmpty();
|
||||||
|
reader.notifyCheckpointComplete(104L);
|
||||||
|
assertThat(completedCheckpointIds).containsExactly(101L);
|
||||||
|
}
|
||||||
|
|
||||||
|
private PostgresSourceReader createReader(final int lsnCommitCheckpointsDelay)
|
||||||
|
throws Exception {
|
||||||
|
final PostgresOffsetFactory offsetFactory = new PostgresOffsetFactory();
|
||||||
|
final PostgresSourceConfigFactory configFactory = new PostgresSourceConfigFactory();
|
||||||
|
configFactory.hostname("host");
|
||||||
|
configFactory.database("pgdb");
|
||||||
|
configFactory.username("username");
|
||||||
|
configFactory.password("password");
|
||||||
|
configFactory.setLsnCommitCheckpointsDelay(lsnCommitCheckpointsDelay);
|
||||||
|
final TestTable customerTable =
|
||||||
|
new TestTable(
|
||||||
|
"pgdb",
|
||||||
|
"customer",
|
||||||
|
"customers",
|
||||||
|
ResolvedSchema.of(Column.physical("id", BIGINT())));
|
||||||
|
final DebeziumDeserializationSchema<?> deserializer = customerTable.getDeserializer();
|
||||||
|
MockPostgresDialect dialect = new MockPostgresDialect(configFactory.create(0));
|
||||||
|
final PostgresSourceBuilder.PostgresIncrementalSource<?> source =
|
||||||
|
new PostgresSourceBuilder.PostgresIncrementalSource<>(
|
||||||
|
configFactory, checkNotNull(deserializer), offsetFactory, dialect);
|
||||||
|
return source.createReader(new TestingReaderContext());
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue