|
@@ -3,13 +3,15 @@ package org.dbsyncer.manager.puller.impl;
|
|
|
import org.dbsyncer.common.event.Event;
|
|
|
import org.dbsyncer.common.util.CollectionUtils;
|
|
|
import org.dbsyncer.common.util.UUIDUtil;
|
|
|
+import org.dbsyncer.connector.ConnectorFactory;
|
|
|
import org.dbsyncer.connector.config.ConnectorConfig;
|
|
|
import org.dbsyncer.connector.config.Table;
|
|
|
import org.dbsyncer.listener.AbstractExtractor;
|
|
|
import org.dbsyncer.listener.Listener;
|
|
|
import org.dbsyncer.listener.config.ExtractorConfig;
|
|
|
import org.dbsyncer.listener.config.ListenerConfig;
|
|
|
-import org.dbsyncer.listener.config.TableCommandConfig;
|
|
|
+import org.dbsyncer.listener.enums.ListenerTypeEnum;
|
|
|
+import org.dbsyncer.listener.quartz.QuartzExtractor;
|
|
|
import org.dbsyncer.listener.quartz.ScheduledTaskJob;
|
|
|
import org.dbsyncer.listener.quartz.ScheduledTaskService;
|
|
|
import org.dbsyncer.manager.Manager;
|
|
@@ -27,10 +29,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
import org.springframework.util.Assert;
|
|
|
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.LinkedHashMap;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Map;
|
|
|
+import java.util.*;
|
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
|
import java.util.concurrent.atomic.AtomicBoolean;
|
|
|
import java.util.stream.Collectors;
|
|
@@ -62,6 +61,9 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob,
|
|
|
@Autowired
|
|
|
private ScheduledTaskService scheduledTaskService;
|
|
|
|
|
|
+ @Autowired
|
|
|
+ private ConnectorFactory connectorFactory;
|
|
|
+
|
|
|
private String key;
|
|
|
|
|
|
private Map<String, AbstractExtractor> map = new ConcurrentHashMap<>();
|
|
@@ -77,21 +79,13 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob,
|
|
|
Assert.notEmpty(list, "映射关系不能为空.");
|
|
|
Meta meta = manager.getMeta(metaId);
|
|
|
Assert.notNull(meta, "Meta不能为空.");
|
|
|
- ConnectorConfig connectorConfig = connector.getConfig();
|
|
|
- ListenerConfig listenerConfig = mapping.getListener();
|
|
|
- List<TableCommandConfig> tableCommandConfig = list.stream().map(t ->
|
|
|
- new TableCommandConfig(t.getSourceTable().getName(), t.getCommand())
|
|
|
- ).collect(Collectors.toList());
|
|
|
-
|
|
|
- AbstractExtractor extractor = listener.getExtractor(new ExtractorConfig(connectorConfig, listenerConfig, meta.getMap(), tableCommandConfig));
|
|
|
+ AbstractExtractor extractor = getExtractor(mapping, connector, list, meta);
|
|
|
Assert.notNull(extractor, "未知的监听配置.");
|
|
|
+
|
|
|
long now = System.currentTimeMillis();
|
|
|
meta.setBeginTime(now);
|
|
|
meta.setEndTime(now);
|
|
|
manager.editMeta(meta);
|
|
|
-
|
|
|
- // 监听数据变更事件
|
|
|
- extractor.addListener(new DefaultListener(mapping, list));
|
|
|
map.putIfAbsent(metaId, extractor);
|
|
|
|
|
|
// 执行任务
|
|
@@ -132,55 +126,59 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob,
|
|
|
scheduledTaskService.stop(key);
|
|
|
}
|
|
|
|
|
|
- final class DefaultListener implements Event {
|
|
|
+ private AbstractExtractor getExtractor(Mapping mapping, Connector connector, List<TableGroup> list, Meta meta)
|
|
|
+ throws InstantiationException, IllegalAccessException {
|
|
|
+ ConnectorConfig connectorConfig = connector.getConfig();
|
|
|
+ ListenerConfig listenerConfig = mapping.getListener();
|
|
|
|
|
|
- private Mapping mapping;
|
|
|
- private String metaId;
|
|
|
- private Map<String, List<FieldPicker>> tablePicker;
|
|
|
- private AtomicBoolean changed = new AtomicBoolean();
|
|
|
+ // timing/log
|
|
|
+ final String listenerType = listenerConfig.getListenerType();
|
|
|
|
|
|
- public DefaultListener(Mapping mapping, List<TableGroup> list) {
|
|
|
- this.mapping = mapping;
|
|
|
- this.metaId = mapping.getMetaId();
|
|
|
- this.tablePicker = new LinkedHashMap<>();
|
|
|
- list.forEach(t -> {
|
|
|
- final Table table = t.getSourceTable();
|
|
|
- final String tableName = table.getName();
|
|
|
- tablePicker.putIfAbsent(tableName, new ArrayList<>());
|
|
|
- tablePicker.get(tableName).add(new FieldPicker(t, table.getColumn(), t.getFieldMapping()));
|
|
|
- });
|
|
|
+ // 默认定时抽取
|
|
|
+ if (ListenerTypeEnum.isTiming(listenerType)) {
|
|
|
+ QuartzExtractor extractor = listener.getExtractor(listenerType, QuartzExtractor.class);
|
|
|
+ List<Map<String, String>> commands = list.stream().map(t -> t.getCommand()).collect(Collectors.toList());
|
|
|
+
|
|
|
+ ExtractorConfig config = new ExtractorConfig(connectorConfig, listenerConfig, meta.getMap(), new QuartzListener(mapping, list));
|
|
|
+ setExtractorConfig(extractor, config);
|
|
|
+ extractor.setConnectorFactory(connectorFactory);
|
|
|
+ extractor.setScheduledTaskService(scheduledTaskService);
|
|
|
+ extractor.setCommands(commands);
|
|
|
+ return extractor;
|
|
|
}
|
|
|
|
|
|
- @Override
|
|
|
- public void changedLogEvent(String tableName, String event, List<Object> before, List<Object> after) {
|
|
|
- logger.info("监听数据=> tableName:{}, event:{}, before:{}, after:{}", tableName, event, before, after);
|
|
|
+ // 基于日志抽取
|
|
|
+ if (ListenerTypeEnum.isLog(listenerType)) {
|
|
|
+ final String connectorType = connectorConfig.getConnectorType();
|
|
|
+ AbstractExtractor extractor = listener.getExtractor(connectorType, AbstractExtractor.class);
|
|
|
|
|
|
- // 处理过程有异常向上抛
|
|
|
- List<FieldPicker> pickers = tablePicker.get(tableName);
|
|
|
- if (!CollectionUtils.isEmpty(pickers)) {
|
|
|
- pickers.parallelStream().forEach(p -> {
|
|
|
- DataEvent data = new DataEvent(event, p.getColumns(before), p.getColumns(after));
|
|
|
- parser.execute(mapping, p.getTableGroup(), data);
|
|
|
- });
|
|
|
- }
|
|
|
-
|
|
|
- // 标记有变更记录
|
|
|
- changed.compareAndSet(false, true);
|
|
|
+ ExtractorConfig config = new ExtractorConfig(connectorConfig, listenerConfig, meta.getMap(), new LogListener(mapping, list));
|
|
|
+ setExtractorConfig(extractor, config);
|
|
|
+ return extractor;
|
|
|
}
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ private void setExtractorConfig(AbstractExtractor extractor, ExtractorConfig config) {
|
|
|
+ extractor.setConnectorConfig(config.getConnectorConfig());
|
|
|
+ extractor.setListenerConfig(config.getListenerConfig());
|
|
|
+ extractor.setMap(config.getMap());
|
|
|
+ extractor.addListener(config.getEvent());
|
|
|
+ }
|
|
|
+
|
|
|
+ abstract class AbstractListener implements Event {
|
|
|
+ protected Mapping mapping;
|
|
|
+ protected String metaId;
|
|
|
+ protected AtomicBoolean changed = new AtomicBoolean();
|
|
|
|
|
|
@Override
|
|
|
- public void changedQuartzEvent(String tableName, String event, Map<String, Object> before, Map<String, Object> after) {
|
|
|
- // 处理过程有异常向上抛
|
|
|
- List<FieldPicker> pickers = tablePicker.get(tableName);
|
|
|
- if (!CollectionUtils.isEmpty(pickers)) {
|
|
|
- pickers.parallelStream().forEach(p -> {
|
|
|
- DataEvent data = new DataEvent(event, before, after);
|
|
|
- parser.execute(mapping, p.getTableGroup(), data);
|
|
|
- });
|
|
|
- }
|
|
|
+ public void changedLogEvent(String tableName, String event, List<Object> before, List<Object> after) {
|
|
|
+ // nothing to do
|
|
|
+ }
|
|
|
|
|
|
- // 标记有变更记录
|
|
|
- changed.compareAndSet(false, true);
|
|
|
+ @Override
|
|
|
+ public void changedQuartzEvent(int tableGroupIndex, String event, Map<String, Object> before, Map<String, Object> after) {
|
|
|
+ // nothing to do
|
|
|
}
|
|
|
|
|
|
@Override
|
|
@@ -203,4 +201,106 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob,
|
|
|
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * </p>定时模式
|
|
|
+ * <ol>
|
|
|
+ * <li>获取映射关系增量数据</li>
|
|
|
+ * <li>根据过滤条件筛选</li>
|
|
|
+ * </ol>
|
|
|
+ * </p>同步关系:
|
|
|
+ * </p>数据源表 >> 目标源表
|
|
|
+ * <ul>
|
|
|
+ * <li>A >> B</li>
|
|
|
+ * <li>A >> C</li>
|
|
|
+ * <li>E >> F</li>
|
|
|
+ * </ul>
|
|
|
+ * </p>PS:
|
|
|
+ * <ol>
|
|
|
+ * <li>依次执行同步关系A >> B 然后 A >> C ...</li>
|
|
|
+ * </ol>
|
|
|
+ */
|
|
|
+ final class QuartzListener extends AbstractListener {
|
|
|
+
|
|
|
+ private List<FieldPicker> tablePicker;
|
|
|
+
|
|
|
+ public QuartzListener(Mapping mapping, List<TableGroup> list) {
|
|
|
+ this.mapping = mapping;
|
|
|
+ this.metaId = mapping.getMetaId();
|
|
|
+ this.tablePicker = new LinkedList<>();
|
|
|
+ list.forEach(t -> tablePicker.add(new FieldPicker(t, t.getFilter())));
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void changedQuartzEvent(int tableGroupIndex, String event, Map<String, Object> before, Map<String, Object> after) {
|
|
|
+ final FieldPicker picker = tablePicker.get(tableGroupIndex);
|
|
|
+ logger.info("监听数据=> tableName:{}, event:{}, before:{}, after:{}", picker.getTableGroup().getSourceTable().getName(), event, before, after);
|
|
|
+
|
|
|
+ // 处理过程有异常向上抛
|
|
|
+ DataEvent data = new DataEvent(event, before, after);
|
|
|
+ if(picker.filter(data)){
|
|
|
+ parser.execute(mapping, picker.getTableGroup(), data);
|
|
|
+ }
|
|
|
+
|
|
|
+ // 标记有变更记录
|
|
|
+ changed.compareAndSet(false, true);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * </p>日志模式
|
|
|
+ * <ol>
|
|
|
+ * <li>监听表增量数据</li>
|
|
|
+ * <li>根据过滤条件筛选</li>
|
|
|
+ * </ol>
|
|
|
+ * </p>同步关系:
|
|
|
+ * </p>数据源表 >> 目标源表
|
|
|
+ * <ul>
|
|
|
+ * <li>A >> B</li>
|
|
|
+ * <li>A >> C</li>
|
|
|
+ * <li>E >> F</li>
|
|
|
+ * </ul>
|
|
|
+ * </p>PS:
|
|
|
+ * <ol>
|
|
|
+ * <li>为减少开销而选择复用监听器实例, 启动时只需创建一个数据源连接器.</li>
|
|
|
+ * <li>关系A >> B和A >> C会复用A监听的数据, A监听到增量数据,会发送给B和C.</li>
|
|
|
+ * <li>该模式下,会监听表所有字段.</li>
|
|
|
+ * </ol>
|
|
|
+ */
|
|
|
+ final class LogListener extends AbstractListener {
|
|
|
+
|
|
|
+ private Map<String, List<FieldPicker>> tablePicker;
|
|
|
+
|
|
|
+ public LogListener(Mapping mapping, List<TableGroup> list) {
|
|
|
+ this.mapping = mapping;
|
|
|
+ this.metaId = mapping.getMetaId();
|
|
|
+ this.tablePicker = new LinkedHashMap<>();
|
|
|
+ list.forEach(t -> {
|
|
|
+ final Table table = t.getSourceTable();
|
|
|
+ final String tableName = table.getName();
|
|
|
+ tablePicker.putIfAbsent(tableName, new ArrayList<>());
|
|
|
+ tablePicker.get(tableName).add(new FieldPicker(t, t.getFilter(), table.getColumn(), t.getFieldMapping()));
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void changedLogEvent(String tableName, String event, List<Object> before, List<Object> after) {
|
|
|
+ logger.info("监听数据=> tableName:{}, event:{}, before:{}, after:{}", tableName, event, before, after);
|
|
|
+
|
|
|
+ // 处理过程有异常向上抛
|
|
|
+ List<FieldPicker> pickers = tablePicker.get(tableName);
|
|
|
+ if (!CollectionUtils.isEmpty(pickers)) {
|
|
|
+ pickers.parallelStream().forEach(picker -> {
|
|
|
+ DataEvent data = new DataEvent(event, picker.getColumns(before), picker.getColumns(after));
|
|
|
+ if(picker.filter(data)){
|
|
|
+ parser.execute(mapping, picker.getTableGroup(), data);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ // 标记有变更记录
|
|
|
+ changed.compareAndSet(false, true);
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
}
|