!66 merge

Merge pull request !66 from AE86/V_1.0.0_Beta

dbsyncer-biz/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>dbsyncer-biz</artifactId>

dbsyncer-biz/src/main/java/org/dbsyncer/biz/ConfigService.java

@@ -3,6 +3,7 @@ package org.dbsyncer.biz;
 import org.dbsyncer.biz.vo.ConfigVo;
 import org.dbsyncer.parser.model.ConfigModel;
 
+import java.io.File;
 import java.util.List;
 import java.util.Map;
 
@@ -48,4 +49,17 @@ public interface ConfigService {
      */
     List<ConfigModel> getConfigModelAll();
 
+    /**
+     * 校验文件格式
+     *
+     * @param filename
+     */
+    void checkFileSuffix(String filename);
+
+    /**
+     * 更新配置
+     *
+     * @param file
+     */
+    void refreshConfig(File file);
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java

@@ -1,5 +1,6 @@
 package org.dbsyncer.biz.checker.impl.connector;
 
+import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.config.DatabaseConfig;
 import org.springframework.stereotype.Component;
 
@@ -18,5 +19,8 @@ public class DqlPostgreSQLConfigChecker extends AbstractDataBaseConfigChecker {
         super.modify(connectorConfig, params);
         super.modifyDql(connectorConfig, params);
         super.modifySchema(connectorConfig, params);
+
+        connectorConfig.getProperties().put("dropSlotOnClose", StringUtil.isNotBlank(params.get("dropSlotOnClose")) ? "true" : "false");
+        connectorConfig.getProperties().put("pluginName", params.get("pluginName"));
     }
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConfigServiceImpl.java

@@ -1,17 +1,26 @@
 package org.dbsyncer.biz.impl;
 
+import org.apache.commons.io.FileUtils;
 import org.dbsyncer.biz.ConfigService;
-import org.dbsyncer.biz.checker.impl.config.ConfigChecker;
+import org.dbsyncer.biz.checker.Checker;
 import org.dbsyncer.biz.vo.ConfigVo;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.manager.Manager;
+import org.dbsyncer.manager.template.impl.PreloadTemplate;
+import org.dbsyncer.parser.logger.LogService;
+import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.Config;
 import org.dbsyncer.parser.model.ConfigModel;
+import org.dbsyncer.plugin.enums.FileSuffixEnum;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import org.springframework.util.Assert;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -30,7 +39,13 @@ public class ConfigServiceImpl implements ConfigService {
     private Manager manager;
 
     @Autowired
-    private ConfigChecker configChecker;
+    private Checker configChecker;
+
+    @Autowired
+    private PreloadTemplate preloadTemplate;
+
+    @Autowired
+    private LogService logService;
 
     @Override
     public String edit(Map<String, String> params) {
@@ -77,6 +92,32 @@ public class ConfigServiceImpl implements ConfigService {
         return list;
     }
 
+    @Override
+    public void checkFileSuffix(String filename) {
+        Assert.hasText(filename, "the config filename is null.");
+        String suffix = filename.substring(filename.lastIndexOf(".") + 1, filename.length());
+        FileSuffixEnum fileSuffix = FileSuffixEnum.getFileSuffix(suffix);
+        Assert.notNull(fileSuffix, "Illegal file suffix");
+        Assert.isTrue(FileSuffixEnum.JSON == fileSuffix, String.format("不正确的文件扩展名 \"%s\"，只支持 \"%s\" 的文件扩展名。", filename, FileSuffixEnum.JSON.getName()));
+    }
+
+    @Override
+    public void refreshConfig(File file) {
+        Assert.notNull(file, "the config file is null.");
+        try {
+            List<String> lines = FileUtils.readLines(file, Charset.defaultCharset());
+            if (!CollectionUtils.isEmpty(lines)) {
+                StringBuilder json = new StringBuilder();
+                lines.forEach(line -> json.append(line));
+                preloadTemplate.reload(json.toString());
+            }
+        } catch (IOException e) {
+            logService.log(LogType.CacheLog.IMPORT_ERROR);
+        } finally {
+            FileUtils.deleteQuietly(file);
+        }
+    }
+
     private ConfigVo convertConfig2Vo(Config config) {
         ConfigVo configVo = new ConfigVo();
         BeanUtils.copyProperties(config, configVo);

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java

@@ -10,19 +10,19 @@ import org.dbsyncer.biz.vo.MappingVo;
 import org.dbsyncer.biz.vo.MetaVo;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.monitor.Monitor;
 import org.dbsyncer.parser.enums.ModelEnum;
 import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.*;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
-import java.util.*;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -33,11 +33,6 @@ import java.util.stream.Collectors;
 @Service
 public class MappingServiceImpl extends BaseServiceImpl implements MappingService {
 
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
-    @Autowired
-    private Monitor monitor;
-
     @Autowired
     private MappingChecker mappingChecker;
 
@@ -56,7 +51,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
 
         // 匹配相似表 on
         String autoMatchTable = params.get("autoMatchTable");
-        if(StringUtil.isNotBlank(autoMatchTable)){
+        if (StringUtil.isNotBlank(autoMatchTable)) {
             matchSimilarTable(model);
         }
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java

@@ -12,6 +12,7 @@ import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.monitor.Monitor;
 import org.dbsyncer.monitor.enums.DiskMetricEnum;
 import org.dbsyncer.monitor.enums.MetricEnum;
+import org.dbsyncer.monitor.enums.TaskMetricEnum;
 import org.dbsyncer.monitor.enums.ThreadPoolMetricEnum;
 import org.dbsyncer.monitor.model.AppReportMetric;
 import org.dbsyncer.monitor.model.MetricResponse;
@@ -47,6 +48,8 @@ public class MonitorServiceImpl implements MonitorService {
 
     @PostConstruct
     private void init() {
+        metricDetailFormatterMap.putIfAbsent(TaskMetricEnum.STORAGE_ACTIVE.getCode(), new ValueMetricDetailFormatter());
+        metricDetailFormatterMap.putIfAbsent(TaskMetricEnum.STORAGE_REMAINING_CAPACITY.getCode(), new ValueMetricDetailFormatter());
         metricDetailFormatterMap.putIfAbsent(ThreadPoolMetricEnum.TASK_SUBMITTED.getCode(), new ValueMetricDetailFormatter());
         metricDetailFormatterMap.putIfAbsent(ThreadPoolMetricEnum.QUEUE_UP.getCode(), new ValueMetricDetailFormatter());
         metricDetailFormatterMap.putIfAbsent(ThreadPoolMetricEnum.ACTIVE.getCode(), new ValueMetricDetailFormatter());
@@ -167,7 +170,7 @@ public class MonitorServiceImpl implements MonitorService {
 
     private List<MetricResponseVo> getMetrics(List<MetricResponse> metrics) {
         // 线程池状态
-        List<MetricResponse> metricList = monitor.getThreadPoolInfo();
+        List<MetricResponse> metricList = monitor.getMetricInfo();
         // 系统指标
         metricList.addAll(metrics);
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java

@@ -15,6 +15,7 @@ import org.dbsyncer.plugin.enums.FileSuffixEnum;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
+import org.springframework.util.Assert;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -70,14 +71,14 @@ public class PluginServiceImpl implements PluginService {
 
     @Override
     public void checkFileSuffix(String filename) {
-        if (StringUtil.isNotBlank(filename)) {
-            String suffix = filename.substring(filename.lastIndexOf(".") + 1, filename.length());
-            if (null == FileSuffixEnum.getFileSuffix(suffix)) {
-                suffix = StringUtil.join(FileSuffixEnum.values(), ",").toLowerCase();
-                String msg = String.format("不正确的文件扩展名 \"%s\"，只支持 \"%s\" 的文件扩展名。", filename, suffix);
-                logService.log(LogType.PluginLog.CHECK_ERROR, msg);
-                throw new BizException(msg);
-            }
+        Assert.hasText(filename, "the plugin filename is null.");
+        String suffix = filename.substring(filename.lastIndexOf(".") + 1, filename.length());
+        FileSuffixEnum fileSuffix = FileSuffixEnum.getFileSuffix(suffix);
+        Assert.notNull(fileSuffix, "Illegal file suffix");
+        if (FileSuffixEnum.JAR != fileSuffix) {
+            String msg = String.format("不正确的文件扩展名 \"%s\"，只支持 \"%s\" 的文件扩展名。", filename, FileSuffixEnum.JAR.getName());
+            logService.log(LogType.PluginLog.CHECK_ERROR, msg);
+            throw new BizException(msg);
         }
     }
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java

@@ -9,8 +9,6 @@ import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
@@ -27,8 +25,6 @@ import java.util.stream.Stream;
 @Service
 public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroupService {
 
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
     @Autowired
     private Checker tableGroupChecker;
 

dbsyncer-cache/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>dbsyncer-cache</artifactId>

dbsyncer-cluster/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-cluster</artifactId>

dbsyncer-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-common</artifactId>

dbsyncer-common/src/main/java/org/dbsyncer/common/column/AbstractColumnValue.java

@@ -0,0 +1,20 @@
+package org.dbsyncer.common.column;
+
+public abstract class AbstractColumnValue<T> implements ColumnValue {
+
+    protected Object value;
+
+    protected T getValue() {
+        return (T) value;
+    }
+
+    public void setValue(T value) {
+        this.value = value;
+    }
+
+    @Override
+    public boolean isNull() {
+        return value == null;
+    }
+
+}

dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java

@@ -0,0 +1,40 @@
+package org.dbsyncer.common.column;
+
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/4/22 22:39
+ */
+public interface ColumnValue {
+
+    boolean isNull();
+
+    String asString();
+
+    byte[] asByteArray();
+
+    Short asShort();
+
+    Integer asInteger();
+
+    Long asLong();
+
+    Float asFloat();
+
+    Double asDouble();
+
+    Boolean asBoolean();
+
+    BigDecimal asBigDecimal();
+
+    Date asDate();
+
+    Timestamp asTimestamp();
+
+    Time asTime();
+}

dbsyncer-common/src/main/java/org/dbsyncer/common/config/ScheduleConfig.java

@@ -22,7 +22,7 @@ public class ScheduleConfig implements SchedulingConfigurer {
 
     @Bean(name = "taskScheduler", destroyMethod = "shutdown")
     public ThreadPoolTaskScheduler taskScheduler() {
-        int poolSize = Runtime.getRuntime().availableProcessors() * 2;
+        int poolSize = Runtime.getRuntime().availableProcessors();
         ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
         //核心线程池大小
         scheduler.setPoolSize(poolSize);

dbsyncer-common/src/main/java/org/dbsyncer/common/event/Event.java

@@ -30,6 +30,11 @@ public interface Event {
      */
     void forceFlushEvent(Map<String,String> map);
 
+    /**
+     * 刷新事件变更时间
+     */
+    void refreshFlushEventUpdateTime();
+
     /**
      * 异常事件
      *

dbsyncer-common/src/main/java/org/dbsyncer/common/event/RowChangedEvent.java

@@ -19,25 +19,20 @@ public class RowChangedEvent {
 
     private int tableGroupIndex;
     private String sourceTableName;
-    private String targetTableName;
     private String event;
-    private List<Object> beforeData;
-    private List<Object> afterData;
-    private Map<String, Object> before;
-    private Map<String, Object> after;
+    private List<Object> dataList;
+    private Map<String, Object> dataMap;
 
-    public RowChangedEvent(int tableGroupIndex, String event, Map<String, Object> before, Map<String, Object> after) {
+    public RowChangedEvent(int tableGroupIndex, String event, Map<String, Object> data) {
         this.tableGroupIndex = tableGroupIndex;
         this.event = event;
-        this.before = before;
-        this.after = after;
+        this.dataMap = data;
     }
 
-    public RowChangedEvent(String sourceTableName, String event, List<Object> beforeData, List<Object> afterData) {
+    public RowChangedEvent(String sourceTableName, String event, List<Object> data) {
         this.sourceTableName = sourceTableName;
         this.event = event;
-        this.beforeData = beforeData;
-        this.afterData = afterData;
+        this.dataList = data;
     }
 
     public int getTableGroupIndex() {
@@ -52,48 +47,24 @@ public class RowChangedEvent {
         this.sourceTableName = sourceTableName;
     }
 
-    public String getTargetTableName() {
-        return targetTableName;
-    }
-
-    public void setTargetTableName(String targetTableName) {
-        this.targetTableName = targetTableName;
-    }
-
     public String getEvent() {
         return event;
     }
 
-    public List<Object> getBeforeData() {
-        return beforeData;
-    }
-
-    public void setBeforeData(List<Object> beforeData) {
-        this.beforeData = beforeData;
-    }
-
-    public List<Object> getAfterData() {
-        return afterData;
-    }
-
-    public void setAfterData(List<Object> afterData) {
-        this.afterData = afterData;
-    }
-
-    public Map<String, Object> getBefore() {
-        return before;
+    public List<Object> getDataList() {
+        return dataList;
     }
 
-    public void setBefore(Map<String, Object> before) {
-        this.before = before;
+    public void setDataList(List<Object> dataList) {
+        this.dataList = dataList;
     }
 
-    public Map<String, Object> getAfter() {
-        return after;
+    public Map<String, Object> getDataMap() {
+        return dataMap;
     }
 
-    public void setAfter(Map<String, Object> after) {
-        this.after = after;
+    public void setDataMap(Map<String, Object> dataMap) {
+        this.dataMap = dataMap;
     }
 
     @Override

dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/BufferedRandomAccessFile.java → dbsyncer-common/src/main/java/org/dbsyncer/common/file/BufferedRandomAccessFile.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.listener.file;
+package org.dbsyncer.common.file;
 
 import java.io.File;
 import java.io.FileNotFoundException;

dbsyncer-common/src/main/java/org/dbsyncer/common/model/ConvertContext.java

@@ -0,0 +1,21 @@
+package org.dbsyncer.common.model;
+
+import org.dbsyncer.common.spi.ProxyApplicationContext;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/30 16:00
+ */
+public class ConvertContext {
+
+    /**
+     * Spring上下文
+     */
+    protected ProxyApplicationContext context;
+
+    public ProxyApplicationContext getContext() {
+        return context;
+    }
+
+}

dbsyncer-common/src/main/java/org/dbsyncer/common/model/FullConvertContext.java

@@ -0,0 +1,38 @@
+package org.dbsyncer.common.model;
+
+import org.dbsyncer.common.spi.ProxyApplicationContext;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/30 16:04
+ */
+public class FullConvertContext extends ConvertContext {
+
+    /**
+     * 全量同步，数据源数据集合
+     */
+    private List<Map> sourceList;
+
+    /**
+     * 全量同步，目标源源数据集合
+     */
+    private List<Map> targetList;
+
+    public FullConvertContext(ProxyApplicationContext context, List<Map> sourceList, List<Map> targetList) {
+        this.context = context;
+        this.sourceList = sourceList;
+        this.targetList = targetList;
+    }
+
+    public List<Map> getSourceList() {
+        return sourceList;
+    }
+
+    public List<Map> getTargetList() {
+        return targetList;
+    }
+}

dbsyncer-common/src/main/java/org/dbsyncer/common/model/IncrementConvertContext.java

@@ -0,0 +1,47 @@
+package org.dbsyncer.common.model;
+
+import org.dbsyncer.common.spi.ProxyApplicationContext;
+
+import java.util.Map;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/30 16:06
+ */
+public class IncrementConvertContext extends ConvertContext {
+
+    /**
+     * 增量同步，事件（INSERT/UPDATE/DELETE）
+     */
+    private String event;
+
+    /**
+     * 增量同步，数据源数据
+     */
+    private Map source;
+
+    /**
+     * 增量同步，目标源数据
+     */
+    private Map target;
+
+    public IncrementConvertContext(ProxyApplicationContext context, String event, Map source, Map target) {
+        this.context = context;
+        this.event = event;
+        this.source = source;
+        this.target = target;
+    }
+
+    public String getEvent() {
+        return event;
+    }
+
+    public Map getSource() {
+        return source;
+    }
+
+    public Map getTarget() {
+        return target;
+    }
+}

dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskService.java

@@ -11,9 +11,9 @@ public interface ScheduledTaskService {
      * 第六位，星期，取值1-7
      * [秒 分 时 日 月 星期]
      *
-     * @param key 任务唯一key
+     * @param key  任务唯一key
      * @param cron 任务表达式
-     * @param job 任务实现
+     * @param job  任务实现
      */
     void start(String key, String cron, ScheduledTaskJob job);
 

dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConvertService.java

@@ -1,7 +1,7 @@
 package org.dbsyncer.common.spi;
 
-import java.util.List;
-import java.util.Map;
+import org.dbsyncer.common.model.FullConvertContext;
+import org.dbsyncer.common.model.IncrementConvertContext;
 
 /**
  * 插件扩展服务接口
@@ -16,19 +16,16 @@ public interface ConvertService {
     /**
      * 全量同步
      *
-     * @param source 数据源
-     * @param target 目标源
+     * @param context 上下文
      */
-    void convert(List<Map> source, List<Map> target);
+    void convert(FullConvertContext context);
 
     /**
      * 增量同步
      *
-     * @param event  事件（INSERT/UPDATE/DELETE）
-     * @param source 数据源
-     * @param target 目标源
+     * @param context 上下文
      */
-    void convert(String event, Map source, Map target);
+    void convert(IncrementConvertContext context);
 
     /**
      * 版本号

dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ProxyApplicationContext.java

@@ -0,0 +1,13 @@
+package org.dbsyncer.common.spi;
+
+import org.springframework.context.ApplicationContext;
+
+/**
+ * Spring上下文代理对象
+ *
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/30 15:13
+ */
+public interface ProxyApplicationContext extends ApplicationContext {
+}

dbsyncer-common/src/main/java/org/dbsyncer/common/util/DateFormatUtil.java

@@ -65,18 +65,16 @@ public abstract class DateFormatUtil {
             .optionalEnd()
             .toFormatter();
 
-    private static ZoneId zoneId = ZoneId.systemDefault();
-
     public static String getCurrentTime() {
         return LocalDateTime.now().format(TIME_FORMATTER);
     }
 
     public static String dateToString(Date date) {
-        return date.toInstant().atZone(zoneId).toLocalDate().format(DATE_FORMATTER);
+        return date.toLocalDate().format(DATE_FORMATTER);
     }
 
-    public static String dateToChineseStandardTimeString(Date date) {
-        return date.toInstant().atZone(zoneId).toLocalDateTime().format(CHINESE_STANDARD_TIME_FORMATTER);
+    public static String timestampToString(Timestamp timestamp) {
+        return timestamp.toLocalDateTime().format(CHINESE_STANDARD_TIME_FORMATTER);
     }
 
     public static Date stringToDate(String s) {

dbsyncer-connector/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-connector</artifactId>

dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/SqlBuilderConfig.java

@@ -8,6 +8,8 @@ import java.util.List;
 public class SqlBuilderConfig {
 
     private Database database;
+    // 架构名
+    private String schema;
     // 表名
     private String tableName;
     // 主键
@@ -19,8 +21,9 @@ public class SqlBuilderConfig {
     // 引号
     private String quotation;
 
-    public SqlBuilderConfig(Database database, String tableName, String pk, List<Field> fields, String queryFilter, String quotation) {
+    public SqlBuilderConfig(Database database, String schema, String tableName, String pk, List<Field> fields, String queryFilter, String quotation) {
         this.database = database;
+        this.schema = schema;
         this.tableName = tableName;
         this.pk = pk;
         this.fields = fields;
@@ -32,6 +35,10 @@ public class SqlBuilderConfig {
         return database;
     }
 
+    public String getSchema() {
+        return schema;
+    }
+
     public String getTableName() {
         return tableName;
     }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/WriterBatchConfig.java

@@ -27,22 +27,13 @@ public class WriterBatchConfig {
      * 集合数据
      */
     private List<Map> data;
-    /**
-     * 强制更新
-     */
-    private boolean isForceUpdate;
 
     public WriterBatchConfig(String tableName, String event, Map<String, String> command, List<Field> fields, List<Map> data) {
-        this(tableName, event, command, fields, data, false);
-    }
-
-    public WriterBatchConfig(String tableName, String event, Map<String, String> command, List<Field> fields, List<Map> data, boolean isForceUpdate) {
         this.tableName = tableName;
         this.event = event;
         this.command = command;
         this.fields = fields;
         this.data = data;
-        this.isForceUpdate = isForceUpdate;
     }
 
     public String getTableName() {
@@ -65,7 +56,4 @@ public class WriterBatchConfig {
         return data;
     }
 
-    public boolean isForceUpdate() {
-        return isForceUpdate;
-    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java

@@ -63,9 +63,11 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
     @Override
     public MetaInfo getMetaInfo(DatabaseConnectorMapper connectorMapper, String tableName) {
         String quotation = buildSqlWithQuotation();
-        StringBuilder queryMetaSql = new StringBuilder("SELECT * FROM ").append(quotation).append(tableName).append(quotation).append(
-                " WHERE 1 != 1");
-        return connectorMapper.execute(databaseTemplate -> getMetaInfo(databaseTemplate, queryMetaSql.toString(), tableName));
+        DatabaseConfig config = connectorMapper.getConfig();
+        String queryMetaSql = new StringBuilder("SELECT * FROM ").append(getSchema(config, quotation)).append(quotation).append(tableName)
+                .append(quotation).append(" WHERE 1!=1").toString();
+
+        return connectorMapper.execute(databaseTemplate -> getMetaInfo(databaseTemplate, queryMetaSql, config.getSchema(), tableName));
     }
 
     @Override
@@ -92,7 +94,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         Collections.addAll(config.getArgs(), getPageArgs(config.getPageIndex(), config.getPageSize()));
 
         // 3、执行SQL
-        List<Map<String, Object>> list = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(querySql, config.getArgs().toArray()));
+        List<Map<String, Object>> list = connectorMapper.execute(
+                databaseTemplate -> databaseTemplate.queryForList(querySql, config.getArgs().toArray()));
 
         // 4、返回结果集
         return new Result(list);
@@ -120,6 +123,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         if (!isInsert(event)) {
             if (isDelete(event)) {
                 fields.clear();
+            } else if (isUpdate(event)) {
+                fields.remove(pkField);
             }
             fields.add(pkField);
         }
@@ -142,19 +147,15 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
                     })
             );
         } catch (Exception e) {
-            result.addFailData(data);
-            result.getError().append(e.getMessage());
+            logger.error(e.getMessage());
+            data.forEach(row -> forceUpdate(result, connectorMapper, config, pkField, row));
         }
 
         if (null != execute) {
             int batchSize = execute.length;
             for (int i = 0; i < batchSize; i++) {
                 if (execute[i] == 0) {
-                    if (config.isForceUpdate()) {
-                        forceUpdate(result, connectorMapper, config, pkField, data.get(i));
-                    } else {
-                        result.getFailData().add(data.get(i));
-                    }
+                    forceUpdate(result, connectorMapper, config, pkField, data.get(i));
                     continue;
                 }
                 result.getSuccessData().add(data.get(i));
@@ -166,47 +167,38 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
     @Override
     public Map<String, String> getSourceCommand(CommandConfig commandConfig) {
         // 获取过滤SQL
-        List<Filter> filter = commandConfig.getFilter();
-        String queryFilterSql = getQueryFilterSql(filter);
+        final String queryFilterSql = getQueryFilterSql(commandConfig.getFilter());
+        final String quotation = buildSqlWithQuotation();
 
         // 获取查询SQL
-        Table table = commandConfig.getTable();
         Map<String, String> map = new HashMap<>();
-
-        String query = ConnectorConstant.OPERTION_QUERY;
-        map.put(query, buildSql(query, commandConfig, queryFilterSql));
-
+        String schema = getSchema((DatabaseConfig) commandConfig.getConnectorConfig(), quotation);
+        map.put(ConnectorConstant.OPERTION_QUERY, buildSql(ConnectorConstant.OPERTION_QUERY, commandConfig, schema, queryFilterSql));
         // 获取查询总数SQL
-        String quotation = buildSqlWithQuotation();
-        String pk = findOriginalTablePrimaryKey(commandConfig, quotation);
-        StringBuilder queryCount = new StringBuilder();
-        queryCount.append("SELECT COUNT(1) FROM (SELECT 1 FROM ").append(quotation).append(table.getName()).append(quotation);
-        if (StringUtil.isNotBlank(queryFilterSql)) {
-            queryCount.append(queryFilterSql);
-        }
-        queryCount.append(" GROUP BY ").append(pk).append(") DBSYNCER_T");
-        map.put(ConnectorConstant.OPERTION_QUERY_COUNT, queryCount.toString());
+        map.put(ConnectorConstant.OPERTION_QUERY_COUNT, getQueryCountSql(commandConfig, schema, quotation, queryFilterSql));
         return map;
     }
 
     @Override
     public Map<String, String> getTargetCommand(CommandConfig commandConfig) {
+        String quotation = buildSqlWithQuotation();
+        String schema = getSchema((DatabaseConfig) commandConfig.getConnectorConfig(), quotation);
+
         // 获取增删改SQL
         Map<String, String> map = new HashMap<>();
         String insert = SqlBuilderEnum.INSERT.getName();
-        map.put(insert, buildSql(insert, commandConfig, null));
+        map.put(insert, buildSql(insert, commandConfig, schema, null));
 
         String update = SqlBuilderEnum.UPDATE.getName();
-        map.put(update, buildSql(update, commandConfig, null));
+        map.put(update, buildSql(update, commandConfig, schema, null));
 
         String delete = SqlBuilderEnum.DELETE.getName();
-        map.put(delete, buildSql(delete, commandConfig, null));
+        map.put(delete, buildSql(delete, commandConfig, schema, null));
 
         // 获取查询数据行是否存在
-        String quotation = buildSqlWithQuotation();
         String pk = findOriginalTablePrimaryKey(commandConfig, quotation);
-        StringBuilder queryCount = new StringBuilder().append("SELECT COUNT(1) FROM ").append(quotation).append(commandConfig.getTable().getName()).append(
-                quotation).append(" WHERE ").append(pk).append(" = ?");
+        StringBuilder queryCount = new StringBuilder().append("SELECT COUNT(1) FROM ").append(schema).append(quotation).append(
+                commandConfig.getTable().getName()).append(quotation).append(" WHERE ").append(pk).append(" = ?");
         String queryCountExist = ConnectorConstant.OPERTION_QUERY_COUNT_EXIST;
         map.put(queryCountExist, queryCount.toString());
         return map;
@@ -230,6 +222,21 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         return "";
     }
 
+    /**
+     * 获取架构名
+     *
+     * @param config
+     * @param quotation
+     * @return
+     */
+    protected String getSchema(DatabaseConfig config, String quotation) {
+        StringBuilder schema = new StringBuilder();
+        if (StringUtil.isNotBlank(config.getSchema())) {
+            schema.append(quotation).append(config.getSchema()).append(quotation).append(".");
+        }
+        return schema.toString();
+    }
+
     /**
      * 获取表列表
      *
@@ -245,6 +252,27 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         return Collections.EMPTY_LIST;
     }
 
+    /**
+     * 获取查询总数SQL
+     *
+     * @param commandConfig
+     * @param schema
+     * @param quotation
+     * @param queryFilterSql
+     * @return
+     */
+    protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
+        String table = commandConfig.getTable().getName();
+        String pk = findOriginalTablePrimaryKey(commandConfig, quotation);
+        StringBuilder queryCount = new StringBuilder();
+        queryCount.append("SELECT COUNT(1) FROM (SELECT 1 FROM ").append(schema).append(quotation).append(table).append(quotation);
+        if (StringUtil.isNotBlank(queryFilterSql)) {
+            queryCount.append(queryFilterSql);
+        }
+        queryCount.append(" GROUP BY ").append(pk).append(") DBSYNCER_T");
+        return queryCount.toString();
+    }
+
     /**
      * 获取查询条件SQL
      *
@@ -317,10 +345,11 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
      *
      * @param type           {@link SqlBuilderEnum}
      * @param commandConfig
+     * @param schema
      * @param queryFilterSQL
      * @return
      */
-    protected String buildSql(String type, CommandConfig commandConfig, String queryFilterSQL) {
+    protected String buildSql(String type, CommandConfig commandConfig, String schema, String queryFilterSQL) {
         Table table = commandConfig.getTable();
         if (null == table) {
             logger.error("Table can not be null.");
@@ -359,7 +388,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
             pk = findOriginalTablePrimaryKey(commandConfig, "");
         }
 
-        SqlBuilderConfig config = new SqlBuilderConfig(this, tableName, pk, fields, queryFilterSQL, buildSqlWithQuotation());
+        SqlBuilderConfig config = new SqlBuilderConfig(this, schema, tableName, pk, fields, queryFilterSQL, buildSqlWithQuotation());
         return SqlBuilderEnum.getSqlBuilder(type).buildSql(config);
     }
 
@@ -368,10 +397,11 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
      *
      * @param databaseTemplate
      * @param metaSql          查询元数据
+     * @param schema           架构名
      * @param tableName        表名
      * @return
      */
-    protected MetaInfo getMetaInfo(DatabaseTemplate databaseTemplate, String metaSql, String tableName) throws SQLException {
+    protected MetaInfo getMetaInfo(DatabaseTemplate databaseTemplate, String metaSql, String schema, String tableName) throws SQLException {
         SqlRowSet sqlRowSet = databaseTemplate.queryForRowSet(metaSql);
         ResultSetWrappingSqlRowSet rowSet = (ResultSetWrappingSqlRowSet) sqlRowSet;
         SqlRowSetMetaData metaData = rowSet.getMetaData();
@@ -384,7 +414,10 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         List<Field> fields = new ArrayList<>(columnCount);
         Map<String, List<String>> tables = new HashMap<>();
         try {
-            DatabaseMetaData md = databaseTemplate.getConnection().getMetaData();
+            Connection connection = databaseTemplate.getConnection();
+            DatabaseMetaData md = connection.getMetaData();
+            final String catalog = connection.getCatalog();
+            schema = StringUtil.isNotBlank(schema) ? schema : null;
             String name = null;
             String label = null;
             String typeName = null;
@@ -394,7 +427,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
             for (int i = 1; i <= columnCount; i++) {
                 table = StringUtil.isNotBlank(tableName) ? tableName : metaData.getTableName(i);
                 if (null == tables.get(table)) {
-                    tables.putIfAbsent(table, findTablePrimaryKeys(md, table));
+                    tables.putIfAbsent(table, findTablePrimaryKeys(md, catalog, schema, table));
                 }
                 name = metaData.getColumnName(i);
                 label = metaData.getColumnLabel(i);
@@ -441,16 +474,18 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
      * 返回表主键
      *
      * @param md
+     * @param catalog
+     * @param schema
      * @param tableName
      * @return
      * @throws SQLException
      */
-    private List<String> findTablePrimaryKeys(DatabaseMetaData md, String tableName) throws SQLException {
+    private List<String> findTablePrimaryKeys(DatabaseMetaData md, String catalog, String schema, String tableName) throws SQLException {
         //根据表名获得主键结果集
         ResultSet rs = null;
         List<String> primaryKeys = new ArrayList<>();
         try {
-            rs = md.getPrimaryKeys(null, null, tableName);
+            rs = md.getPrimaryKeys(catalog, schema, tableName);
             while (rs.next()) {
                 primaryKeys.add(rs.getString("COLUMN_NAME"));
             }
@@ -509,6 +544,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         if (!isInsert(event)) {
             if (isDelete(event)) {
                 fields.clear();
+            } else if (isUpdate(event)) {
+                fields.remove(pkField);
             }
             fields.add(pkField);
         }
@@ -534,7 +571,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
     private boolean existRow(DatabaseConnectorMapper connectorMapper, String sql, Object value) {
         int rowNum = 0;
         try {
-            rowNum = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(sql, new Object[]{value}, Integer.class));
+            rowNum = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(sql, new Object[] {value}, Integer.class));
         } catch (Exception e) {
             logger.error("检查数据行存在异常:{}，SQL:{},参数:{}", e.getMessage(), sql, value);
         }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/setter/BitSetter.java

@@ -26,6 +26,11 @@ public class BitSetter extends AbstractSetter<byte[]> {
             ps.setInt(i, integer);
             return;
         }
+        if (val instanceof Boolean) {
+            Boolean b = (Boolean) val;
+            ps.setBoolean(i, b);
+            return;
+        }
         throw new ConnectorException(String.format("BitSetter can not find type [%s], val [%s]", type, val));
     }
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderDelete.java

@@ -2,8 +2,6 @@ package org.dbsyncer.connector.database.sqlbuilder;
 
 import org.dbsyncer.connector.config.SqlBuilderConfig;
 import org.dbsyncer.connector.database.AbstractSqlBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * @author AE86
@@ -12,14 +10,12 @@ import org.slf4j.LoggerFactory;
  */
 public class SqlBuilderDelete extends AbstractSqlBuilder {
 
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
     @Override
     public String buildSql(SqlBuilderConfig config) {
         String tableName = config.getTableName();
         String quotation = config.getQuotation();
         // DELETE FROM "USER" WHERE "ID"=?
-        return new StringBuilder().append("DELETE FROM ").append(quotation).append(tableName).append(quotation).append(" WHERE ").append(quotation).append(config.getPk()).append(quotation)
+        return new StringBuilder().append("DELETE FROM ").append(config.getSchema()).append(quotation).append(tableName).append(quotation).append(" WHERE ").append(quotation).append(config.getPk()).append(quotation)
                 .append("=?").toString();
     }
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderInsert.java

@@ -35,7 +35,7 @@ public class SqlBuilderInsert extends AbstractSqlBuilder {
             }
         }
         // INSERT INTO "USER"("USERNAME","AGE") VALUES (?,?)
-        sql.insert(0, "INSERT INTO ").append(quotation).append(tableName).append(quotation).append("(").append(fs).append(") VALUES (")
+        sql.insert(0, "INSERT INTO ").append(config.getSchema()).append(quotation).append(tableName).append(quotation).append("(").append(fs).append(") VALUES (")
                 .append(vs).append(")");
         return sql.toString();
     }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQuery.java

@@ -53,7 +53,7 @@ public class SqlBuilderQuery extends AbstractSqlBuilder {
             }
         }
         // SELECT "ID","NAME" FROM "USER"
-        sql.insert(0, "SELECT ").append(" FROM ").append(quotation).append(tableName).append(quotation);
+        sql.insert(0, "SELECT ").append(" FROM ").append(config.getSchema()).append(quotation).append(tableName).append(quotation);
         // 解析查询条件
         if (StringUtil.isNotBlank(queryFilter)) {
             sql.append(queryFilter);

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderUpdate.java

@@ -1,10 +1,8 @@
 package org.dbsyncer.connector.database.sqlbuilder;
 
-import org.dbsyncer.connector.model.Field;
 import org.dbsyncer.connector.config.SqlBuilderConfig;
 import org.dbsyncer.connector.database.AbstractSqlBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.dbsyncer.connector.model.Field;
 
 import java.util.List;
 
@@ -15,8 +13,6 @@ import java.util.List;
  */
 public class SqlBuilderUpdate extends AbstractSqlBuilder {
 
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
     @Override
     public String buildSql(SqlBuilderConfig config) {
         String tableName = config.getTableName();
@@ -25,8 +21,13 @@ public class SqlBuilderUpdate extends AbstractSqlBuilder {
         StringBuilder sql = new StringBuilder();
         int size = fields.size();
         int end = size - 1;
-        sql.append("UPDATE ").append(quotation).append(tableName).append(quotation).append(" SET ");
+        sql.append("UPDATE ").append(config.getSchema()).append(quotation).append(tableName).append(quotation).append(" SET ");
         for (int i = 0; i < size; i++) {
+            // skip pk
+            if(fields.get(i).isPk()){
+               continue;
+            }
+
             // "USERNAME"=?
             sql.append(quotation).append(fields.get(i).getName()).append(quotation).append("=?");
             //如果不是最后一个字段

dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MysqlConnector.java

@@ -1,15 +1,20 @@
 package org.dbsyncer.connector.mysql;
 
-import org.dbsyncer.connector.model.PageSql;
-import org.dbsyncer.connector.model.Table;
 import org.dbsyncer.connector.constant.DatabaseConstant;
 import org.dbsyncer.connector.database.AbstractDatabaseConnector;
 import org.dbsyncer.connector.database.DatabaseConnectorMapper;
+import org.dbsyncer.connector.model.PageSql;
+import org.dbsyncer.connector.model.Table;
 
 import java.util.List;
 
 public final class MysqlConnector extends AbstractDatabaseConnector {
 
+    @Override
+    protected String buildSqlWithQuotation() {
+        return "`";
+    }
+
     @Override
     public String getPageSql(PageSql config) {
         return config.getQuerySql() + DatabaseConstant.MYSQL_PAGE_SQL;
@@ -17,7 +22,7 @@ public final class MysqlConnector extends AbstractDatabaseConnector {
 
     @Override
     public Object[] getPageArgs(int pageIndex, int pageSize) {
-        return new Object[]{(pageIndex - 1) * pageSize, pageSize};
+        return new Object[] {(pageIndex - 1) * pageSize, pageSize};
     }
 
     @Override

dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java

@@ -1,6 +1,9 @@
 package org.dbsyncer.connector.oracle;
 
 import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.config.CommandConfig;
+import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.model.PageSql;
 import org.dbsyncer.connector.model.Table;
 import org.dbsyncer.connector.constant.DatabaseConstant;
@@ -43,4 +46,17 @@ public final class OracleConnector extends AbstractDatabaseConnector {
     protected String getValidationQuery() {
         return "select 1 from dual";
     }
+
+    @Override
+    protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
+        // 有过滤条件，走默认方式
+        if (StringUtil.isNotBlank(queryFilterSql)) {
+            return super.getQueryCountSql(commandConfig, schema, quotation, queryFilterSql);
+        }
+
+        // 从系统表查询
+        final String table = commandConfig.getTable().getName();
+        DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
+        return String.format("SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = '%s' AND TABLE_NAME = '%s'", cfg.getUsername().toUpperCase(), table);
+    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java

@@ -1,6 +1,8 @@
 package org.dbsyncer.connector.postgresql;
 
 import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.config.CommandConfig;
 import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.model.PageSql;
 import org.dbsyncer.connector.model.Table;
@@ -46,4 +48,17 @@ public final class PostgreSQLConnector extends AbstractDatabaseConnector {
     protected String buildSqlWithQuotation() {
         return "\"";
     }
-}
+
+    @Override
+    protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
+        // 有过滤条件，走默认方式
+        if (StringUtil.isNotBlank(queryFilterSql)) {
+            return super.getQueryCountSql(commandConfig, schema, quotation, queryFilterSql);
+        }
+
+        // 从系统表查询
+        final String table = commandConfig.getTable().getName();
+        DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
+        return String.format("SELECT N_LIVE_TUP FROM PG_STAT_USER_TABLES WHERE SCHEMANAME='%s' AND RELNAME='%s'", cfg.getSchema(), table);
+    }
+}

dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/AbstractDQLConnector.java

@@ -43,7 +43,7 @@ public abstract class AbstractDQLConnector extends AbstractDatabaseConnector {
         sql = sql.replace("\r", " ");
         sql = sql.replace("\n", " ");
         String queryMetaSql = StringUtil.contains(sql, " WHERE ") ? cfg.getSql() + " AND 1!=1 " : cfg.getSql() + " WHERE 1!=1 ";
-        return connectorMapper.execute(databaseTemplate -> super.getMetaInfo(databaseTemplate, queryMetaSql, cfg.getTable()));
+        return connectorMapper.execute(databaseTemplate -> super.getMetaInfo(databaseTemplate, queryMetaSql, cfg.getSchema(), cfg.getTable()));
     }
 
     /**
@@ -82,4 +82,4 @@ public abstract class AbstractDQLConnector extends AbstractDatabaseConnector {
         map.put(ConnectorConstant.OPERTION_QUERY_COUNT, queryCount.toString());
         return map;
     }
-}
+}

dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java

@@ -3,23 +3,16 @@ package org.dbsyncer.connector.sqlserver;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.config.CommandConfig;
 import org.dbsyncer.connector.config.DatabaseConfig;
-import org.dbsyncer.connector.constant.ConnectorConstant;
 import org.dbsyncer.connector.constant.DatabaseConstant;
 import org.dbsyncer.connector.database.AbstractDatabaseConnector;
 import org.dbsyncer.connector.database.DatabaseConnectorMapper;
 import org.dbsyncer.connector.model.PageSql;
 import org.dbsyncer.connector.model.Table;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 public final class SqlServerConnector extends AbstractDatabaseConnector {
 
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
     @Override
     public List<Table> getTable(DatabaseConnectorMapper connectorMapper) {
         DatabaseConfig config = connectorMapper.getConfig();
@@ -33,33 +26,19 @@ public final class SqlServerConnector extends AbstractDatabaseConnector {
 
     @Override
     public Object[] getPageArgs(int pageIndex, int pageSize) {
-        return new Object[]{(pageIndex - 1) * pageSize + 1, pageIndex * pageSize};
+        return new Object[] {(pageIndex - 1) * pageSize + 1, pageIndex * pageSize};
     }
 
     @Override
-    public Map<String, String> getSourceCommand(CommandConfig commandConfig) {
-        // 获取过滤SQL
-        String queryFilterSql = this.getQueryFilterSql(commandConfig.getFilter());
-
-        // 获取查询SQL
-        Table table = commandConfig.getTable();
-        Map<String, String> map = new HashMap<>();
-
-        String query = ConnectorConstant.OPERTION_QUERY;
-        map.put(query, this.buildSql(query, commandConfig, queryFilterSql));
-
-        // 获取查询总数SQL
-        StringBuilder queryCount = new StringBuilder();
+    protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
+        // 有过滤条件，走默认方式
         if (StringUtil.isNotBlank(queryFilterSql)) {
-            queryCount.append("SELECT COUNT(*) FROM ").append(table.getName()).append(queryFilterSql);
-        } else {
-            DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
-            // 从存储过程查询（定时更新总数，可能存在误差）
-            queryCount.append("SELECT ROWS FROM SYSINDEXES WHERE ID = OBJECT_ID('").append(cfg.getSchema()).append(".").append(table.getName()).append(
-                    "') AND INDID IN (0, 1)");
+            return super.getQueryCountSql(commandConfig, schema, quotation, queryFilterSql);
         }
-        map.put(ConnectorConstant.OPERTION_QUERY_COUNT, queryCount.toString());
-        return map;
-    }
 
+        String table = commandConfig.getTable().getName();
+        DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
+        // 从存储过程查询（定时更新总数，可能存在误差）
+        return String.format("SELECT ROWS FROM SYSINDEXES WHERE ID = OBJECT_ID('%s.%s') AND INDID IN (0, 1)", cfg.getSchema(), table);
+    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/DatabaseUtil.java

@@ -6,6 +6,9 @@ import org.dbsyncer.connector.ConnectorException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.regex.Matcher;
+
+import static java.util.regex.Pattern.compile;
 
 public abstract class DatabaseUtil {
 
@@ -33,4 +36,19 @@ public abstract class DatabaseUtil {
         }
     }
 
+    public static String getDatabaseName(String url) {
+        Matcher matcher = compile("(//)(?!(\\?)).+?(\\?)").matcher(url);
+        while (matcher.find()) {
+            url = matcher.group(0);
+            break;
+        }
+        int s = url.lastIndexOf("/");
+        int e = url.lastIndexOf("?");
+        if (s > 0 && e > 0) {
+            return StringUtil.substring(url, s + 1, e);
+        }
+
+        throw new ConnectorException("database is invalid");
+    }
+
 }

dbsyncer-listener/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-listener</artifactId>

dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java

@@ -2,7 +2,6 @@ package org.dbsyncer.listener;
 
 import org.dbsyncer.common.event.RowChangedEvent;
 import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.constant.ConnectorConstant;
 import org.dbsyncer.connector.database.DatabaseConnectorMapper;
@@ -39,7 +38,7 @@ public abstract class AbstractDatabaseExtractor extends AbstractExtractor {
             switch (event.getEvent()){
                 case ConnectorConstant.OPERTION_UPDATE:
                 case ConnectorConstant.OPERTION_INSERT:
-                    event.setAfterData(queryData(event.getAfterData()));
+                    event.setDataList(queryData(event.getDataList()));
                     break;
                 default:
                     break;

dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java

@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
 public abstract class AbstractExtractor implements Extractor {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
+    protected String metaId;
     protected Executor taskExecutor;
     protected ConnectorFactory connectorFactory;
     protected ScheduledTaskService scheduledTaskService;
@@ -81,6 +82,10 @@ public abstract class AbstractExtractor implements Extractor {
         }
     }
 
+    public void setMetaId(String metaId) {
+        this.metaId = metaId;
+    }
+
     public void setTaskExecutor(Executor taskExecutor) {
         this.taskExecutor = taskExecutor;
     }

dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java

@@ -2,7 +2,7 @@ package org.dbsyncer.listener.file;
 
 import org.apache.commons.io.IOUtils;
 import org.dbsyncer.common.event.RowChangedEvent;
-import org.dbsyncer.common.util.JsonUtil;
+import org.dbsyncer.common.file.BufferedRandomAccessFile;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.RandomUtil;
 import org.dbsyncer.common.util.StringUtil;
@@ -24,7 +24,6 @@ import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.file.*;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -149,7 +148,7 @@ public class FileExtractor extends AbstractExtractor {
                 snapshot.put(filePosKey, String.valueOf(raf.getFilePointer()));
                 if (StringUtil.isNotBlank(line)) {
                     List<Object> row = fileResolver.parseList(pipelineResolver.fields, separator, line);
-                    changedEvent(new RowChangedEvent(fileName, ConnectorConstant.OPERTION_UPDATE, Collections.EMPTY_LIST, row));
+                    changedEvent(new RowChangedEvent(fileName, ConnectorConstant.OPERTION_UPDATE, row));
                 }
             }
 

dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MysqlExtractor.java

@@ -6,6 +6,7 @@ import org.dbsyncer.common.event.RowChangedEvent;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.constant.ConnectorConstant;
+import org.dbsyncer.connector.util.DatabaseUtil;
 import org.dbsyncer.listener.AbstractDatabaseExtractor;
 import org.dbsyncer.listener.ListenerException;
 import org.dbsyncer.listener.config.Host;
@@ -86,7 +87,7 @@ public class MysqlExtractor extends AbstractDatabaseExtractor {
         if (StringUtil.isBlank(config.getUrl())) {
             throw new ListenerException("url is invalid");
         }
-        database = readDatabaseName(config.getUrl());
+        database = DatabaseUtil.getDatabaseName(config.getUrl());
         cluster = readNodes(config.getUrl());
         Assert.notEmpty(cluster, "Mysql连接地址有误.");
 
@@ -104,21 +105,6 @@ public class MysqlExtractor extends AbstractDatabaseExtractor {
         client.connect();
     }
 
-    private String readDatabaseName(String url) {
-        Matcher matcher = compile("(//)(?!(\\?)).+?(\\?)").matcher(url);
-        while (matcher.find()) {
-            url = matcher.group(0);
-            break;
-        }
-        int s = url.lastIndexOf("/");
-        int e = url.lastIndexOf("?");
-        if (s > 0 && e > 0) {
-            return StringUtil.substring(url, s + 1, e);
-        }
-
-        throw new ListenerException("database is invalid");
-    }
-
     private List<Host> readNodes(String url) {
         Matcher matcher = compile("(//)(?!(/)).+?(/)").matcher(url);
         while (matcher.find()) {
@@ -242,9 +228,8 @@ public class MysqlExtractor extends AbstractDatabaseExtractor {
                 UpdateRowsEventData data = event.getData();
                 if (isFilterTable(data.getTableId())) {
                     data.getRows().forEach(m -> {
-                        List<Object> before = Stream.of(m.getKey()).collect(Collectors.toList());
                         List<Object> after = Stream.of(m.getValue()).collect(Collectors.toList());
-                        sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_UPDATE, before, after));
+                        sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_UPDATE, after));
                     });
                 }
                 return;
@@ -255,7 +240,7 @@ public class MysqlExtractor extends AbstractDatabaseExtractor {
                 if (isFilterTable(data.getTableId())) {
                     data.getRows().forEach(m -> {
                         List<Object> after = Stream.of(m).collect(Collectors.toList());
-                        sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_INSERT, Collections.EMPTY_LIST, after));
+                        sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_INSERT, after));
                     });
                 }
                 return;
@@ -266,7 +251,7 @@ public class MysqlExtractor extends AbstractDatabaseExtractor {
                 if (isFilterTable(data.getTableId())) {
                     data.getRows().forEach(m -> {
                         List<Object> before = Stream.of(m).collect(Collectors.toList());
-                        sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_DELETE, before, Collections.EMPTY_LIST));
+                        sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_DELETE, before));
                     });
                 }
                 return;

dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java

@@ -353,15 +353,15 @@ public class DBChangeNotification {
             List<Object> data = new ArrayList<>();
             if (event.getCode() == TableChangeDescription.TableOperation.UPDATE.getCode()) {
                 read(event.getTableName(), event.getRowId(), data);
-                listeners.forEach(listener -> listener.onEvents(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_UPDATE, Collections.EMPTY_LIST, data)));
+                listeners.forEach(listener -> listener.onEvents(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_UPDATE, data)));
 
             } else if (event.getCode() == TableChangeDescription.TableOperation.INSERT.getCode()) {
                 read(event.getTableName(), event.getRowId(), data);
-                listeners.forEach(listener -> listener.onEvents(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_INSERT, Collections.EMPTY_LIST, data)));
+                listeners.forEach(listener -> listener.onEvents(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_INSERT, data)));
 
             } else {
                 data.add(event.getRowId());
-                listeners.forEach(listener -> listener.onEvents(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_DELETE, data, Collections.EMPTY_LIST)));
+                listeners.forEach(listener -> listener.onEvents(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_DELETE, data)));
             }
         }
     }

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java

@@ -1,7 +1,6 @@
 package org.dbsyncer.listener.postgresql;
 
 import org.dbsyncer.connector.config.DatabaseConfig;
-import org.dbsyncer.listener.postgresql.column.ColumnValue;
 import org.dbsyncer.listener.postgresql.column.PgColumnValue;
 import org.dbsyncer.listener.postgresql.enums.MessageTypeEnum;
 import org.postgresql.replication.LogSequenceNumber;
@@ -16,9 +15,11 @@ import java.nio.ByteBuffer;
  */
 public abstract class AbstractMessageDecoder implements MessageDecoder {
 
+    protected String metaId;
+
     protected DatabaseConfig config;
 
-    private ColumnValue value = new PgColumnValue();
+    private static final PgColumnValue value = new PgColumnValue();
 
     @Override
     public boolean skipMessage(ByteBuffer buffer, LogSequenceNumber startLsn, LogSequenceNumber lastReceiveLsn) {
@@ -49,7 +50,12 @@ public abstract class AbstractMessageDecoder implements MessageDecoder {
 
     @Override
     public String getSlotName() {
-        return String.format("dbs_slot_%s_%s", config.getSchema(), config.getUsername());
+        return String.format("dbs_slot_%s_%s_%s", config.getSchema(), config.getUsername(), metaId);
+    }
+
+    @Override
+    public void setMetaId(String metaId) {
+        this.metaId = metaId;
     }
 
     @Override
@@ -58,7 +64,7 @@ public abstract class AbstractMessageDecoder implements MessageDecoder {
     }
 
     /**
-     * Resolve the value of a {@link ColumnValue}.
+     * Resolve value
      *
      * @param typeName
      * @param columnValue
@@ -105,7 +111,7 @@ public abstract class AbstractMessageDecoder implements MessageDecoder {
 
             case "numeric":
             case "decimal":
-                return value.asDecimal();
+                return value.asBigDecimal();
 
             case "character":
             case "char":

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java

@@ -29,6 +29,8 @@ public interface MessageDecoder {
 
     void withSlotOption(ChainedLogicalStreamBuilder builder);
 
+    void setMetaId(String metaId);
+
     void setConfig(DatabaseConfig config);
 
 }

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java

@@ -93,6 +93,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor {
 
             database = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_DATABASE, String.class));
             messageDecoder = MessageDecoderEnum.getMessageDecoder(config.getProperty(PLUGIN_NAME));
+            messageDecoder.setMetaId(metaId);
             messageDecoder.setConfig(config);
             messageDecoder.postProcessBeforeInitialization(connectorFactory, connectorMapper);
             dropSlotOnClose = BooleanUtil.toBoolean(config.getProperty(DROP_SLOT_ON_CLOSE, "true"));

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/column/AbstractColumnValue.java

@@ -1,182 +0,0 @@
-package org.dbsyncer.listener.postgresql.column;
-
-import org.dbsyncer.common.util.DateFormatUtil;
-import org.dbsyncer.listener.ListenerException;
-import org.postgresql.PGStatement;
-import org.postgresql.geometric.*;
-import org.postgresql.util.PGInterval;
-import org.postgresql.util.PGmoney;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.SQLException;
-import java.sql.Date;
-import java.sql.Timestamp;
-import java.time.Instant;
-import java.time.OffsetDateTime;
-import java.time.OffsetTime;
-import java.time.ZoneOffset;
-import java.util.concurrent.TimeUnit;
-
-public abstract class AbstractColumnValue implements ColumnValue {
-
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
-    @Override
-    public Date asDate() {
-        return DateFormatUtil.stringToDate(asString());
-    }
-
-    @Override
-    public Object asTime() {
-        return asString();
-    }
-
-    @Override
-    public Object asLocalTime() {
-        return DateFormatUtil.stringToLocalTime(asString());
-    }
-
-    @Override
-    public OffsetTime asOffsetTimeUtc() {
-        return DateFormatUtil.timeWithTimeZone(asString());
-    }
-
-    @Override
-    public OffsetDateTime asOffsetDateTimeAtUtc() {
-        if ("infinity".equals(asString())) {
-            return OffsetDateTime.ofInstant(toInstantFromMillis(PGStatement.DATE_POSITIVE_INFINITY), ZoneOffset.UTC);
-        } else if ("-infinity".equals(asString())) {
-            return OffsetDateTime.ofInstant(toInstantFromMillis(PGStatement.DATE_NEGATIVE_INFINITY), ZoneOffset.UTC);
-        }
-        return DateFormatUtil.timestampWithTimeZoneToOffsetDateTime(asString());
-    }
-
-    @Override
-    public Timestamp asTimestamp() {
-        if ("infinity".equals(asString())) {
-            return Timestamp.from(toInstantFromMicros(PGStatement.DATE_POSITIVE_INFINITY));
-        } else if ("-infinity".equals(asString())) {
-            return Timestamp.from(toInstantFromMicros(PGStatement.DATE_NEGATIVE_INFINITY));
-        }
-        return DateFormatUtil.stringToTimestamp(asString());
-    }
-
-    @Override
-    public PGbox asBox() {
-        try {
-            return new PGbox(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse point {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public PGcircle asCircle() {
-        try {
-            return new PGcircle(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse circle {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public Object asInterval() {
-        try {
-            return new PGInterval(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse point {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public PGline asLine() {
-        try {
-            return new PGline(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse point {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public PGlseg asLseg() {
-        try {
-            return new PGlseg(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse point {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public PGmoney asMoney() {
-        try {
-            final String value = asString();
-            if (value != null && value.startsWith("-")) {
-                final String negativeMoney = "(" + value.substring(1) + ")";
-                return new PGmoney(negativeMoney);
-            }
-            return new PGmoney(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse money {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public PGpath asPath() {
-        try {
-            return new PGpath(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse point {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public PGpoint asPoint() {
-        try {
-            return new PGpoint(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse point {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public PGpolygon asPolygon() {
-        try {
-            return new PGpolygon(asString());
-        } catch (final SQLException e) {
-            logger.error("Failed to parse point {}, {}", asString(), e);
-            throw new ListenerException(e);
-        }
-    }
-
-    @Override
-    public boolean isArray() {
-        return false;
-    }
-
-    @Override
-    public Object asArray() {
-        return null;
-    }
-
-    private Instant toInstantFromMicros(long microsSinceEpoch) {
-        return Instant.ofEpochSecond(
-                TimeUnit.MICROSECONDS.toSeconds(microsSinceEpoch),
-                TimeUnit.MICROSECONDS.toNanos(microsSinceEpoch % TimeUnit.SECONDS.toMicros(1)));
-    }
-
-    private Instant toInstantFromMillis(long millisecondSinceEpoch) {
-        return Instant.ofEpochSecond(
-                TimeUnit.MILLISECONDS.toSeconds(millisecondSinceEpoch),
-                TimeUnit.MILLISECONDS.toNanos(millisecondSinceEpoch % TimeUnit.SECONDS.toMillis(1)));
-    }
-
-}

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/column/ColumnValue.java

@@ -1,73 +0,0 @@
-package org.dbsyncer.listener.postgresql.column;
-
-import org.postgresql.geometric.*;
-import org.postgresql.util.PGmoney;
-
-import java.sql.Date;
-import java.sql.Timestamp;
-import java.time.OffsetDateTime;
-import java.time.OffsetTime;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2022/4/22 22:39
- * @see org.postgresql.jdbc.TypeInfoCache
- */
-public interface ColumnValue {
-
-    void setValue(String value);
-
-    boolean isNull();
-
-    String asString();
-
-    Boolean asBoolean();
-
-    Integer asInteger();
-
-    Long asLong();
-
-    Float asFloat();
-
-    Double asDouble();
-
-    Object asDecimal();
-
-    Date asDate();
-
-    OffsetDateTime asOffsetDateTimeAtUtc();
-
-    Timestamp asTimestamp();
-
-    Object asTime();
-
-    Object asLocalTime();
-
-    OffsetTime asOffsetTimeUtc();
-
-    byte[] asByteArray();
-
-    PGbox asBox();
-
-    PGcircle asCircle();
-
-    Object asInterval();
-
-    PGline asLine();
-
-    Object asLseg();
-
-    PGmoney asMoney();
-
-    PGpath asPath();
-
-    PGpoint asPoint();
-
-    PGpolygon asPolygon();
-
-    boolean isArray();
-
-    Object asArray();
-
-}

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/column/PgColumnValue.java

@@ -1,59 +1,206 @@
 package org.dbsyncer.listener.postgresql.column;
 
+import org.dbsyncer.common.column.AbstractColumnValue;
+import org.dbsyncer.common.util.DateFormatUtil;
 import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.listener.ListenerException;
+import org.postgresql.PGStatement;
+import org.postgresql.geometric.*;
+import org.postgresql.util.PGInterval;
+import org.postgresql.util.PGmoney;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.SQLException;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.time.*;
+import java.util.concurrent.TimeUnit;
 
-public final class PgColumnValue extends AbstractColumnValue {
+public final class PgColumnValue extends AbstractColumnValue<String> {
 
-    private String value;
-
-    public void setValue(String value) {
-        this.value = value;
-    }
+    private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Override
-    public boolean isNull() {
-        return value == null;
+    public String asString() {
+        return getValue();
     }
 
     @Override
-    public String asString() {
-        return value;
+    public byte[] asByteArray() {
+        return StringUtil.hexStringToByteArray(getValue().substring(2));
     }
 
     @Override
-    public Boolean asBoolean() {
-        return "t".equalsIgnoreCase(value);
+    public Short asShort() {
+        return Short.valueOf(getValue());
     }
 
     @Override
     public Integer asInteger() {
-        return Integer.valueOf(value);
+        return Integer.valueOf(getValue());
     }
 
     @Override
     public Long asLong() {
-        return Long.valueOf(value);
+        return Long.valueOf(getValue());
     }
 
     @Override
     public Float asFloat() {
-        return Float.valueOf(value);
+        return Float.valueOf(getValue());
     }
 
     @Override
     public Double asDouble() {
-        return Double.valueOf(value);
+        return Double.valueOf(getValue());
     }
 
     @Override
-    public Object asDecimal() {
-        return new BigDecimal(value);
+    public Boolean asBoolean() {
+        return "t".equalsIgnoreCase(getValue());
     }
 
     @Override
-    public byte[] asByteArray() {
-        return StringUtil.hexStringToByteArray(value.substring(2));
+    public BigDecimal asBigDecimal() {
+        return new BigDecimal(getValue());
+    }
+
+    @Override
+    public Date asDate() {
+        return DateFormatUtil.stringToDate(asString());
+    }
+
+    @Override
+    public Timestamp asTimestamp() {
+        if ("infinity".equals(asString())) {
+            return Timestamp.from(toInstantFromMicros(PGStatement.DATE_POSITIVE_INFINITY));
+        } else if ("-infinity".equals(asString())) {
+            return Timestamp.from(toInstantFromMicros(PGStatement.DATE_NEGATIVE_INFINITY));
+        }
+        return DateFormatUtil.stringToTimestamp(asString());
+    }
+
+    @Override
+    public Time asTime() {
+        return Time.valueOf(getValue());
+    }
+
+    public LocalTime asLocalTime() {
+        return DateFormatUtil.stringToLocalTime(asString());
+    }
+
+    public OffsetTime asOffsetTimeUtc() {
+        return DateFormatUtil.timeWithTimeZone(asString());
+    }
+
+    public OffsetDateTime asOffsetDateTimeAtUtc() {
+        if ("infinity".equals(asString())) {
+            return OffsetDateTime.ofInstant(toInstantFromMillis(PGStatement.DATE_POSITIVE_INFINITY), ZoneOffset.UTC);
+        } else if ("-infinity".equals(asString())) {
+            return OffsetDateTime.ofInstant(toInstantFromMillis(PGStatement.DATE_NEGATIVE_INFINITY), ZoneOffset.UTC);
+        }
+        return DateFormatUtil.timestampWithTimeZoneToOffsetDateTime(asString());
     }
+
+    public PGbox asBox() {
+        try {
+            return new PGbox(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse point {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    public PGcircle asCircle() {
+        try {
+            return new PGcircle(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse circle {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    public Object asInterval() {
+        try {
+            return new PGInterval(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse point {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    public PGline asLine() {
+        try {
+            return new PGline(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse point {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    public PGlseg asLseg() {
+        try {
+            return new PGlseg(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse point {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    public PGmoney asMoney() {
+        try {
+            final String value = asString();
+            if (value != null && value.startsWith("-")) {
+                final String negativeMoney = "(" + value.substring(1) + ")";
+                return new PGmoney(negativeMoney);
+            }
+            return new PGmoney(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse money {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    public PGpath asPath() {
+        try {
+            return new PGpath(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse point {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    public PGpoint asPoint() {
+        try {
+            return new PGpoint(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse point {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    public PGpolygon asPolygon() {
+        try {
+            return new PGpolygon(asString());
+        } catch (final SQLException e) {
+            logger.error("Failed to parse point {}, {}", asString(), e);
+            throw new ListenerException(e);
+        }
+    }
+
+    private Instant toInstantFromMicros(long microsSinceEpoch) {
+        return Instant.ofEpochSecond(
+                TimeUnit.MICROSECONDS.toSeconds(microsSinceEpoch),
+                TimeUnit.MICROSECONDS.toNanos(microsSinceEpoch % TimeUnit.SECONDS.toMicros(1)));
+    }
+
+    private Instant toInstantFromMillis(long millisecondSinceEpoch) {
+        return Instant.ofEpochSecond(
+                TimeUnit.MILLISECONDS.toSeconds(millisecondSinceEpoch),
+                TimeUnit.MILLISECONDS.toNanos(millisecondSinceEpoch % TimeUnit.SECONDS.toMillis(1)));
+    }
+
 }

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java

@@ -29,7 +29,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     private static final LocalDateTime PG_EPOCH = LocalDateTime.of(2000, 1, 1, 0, 0, 0);
-    private static final String GET_TABLE_SCHEMA = "select oid,relname as tableName from pg_class t inner join (select ns.oid as nspoid, ns.nspname from pg_namespace ns where ns.nspname = (select (current_schemas(false))[s.r] from generate_series(1, array_upper(current_schemas(false), 1)) as s(r))) as n on n.nspoid = t.relnamespace where relkind = 'r'";
+    private static final String GET_TABLE_SCHEMA = "select oid,relname as tableName from pg_class t inner join (select ns.oid as nspoid, ns.nspname from pg_namespace ns where ns.nspname = '%s') as n on n.nspoid = t.relnamespace where relkind = 'r'";
     private static final Map<Integer, TableId> tables = new LinkedHashMap<>();
     private ConnectorFactory connectorFactory;
     private DatabaseConnectorMapper connectorMapper;
@@ -114,7 +114,8 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder {
     }
 
     private void readSchema() {
-        List<Map> schemas = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(GET_TABLE_SCHEMA));
+        final String querySchema = String.format(GET_TABLE_SCHEMA, config.getSchema());
+        List<Map> schemas = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(querySchema));
         if (!CollectionUtils.isEmpty(schemas)) {
             schemas.forEach(map -> {
                 Long oid = (Long) map.get("oid");
@@ -137,10 +138,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder {
                 case "O":
                     List<Object> data = new ArrayList<>();
                     readTupleData(tableId, buffer, data);
-                    if (MessageTypeEnum.DELETE == type) {
-                        return new RowChangedEvent(tableId.tableName, type.name(), data, Collections.EMPTY_LIST);
-                    }
-                    return new RowChangedEvent(tableId.tableName, type.name(), Collections.EMPTY_LIST, data);
+                    return new RowChangedEvent(tableId.tableName, type.name(), data);
 
                 default:
                     logger.info("N, K, O not set, got instead {}", newTuple);

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java

@@ -81,11 +81,8 @@ public class TestDecodingMessageDecoder extends AbstractMessageDecoder {
         switch (eventType) {
             case ConnectorConstant.OPERTION_UPDATE:
             case ConnectorConstant.OPERTION_INSERT:
-                event = new RowChangedEvent(table, eventType, Collections.EMPTY_LIST, data);
-                break;
-
             case ConnectorConstant.OPERTION_DELETE:
-                event = new RowChangedEvent(table, eventType, data, Collections.EMPTY_LIST);
+                event = new RowChangedEvent(table, eventType, data);
                 break;
 
             default:

dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java

@@ -13,7 +13,6 @@ import org.dbsyncer.listener.AbstractExtractor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -113,21 +112,21 @@ public abstract class AbstractQuartzExtractor extends AbstractExtractor implemen
             Object event = null;
             for (Map<String, Object> row : data) {
                 if(StringUtil.isBlank(eventFieldName)){
-                    changedEvent(new RowChangedEvent(index, ConnectorConstant.OPERTION_UPDATE, Collections.EMPTY_MAP, row));
+                    changedEvent(new RowChangedEvent(index, ConnectorConstant.OPERTION_UPDATE, row));
                     continue;
                 }
 
                 event = row.get(eventFieldName);
                 if (update.contains(event)) {
-                    changedEvent(new RowChangedEvent(index, ConnectorConstant.OPERTION_UPDATE, Collections.EMPTY_MAP, row));
+                    changedEvent(new RowChangedEvent(index, ConnectorConstant.OPERTION_UPDATE, row));
                     continue;
                 }
                 if (insert.contains(event)) {
-                    changedEvent(new RowChangedEvent(index, ConnectorConstant.OPERTION_INSERT, Collections.EMPTY_MAP, row));
+                    changedEvent(new RowChangedEvent(index, ConnectorConstant.OPERTION_INSERT, row));
                     continue;
                 }
                 if (delete.contains(event)) {
-                    changedEvent(new RowChangedEvent(index, ConnectorConstant.OPERTION_DELETE, row, Collections.EMPTY_MAP));
+                    changedEvent(new RowChangedEvent(index, ConnectorConstant.OPERTION_DELETE, row));
                     continue;
                 }
 

dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java

@@ -34,7 +34,6 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor {
     private static final String STATEMENTS_PLACEHOLDER = "#";
     private static final String GET_DATABASE_NAME = "SELECT db_name()";
     private static final String GET_TABLE_LIST = "SELECT NAME FROM SYS.TABLES WHERE SCHEMA_ID = SCHEMA_ID('#') AND IS_MS_SHIPPED = 0";
-    private static final String IS_SERVER_AGENT_RUNNING = "EXEC master.#.xp_servicecontrol N'QUERYSTATE', N'SQLSERVERAGENT'";
     private static final String IS_DB_CDC_ENABLED = "SELECT is_cdc_enabled FROM sys.databases WHERE name = '#'";
     private static final String IS_TABLE_CDC_ENABLED = "SELECT COUNT(*) FROM sys.tables tb WHERE tb.is_tracked_by_cdc = 1 AND tb.name='#'";
     private static final String ENABLE_DB_CDC = "IF EXISTS(select 1 from sys.databases where name = '#' AND is_cdc_enabled=0) EXEC sys.sp_cdc_enable_db";
@@ -49,8 +48,8 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor {
     private static final int OFFSET_COLUMNS = 4;
     private final Lock connectLock = new ReentrantLock();
     private volatile boolean connected;
-    private Set<String> tables;
-    private Set<SqlServerChangeTable> changeTables;
+    private static Set<String> tables;
+    private static Set<SqlServerChangeTable> changeTables;
     private DatabaseConnectorMapper connectorMapper;
     private Worker worker;
     private Lsn lastLsn;
@@ -70,9 +69,6 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor {
             readTables();
             Assert.notEmpty(tables, "No tables available");
 
-            boolean enabledServerAgent = queryAndMap(IS_SERVER_AGENT_RUNNING.replace(STATEMENTS_PLACEHOLDER, schema), rs -> "Running.".equals(rs.getString(1)));
-            Assert.isTrue(enabledServerAgent, "Please ensure that the SQL Server Agent is running");
-
             enableDBCDC();
             enableTableCDC();
             readChangeTables();
@@ -250,17 +246,17 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor {
         for (CDCEvent event : list) {
             int code = event.getCode();
             if (TableOperationEnum.isUpdateAfter(code)) {
-                sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_UPDATE, Collections.EMPTY_LIST, event.getRow()));
+                sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_UPDATE, event.getRow()));
                 continue;
             }
 
             if (TableOperationEnum.isInsert(code)) {
-                sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_INSERT, Collections.EMPTY_LIST, event.getRow()));
+                sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_INSERT, event.getRow()));
                 continue;
             }
 
             if (TableOperationEnum.isDelete(code)) {
-                sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_DELETE, event.getRow(), Collections.EMPTY_LIST));
+                sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_DELETE, event.getRow()));
             }
         }
     }
@@ -329,10 +325,10 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor {
                         continue;
                     }
 
+                    pull(stopLsn);
+
                     lastLsn = stopLsn;
                     snapshot.put(LSN_POSITION, lastLsn.toString());
-
-                    pull(stopLsn);
                 } catch (Exception e) {
                     logger.error(e.getMessage());
                     sleepInMills(1000L);

dbsyncer-listener/src/main/test/DBChangeNotificationTest.java

@@ -28,7 +28,7 @@ public class DBChangeNotificationTest {
 
         final DBChangeNotification dcn = new DBChangeNotification(username, password, url);
         dcn.addRowEventListener((e) ->
-            logger.info("{}触发{}, before:{}, after:{}", e.getSourceTableName(), e.getEvent(), e.getBeforeData(), e.getAfterData())
+            logger.info("{}触发{}, data:{}", e.getSourceTableName(), e.getEvent(), e.getDataList())
         );
         dcn.start();
 

dbsyncer-manager/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-manager</artifactId>

dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java

@@ -93,12 +93,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addConnector(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD));
     }
 
     @Override
     public String editConnector(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT));
     }
 
     @Override
@@ -137,12 +137,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addMapping(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD));
     }
 
     @Override
     public String editMapping(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT));
     }
 
     @Override
@@ -166,12 +166,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addTableGroup(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, GroupStrategyEnum.TABLE, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD, GroupStrategyEnum.TABLE));
     }
 
     @Override
     public String editTableGroup(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, GroupStrategyEnum.TABLE, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT, GroupStrategyEnum.TABLE));
     }
 
     @Override
@@ -206,12 +206,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addMeta(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD));
     }
 
     @Override
     public String editMeta(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT));
     }
 
     @Override
@@ -234,12 +234,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addConfig(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD));
     }
 
     @Override
     public String editConfig(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT));
     }
 
     @Override

dbsyncer-manager/src/main/java/org/dbsyncer/manager/config/OperationConfig.java

@@ -1,6 +1,7 @@
 package org.dbsyncer.manager.config;
 
 import org.dbsyncer.manager.enums.GroupStrategyEnum;
+import org.dbsyncer.manager.enums.HandlerEnum;
 import org.dbsyncer.manager.template.Handler;
 import org.dbsyncer.parser.model.ConfigModel;
 
@@ -10,9 +11,9 @@ public class OperationConfig {
 
     private ConfigModel model;
 
-    private GroupStrategyEnum groupStrategyEnum;
+    private GroupStrategyEnum groupStrategyEnum = GroupStrategyEnum.DEFAULT;
 
-    private Handler handler;
+    private HandlerEnum handlerEnum;
 
     public OperationConfig(String id) {
         this.id = id;
@@ -23,15 +24,15 @@ public class OperationConfig {
         this.groupStrategyEnum = groupStrategyEnum;
     }
 
-    public OperationConfig(ConfigModel model, Handler handler) {
+    public OperationConfig(ConfigModel model, HandlerEnum handlerEnum) {
         this.model = model;
-        this.handler = handler;
+        this.handlerEnum = handlerEnum;
     }
 
-    public OperationConfig(ConfigModel model, GroupStrategyEnum groupStrategyEnum, Handler handler) {
+    public OperationConfig(ConfigModel model, HandlerEnum handlerEnum, GroupStrategyEnum groupStrategyEnum) {
         this.model = model;
+        this.handlerEnum = handlerEnum;
         this.groupStrategyEnum = groupStrategyEnum;
-        this.handler = handler;
     }
 
     public String getId() {
@@ -47,6 +48,6 @@ public class OperationConfig {
     }
 
     public Handler getHandler() {
-        return handler;
+        return handlerEnum.getHandler();
     }
 }

dbsyncer-manager/src/main/java/org/dbsyncer/manager/config/PreloadConfig.java

@@ -1,37 +0,0 @@
-package org.dbsyncer.manager.config;
-
-import org.dbsyncer.manager.enums.GroupStrategyEnum;
-import org.dbsyncer.manager.enums.HandlerEnum;
-import org.dbsyncer.manager.template.Handler;
-
-public class PreloadConfig {
-
-    private String filterType;
-
-    private GroupStrategyEnum groupStrategyEnum;
-
-    private HandlerEnum handlerEnum;
-
-    public PreloadConfig(String filterType, HandlerEnum handlerEnum) {
-        this.filterType = filterType;
-        this.handlerEnum = handlerEnum;
-    }
-
-    public PreloadConfig(String filterType, GroupStrategyEnum groupStrategyEnum, HandlerEnum handlerEnum) {
-        this.filterType = filterType;
-        this.groupStrategyEnum = groupStrategyEnum;
-        this.handlerEnum = handlerEnum;
-    }
-
-    public String getFilterType() {
-        return filterType;
-    }
-
-    public GroupStrategyEnum getGroupStrategyEnum() {
-        return groupStrategyEnum;
-    }
-
-    public HandlerEnum getHandlerEnum() {
-        return handlerEnum;
-    }
-}

dbsyncer-manager/src/main/java/org/dbsyncer/manager/config/QueryConfig.java

@@ -7,7 +7,7 @@ public class QueryConfig<T> {
 
     private ConfigModel configModel;
 
-    private GroupStrategyEnum groupStrategyEnum;
+    private GroupStrategyEnum groupStrategyEnum = GroupStrategyEnum.DEFAULT;
 
     public QueryConfig(ConfigModel configModel) {
         this.configModel = configModel;

dbsyncer-manager/src/main/java/org/dbsyncer/manager/enums/HandlerEnum.java

@@ -5,6 +5,7 @@ import org.dbsyncer.manager.config.PreloadCallBack;
 import org.dbsyncer.manager.handler.AbstractOperationHandler;
 import org.dbsyncer.manager.handler.AbstractPreloadHandler;
 import org.dbsyncer.manager.template.Handler;
+import org.dbsyncer.storage.constant.ConfigConstant;
 
 /**
  * @author AE86
@@ -16,74 +17,106 @@ public enum HandlerEnum {
     /**
      * 添加
      */
-    OPR_ADD(new AbstractOperationHandler(){
+    OPR_ADD("add", new AbstractOperationHandler() {
         @Override
         protected void handle(OperationCallBack operationCallBack) {
             operationCallBack.add();
         }
     }),
+
     /**
      * 修改
      */
-    OPR_EDIT(new AbstractOperationHandler(){
+    OPR_EDIT("edit", new AbstractOperationHandler() {
         @Override
         protected void handle(OperationCallBack operationCallBack) {
             operationCallBack.edit();
         }
     }),
+
     /**
      * 预加载Connector
      */
-    PRELOAD_CONNECTOR(new AbstractPreloadHandler(){
+    PRELOAD_CONNECTOR(ConfigConstant.CONNECTOR, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseConnector();
         }
     }),
+
     /**
      * 预加载Mapping
      */
-    PRELOAD_MAPPING(new AbstractPreloadHandler(){
+    PRELOAD_MAPPING(ConfigConstant.MAPPING, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseMapping();
         }
     }),
+
     /**
      * 预加载TableGroup
      */
-    PRELOAD_TABLE_GROUP(new AbstractPreloadHandler(){
+    PRELOAD_TABLE_GROUP(ConfigConstant.TABLE_GROUP, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseTableGroup();
         }
-    }),
+    }, GroupStrategyEnum.TABLE),
+
     /**
      * 预加载Meta
      */
-    PRELOAD_META(new AbstractPreloadHandler(){
+    PRELOAD_META(ConfigConstant.META, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseMeta();
         }
     }),
+
     /**
      * 预加载Config
      */
-    PRELOAD_CONFIG(new AbstractPreloadHandler(){
+    PRELOAD_CONFIG(ConfigConstant.CONFIG, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseConfig();
         }
     });
 
+    private String modelType;
+    private boolean preload;
     private Handler handler;
+    private GroupStrategyEnum groupStrategyEnum;
+
+    HandlerEnum(String modelType, Handler handler) {
+        this(modelType, false, handler, null);
+    }
 
-    HandlerEnum(Handler handler) {
+    HandlerEnum(String modelType, boolean preload, Handler handler) {
+        this(modelType, preload, handler, GroupStrategyEnum.DEFAULT);
+    }
+
+    HandlerEnum(String modelType, boolean preload, Handler handler, GroupStrategyEnum groupStrategyEnum) {
+        this.modelType = modelType;
+        this.preload = preload;
         this.handler = handler;
+        this.groupStrategyEnum = groupStrategyEnum;
+    }
+
+    public String getModelType() {
+        return modelType;
+    }
+
+    public boolean isPreload() {
+        return preload;
     }
 
     public Handler getHandler() {
         return handler;
     }
-}
+
+    public GroupStrategyEnum getGroupStrategyEnum() {
+        return groupStrategyEnum;
+    }
+}

dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java

@@ -37,10 +37,10 @@ import org.springframework.util.Assert;
 
 import javax.annotation.PostConstruct;
 import java.time.Instant;
+import java.time.LocalDateTime;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
@@ -164,7 +164,7 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob
     }
 
     private void setExtractorConfig(AbstractExtractor extractor, ConnectorConfig connector, ListenerConfig listener,
-                                    Map<String, String> snapshot, Event event) {
+                                    Map<String, String> snapshot, AbstractListener event) {
         extractor.setTaskExecutor(taskExecutor);
         extractor.setConnectorFactory(connectorFactory);
         extractor.setScheduledTaskService(scheduledTaskService);
@@ -172,19 +172,21 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob
         extractor.setListenerConfig(listener);
         extractor.setSnapshot(snapshot);
         extractor.addListener(event);
+        extractor.setMetaId(event.metaId);
     }
 
     abstract class AbstractListener implements Event {
+        private static final int FLUSH_DELAYED_SECONDS = 30;
         protected Mapping mapping;
         protected String metaId;
-        protected AtomicBoolean changed = new AtomicBoolean();
+        private LocalDateTime updateTime = LocalDateTime.now();
 
         @Override
         public void flushEvent(Map<String, String> map) {
-            // 如果有变更，执行更新
-            if (changed.compareAndSet(true, false)) {
+            // 30s内更新，执行写入
+            if (updateTime.isAfter(LocalDateTime.now().minusSeconds(FLUSH_DELAYED_SECONDS))) {
                 if (!CollectionUtils.isEmpty(map)) {
-                    logger.info("{}", map);
+                    logger.debug("{}", map);
                 }
                 forceFlushEvent(map);
             }
@@ -199,6 +201,11 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob
             }
         }
 
+        @Override
+        public void refreshFlushEventUpdateTime() {
+            updateTime = LocalDateTime.now();
+        }
+
         @Override
         public void errorEvent(Exception e) {
             logService.log(LogType.TableGroupLog.INCREMENT_FAILED, e.getMessage());
@@ -244,13 +251,12 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob
             final FieldPicker picker = tablePicker.get(rowChangedEvent.getTableGroupIndex());
             TableGroup tableGroup = picker.getTableGroup();
             rowChangedEvent.setSourceTableName(tableGroup.getSourceTable().getName());
-            rowChangedEvent.setTargetTableName(tableGroup.getTargetTable().getName());
 
             // 处理过程有异常向上抛
             parser.execute(mapping, tableGroup, rowChangedEvent);
 
             // 标记有变更记录
-            changed.compareAndSet(false, true);
+            refreshFlushEventUpdateTime();
         }
     }
 
@@ -303,17 +309,14 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob
             List<FieldPicker> pickers = tablePicker.get(rowChangedEvent.getSourceTableName());
             if (!CollectionUtils.isEmpty(pickers)) {
                 pickers.forEach(picker -> {
-                    final Map<String, Object> before = picker.getColumns(rowChangedEvent.getBeforeData());
-                    final Map<String, Object> after = picker.getColumns(rowChangedEvent.getAfterData());
-                    if (picker.filter(StringUtil.equals(ConnectorConstant.OPERTION_DELETE, rowChangedEvent.getEvent()) ? before : after)) {
-                        rowChangedEvent.setBefore(before);
-                        rowChangedEvent.setAfter(after);
-                        rowChangedEvent.setTargetTableName(picker.getTableGroup().getTargetTable().getName());
+                    final Map<String, Object> dataMap = picker.getColumns(rowChangedEvent.getDataList());
+                    if (picker.filter(dataMap)) {
+                        rowChangedEvent.setDataMap(dataMap);
                         parser.execute(mapping, picker.getTableGroup(), rowChangedEvent);
                     }
                 });
                 // 标记有变更记录
-                changed.compareAndSet(false, true);
+                refreshFlushEventUpdateTime();
                 eventCounter.set(0);
                 return;
             }

dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/TableGroupStrategy.java

@@ -3,7 +3,9 @@ package org.dbsyncer.manager.strategy;
 import org.dbsyncer.manager.ManagerException;
 import org.dbsyncer.manager.template.GroupStrategy;
 import org.dbsyncer.parser.model.ConfigModel;
+import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.storage.constant.ConfigConstant;
 
 /**
  * @author AE86
@@ -21,6 +23,10 @@ public class TableGroupStrategy implements GroupStrategy {
             // 格式：${type} + "_" + ${mappingId}
             return new StringBuilder(type).append("_").append(mappingId).toString();
         }
+        if (model instanceof Mapping) {
+            Mapping m = (Mapping) model;
+            return new StringBuilder(ConfigConstant.TABLE_GROUP).append("_").append(m.getId()).toString();
+        }
         throw new ManagerException(String.format("Not support config model \"%s\".", model));
     }
 

dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/AbstractTemplate.java

@@ -1,26 +0,0 @@
-package org.dbsyncer.manager.template;
-
-import org.dbsyncer.manager.config.OperationConfig;
-import org.dbsyncer.manager.config.PreloadConfig;
-import org.dbsyncer.manager.config.QueryConfig;
-import org.dbsyncer.manager.enums.GroupStrategyEnum;
-
-public abstract class AbstractTemplate {
-
-    protected GroupStrategyEnum getDefaultStrategy(PreloadConfig config){
-        return getDefaultStrategy(config.getGroupStrategyEnum());
-    }
-
-    protected GroupStrategyEnum getDefaultStrategy(OperationConfig config){
-        return getDefaultStrategy(config.getGroupStrategyEnum());
-    }
-
-    protected GroupStrategyEnum getDefaultStrategy(QueryConfig query){
-        return getDefaultStrategy(query.getGroupStrategyEnum());
-    }
-
-    private GroupStrategyEnum getDefaultStrategy(GroupStrategyEnum strategy){
-        return null != strategy ? strategy : GroupStrategyEnum.DEFAULT;
-    }
-
-}

dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/impl/OperationTemplate.java

@@ -8,7 +8,6 @@ import org.dbsyncer.manager.config.OperationCallBack;
 import org.dbsyncer.manager.config.OperationConfig;
 import org.dbsyncer.manager.config.QueryConfig;
 import org.dbsyncer.manager.enums.GroupStrategyEnum;
-import org.dbsyncer.manager.template.AbstractTemplate;
 import org.dbsyncer.manager.template.GroupStrategy;
 import org.dbsyncer.manager.template.Handler;
 import org.dbsyncer.parser.model.ConfigModel;
@@ -22,7 +21,11 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 
 /**
  * 操作配置模板
@@ -32,7 +35,7 @@ import java.util.*;
  * @date 2019/9/16 23:59
  */
 @Component
-public final class OperationTemplate extends AbstractTemplate {
+public final class OperationTemplate {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -44,10 +47,10 @@ public final class OperationTemplate extends AbstractTemplate {
 
     public <T> List<T> queryAll(QueryConfig<T> query) {
         ConfigModel model = query.getConfigModel();
-        String groupId = getGroupId(model, getDefaultStrategy(query));
+        String groupId = getGroupId(model, query.getGroupStrategyEnum());
         Group group = cacheService.get(groupId, Group.class);
         if (null != group) {
-            List<String> index = group.getAll();
+            List<String> index = group.getIndex();
             if (!CollectionUtils.isEmpty(index)) {
                 List<T> list = new ArrayList<>();
                 Class<? extends ConfigModel> clazz = model.getClass();
@@ -84,8 +87,7 @@ public final class OperationTemplate extends AbstractTemplate {
         handler.execute(new OperationCallBack(storageService, StorageEnum.CONFIG, params));
 
         // 3、缓存
-        GroupStrategyEnum strategy = getDefaultStrategy(config);
-        cache(model, strategy);
+        cache(model, config.getGroupStrategyEnum());
         return model.getId();
     }
 
@@ -108,7 +110,7 @@ public final class OperationTemplate extends AbstractTemplate {
         Assert.hasText(id, "ID can not be empty.");
         // 删除分组
         ConfigModel model = cacheService.get(id, ConfigModel.class);
-        String groupId = getGroupId(model, getDefaultStrategy(config));
+        String groupId = getGroupId(model, config.getGroupStrategyEnum());
         Group group = cacheService.get(groupId, Group.class);
         if (null != group) {
             group.remove(id);
@@ -120,7 +122,7 @@ public final class OperationTemplate extends AbstractTemplate {
         storageService.remove(StorageEnum.CONFIG, id);
     }
 
-    private String getGroupId(ConfigModel model, GroupStrategyEnum strategy) {
+    public String getGroupId(ConfigModel model, GroupStrategyEnum strategy) {
         Assert.notNull(model, "ConfigModel can not be null.");
         Assert.notNull(strategy, "GroupStrategyEnum can not be null.");
         GroupStrategy groupStrategy = strategy.getGroupStrategy();
@@ -146,7 +148,7 @@ public final class OperationTemplate extends AbstractTemplate {
         }
     }
 
-    class Group {
+    static class Group {
 
         private List<String> index;
 
@@ -168,10 +170,13 @@ public final class OperationTemplate extends AbstractTemplate {
             return index.size();
         }
 
-        public List<String> getAll() {
+        public List<String> getIndex() {
             return Collections.unmodifiableList(index);
         }
 
+        public void setIndex(List<String> index) {
+            this.index = index;
+        }
     }
 
 }

dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/impl/PreloadTemplate.java

@@ -1,15 +1,16 @@
 package org.dbsyncer.manager.template.impl;
 
+import com.alibaba.fastjson.JSONObject;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.manager.Manager;
+import org.dbsyncer.manager.config.OperationConfig;
 import org.dbsyncer.manager.config.PreloadCallBack;
-import org.dbsyncer.manager.config.PreloadConfig;
 import org.dbsyncer.manager.config.QueryConfig;
-import org.dbsyncer.manager.enums.GroupStrategyEnum;
 import org.dbsyncer.manager.enums.HandlerEnum;
-import org.dbsyncer.manager.template.AbstractTemplate;
 import org.dbsyncer.manager.template.Handler;
+import org.dbsyncer.manager.template.impl.OperationTemplate.Group;
 import org.dbsyncer.parser.Parser;
 import org.dbsyncer.parser.enums.MetaEnum;
 import org.dbsyncer.parser.model.ConfigModel;
@@ -26,6 +27,7 @@ import org.springframework.context.ApplicationListener;
 import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.stereotype.Component;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -37,7 +39,7 @@ import java.util.Map;
  * @date 2019/9/16 23:59
  */
 @Component
-public final class PreloadTemplate extends AbstractTemplate implements ApplicationListener<ContextRefreshedEvent> {
+public final class PreloadTemplate implements ApplicationListener<ContextRefreshedEvent> {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -53,16 +55,16 @@ public final class PreloadTemplate extends AbstractTemplate implements Applicati
     @Autowired
     private OperationTemplate operationTemplate;
 
-    public void execute(PreloadConfig config) {
+    public void execute(HandlerEnum handlerEnum) {
         Query query = new Query();
         query.setType(StorageEnum.CONFIG);
-        String filterType = config.getFilterType();
-        query.addFilter(ConfigConstant.CONFIG_MODEL_TYPE, filterType);
+        String modelType = handlerEnum.getModelType();
+        query.addFilter(ConfigConstant.CONFIG_MODEL_TYPE, modelType);
 
         int pageNum = 1;
         int pageSize = 20;
         long total = 0;
-        for(;;){
+        for (; ; ) {
             query.setPageNum(pageNum);
             query.setPageSize(pageSize);
             Paging paging = storageService.query(query);
@@ -70,42 +72,69 @@ public final class PreloadTemplate extends AbstractTemplate implements Applicati
             if (CollectionUtils.isEmpty(data)) {
                 break;
             }
-            Handler handler = config.getHandlerEnum().getHandler();
-            GroupStrategyEnum strategy = getDefaultStrategy(config);
+            Handler handler = handlerEnum.getHandler();
             data.forEach(map -> {
                 String json = (String) map.get(ConfigConstant.CONFIG_MODEL_JSON);
                 ConfigModel model = (ConfigModel) handler.execute(new PreloadCallBack(parser, json));
                 if (null != model) {
-                    operationTemplate.cache(model, strategy);
+                    operationTemplate.cache(model, handlerEnum.getGroupStrategyEnum());
                 }
             });
             total += paging.getTotal();
-            pageNum ++;
+            pageNum++;
         }
 
-        logger.info("PreLoad {}:{}", filterType, total);
+        logger.info("PreLoad {}:{}", modelType, total);
     }
 
-    @Override
-    public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
+    public void reload(String json) {
+        Map<String, JSONObject> map = JsonUtil.jsonToObj(json, Map.class);
+        if (CollectionUtils.isEmpty(map)) {
+            return;
+        }
+
+        // Load configs
+        reload(map, HandlerEnum.PRELOAD_CONFIG);
         // Load connectors
-        execute(new PreloadConfig(ConfigConstant.CONNECTOR, HandlerEnum.PRELOAD_CONNECTOR));
+        reload(map, HandlerEnum.PRELOAD_CONNECTOR);
         // Load mappings
-        execute(new PreloadConfig(ConfigConstant.MAPPING, HandlerEnum.PRELOAD_MAPPING));
-        // Load tableGroups
-        execute(new PreloadConfig(ConfigConstant.TABLE_GROUP, GroupStrategyEnum.TABLE, HandlerEnum.PRELOAD_TABLE_GROUP));
+        reload(map, HandlerEnum.PRELOAD_MAPPING);
         // Load metas
-        execute(new PreloadConfig(ConfigConstant.META, HandlerEnum.PRELOAD_META));
-        // Load configs
-        execute(new PreloadConfig(ConfigConstant.CONFIG, HandlerEnum.PRELOAD_CONFIG));
+        reload(map, HandlerEnum.PRELOAD_META);
 
-        // Load plugins
-        manager.loadPlugins();
+        launch();
+    }
 
-        // Check connectors status
-        manager.checkAllConnectorStatus();
+    private void reload(Map<String, JSONObject> map, HandlerEnum handlerEnum) {
+        reload(map, handlerEnum, handlerEnum.getModelType());
+    }
 
-        // Launch drivers
+    private void reload(Map<String, JSONObject> map, HandlerEnum handlerEnum, String groupId) {
+        JSONObject config = map.get(groupId);
+        Group group = JsonUtil.jsonToObj(config.toJSONString(), Group.class);
+        if (null == group) {
+            return;
+        }
+
+        List<String> index = group.getIndex();
+        if (CollectionUtils.isEmpty(index)) {
+            return;
+        }
+
+        Handler handler = handlerEnum.getHandler();
+        for (String e : index) {
+            JSONObject m = map.get(e);
+            ConfigModel model = (ConfigModel) handler.execute(new PreloadCallBack(parser, m.toJSONString()));
+            operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD, handlerEnum.getGroupStrategyEnum()));
+            // Load tableGroups
+            if (HandlerEnum.PRELOAD_MAPPING == handlerEnum) {
+                handlerEnum = HandlerEnum.PRELOAD_TABLE_GROUP;
+                reload(map, handlerEnum, operationTemplate.getGroupId(model, handlerEnum.getGroupStrategyEnum()));
+            }
+        }
+    }
+
+    private void launch() {
         Meta meta = new Meta();
         meta.setType(ConfigConstant.META);
         QueryConfig<Meta> queryConfig = new QueryConfig<>(meta);
@@ -123,4 +152,19 @@ public final class PreloadTemplate extends AbstractTemplate implements Applicati
         }
     }
 
+    @Override
+    public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
+        // Load configModels
+        Arrays.stream(HandlerEnum.values()).filter(handlerEnum -> handlerEnum.isPreload()).forEach(handlerEnum -> execute(handlerEnum));
+
+        // Load plugins
+        manager.loadPlugins();
+
+        // Check connectors status
+        manager.checkAllConnectorStatus();
+
+        // Launch drivers
+        launch();
+    }
+
 }

dbsyncer-monitor/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-monitor</artifactId>

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java

@@ -33,7 +33,7 @@ public interface Monitor {
 
     List<MetricEnum> getMetricEnumAll();
 
-    List<MetricResponse> getThreadPoolInfo();
+    List<MetricResponse> getMetricInfo();
 
     AppReportMetric getAppReportMetric();
 

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java

@@ -1,6 +1,5 @@
 package org.dbsyncer.monitor;
 
-import org.dbsyncer.common.config.ThreadPoolConfig;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
@@ -8,10 +7,12 @@ import org.dbsyncer.connector.constant.ConnectorConstant;
 import org.dbsyncer.manager.Manager;
 import org.dbsyncer.monitor.enums.MetricEnum;
 import org.dbsyncer.monitor.enums.StatisticEnum;
+import org.dbsyncer.monitor.enums.TaskMetricEnum;
 import org.dbsyncer.monitor.enums.ThreadPoolMetricEnum;
 import org.dbsyncer.monitor.model.AppReportMetric;
 import org.dbsyncer.monitor.model.MetricResponse;
 import org.dbsyncer.monitor.model.Sample;
+import org.dbsyncer.parser.flush.BufferActuator;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.storage.constant.ConfigConstant;
@@ -25,7 +26,6 @@ import org.springframework.stereotype.Component;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicLong;
@@ -46,7 +46,10 @@ public class MonitorFactory implements Monitor {
     private Executor taskExecutor;
 
     @Autowired
-    private ThreadPoolConfig threadPoolConfig;
+    private BufferActuator writerBufferActuator;
+
+    @Autowired
+    private BufferActuator storageBufferActuator;
 
     @Override
     public Mapping getMapping(String mappingId) {
@@ -105,16 +108,18 @@ public class MonitorFactory implements Monitor {
     }
 
     @Override
-    public List<MetricResponse> getThreadPoolInfo() {
+    public List<MetricResponse> getMetricInfo() {
         ThreadPoolTaskExecutor threadTask = (ThreadPoolTaskExecutor) taskExecutor;
         ThreadPoolExecutor pool = threadTask.getThreadPoolExecutor();
 
         List<MetricResponse> list = new ArrayList<>();
-        list.add(createMetricResponse(ThreadPoolMetricEnum.TASK_SUBMITTED, pool.getTaskCount()));
-        list.add(createMetricResponse(ThreadPoolMetricEnum.QUEUE_UP, pool.getQueue().size()));
-        list.add(createMetricResponse(ThreadPoolMetricEnum.ACTIVE, pool.getActiveCount()));
-        list.add(createMetricResponse(ThreadPoolMetricEnum.COMPLETED, pool.getCompletedTaskCount()));
-        list.add(createMetricResponse(ThreadPoolMetricEnum.REMAINING_CAPACITY, pool.getQueue().remainingCapacity()));
+        list.add(createTaskMetricResponse(TaskMetricEnum.STORAGE_ACTIVE, storageBufferActuator.getQueue().size()));
+        list.add(createTaskMetricResponse(TaskMetricEnum.STORAGE_REMAINING_CAPACITY, storageBufferActuator.getQueueCapacity() - storageBufferActuator.getQueue().size()));
+        list.add(createThreadPoolMetricResponse(ThreadPoolMetricEnum.TASK_SUBMITTED, pool.getTaskCount()));
+        list.add(createThreadPoolMetricResponse(ThreadPoolMetricEnum.QUEUE_UP, pool.getQueue().size()));
+        list.add(createThreadPoolMetricResponse(ThreadPoolMetricEnum.ACTIVE, pool.getActiveCount()));
+        list.add(createThreadPoolMetricResponse(ThreadPoolMetricEnum.COMPLETED, pool.getCompletedTaskCount()));
+        list.add(createThreadPoolMetricResponse(ThreadPoolMetricEnum.REMAINING_CAPACITY, pool.getQueue().remainingCapacity()));
         return list;
     }
 
@@ -127,13 +132,8 @@ public class MonitorFactory implements Monitor {
         report.setInsert(getMappingInsert(metaAll));
         report.setUpdate(getMappingUpdate(metaAll));
         report.setDelete(getMappingDelete(metaAll));
-
-        // 线程池使用情况
-        ThreadPoolTaskExecutor threadTask = (ThreadPoolTaskExecutor) taskExecutor;
-        ThreadPoolExecutor pool = threadTask.getThreadPoolExecutor();
-        BlockingQueue<Runnable> queue = pool.getQueue();
-        report.setQueueUp(queue.size());
-        report.setQueueCapacity(threadPoolConfig.getQueueCapacity());
+        report.setQueueUp(writerBufferActuator.getQueue().size());
+        report.setQueueCapacity(writerBufferActuator.getQueueCapacity());
         return report;
     }
 
@@ -187,7 +187,11 @@ public class MonitorFactory implements Monitor {
         return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_DELETE));
     }
 
-    private MetricResponse createMetricResponse(ThreadPoolMetricEnum metricEnum, Object value) {
+    private MetricResponse createThreadPoolMetricResponse(ThreadPoolMetricEnum metricEnum, Object value) {
+        return new MetricResponse(metricEnum.getCode(), metricEnum.getGroup(), metricEnum.getMetricName(), Arrays.asList(new Sample(StatisticEnum.COUNT.getTagValueRepresentation(), value)));
+    }
+
+    private MetricResponse createTaskMetricResponse(TaskMetricEnum metricEnum, Object value) {
         return new MetricResponse(metricEnum.getCode(), metricEnum.getGroup(), metricEnum.getMetricName(), Arrays.asList(new Sample(StatisticEnum.COUNT.getTagValueRepresentation(), value)));
     }
 

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/TaskMetricEnum.java

@@ -0,0 +1,43 @@
+package org.dbsyncer.monitor.enums;
+
+/**
+ * 执行任务指标
+ *
+ * @author AE86
+ * @version 1.0.0
+ * @date 2021/07/23 0:19
+ */
+public enum TaskMetricEnum {
+
+    /**
+     * 处理中
+     */
+    STORAGE_ACTIVE("parser.storage.buffer.actuator.active", "持久化", "处理中"),
+
+    /**
+     * 空闲队列
+     */
+    STORAGE_REMAINING_CAPACITY("parser.storage.buffer.actuator.capacity", "持久化", "空闲队列");
+
+    private String code;
+    private String group;
+    private String metricName;
+
+    TaskMetricEnum(String code, String group, String metricName) {
+        this.code = code;
+        this.group = group;
+        this.metricName = metricName;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public String getGroup() {
+        return group;
+    }
+
+    public String getMetricName() {
+        return metricName;
+    }
+}

dbsyncer-parser/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-parser</artifactId>
@@ -38,7 +38,7 @@
             <artifactId>dbsyncer-storage</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
-        
+
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserFactory.java

@@ -8,7 +8,10 @@ import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.ConnectorFactory;
 import org.dbsyncer.connector.ConnectorMapper;
-import org.dbsyncer.connector.config.*;
+import org.dbsyncer.connector.config.CommandConfig;
+import org.dbsyncer.connector.config.ConnectorConfig;
+import org.dbsyncer.connector.config.ReaderConfig;
+import org.dbsyncer.connector.config.WriterBatchConfig;
 import org.dbsyncer.connector.constant.ConnectorConstant;
 import org.dbsyncer.connector.enums.ConnectorEnum;
 import org.dbsyncer.connector.enums.FilterEnum;
@@ -20,12 +23,11 @@ import org.dbsyncer.listener.enums.QuartzFilterEnum;
 import org.dbsyncer.parser.enums.ConvertEnum;
 import org.dbsyncer.parser.enums.ParserEnum;
 import org.dbsyncer.parser.event.FullRefreshEvent;
-import org.dbsyncer.parser.flush.BufferActuator;
-import org.dbsyncer.parser.flush.FlushStrategy;
-import org.dbsyncer.parser.flush.model.WriterRequest;
 import org.dbsyncer.parser.logger.LogService;
 import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.*;
+import org.dbsyncer.parser.strategy.FlushStrategy;
+import org.dbsyncer.parser.strategy.ParserStrategy;
 import org.dbsyncer.parser.util.ConvertUtil;
 import org.dbsyncer.parser.util.PickerUtil;
 import org.dbsyncer.plugin.PluginFactory;
@@ -81,7 +83,7 @@ public class ParserFactory implements Parser {
     private ApplicationContext applicationContext;
 
     @Autowired
-    private BufferActuator writerBufferActuator;
+    private ParserStrategy parserStrategy;
 
     @Override
     public ConnectorMapper connect(ConnectorConfig config) {
@@ -297,23 +299,20 @@ public class ParserFactory implements Parser {
 
     @Override
     public void execute(Mapping mapping, TableGroup tableGroup, RowChangedEvent event) {
-        logger.info("Table[{}] {}, before:{}, after:{}", event.getSourceTableName(), event.getEvent(),
-                event.getBefore(), event.getAfter());
+        logger.debug("Table[{}] {}, data:{}", event.getSourceTableName(), event.getEvent(), event.getDataMap());
 
         // 1、获取映射字段
-        final String eventName = event.getEvent();
-        Map<String, Object> data = StringUtil.equals(ConnectorConstant.OPERTION_DELETE, eventName) ? event.getBefore() : event.getAfter();
-        Picker picker = new Picker(tableGroup.getFieldMapping(), data);
-        Map target = picker.getTargetMap();
+        final Picker picker = new Picker(tableGroup.getFieldMapping());
+        final Map target = picker.pickData(event.getDataMap());
 
         // 2、参数转换
         ConvertUtil.convert(tableGroup.getConvert(), target);
 
         // 3、插件转换
-        pluginFactory.convert(tableGroup.getPlugin(), eventName, data, target);
+        pluginFactory.convert(tableGroup.getPlugin(), event.getEvent(), event.getDataMap(), target);
 
-        // 4、写入缓冲执行器
-        writerBufferActuator.offer(new WriterRequest(tableGroup.getId(), target, mapping.getMetaId(), mapping.getTargetConnectorId(), event.getSourceTableName(), event.getTargetTableName(), eventName, picker.getTargetFields(), tableGroup.getCommand()));
+        // 4、处理数据
+        parserStrategy.execute(tableGroup.getId(), event.getEvent(), target);
     }
 
     /**
@@ -330,12 +329,11 @@ public class ParserFactory implements Parser {
         String event = batchWriter.getEvent();
         Map<String, String> command = batchWriter.getCommand();
         List<Field> fields = batchWriter.getFields();
-        boolean forceUpdate = batchWriter.isForceUpdate();
         // 总数
         int total = dataList.size();
         // 单次任务
         if (total <= batchSize) {
-            return connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, dataList, forceUpdate));
+            return connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, dataList));
         }
 
         // 批量任务, 拆分
@@ -358,7 +356,7 @@ public class ParserFactory implements Parser {
 
             taskExecutor.execute(() -> {
                 try {
-                    Result w = connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, data, forceUpdate));
+                    Result w = connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, data));
                     result.addSuccessData(w.getSuccessData());
                     result.addFailData(w.getFailData());
                     result.getError().append(w.getError());

dbsyncer-parser/src/main/java/org/dbsyncer/parser/config/ParserFlushStrategyConfiguration.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/config/ParserStrategyConfiguration.java

@@ -1,7 +1,9 @@
 package org.dbsyncer.parser.config;
 
-import org.dbsyncer.parser.flush.FlushStrategy;
-import org.dbsyncer.parser.flush.impl.DisableFullFlushStrategy;
+import org.dbsyncer.parser.strategy.FlushStrategy;
+import org.dbsyncer.parser.strategy.ParserStrategy;
+import org.dbsyncer.parser.strategy.impl.DisableFullFlushStrategy;
+import org.dbsyncer.parser.strategy.impl.DisableWriterBufferActuatorStrategy;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -12,7 +14,7 @@ import org.springframework.context.annotation.Configuration;
  * @date 2021/11/18 21:36
  */
 @Configuration
-public class ParserFlushStrategyConfiguration {
+public class ParserStrategyConfiguration {
 
     @Bean
     @ConditionalOnMissingBean
@@ -20,4 +22,10 @@ public class ParserFlushStrategyConfiguration {
         return new DisableFullFlushStrategy();
     }
 
+    @Bean
+    @ConditionalOnMissingBean
+    public ParserStrategy parserStrategy() {
+        return new DisableWriterBufferActuatorStrategy();
+    }
+
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/DateToChineseStandardTimeHandler.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/TimestampToChineseStandardTimeHandler.java

@@ -3,22 +3,22 @@ package org.dbsyncer.parser.convert.handler;
 import org.dbsyncer.common.util.DateFormatUtil;
 import org.dbsyncer.parser.convert.AbstractHandler;
 
-import java.sql.Date;
+import java.sql.Timestamp;
 
 /**
- * Date转中国标准时间
+ * Timestamp转中国标准时间
  *
  * @author AE86
  * @version 1.0.0
  * @date 2021/12/20 23:04
  */
-public class DateToChineseStandardTimeHandler extends AbstractHandler {
+public class TimestampToChineseStandardTimeHandler extends AbstractHandler {
 
     @Override
     public Object convert(String args, Object value) {
-        if (value instanceof Date) {
-            Date d = (Date) value;
-            value = DateFormatUtil.dateToChineseStandardTimeString(d);
+        if (value instanceof Timestamp) {
+            Timestamp t = (Timestamp) value;
+            value = DateFormatUtil.timestampToString(t);
         }
         return value;
     }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/ConvertEnum.java

@@ -30,6 +30,10 @@ public enum ConvertEnum {
      * Timestamp转Date
      */
     TIMESTAMP_TO_DATE("TIMESTAMP_TO_DATE", "Timestamp转Date", 0, new TimestampToDateHandler()),
+    /**
+     * Timestamp转中国标准时间
+     */
+    TIMESTAMP_TO_CHINESE_STANDARD_TIME("TIMESTAMP_TO_CHINESE_STANDARD_TIME", "Timestamp转yyyy-MM-dd HH:mm:ss", 0, new TimestampToChineseStandardTimeHandler()),
     /**
      * Timestamp转Long
      */
@@ -38,10 +42,6 @@ public enum ConvertEnum {
      * Long转Timestamp
      */
     LONG_TO_TIMESTAMP("LONG_TO_TIMESTAMP", "Long转Timestamp", 0, new LongToTimestampHandler()),
-    /**
-     * Date转中国标准时间
-     */
-    DATE_TO_CHINESE_STANDARD_TIME("DATE_TO_CHINESE_STANDARD_TIME", "Date转yyyy-MM-dd HH:mm:ss", 0, new DateToChineseStandardTimeHandler()),
     /**
      * Byte[]转String
      */

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java

@@ -7,17 +7,21 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import javax.annotation.PostConstruct;
+import java.lang.reflect.ParameterizedType;
 import java.time.Instant;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
+ * 任务缓存执行器
+ * <p>1. 任务优先进入缓存队列
+ * <p>2. 将任务分区合并，批量执行
+ *
  * @author AE86
  * @version 1.0.0
  * @date 2022/3/27 17:36
@@ -31,35 +35,25 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
 
     private static final int CAPACITY = 10_0000;
 
-    private Queue<Request> buffer = new LinkedBlockingQueue(CAPACITY);
+    private static final int MAX_BATCH_COUNT = 2000;
 
-    private Queue<Request> temp = new LinkedBlockingQueue(CAPACITY);
+    private static final int PERIOD = 300;
+
+    private Queue<Request> buffer;
 
     private final Lock lock = new ReentrantLock(true);
 
     private volatile boolean running;
 
-    private final static long MAX_BATCH_COUNT = 1000L;
+    private Class<Response> responseClazz;
 
     @PostConstruct
     private void init() {
-        scheduledTaskService.start(getPeriod(), this);
+        responseClazz = (Class<Response>) ((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[1];
+        buffer = new LinkedBlockingQueue(getQueueCapacity());
+        scheduledTaskService.start(PERIOD, this);
     }
 
-    /**
-     * 获取定时间隔（毫秒）
-     *
-     * @return
-     */
-    protected abstract long getPeriod();
-
-    /**
-     * 生成缓存value
-     *
-     * @return
-     */
-    protected abstract BufferResponse getValue();
-
     /**
      * 生成分区key
      *
@@ -84,23 +78,18 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
     protected abstract void pull(Response response);
 
     @Override
-    public void offer(BufferRequest request) {
-        if (running) {
-            temp.offer((Request) request);
-        } else {
-            buffer.offer((Request) request);
-        }
+    public Queue getQueue() {
+        return buffer;
+    }
 
-        // TODO 临时解决方案：生产大于消费问题，限制生产速度
-        int size = temp.size() + buffer.size();
-        if (size >= CAPACITY) {
-            try {
-                TimeUnit.SECONDS.sleep(30);
-                logger.warn("当前任务队列大小{}已达上限{}，请稍等{}秒", size, CAPACITY, 30);
-            } catch (InterruptedException e) {
-                logger.error(e.getMessage());
-            }
-        }
+    @Override
+    public int getQueueCapacity() {
+        return CAPACITY;
+    }
+
+    @Override
+    public void offer(BufferRequest request) {
+        buffer.offer((Request) request);
     }
 
     @Override
@@ -116,8 +105,6 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
             if (locked) {
                 running = true;
                 flush(buffer);
-                running = false;
-                flush(temp);
             }
         } catch (Exception e) {
             logger.error(e.getMessage());
@@ -129,7 +116,7 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
         }
     }
 
-    private void flush(Queue<Request> queue) {
+    private void flush(Queue<Request> queue) throws IllegalAccessException, InstantiationException {
         if (!queue.isEmpty()) {
             AtomicLong batchCounter = new AtomicLong();
             final Map<String, BufferResponse> map = new LinkedHashMap<>();
@@ -137,7 +124,7 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
                 Request poll = queue.poll();
                 String key = getPartitionKey(poll);
                 if (!map.containsKey(key)) {
-                    map.putIfAbsent(key, getValue());
+                    map.putIfAbsent(key, (BufferResponse) responseClazz.newInstance());
                 }
                 partition(poll, (Response) map.get(key));
                 batchCounter.incrementAndGet();
@@ -150,7 +137,7 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
                 } catch (Exception e) {
                     logger.error("[{}]异常{}", key);
                 }
-                logger.info("[{}]{}条，耗时{}秒", key, flushTask.getTaskSize(), (Instant.now().toEpochMilli() - now) / 1000);
+                logger.info("[{}]{}条，耗时{}毫秒", key, flushTask.getTaskSize(), (Instant.now().toEpochMilli() - now));
             });
             map.clear();
         }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java

@@ -5,6 +5,7 @@ import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.parser.model.Meta;
+import org.dbsyncer.parser.strategy.FlushStrategy;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.util.Assert;
 
@@ -46,16 +47,11 @@ public abstract class AbstractFlushStrategy implements FlushStrategy {
     }
 
     protected void refreshTotal(String metaId, Result writer) {
-        Meta meta = getMeta(metaId);
-        meta.getFail().getAndAdd(writer.getFailData().size());
-        meta.getSuccess().getAndAdd(writer.getSuccessData().size());
-    }
-
-    protected Meta getMeta(String metaId) {
         Assert.hasText(metaId, "Meta id can not be empty.");
         Meta meta = cacheService.get(metaId, Meta.class);
         Assert.notNull(meta, "Meta can not be null.");
-        return meta;
+        meta.getFail().getAndAdd(writer.getFailData().size());
+        meta.getSuccess().getAndAdd(writer.getSuccessData().size());
     }
 
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/BufferActuator.java

@@ -1,5 +1,7 @@
 package org.dbsyncer.parser.flush;
 
+import java.util.Queue;
+
 /**
  * @author AE86
  * @version 1.0.0
@@ -7,6 +9,25 @@ package org.dbsyncer.parser.flush;
  */
 public interface BufferActuator {
 
+    /**
+     * 获取缓存队列
+     *
+     * @return
+     */
+    Queue getQueue();
+
+    /**
+     * 获取缓存队列容量
+     *
+     * @return
+     */
+    int getQueueCapacity();
+
+    /**
+     * 提交任务
+     *
+     * @param request
+     */
     void offer(BufferRequest request);
 
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java

@@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSONException;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.parser.flush.BufferActuator;
 import org.dbsyncer.parser.flush.FlushService;
-import org.dbsyncer.parser.flush.model.StorageRequest;
+import org.dbsyncer.parser.model.StorageRequest;
 import org.dbsyncer.common.snowflake.SnowflakeIdWorker;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java

@@ -1,9 +1,8 @@
 package org.dbsyncer.parser.flush.impl;
 
 import org.dbsyncer.parser.flush.AbstractBufferActuator;
-import org.dbsyncer.parser.flush.BufferResponse;
-import org.dbsyncer.parser.flush.model.StorageRequest;
-import org.dbsyncer.parser.flush.model.StorageResponse;
+import org.dbsyncer.parser.model.StorageRequest;
+import org.dbsyncer.parser.model.StorageResponse;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.enums.StorageEnum;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -21,13 +20,8 @@ public class StorageBufferActuator extends AbstractBufferActuator<StorageRequest
     private StorageService storageService;
 
     @Override
-    protected long getPeriod() {
-        return 500;
-    }
-
-    @Override
-    protected BufferResponse getValue() {
-        return new StorageResponse();
+    public int getQueueCapacity() {
+        return super.getQueueCapacity() / 4;
     }
 
     @Override

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/WriterBufferActuator.java

@@ -7,18 +7,12 @@ import org.dbsyncer.connector.ConnectorMapper;
 import org.dbsyncer.connector.config.ConnectorConfig;
 import org.dbsyncer.parser.ParserFactory;
 import org.dbsyncer.parser.flush.AbstractBufferActuator;
-import org.dbsyncer.parser.flush.BufferResponse;
-import org.dbsyncer.parser.flush.FlushStrategy;
-import org.dbsyncer.parser.flush.model.WriterRequest;
-import org.dbsyncer.parser.flush.model.WriterResponse;
-import org.dbsyncer.parser.model.BatchWriter;
-import org.dbsyncer.parser.model.Connector;
+import org.dbsyncer.parser.model.*;
+import org.dbsyncer.parser.strategy.FlushStrategy;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
-import java.util.Collections;
-
 /**
  * @author AE86
  * @version 1.0.0
@@ -41,16 +35,6 @@ public class WriterBufferActuator extends AbstractBufferActuator<WriterRequest,
 
     private final static int BATCH_SIZE = 100;
 
-    @Override
-    protected long getPeriod() {
-        return 300;
-    }
-
-    @Override
-    protected BufferResponse getValue() {
-        return new WriterResponse();
-    }
-
     @Override
     protected String getPartitionKey(WriterRequest request) {
         return new StringBuilder(request.getTableGroupId()).append("-").append(request.getEvent()).toString();
@@ -62,22 +46,26 @@ public class WriterBufferActuator extends AbstractBufferActuator<WriterRequest,
         if (response.isMerged()) {
             return;
         }
-        response.setMetaId(request.getMetaId());
-        response.setTargetConnectorId(request.getTargetConnectorId());
-        response.setSourceTableName(request.getSourceTableName());
-        response.setTargetTableName(request.getTargetTableName());
+        response.setTableGroupId(request.getTableGroupId());
         response.setEvent(request.getEvent());
-        response.setFields(Collections.unmodifiableList(request.getFields()));
-        response.setCommand(request.getCommand());
         response.setMerged(true);
     }
 
     @Override
     protected void pull(WriterResponse response) {
-        ConnectorMapper targetConnectorMapper = connectorFactory.connect(getConnectorConfig(response.getTargetConnectorId()));
-        Result result = parserFactory.writeBatch(new BatchWriter(targetConnectorMapper, response.getCommand(), response.getTargetTableName(), response.getEvent(),
-                response.getFields(), response.getDataList(), BATCH_SIZE, true));
-        flushStrategy.flushIncrementData(response.getMetaId(), result, response.getEvent());
+        // 1、获取配置信息
+        final TableGroup tableGroup = cacheService.get(response.getTableGroupId(), TableGroup.class);
+        final Mapping mapping = cacheService.get(tableGroup.getMappingId(), Mapping.class);
+        final String targetTableName = tableGroup.getTargetTable().getName();
+        final Picker picker = new Picker(tableGroup.getFieldMapping());
+
+        // 2、批量执行同步
+        ConnectorMapper targetConnectorMapper = connectorFactory.connect(getConnectorConfig(mapping.getTargetConnectorId()));
+        Result result = parserFactory.writeBatch(new BatchWriter(targetConnectorMapper, tableGroup.getCommand(), targetTableName, response.getEvent(),
+                picker.getTargetFields(), response.getDataList(), BATCH_SIZE));
+
+        // 3、持久化同步结果
+        flushStrategy.flushIncrementData(mapping.getMetaId(), result, response.getEvent());
     }
 
     /**

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/AbstractWriter.java

@@ -1,84 +0,0 @@
-package org.dbsyncer.parser.flush.model;
-
-import org.dbsyncer.connector.model.Field;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2022/4/4 23:02
- */
-public abstract class AbstractWriter {
-
-    private String metaId;
-
-    private String targetConnectorId;
-
-    private String sourceTableName;
-
-    private String targetTableName;
-
-    private List<Field> fields;
-
-    private Map<String, String> command;
-
-    private String event;
-
-    public String getMetaId() {
-        return metaId;
-    }
-
-    public void setMetaId(String metaId) {
-        this.metaId = metaId;
-    }
-
-    public String getTargetConnectorId() {
-        return targetConnectorId;
-    }
-
-    public void setTargetConnectorId(String targetConnectorId) {
-        this.targetConnectorId = targetConnectorId;
-    }
-
-    public String getSourceTableName() {
-        return sourceTableName;
-    }
-
-    public void setSourceTableName(String sourceTableName) {
-        this.sourceTableName = sourceTableName;
-    }
-
-    public String getTargetTableName() {
-        return targetTableName;
-    }
-
-    public void setTargetTableName(String targetTableName) {
-        this.targetTableName = targetTableName;
-    }
-
-    public List<Field> getFields() {
-        return fields;
-    }
-
-    public void setFields(List<Field> fields) {
-        this.fields = fields;
-    }
-
-    public Map<String, String> getCommand() {
-        return command;
-    }
-
-    public void setCommand(Map<String, String> command) {
-        this.command = command;
-    }
-
-    public String getEvent() {
-        return event;
-    }
-
-    public void setEvent(String event) {
-        this.event = event;
-    }
-}

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/WriterRequest.java

@@ -1,40 +0,0 @@
-package org.dbsyncer.parser.flush.model;
-
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.parser.flush.BufferRequest;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2022/3/27 16:57
- */
-public class WriterRequest extends AbstractWriter implements BufferRequest {
-
-    private String tableGroupId;
-
-    private Map row;
-
-    public WriterRequest(String tableGroupId, Map row, String metaId, String targetConnectorId, String sourceTableName, String targetTableName, String event, List<Field> fields, Map<String, String> command) {
-        setMetaId(metaId);
-        setTargetConnectorId(targetConnectorId);
-        setSourceTableName(sourceTableName);
-        setTargetTableName(targetTableName);
-        setEvent(event);
-        setFields(fields);
-        setCommand(command);
-        this.tableGroupId = tableGroupId;
-        this.row = row;
-    }
-
-    public String getTableGroupId() {
-        return tableGroupId;
-    }
-
-    public Map getRow() {
-        return row;
-    }
-
-}

dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogType.java

@@ -242,7 +242,8 @@ public interface LogType {
      */
     enum CacheLog implements LogType {
         IMPORT("70", "导入配置"),
-        EXPORT("71", "导出配置");
+        IMPORT_ERROR("71", "导入配置异常"),
+        EXPORT("72", "导出配置");
 
         private String type;
         private String message;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractWriter.java

@@ -0,0 +1,29 @@
+package org.dbsyncer.parser.model;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/4/4 23:02
+ */
+public abstract class AbstractWriter {
+
+    private String tableGroupId;
+
+    private String event;
+
+    public String getTableGroupId() {
+        return tableGroupId;
+    }
+
+    public void setTableGroupId(String tableGroupId) {
+        this.tableGroupId = tableGroupId;
+    }
+
+    public String getEvent() {
+        return event;
+    }
+
+    public void setEvent(String event) {
+        this.event = event;
+    }
+}

dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java

@@ -8,22 +8,16 @@ import java.util.Map;
 
 public final class BatchWriter {
 
-    private ConnectorMapper     connectorMapper;
+    private ConnectorMapper connectorMapper;
     private Map<String, String> command;
-    private String              tableName;
-    private String              event;
-    private List<Field>         fields;
-    private List<Map>           dataList;
-    private int                 batchSize;
-    private boolean             isForceUpdate;
+    private String tableName;
+    private String event;
+    private List<Field> fields;
+    private List<Map> dataList;
+    private int batchSize;
 
     public BatchWriter(ConnectorMapper connectorMapper, Map<String, String> command, String tableName, String event,
                        List<Field> fields, List<Map> dataList, int batchSize) {
-        this(connectorMapper, command, tableName, event, fields, dataList, batchSize, false);
-    }
-
-    public BatchWriter(ConnectorMapper connectorMapper, Map<String, String> command, String tableName, String event,
-                       List<Field> fields, List<Map> dataList, int batchSize, boolean isForceUpdate) {
         this.connectorMapper = connectorMapper;
         this.command = command;
         this.tableName = tableName;
@@ -31,7 +25,6 @@ public final class BatchWriter {
         this.fields = fields;
         this.dataList = dataList;
         this.batchSize = batchSize;
-        this.isForceUpdate = isForceUpdate;
     }
 
     public ConnectorMapper getConnectorMapper() {
@@ -62,7 +55,4 @@ public final class BatchWriter {
         return batchSize;
     }
 
-    public boolean isForceUpdate() {
-        return isForceUpdate;
-    }
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Picker.java

@@ -3,24 +3,18 @@ package org.dbsyncer.parser.model;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.connector.model.Field;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class Picker {
 
     private List<Field> sourceFields;
     private List<Field> targetFields;
-    private List<Map>   targetMapList;
 
     public Picker(List<FieldMapping> fieldMapping) {
-        pickFields(fieldMapping);
-    }
-
-    public Picker(List<FieldMapping> fieldMapping, Map<String, Object> data) {
-        pickFields(fieldMapping);
-        pickData(data);
-    }
-
-    public void pickFields(List<FieldMapping> fieldMapping) {
         sourceFields = new ArrayList<>();
         targetFields = new ArrayList<>();
         if (!CollectionUtils.isEmpty(fieldMapping)) {
@@ -32,7 +26,7 @@ public class Picker {
     }
 
     public List<Map> pickData(List<Map> data) {
-        targetMapList = new ArrayList<>();
+        List<Map> targetMapList = new ArrayList<>();
         if (!CollectionUtils.isEmpty(data)) {
             final int size = data.size();
             final int sFieldSize = sourceFields.size();
@@ -46,13 +40,12 @@ public class Picker {
         return targetMapList;
     }
 
-    public void pickData(Map<String, Object> data) {
-        targetMapList = new ArrayList<>();
+    public Map pickData(Map<String, Object> data) {
+        Map targetMap = new HashMap<>();
         if (!CollectionUtils.isEmpty(data)) {
-            Map targetMap = new HashMap<>();
             exchange(sourceFields.size(), sourceFields, targetFields, data, targetMap);
-            targetMapList.add(targetMap);
         }
+        return targetMap;
     }
 
     private void exchange(int sFieldSize, List<Field> sFields, List<Field> tFields, Map<String, Object> source,
@@ -70,16 +63,11 @@ public class Picker {
         }
     }
 
-    public Map getTargetMap() {
-        return CollectionUtils.isEmpty(targetMapList) ? Collections.EMPTY_MAP : targetMapList.get(0);
-    }
-
     public List<Field> getTargetFields() {
         return targetFields;
     }
 
-    public List<Map> getTargetMapList() {
-        return targetMapList;
+    public Map<String, Field> getSourceFieldMap() {
+        return sourceFields.stream().collect(Collectors.toMap(Field::getName, f -> f, (k1, k2) -> k1));
     }
-
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/StorageRequest.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/StorageRequest.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.model;
+package org.dbsyncer.parser.model;
 
 import org.dbsyncer.parser.flush.BufferRequest;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/StorageResponse.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/StorageResponse.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.model;
+package org.dbsyncer.parser.model;
 
 import org.dbsyncer.parser.flush.BufferResponse;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java

@@ -0,0 +1,26 @@
+package org.dbsyncer.parser.model;
+
+import org.dbsyncer.parser.flush.BufferRequest;
+
+import java.util.Map;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/3/27 16:57
+ */
+public class WriterRequest extends AbstractWriter implements BufferRequest {
+
+    private Map row;
+
+    public WriterRequest(String tableGroupId, String event, Map row) {
+        setTableGroupId(tableGroupId);
+        setEvent(event);
+        this.row = row;
+    }
+
+    public Map getRow() {
+        return row;
+    }
+
+}

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/WriterResponse.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.model;
+package org.dbsyncer.parser.model;
 
 import org.dbsyncer.parser.flush.BufferResponse;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/FlushStrategy.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/FlushStrategy.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush;
+package org.dbsyncer.parser.strategy;
 
 import org.dbsyncer.common.model.Result;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/ParserStrategy.java

@@ -0,0 +1,9 @@
+package org.dbsyncer.parser.strategy;
+
+import java.util.Map;
+
+public interface ParserStrategy {
+
+    void execute(String tableGroupId, String event, Map<String, Object> data);
+
+}

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/DisableFullFlushStrategy.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.impl;
+package org.dbsyncer.parser.strategy.impl;
 
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableWriterBufferActuatorStrategy.java

@@ -0,0 +1,102 @@
+package org.dbsyncer.parser.strategy.impl;
+
+import com.google.protobuf.ByteString;
+import org.dbsyncer.cache.CacheService;
+import org.dbsyncer.common.event.RowChangedEvent;
+import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.constant.ConnectorConstant;
+import org.dbsyncer.connector.model.Field;
+import org.dbsyncer.parser.flush.BufferActuator;
+import org.dbsyncer.parser.model.Mapping;
+import org.dbsyncer.parser.model.Picker;
+import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.parser.model.WriterRequest;
+import org.dbsyncer.parser.strategy.ParserStrategy;
+import org.dbsyncer.parser.util.ConvertUtil;
+import org.dbsyncer.plugin.PluginFactory;
+import org.dbsyncer.storage.binlog.AbstractBinlogRecorder;
+import org.dbsyncer.storage.binlog.proto.BinlogMap;
+import org.dbsyncer.storage.binlog.proto.BinlogMessage;
+import org.dbsyncer.storage.binlog.proto.EventEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Queue;
+
+public final class DisableWriterBufferActuatorStrategy extends AbstractBinlogRecorder<WriterRequest> implements ParserStrategy {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    @Autowired
+    private BufferActuator writerBufferActuator;
+
+    @Autowired
+    private CacheService cacheService;
+
+    @Override
+    public void execute(String tableGroupId, String event, Map<String, Object> data) {
+        try {
+            BinlogMap.Builder dataBuilder = BinlogMap.newBuilder();
+            data.forEach((k, v) -> {
+                if (null != v) {
+                    ByteString bytes = serializeValue(v);
+                    if (null != bytes) {
+                        dataBuilder.putRow(k, bytes);
+                    }
+                }
+            });
+
+            BinlogMessage builder = BinlogMessage.newBuilder()
+                    .setTableGroupId(tableGroupId)
+                    .setEvent(EventEnum.valueOf(event))
+                    .setData(dataBuilder.build())
+                    .build();
+            flush(builder);
+        } catch (Exception e) {
+            logger.error(e.getMessage());
+        }
+    }
+
+    @Override
+    public Queue getQueue() {
+        return writerBufferActuator.getQueue();
+    }
+
+    @Override
+    public int getQueueCapacity() {
+        return writerBufferActuator.getQueueCapacity();
+    }
+
+    @Override
+    protected String getTaskName() {
+        return "WriterBinlog";
+    }
+
+    @Override
+    protected WriterRequest deserialize(BinlogMessage message) {
+        if (CollectionUtils.isEmpty(message.getData().getRowMap())) {
+            return null;
+        }
+
+        // 1、获取配置信息
+        final String tableGroupId = message.getTableGroupId();
+        final TableGroup tableGroup = cacheService.get(tableGroupId, TableGroup.class);
+
+        // 2、反序列数据
+        final Picker picker = new Picker(tableGroup.getFieldMapping());
+        final Map<String, Field> fieldMap = picker.getSourceFieldMap();
+        Map<String, Object> data = new HashMap<>();
+        message.getData().getRowMap().forEach((k, v) -> {
+            if (fieldMap.containsKey(k)) {
+                data.put(k, resolveValue(fieldMap.get(k).getType(), v));
+            }
+        });
+
+        return new WriterRequest(message.getTableGroupId(), message.getEvent().name(), data);
+    }
+
+}

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/EnableFlushStrategy.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/EnableFlushStrategy.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.impl;
+package org.dbsyncer.parser.strategy.impl;
 
 import org.dbsyncer.parser.flush.AbstractFlushStrategy;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/EnableWriterBufferActuatorStrategy.java

@@ -0,0 +1,50 @@
+package org.dbsyncer.parser.strategy.impl;
+
+import org.dbsyncer.parser.flush.BufferActuator;
+import org.dbsyncer.parser.model.WriterRequest;
+import org.dbsyncer.parser.strategy.ParserStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+@Component
+@ConditionalOnProperty(value = "dbsyncer.parser.writer.buffer.actuator.enabled", havingValue = "true")
+public final class EnableWriterBufferActuatorStrategy implements ParserStrategy {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    private static final double BUFFER_THRESHOLD = 0.8;
+
+    @Autowired
+    private BufferActuator writerBufferActuator;
+
+    private double limit;
+
+    @PostConstruct
+    private void init() {
+        limit = Math.ceil(writerBufferActuator.getQueueCapacity() * BUFFER_THRESHOLD);
+    }
+
+    @Override
+    public void execute(String tableGroupId, String event, Map<String, Object> data) {
+        writerBufferActuator.offer(new WriterRequest(tableGroupId, event, data));
+
+        // 超过容量限制，限制生产速度
+        final int size = writerBufferActuator.getQueue().size();
+        if (size >= limit) {
+            try {
+                TimeUnit.SECONDS.sleep(30);
+                logger.warn("当前任务队列大小{}已达上限{}，请稍等{}秒", size, limit, 30);
+            } catch (InterruptedException e) {
+                logger.error(e.getMessage());
+            }
+        }
+    }
+
+}

dbsyncer-plugin/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-plugin</artifactId>

dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java

@@ -1,7 +1,10 @@
 package org.dbsyncer.plugin;
 
 import org.apache.commons.io.FileUtils;
+import org.dbsyncer.common.model.FullConvertContext;
+import org.dbsyncer.common.model.IncrementConvertContext;
 import org.dbsyncer.common.spi.ConvertService;
+import org.dbsyncer.common.spi.ProxyApplicationContext;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.plugin.config.Plugin;
 import org.slf4j.Logger;
@@ -45,6 +48,9 @@ public class PluginFactory {
     @Autowired
     private Map<String, ConvertService> service;
 
+    @Autowired
+    private ProxyApplicationContext applicationContextProxy;
+
     @PostConstruct
     private void init() {
         Map<String, ConvertService> unmodifiable = new LinkedHashMap<>();
@@ -92,13 +98,13 @@ public class PluginFactory {
 
     public void convert(Plugin plugin, List<Map> source, List<Map> target) {
         if (null != plugin && service.containsKey(plugin.getClassName())) {
-            service.get(plugin.getClassName()).convert(source, target);
+            service.get(plugin.getClassName()).convert(new FullConvertContext(applicationContextProxy, source, target));
         }
     }
 
     public void convert(Plugin plugin, String event, Map<String, Object> source, Map<String, Object> target) {
         if (null != plugin && service.containsKey(plugin.getClassName())) {
-            service.get(plugin.getClassName()).convert(event, source, target);
+            service.get(plugin.getClassName()).convert(new IncrementConvertContext(applicationContextProxy, event, source, target));
         }
     }