Merge branch 'V_1.0.0_Beta' into fuxinpeng-beta

# Conflicts:
#	dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/impl/PreloadTemplate.java

dbsyncer-biz/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>dbsyncer-biz</artifactId>

dbsyncer-biz/src/main/java/org/dbsyncer/biz/ConfigService.java

@@ -3,6 +3,7 @@ package org.dbsyncer.biz;
 import org.dbsyncer.biz.vo.ConfigVo;
 import org.dbsyncer.parser.model.ConfigModel;
 
+import java.io.File;
 import java.util.List;
 import java.util.Map;
 
@@ -48,4 +49,17 @@ public interface ConfigService {
      */
     List<ConfigModel> getConfigModelAll();
 
+    /**
+     * 校验文件格式
+     *
+     * @param filename
+     */
+    void checkFileSuffix(String filename);
+
+    /**
+     * 更新配置
+     *
+     * @param file
+     */
+    void refreshConfig(File file);
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConfigServiceImpl.java

@@ -1,17 +1,26 @@
 package org.dbsyncer.biz.impl;
 
+import org.apache.commons.io.FileUtils;
 import org.dbsyncer.biz.ConfigService;
 import org.dbsyncer.biz.checker.impl.config.ConfigChecker;
 import org.dbsyncer.biz.vo.ConfigVo;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.manager.Manager;
+import org.dbsyncer.manager.template.impl.PreloadTemplate;
+import org.dbsyncer.parser.logger.LogService;
+import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.Config;
 import org.dbsyncer.parser.model.ConfigModel;
+import org.dbsyncer.plugin.enums.FileSuffixEnum;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import org.springframework.util.Assert;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -32,6 +41,12 @@ public class ConfigServiceImpl implements ConfigService {
     @Autowired
     private ConfigChecker configChecker;
 
+    @Autowired
+    private PreloadTemplate preloadTemplate;
+
+    @Autowired
+    private LogService logService;
+
     @Override
     public String edit(Map<String, String> params) {
         synchronized (this) {
@@ -77,6 +92,32 @@ public class ConfigServiceImpl implements ConfigService {
         return list;
     }
 
+    @Override
+    public void checkFileSuffix(String filename) {
+        Assert.hasText(filename, "the config filename is null.");
+        String suffix = filename.substring(filename.lastIndexOf(".") + 1, filename.length());
+        FileSuffixEnum fileSuffix = FileSuffixEnum.getFileSuffix(suffix);
+        Assert.notNull(fileSuffix, "Illegal file suffix");
+        Assert.isTrue(FileSuffixEnum.JSON == fileSuffix, String.format("不正确的文件扩展名 \"%s\"，只支持 \"%s\" 的文件扩展名。", filename, FileSuffixEnum.JSON.getName()));
+    }
+
+    @Override
+    public void refreshConfig(File file) {
+        Assert.notNull(file, "the config file is null.");
+        try {
+            List<String> lines = FileUtils.readLines(file, Charset.defaultCharset());
+            if (!CollectionUtils.isEmpty(lines)) {
+                StringBuilder json = new StringBuilder();
+                lines.forEach(line -> json.append(line));
+                preloadTemplate.reload(json.toString());
+            }
+        } catch (IOException e) {
+            logService.log(LogType.CacheLog.IMPORT_ERROR);
+        } finally {
+            FileUtils.deleteQuietly(file);
+        }
+    }
+
     private ConfigVo convertConfig2Vo(Config config) {
         ConfigVo configVo = new ConfigVo();
         BeanUtils.copyProperties(config, configVo);

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java

@@ -10,19 +10,19 @@ import org.dbsyncer.biz.vo.MappingVo;
 import org.dbsyncer.biz.vo.MetaVo;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.monitor.Monitor;
 import org.dbsyncer.parser.enums.ModelEnum;
 import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.*;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
-import java.util.*;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -33,11 +33,6 @@ import java.util.stream.Collectors;
 @Service
 public class MappingServiceImpl extends BaseServiceImpl implements MappingService {
 
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
-    @Autowired
-    private Monitor monitor;
-
     @Autowired
     private MappingChecker mappingChecker;
 
@@ -56,7 +51,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
 
         // 匹配相似表 on
         String autoMatchTable = params.get("autoMatchTable");
-        if(StringUtil.isNotBlank(autoMatchTable)){
+        if (StringUtil.isNotBlank(autoMatchTable)) {
             matchSimilarTable(model);
         }
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java

@@ -15,6 +15,7 @@ import org.dbsyncer.plugin.enums.FileSuffixEnum;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
+import org.springframework.util.Assert;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -70,14 +71,14 @@ public class PluginServiceImpl implements PluginService {
 
     @Override
     public void checkFileSuffix(String filename) {
-        if (StringUtil.isNotBlank(filename)) {
-            String suffix = filename.substring(filename.lastIndexOf(".") + 1, filename.length());
-            if (null == FileSuffixEnum.getFileSuffix(suffix)) {
-                suffix = StringUtil.join(FileSuffixEnum.values(), ",").toLowerCase();
-                String msg = String.format("不正确的文件扩展名 \"%s\"，只支持 \"%s\" 的文件扩展名。", filename, suffix);
-                logService.log(LogType.PluginLog.CHECK_ERROR, msg);
-                throw new BizException(msg);
-            }
+        Assert.hasText(filename, "the plugin filename is null.");
+        String suffix = filename.substring(filename.lastIndexOf(".") + 1, filename.length());
+        FileSuffixEnum fileSuffix = FileSuffixEnum.getFileSuffix(suffix);
+        Assert.notNull(fileSuffix, "Illegal file suffix");
+        if (FileSuffixEnum.JAR != fileSuffix) {
+            String msg = String.format("不正确的文件扩展名 \"%s\"，只支持 \"%s\" 的文件扩展名。", filename, FileSuffixEnum.JAR.getName());
+            logService.log(LogType.PluginLog.CHECK_ERROR, msg);
+            throw new BizException(msg);
         }
     }
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java

@@ -9,8 +9,6 @@ import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
@@ -27,8 +25,6 @@ import java.util.stream.Stream;
 @Service
 public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroupService {
 
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
     @Autowired
     private Checker tableGroupChecker;
 

dbsyncer-cache/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>dbsyncer-cache</artifactId>

dbsyncer-cluster/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-cluster</artifactId>

dbsyncer-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-common</artifactId>

dbsyncer-common/src/main/java/org/dbsyncer/common/config/ScheduleConfig.java

@@ -22,7 +22,7 @@ public class ScheduleConfig implements SchedulingConfigurer {
 
     @Bean(name = "taskScheduler", destroyMethod = "shutdown")
     public ThreadPoolTaskScheduler taskScheduler() {
-        int poolSize = Runtime.getRuntime().availableProcessors() * 2;
+        int poolSize = Runtime.getRuntime().availableProcessors();
         ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
         //核心线程池大小
         scheduler.setPoolSize(poolSize);

dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/BufferedRandomAccessFile.java → dbsyncer-common/src/main/java/org/dbsyncer/common/file/BufferedRandomAccessFile.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.listener.file;
+package org.dbsyncer.common.file;
 
 import java.io.File;
 import java.io.FileNotFoundException;

dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskService.java

@@ -11,9 +11,9 @@ public interface ScheduledTaskService {
      * 第六位，星期，取值1-7
      * [秒 分 时 日 月 星期]
      *
-     * @param key 任务唯一key
+     * @param key  任务唯一key
      * @param cron 任务表达式
-     * @param job 任务实现
+     * @param job  任务实现
      */
     void start(String key, String cron, ScheduledTaskJob job);
 

dbsyncer-connector/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-connector</artifactId>

dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/WriterBatchConfig.java

@@ -27,22 +27,13 @@ public class WriterBatchConfig {
      * 集合数据
      */
     private List<Map> data;
-    /**
-     * 强制更新
-     */
-    private boolean isForceUpdate;
 
     public WriterBatchConfig(String tableName, String event, Map<String, String> command, List<Field> fields, List<Map> data) {
-        this(tableName, event, command, fields, data, false);
-    }
-
-    public WriterBatchConfig(String tableName, String event, Map<String, String> command, List<Field> fields, List<Map> data, boolean isForceUpdate) {
         this.tableName = tableName;
         this.event = event;
         this.command = command;
         this.fields = fields;
         this.data = data;
-        this.isForceUpdate = isForceUpdate;
     }
 
     public String getTableName() {
@@ -65,7 +56,4 @@ public class WriterBatchConfig {
         return data;
     }
 
-    public boolean isForceUpdate() {
-        return isForceUpdate;
-    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java

@@ -92,7 +92,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         Collections.addAll(config.getArgs(), getPageArgs(config.getPageIndex(), config.getPageSize()));
 
         // 3、执行SQL
-        List<Map<String, Object>> list = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(querySql, config.getArgs().toArray()));
+        List<Map<String, Object>> list = connectorMapper.execute(
+                databaseTemplate -> databaseTemplate.queryForList(querySql, config.getArgs().toArray()));
 
         // 4、返回结果集
         return new Result(list);
@@ -120,6 +121,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         if (!isInsert(event)) {
             if (isDelete(event)) {
                 fields.clear();
+            } else if (isUpdate(event)) {
+                fields.remove(pkField);
             }
             fields.add(pkField);
         }
@@ -142,19 +145,15 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
                     })
             );
         } catch (Exception e) {
-            result.addFailData(data);
-            result.getError().append(e.getMessage());
+            logger.error(e.getMessage());
+            data.forEach(row -> forceUpdate(result, connectorMapper, config, pkField, row));
         }
 
         if (null != execute) {
             int batchSize = execute.length;
             for (int i = 0; i < batchSize; i++) {
                 if (execute[i] == 0) {
-                    if (config.isForceUpdate()) {
-                        forceUpdate(result, connectorMapper, config, pkField, data.get(i));
-                    } else {
-                        result.getFailData().add(data.get(i));
-                    }
+                    forceUpdate(result, connectorMapper, config, pkField, data.get(i));
                     continue;
                 }
                 result.getSuccessData().add(data.get(i));
@@ -205,7 +204,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         // 获取查询数据行是否存在
         String quotation = buildSqlWithQuotation();
         String pk = findOriginalTablePrimaryKey(commandConfig, quotation);
-        StringBuilder queryCount = new StringBuilder().append("SELECT COUNT(1) FROM ").append(quotation).append(commandConfig.getTable().getName()).append(
+        StringBuilder queryCount = new StringBuilder().append("SELECT COUNT(1) FROM ").append(quotation).append(
+                commandConfig.getTable().getName()).append(
                 quotation).append(" WHERE ").append(pk).append(" = ?");
         String queryCountExist = ConnectorConstant.OPERTION_QUERY_COUNT_EXIST;
         map.put(queryCountExist, queryCount.toString());
@@ -509,6 +509,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         if (!isInsert(event)) {
             if (isDelete(event)) {
                 fields.clear();
+            } else if (isUpdate(event)) {
+                fields.remove(pkField);
             }
             fields.add(pkField);
         }
@@ -534,7 +536,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
     private boolean existRow(DatabaseConnectorMapper connectorMapper, String sql, Object value) {
         int rowNum = 0;
         try {
-            rowNum = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(sql, new Object[]{value}, Integer.class));
+            rowNum = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(sql, new Object[] {value}, Integer.class));
         } catch (Exception e) {
             logger.error("检查数据行存在异常:{}，SQL:{},参数:{}", e.getMessage(), sql, value);
         }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/setter/BitSetter.java

@@ -26,6 +26,11 @@ public class BitSetter extends AbstractSetter<byte[]> {
             ps.setInt(i, integer);
             return;
         }
+        if (val instanceof Boolean) {
+            Boolean b = (Boolean) val;
+            ps.setBoolean(i, b);
+            return;
+        }
         throw new ConnectorException(String.format("BitSetter can not find type [%s], val [%s]", type, val));
     }
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderUpdate.java

@@ -1,10 +1,8 @@
 package org.dbsyncer.connector.database.sqlbuilder;
 
-import org.dbsyncer.connector.model.Field;
 import org.dbsyncer.connector.config.SqlBuilderConfig;
 import org.dbsyncer.connector.database.AbstractSqlBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.dbsyncer.connector.model.Field;
 
 import java.util.List;
 
@@ -15,8 +13,6 @@ import java.util.List;
  */
 public class SqlBuilderUpdate extends AbstractSqlBuilder {
 
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
     @Override
     public String buildSql(SqlBuilderConfig config) {
         String tableName = config.getTableName();
@@ -27,6 +23,11 @@ public class SqlBuilderUpdate extends AbstractSqlBuilder {
         int end = size - 1;
         sql.append("UPDATE ").append(quotation).append(tableName).append(quotation).append(" SET ");
         for (int i = 0; i < size; i++) {
+            // skip pk
+            if(fields.get(i).isPk()){
+               continue;
+            }
+
             // "USERNAME"=?
             sql.append(quotation).append(fields.get(i).getName()).append(quotation).append("=?");
             //如果不是最后一个字段

dbsyncer-listener/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-listener</artifactId>

dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java

@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
 public abstract class AbstractExtractor implements Extractor {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
+    protected String metaId;
     protected Executor taskExecutor;
     protected ConnectorFactory connectorFactory;
     protected ScheduledTaskService scheduledTaskService;
@@ -81,6 +82,10 @@ public abstract class AbstractExtractor implements Extractor {
         }
     }
 
+    public void setMetaId(String metaId) {
+        this.metaId = metaId;
+    }
+
     public void setTaskExecutor(Executor taskExecutor) {
         this.taskExecutor = taskExecutor;
     }

dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java

@@ -2,7 +2,7 @@ package org.dbsyncer.listener.file;
 
 import org.apache.commons.io.IOUtils;
 import org.dbsyncer.common.event.RowChangedEvent;
-import org.dbsyncer.common.util.JsonUtil;
+import org.dbsyncer.common.file.BufferedRandomAccessFile;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.RandomUtil;
 import org.dbsyncer.common.util.StringUtil;

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java

@@ -16,6 +16,8 @@ import java.nio.ByteBuffer;
  */
 public abstract class AbstractMessageDecoder implements MessageDecoder {
 
+    protected String metaId;
+
     protected DatabaseConfig config;
 
     private ColumnValue value = new PgColumnValue();
@@ -49,7 +51,12 @@ public abstract class AbstractMessageDecoder implements MessageDecoder {
 
     @Override
     public String getSlotName() {
-        return String.format("dbs_slot_%s_%s", config.getSchema(), config.getUsername());
+        return String.format("dbs_slot_%s_%s_%s", config.getSchema(), config.getUsername(), metaId);
+    }
+
+    @Override
+    public void setMetaId(String metaId) {
+        this.metaId = metaId;
     }
 
     @Override

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java

@@ -29,6 +29,8 @@ public interface MessageDecoder {
 
     void withSlotOption(ChainedLogicalStreamBuilder builder);
 
+    void setMetaId(String metaId);
+
     void setConfig(DatabaseConfig config);
 
 }

dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java

@@ -93,6 +93,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor {
 
             database = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_DATABASE, String.class));
             messageDecoder = MessageDecoderEnum.getMessageDecoder(config.getProperty(PLUGIN_NAME));
+            messageDecoder.setMetaId(metaId);
             messageDecoder.setConfig(config);
             messageDecoder.postProcessBeforeInitialization(connectorFactory, connectorMapper);
             dropSlotOnClose = BooleanUtil.toBoolean(config.getProperty(DROP_SLOT_ON_CLOSE, "true"));

dbsyncer-manager/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-manager</artifactId>

dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java

@@ -122,12 +122,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addConnector(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD));
     }
 
     @Override
     public String editConnector(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT));
     }
 
     @Override
@@ -166,12 +166,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addMapping(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD));
     }
 
     @Override
     public String editMapping(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT));
     }
 
     @Override
@@ -195,12 +195,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addTableGroup(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, GroupStrategyEnum.TABLE, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD, GroupStrategyEnum.TABLE));
     }
 
     @Override
     public String editTableGroup(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, GroupStrategyEnum.TABLE, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT, GroupStrategyEnum.TABLE));
     }
 
     @Override
@@ -235,12 +235,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addMeta(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD));
     }
 
     @Override
     public String editMeta(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT));
     }
 
     @Override
@@ -263,12 +263,12 @@ public class ManagerFactory implements Manager, ApplicationListener<ClosedEvent>
 
     @Override
     public String addConfig(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD));
     }
 
     @Override
     public String editConfig(ConfigModel model) {
-        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT.getHandler()));
+        return operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_EDIT));
     }
 
     @Override

dbsyncer-manager/src/main/java/org/dbsyncer/manager/config/OperationConfig.java

@@ -1,6 +1,7 @@
 package org.dbsyncer.manager.config;
 
 import org.dbsyncer.manager.enums.GroupStrategyEnum;
+import org.dbsyncer.manager.enums.HandlerEnum;
 import org.dbsyncer.manager.template.Handler;
 import org.dbsyncer.parser.model.ConfigModel;
 
@@ -10,9 +11,9 @@ public class OperationConfig {
 
     private ConfigModel model;
 
-    private GroupStrategyEnum groupStrategyEnum;
+    private GroupStrategyEnum groupStrategyEnum = GroupStrategyEnum.DEFAULT;
 
-    private Handler handler;
+    private HandlerEnum handlerEnum;
 
     public OperationConfig(String id) {
         this.id = id;
@@ -23,15 +24,15 @@ public class OperationConfig {
         this.groupStrategyEnum = groupStrategyEnum;
     }
 
-    public OperationConfig(ConfigModel model, Handler handler) {
+    public OperationConfig(ConfigModel model, HandlerEnum handlerEnum) {
         this.model = model;
-        this.handler = handler;
+        this.handlerEnum = handlerEnum;
     }
 
-    public OperationConfig(ConfigModel model, GroupStrategyEnum groupStrategyEnum, Handler handler) {
+    public OperationConfig(ConfigModel model, HandlerEnum handlerEnum, GroupStrategyEnum groupStrategyEnum) {
         this.model = model;
+        this.handlerEnum = handlerEnum;
         this.groupStrategyEnum = groupStrategyEnum;
-        this.handler = handler;
     }
 
     public String getId() {
@@ -47,6 +48,6 @@ public class OperationConfig {
     }
 
     public Handler getHandler() {
-        return handler;
+        return handlerEnum.getHandler();
     }
 }

dbsyncer-manager/src/main/java/org/dbsyncer/manager/config/PreloadConfig.java

@@ -1,37 +0,0 @@
-package org.dbsyncer.manager.config;
-
-import org.dbsyncer.manager.enums.GroupStrategyEnum;
-import org.dbsyncer.manager.enums.HandlerEnum;
-import org.dbsyncer.manager.template.Handler;
-
-public class PreloadConfig {
-
-    private String filterType;
-
-    private GroupStrategyEnum groupStrategyEnum;
-
-    private HandlerEnum handlerEnum;
-
-    public PreloadConfig(String filterType, HandlerEnum handlerEnum) {
-        this.filterType = filterType;
-        this.handlerEnum = handlerEnum;
-    }
-
-    public PreloadConfig(String filterType, GroupStrategyEnum groupStrategyEnum, HandlerEnum handlerEnum) {
-        this.filterType = filterType;
-        this.groupStrategyEnum = groupStrategyEnum;
-        this.handlerEnum = handlerEnum;
-    }
-
-    public String getFilterType() {
-        return filterType;
-    }
-
-    public GroupStrategyEnum getGroupStrategyEnum() {
-        return groupStrategyEnum;
-    }
-
-    public HandlerEnum getHandlerEnum() {
-        return handlerEnum;
-    }
-}

dbsyncer-manager/src/main/java/org/dbsyncer/manager/config/QueryConfig.java

@@ -7,7 +7,7 @@ public class QueryConfig<T> {
 
     private ConfigModel configModel;
 
-    private GroupStrategyEnum groupStrategyEnum;
+    private GroupStrategyEnum groupStrategyEnum = GroupStrategyEnum.DEFAULT;
 
     public QueryConfig(ConfigModel configModel) {
         this.configModel = configModel;

dbsyncer-manager/src/main/java/org/dbsyncer/manager/enums/HandlerEnum.java

@@ -5,6 +5,7 @@ import org.dbsyncer.manager.config.PreloadCallBack;
 import org.dbsyncer.manager.handler.AbstractOperationHandler;
 import org.dbsyncer.manager.handler.AbstractPreloadHandler;
 import org.dbsyncer.manager.template.Handler;
+import org.dbsyncer.storage.constant.ConfigConstant;
 
 /**
  * @author AE86
@@ -16,61 +17,67 @@ public enum HandlerEnum {
     /**
      * 添加
      */
-    OPR_ADD(new AbstractOperationHandler(){
+    OPR_ADD("add", new AbstractOperationHandler() {
         @Override
         protected void handle(OperationCallBack operationCallBack) {
             operationCallBack.add();
         }
     }),
+
     /**
      * 修改
      */
-    OPR_EDIT(new AbstractOperationHandler(){
+    OPR_EDIT("edit", new AbstractOperationHandler() {
         @Override
         protected void handle(OperationCallBack operationCallBack) {
             operationCallBack.edit();
         }
     }),
+
     /**
      * 预加载Connector
      */
-    PRELOAD_CONNECTOR(new AbstractPreloadHandler(){
+    PRELOAD_CONNECTOR(ConfigConstant.CONNECTOR, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseConnector();
         }
     }),
+
     /**
      * 预加载Mapping
      */
-    PRELOAD_MAPPING(new AbstractPreloadHandler(){
+    PRELOAD_MAPPING(ConfigConstant.MAPPING, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseMapping();
         }
     }),
+
     /**
      * 预加载TableGroup
      */
-    PRELOAD_TABLE_GROUP(new AbstractPreloadHandler(){
+    PRELOAD_TABLE_GROUP(ConfigConstant.TABLE_GROUP, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseTableGroup();
         }
-    }),
+    }, GroupStrategyEnum.TABLE),
+
     /**
      * 预加载Meta
      */
-    PRELOAD_META(new AbstractPreloadHandler(){
+    PRELOAD_META(ConfigConstant.META, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseMeta();
         }
     }),
+
     /**
      * 预加载Config
      */
-    PRELOAD_CONFIG(new AbstractPreloadHandler(){
+    PRELOAD_CONFIG(ConfigConstant.CONFIG, true, new AbstractPreloadHandler() {
         @Override
         protected Object preload(PreloadCallBack preloadCallBack) {
             return preloadCallBack.parseConfig();
@@ -87,13 +94,39 @@ public enum HandlerEnum {
         }
     });
 
+    private String modelType;
+    private boolean preload;
     private Handler handler;
+    private GroupStrategyEnum groupStrategyEnum;
+
+    HandlerEnum(String modelType, Handler handler) {
+        this(modelType, false, handler, null);
+    }
 
-    HandlerEnum(Handler handler) {
+    HandlerEnum(String modelType, boolean preload, Handler handler) {
+        this(modelType, preload, handler, GroupStrategyEnum.DEFAULT);
+    }
+
+    HandlerEnum(String modelType, boolean preload, Handler handler, GroupStrategyEnum groupStrategyEnum) {
+        this.modelType = modelType;
+        this.preload = preload;
         this.handler = handler;
+        this.groupStrategyEnum = groupStrategyEnum;
+    }
+
+    public String getModelType() {
+        return modelType;
+    }
+
+    public boolean isPreload() {
+        return preload;
     }
 
     public Handler getHandler() {
         return handler;
     }
-}
+
+    public GroupStrategyEnum getGroupStrategyEnum() {
+        return groupStrategyEnum;
+    }
+}

dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java

@@ -164,7 +164,7 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob
     }
 
     private void setExtractorConfig(AbstractExtractor extractor, ConnectorConfig connector, ListenerConfig listener,
-                                    Map<String, String> snapshot, Event event) {
+                                    Map<String, String> snapshot, AbstractListener event) {
         extractor.setTaskExecutor(taskExecutor);
         extractor.setConnectorFactory(connectorFactory);
         extractor.setScheduledTaskService(scheduledTaskService);
@@ -172,6 +172,7 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob
         extractor.setListenerConfig(listener);
         extractor.setSnapshot(snapshot);
         extractor.addListener(event);
+        extractor.setMetaId(event.metaId);
     }
 
     abstract class AbstractListener implements Event {

dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/TableGroupStrategy.java

@@ -3,7 +3,9 @@ package org.dbsyncer.manager.strategy;
 import org.dbsyncer.manager.ManagerException;
 import org.dbsyncer.manager.template.GroupStrategy;
 import org.dbsyncer.parser.model.ConfigModel;
+import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.storage.constant.ConfigConstant;
 
 /**
  * @author AE86
@@ -21,6 +23,10 @@ public class TableGroupStrategy implements GroupStrategy {
             // 格式：${type} + "_" + ${mappingId}
             return new StringBuilder(type).append("_").append(mappingId).toString();
         }
+        if (model instanceof Mapping) {
+            Mapping m = (Mapping) model;
+            return new StringBuilder(ConfigConstant.TABLE_GROUP).append("_").append(m.getId()).toString();
+        }
         throw new ManagerException(String.format("Not support config model \"%s\".", model));
     }
 

dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/AbstractTemplate.java

@@ -1,26 +0,0 @@
-package org.dbsyncer.manager.template;
-
-import org.dbsyncer.manager.config.OperationConfig;
-import org.dbsyncer.manager.config.PreloadConfig;
-import org.dbsyncer.manager.config.QueryConfig;
-import org.dbsyncer.manager.enums.GroupStrategyEnum;
-
-public abstract class AbstractTemplate {
-
-    protected GroupStrategyEnum getDefaultStrategy(PreloadConfig config){
-        return getDefaultStrategy(config.getGroupStrategyEnum());
-    }
-
-    protected GroupStrategyEnum getDefaultStrategy(OperationConfig config){
-        return getDefaultStrategy(config.getGroupStrategyEnum());
-    }
-
-    protected GroupStrategyEnum getDefaultStrategy(QueryConfig query){
-        return getDefaultStrategy(query.getGroupStrategyEnum());
-    }
-
-    private GroupStrategyEnum getDefaultStrategy(GroupStrategyEnum strategy){
-        return null != strategy ? strategy : GroupStrategyEnum.DEFAULT;
-    }
-
-}

dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/impl/OperationTemplate.java

@@ -8,7 +8,6 @@ import org.dbsyncer.manager.config.OperationCallBack;
 import org.dbsyncer.manager.config.OperationConfig;
 import org.dbsyncer.manager.config.QueryConfig;
 import org.dbsyncer.manager.enums.GroupStrategyEnum;
-import org.dbsyncer.manager.template.AbstractTemplate;
 import org.dbsyncer.manager.template.GroupStrategy;
 import org.dbsyncer.manager.template.Handler;
 import org.dbsyncer.parser.model.ConfigModel;
@@ -22,7 +21,11 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 
 /**
  * 操作配置模板
@@ -32,7 +35,7 @@ import java.util.*;
  * @date 2019/9/16 23:59
  */
 @Component
-public final class OperationTemplate extends AbstractTemplate {
+public final class OperationTemplate {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -44,10 +47,10 @@ public final class OperationTemplate extends AbstractTemplate {
 
     public <T> List<T> queryAll(QueryConfig<T> query) {
         ConfigModel model = query.getConfigModel();
-        String groupId = getGroupId(model, getDefaultStrategy(query));
+        String groupId = getGroupId(model, query.getGroupStrategyEnum());
         Group group = cacheService.get(groupId, Group.class);
         if (null != group) {
-            List<String> index = group.getAll();
+            List<String> index = group.getIndex();
             if (!CollectionUtils.isEmpty(index)) {
                 List<T> list = new ArrayList<>();
                 Class<? extends ConfigModel> clazz = model.getClass();
@@ -84,8 +87,7 @@ public final class OperationTemplate extends AbstractTemplate {
         handler.execute(new OperationCallBack(storageService, StorageEnum.CONFIG, params));
 
         // 3、缓存
-        GroupStrategyEnum strategy = getDefaultStrategy(config);
-        cache(model, strategy);
+        cache(model, config.getGroupStrategyEnum());
         return model.getId();
     }
 
@@ -108,7 +110,7 @@ public final class OperationTemplate extends AbstractTemplate {
         Assert.hasText(id, "ID can not be empty.");
         // 删除分组
         ConfigModel model = cacheService.get(id, ConfigModel.class);
-        String groupId = getGroupId(model, getDefaultStrategy(config));
+        String groupId = getGroupId(model, config.getGroupStrategyEnum());
         Group group = cacheService.get(groupId, Group.class);
         if (null != group) {
             group.remove(id);
@@ -120,7 +122,7 @@ public final class OperationTemplate extends AbstractTemplate {
         storageService.remove(StorageEnum.CONFIG, id);
     }
 
-    private String getGroupId(ConfigModel model, GroupStrategyEnum strategy) {
+    public String getGroupId(ConfigModel model, GroupStrategyEnum strategy) {
         Assert.notNull(model, "ConfigModel can not be null.");
         Assert.notNull(strategy, "GroupStrategyEnum can not be null.");
         GroupStrategy groupStrategy = strategy.getGroupStrategy();
@@ -146,7 +148,7 @@ public final class OperationTemplate extends AbstractTemplate {
         }
     }
 
-    class Group {
+    static class Group {
 
         private List<String> index;
 
@@ -168,10 +170,13 @@ public final class OperationTemplate extends AbstractTemplate {
             return index.size();
         }
 
-        public List<String> getAll() {
+        public List<String> getIndex() {
             return Collections.unmodifiableList(index);
         }
 
+        public void setIndex(List<String> index) {
+            this.index = index;
+        }
     }
 
 }

dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/impl/PreloadTemplate.java

@@ -1,15 +1,16 @@
 package org.dbsyncer.manager.template.impl;
 
+import com.alibaba.fastjson.JSONObject;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.manager.Manager;
+import org.dbsyncer.manager.config.OperationConfig;
 import org.dbsyncer.manager.config.PreloadCallBack;
-import org.dbsyncer.manager.config.PreloadConfig;
 import org.dbsyncer.manager.config.QueryConfig;
-import org.dbsyncer.manager.enums.GroupStrategyEnum;
 import org.dbsyncer.manager.enums.HandlerEnum;
-import org.dbsyncer.manager.template.AbstractTemplate;
 import org.dbsyncer.manager.template.Handler;
+import org.dbsyncer.manager.template.impl.OperationTemplate.Group;
 import org.dbsyncer.parser.Parser;
 import org.dbsyncer.parser.enums.MetaEnum;
 import org.dbsyncer.parser.model.ConfigModel;
@@ -26,6 +27,7 @@ import org.springframework.context.ApplicationListener;
 import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.stereotype.Component;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -37,7 +39,7 @@ import java.util.Map;
  * @date 2019/9/16 23:59
  */
 @Component
-public final class PreloadTemplate extends AbstractTemplate implements ApplicationListener<ContextRefreshedEvent> {
+public final class PreloadTemplate implements ApplicationListener<ContextRefreshedEvent> {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -53,16 +55,16 @@ public final class PreloadTemplate extends AbstractTemplate implements Applicati
     @Autowired
     private OperationTemplate operationTemplate;
 
-    public void execute(PreloadConfig config) {
+    public void execute(HandlerEnum handlerEnum) {
         Query query = new Query();
         query.setType(StorageEnum.CONFIG);
-        String filterType = config.getFilterType();
-        query.addFilter(ConfigConstant.CONFIG_MODEL_TYPE, filterType);
+        String modelType = handlerEnum.getModelType();
+        query.addFilter(ConfigConstant.CONFIG_MODEL_TYPE, modelType);
 
         int pageNum = 1;
         int pageSize = 20;
         long total = 0;
-        for(;;){
+        for (; ; ) {
             query.setPageNum(pageNum);
             query.setPageSize(pageSize);
             Paging paging = storageService.query(query);
@@ -70,44 +72,70 @@ public final class PreloadTemplate extends AbstractTemplate implements Applicati
             if (CollectionUtils.isEmpty(data)) {
                 break;
             }
-            Handler handler = config.getHandlerEnum().getHandler();
-            GroupStrategyEnum strategy = getDefaultStrategy(config);
+            Handler handler = handlerEnum.getHandler();
             data.forEach(map -> {
                 String json = (String) map.get(ConfigConstant.CONFIG_MODEL_JSON);
                 ConfigModel model = (ConfigModel) handler.execute(new PreloadCallBack(parser, json));
                 if (null != model) {
-                    operationTemplate.cache(model, strategy);
+                    operationTemplate.cache(model, handlerEnum.getGroupStrategyEnum());
                 }
             });
             total += paging.getTotal();
-            pageNum ++;
+            pageNum++;
         }
 
-        logger.info("PreLoad {}:{}", filterType, total);
+        logger.info("PreLoad {}:{}", modelType, total);
     }
 
-    @Override
-    public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
+    public void reload(String json) {
+        Map<String, JSONObject> map = JsonUtil.jsonToObj(json, Map.class);
+        if (CollectionUtils.isEmpty(map)) {
+            return;
+        }
+
+        // Load configs
+        reload(map, HandlerEnum.PRELOAD_CONFIG);
         // Load connectors
-        execute(new PreloadConfig(ConfigConstant.CONNECTOR, HandlerEnum.PRELOAD_CONNECTOR));
+        reload(map, HandlerEnum.PRELOAD_CONNECTOR);
         // Load mappings
-        execute(new PreloadConfig(ConfigConstant.MAPPING, HandlerEnum.PRELOAD_MAPPING));
-        // Load tableGroups
-        execute(new PreloadConfig(ConfigConstant.TABLE_GROUP, GroupStrategyEnum.TABLE, HandlerEnum.PRELOAD_TABLE_GROUP));
+        reload(map, HandlerEnum.PRELOAD_MAPPING);
         // Load metas
-        execute(new PreloadConfig(ConfigConstant.META, HandlerEnum.PRELOAD_META));
-        // Load configs
-        execute(new PreloadConfig(ConfigConstant.CONFIG, HandlerEnum.PRELOAD_CONFIG));
+        reload(map, HandlerEnum.PRELOAD_META);
         // Load projectGroups
-        execute(new PreloadConfig(ConfigConstant.PROJECT_GROUP, HandlerEnum.PRELOAD_PROJECT_GROUP));
+        reload(map, HandlerEnum.PRELOAD_PROJECT_GROUP);
+        launch();
+    }
 
-        // Load plugins
-        manager.loadPlugins();
+    private void reload(Map<String, JSONObject> map, HandlerEnum handlerEnum) {
+        reload(map, handlerEnum, handlerEnum.getModelType());
+    }
 
-        // Check connectors status
-        manager.checkAllConnectorStatus();
+    private void reload(Map<String, JSONObject> map, HandlerEnum handlerEnum, String groupId) {
+        JSONObject config = map.get(groupId);
+        Group group = JsonUtil.jsonToObj(config.toJSONString(), Group.class);
+        if (null == group) {
+            return;
+        }
 
-        // Launch drivers
+        List<String> index = group.getIndex();
+        if (CollectionUtils.isEmpty(index)) {
+            return;
+        }
+
+        Handler handler = handlerEnum.getHandler();
+        for (String e : index) {
+            JSONObject m = map.get(e);
+            ConfigModel model = (ConfigModel) handler.execute(new PreloadCallBack(parser, m.toJSONString()));
+            operationTemplate.execute(new OperationConfig(model, HandlerEnum.OPR_ADD, handlerEnum.getGroupStrategyEnum()));
+            // Load tableGroups
+            if (HandlerEnum.PRELOAD_MAPPING == handlerEnum) {
+                handlerEnum = HandlerEnum.PRELOAD_TABLE_GROUP;
+                reload(map, handlerEnum, operationTemplate.getGroupId(model, handlerEnum.getGroupStrategyEnum()));
+            }
+        }
+    }
+
+    private void launch() {
         Meta meta = new Meta();
         meta.setType(ConfigConstant.META);
         QueryConfig<Meta> queryConfig = new QueryConfig<>(meta);
@@ -125,4 +153,19 @@ public final class PreloadTemplate extends AbstractTemplate implements Applicati
         }
     }
 
+    @Override
+    public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
+        // Load configModels
+        Arrays.stream(HandlerEnum.values()).filter(handlerEnum -> handlerEnum.isPreload()).forEach(handlerEnum -> execute(handlerEnum));
+
+        // Load plugins
+        manager.loadPlugins();
+
+        // Check connectors status
+        manager.checkAllConnectorStatus();
+
+        // Launch drivers
+        launch();
+    }
+
 }

dbsyncer-monitor/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-monitor</artifactId>

dbsyncer-parser/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-parser</artifactId>
@@ -38,7 +38,7 @@
             <artifactId>dbsyncer-storage</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
-        
+
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserFactory.java

@@ -5,10 +5,12 @@ import org.dbsyncer.common.event.RowChangedEvent;
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
-import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.ConnectorFactory;
 import org.dbsyncer.connector.ConnectorMapper;
-import org.dbsyncer.connector.config.*;
+import org.dbsyncer.connector.config.CommandConfig;
+import org.dbsyncer.connector.config.ConnectorConfig;
+import org.dbsyncer.connector.config.ReaderConfig;
+import org.dbsyncer.connector.config.WriterBatchConfig;
 import org.dbsyncer.connector.constant.ConnectorConstant;
 import org.dbsyncer.connector.enums.ConnectorEnum;
 import org.dbsyncer.connector.enums.FilterEnum;
@@ -20,12 +22,11 @@ import org.dbsyncer.listener.enums.QuartzFilterEnum;
 import org.dbsyncer.parser.enums.ConvertEnum;
 import org.dbsyncer.parser.enums.ParserEnum;
 import org.dbsyncer.parser.event.FullRefreshEvent;
-import org.dbsyncer.parser.flush.BufferActuator;
-import org.dbsyncer.parser.flush.FlushStrategy;
-import org.dbsyncer.parser.flush.model.WriterRequest;
 import org.dbsyncer.parser.logger.LogService;
 import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.*;
+import org.dbsyncer.parser.strategy.FlushStrategy;
+import org.dbsyncer.parser.strategy.ParserStrategy;
 import org.dbsyncer.parser.util.ConvertUtil;
 import org.dbsyncer.parser.util.PickerUtil;
 import org.dbsyncer.plugin.PluginFactory;
@@ -81,7 +82,7 @@ public class ParserFactory implements Parser {
     private ApplicationContext applicationContext;
 
     @Autowired
-    private BufferActuator writerBufferActuator;
+    private ParserStrategy parserStrategy;
 
     @Override
     public ConnectorMapper connect(ConnectorConfig config) {
@@ -300,20 +301,7 @@ public class ParserFactory implements Parser {
         logger.info("Table[{}] {}, before:{}, after:{}", event.getSourceTableName(), event.getEvent(),
                 event.getBefore(), event.getAfter());
 
-        // 1、获取映射字段
-        final String eventName = event.getEvent();
-        Map<String, Object> data = StringUtil.equals(ConnectorConstant.OPERTION_DELETE, eventName) ? event.getBefore() : event.getAfter();
-        Picker picker = new Picker(tableGroup.getFieldMapping(), data);
-        Map target = picker.getTargetMap();
-
-        // 2、参数转换
-        ConvertUtil.convert(tableGroup.getConvert(), target);
-
-        // 3、插件转换
-        pluginFactory.convert(tableGroup.getPlugin(), eventName, data, target);
-
-        // 4、写入缓冲执行器
-        writerBufferActuator.offer(new WriterRequest(tableGroup.getId(), target, mapping.getMetaId(), mapping.getTargetConnectorId(), event.getSourceTableName(), event.getTargetTableName(), eventName, picker.getTargetFields(), tableGroup.getCommand()));
+        parserStrategy.execute(mapping, tableGroup, event);
     }
 
     /**
@@ -330,12 +318,11 @@ public class ParserFactory implements Parser {
         String event = batchWriter.getEvent();
         Map<String, String> command = batchWriter.getCommand();
         List<Field> fields = batchWriter.getFields();
-        boolean forceUpdate = batchWriter.isForceUpdate();
         // 总数
         int total = dataList.size();
         // 单次任务
         if (total <= batchSize) {
-            return connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, dataList, forceUpdate));
+            return connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, dataList));
         }
 
         // 批量任务, 拆分
@@ -358,7 +345,7 @@ public class ParserFactory implements Parser {
 
             taskExecutor.execute(() -> {
                 try {
-                    Result w = connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, data, forceUpdate));
+                    Result w = connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, data));
                     result.addSuccessData(w.getSuccessData());
                     result.addFailData(w.getFailData());
                     result.getError().append(w.getError());

dbsyncer-parser/src/main/java/org/dbsyncer/parser/config/ParserFlushStrategyConfiguration.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/config/ParserStrategyConfiguration.java

@@ -1,7 +1,9 @@
 package org.dbsyncer.parser.config;
 
-import org.dbsyncer.parser.flush.FlushStrategy;
-import org.dbsyncer.parser.flush.impl.DisableFullFlushStrategy;
+import org.dbsyncer.parser.strategy.FlushStrategy;
+import org.dbsyncer.parser.strategy.ParserStrategy;
+import org.dbsyncer.parser.strategy.impl.DisableFullFlushStrategy;
+import org.dbsyncer.parser.strategy.impl.DisableWriterBufferActuatorStrategy;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -12,7 +14,7 @@ import org.springframework.context.annotation.Configuration;
  * @date 2021/11/18 21:36
  */
 @Configuration
-public class ParserFlushStrategyConfiguration {
+public class ParserStrategyConfiguration {
 
     @Bean
     @ConditionalOnMissingBean
@@ -20,4 +22,10 @@ public class ParserFlushStrategyConfiguration {
         return new DisableFullFlushStrategy();
     }
 
+    @Bean
+    @ConditionalOnMissingBean
+    public ParserStrategy parserStrategy() {
+        return new DisableWriterBufferActuatorStrategy();
+    }
+
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java

@@ -7,6 +7,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import javax.annotation.PostConstruct;
+import java.lang.reflect.ParameterizedType;
 import java.time.Instant;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -18,6 +19,10 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
+ * 任务缓存执行器
+ * <p>1. 任务优先进入缓存队列
+ * <p>2. 将任务分区合并，批量执行
+ *
  * @author AE86
  * @version 1.0.0
  * @date 2022/3/27 17:36
@@ -31,35 +36,26 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
 
     private static final int CAPACITY = 10_0000;
 
-    private Queue<Request> buffer = new LinkedBlockingQueue(CAPACITY);
+    private static final double BUFFER_THRESHOLD = 0.75;
+
+    private static final long MAX_BATCH_COUNT = 1000L;
 
-    private Queue<Request> temp = new LinkedBlockingQueue(CAPACITY);
+    private static final long PERIOD = 300;
+
+    private Queue<Request> buffer = new LinkedBlockingQueue(CAPACITY);
 
     private final Lock lock = new ReentrantLock(true);
 
     private volatile boolean running;
 
-    private final static long MAX_BATCH_COUNT = 1000L;
+    private Class<Response> responseClazz;
 
     @PostConstruct
     private void init() {
-        scheduledTaskService.start(getPeriod(), this);
+        responseClazz = (Class<Response>) ((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[1];
+        scheduledTaskService.start(PERIOD, this);
     }
 
-    /**
-     * 获取定时间隔（毫秒）
-     *
-     * @return
-     */
-    protected abstract long getPeriod();
-
-    /**
-     * 生成缓存value
-     *
-     * @return
-     */
-    protected abstract BufferResponse getValue();
-
     /**
      * 生成分区key
      *
@@ -83,17 +79,18 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
      */
     protected abstract void pull(Response response);
 
+    @Override
+    public Queue getQueue() {
+        return buffer;
+    }
+
     @Override
     public void offer(BufferRequest request) {
-        if (running) {
-            temp.offer((Request) request);
-        } else {
-            buffer.offer((Request) request);
-        }
+        buffer.offer((Request) request);
 
         // TODO 临时解决方案：生产大于消费问题，限制生产速度
-        int size = temp.size() + buffer.size();
-        if (size >= CAPACITY) {
+        int size = buffer.size();
+        if (size >= (CAPACITY * BUFFER_THRESHOLD)) {
             try {
                 TimeUnit.SECONDS.sleep(30);
                 logger.warn("当前任务队列大小{}已达上限{}，请稍等{}秒", size, CAPACITY, 30);
@@ -116,8 +113,6 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
             if (locked) {
                 running = true;
                 flush(buffer);
-                running = false;
-                flush(temp);
             }
         } catch (Exception e) {
             logger.error(e.getMessage());
@@ -129,7 +124,7 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
         }
     }
 
-    private void flush(Queue<Request> queue) {
+    private void flush(Queue<Request> queue) throws IllegalAccessException, InstantiationException {
         if (!queue.isEmpty()) {
             AtomicLong batchCounter = new AtomicLong();
             final Map<String, BufferResponse> map = new LinkedHashMap<>();
@@ -137,7 +132,7 @@ public abstract class AbstractBufferActuator<Request, Response> implements Buffe
                 Request poll = queue.poll();
                 String key = getPartitionKey(poll);
                 if (!map.containsKey(key)) {
-                    map.putIfAbsent(key, getValue());
+                    map.putIfAbsent(key, (BufferResponse) responseClazz.newInstance());
                 }
                 partition(poll, (Response) map.get(key));
                 batchCounter.incrementAndGet();

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java

@@ -5,6 +5,7 @@ import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.parser.model.Meta;
+import org.dbsyncer.parser.strategy.FlushStrategy;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.util.Assert;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/BufferActuator.java

@@ -1,5 +1,7 @@
 package org.dbsyncer.parser.flush;
 
+import java.util.Queue;
+
 /**
  * @author AE86
  * @version 1.0.0
@@ -7,6 +9,18 @@ package org.dbsyncer.parser.flush;
  */
 public interface BufferActuator {
 
+    /**
+     * 获取缓存队列
+     *
+     * @return
+     */
+    Queue getQueue();
+
+    /**
+     * 提交任务
+     *
+     * @param request
+     */
     void offer(BufferRequest request);
 
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java

@@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSONException;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.parser.flush.BufferActuator;
 import org.dbsyncer.parser.flush.FlushService;
-import org.dbsyncer.parser.flush.model.StorageRequest;
+import org.dbsyncer.parser.model.StorageRequest;
 import org.dbsyncer.common.snowflake.SnowflakeIdWorker;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java

@@ -1,9 +1,8 @@
 package org.dbsyncer.parser.flush.impl;
 
 import org.dbsyncer.parser.flush.AbstractBufferActuator;
-import org.dbsyncer.parser.flush.BufferResponse;
-import org.dbsyncer.parser.flush.model.StorageRequest;
-import org.dbsyncer.parser.flush.model.StorageResponse;
+import org.dbsyncer.parser.model.StorageRequest;
+import org.dbsyncer.parser.model.StorageResponse;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.enums.StorageEnum;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -20,16 +19,6 @@ public class StorageBufferActuator extends AbstractBufferActuator<StorageRequest
     @Autowired
     private StorageService storageService;
 
-    @Override
-    protected long getPeriod() {
-        return 500;
-    }
-
-    @Override
-    protected BufferResponse getValue() {
-        return new StorageResponse();
-    }
-
     @Override
     protected String getPartitionKey(StorageRequest bufferTask) {
         return bufferTask.getMetaId();

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/WriterBufferActuator.java

@@ -7,10 +7,9 @@ import org.dbsyncer.connector.ConnectorMapper;
 import org.dbsyncer.connector.config.ConnectorConfig;
 import org.dbsyncer.parser.ParserFactory;
 import org.dbsyncer.parser.flush.AbstractBufferActuator;
-import org.dbsyncer.parser.flush.BufferResponse;
-import org.dbsyncer.parser.flush.FlushStrategy;
-import org.dbsyncer.parser.flush.model.WriterRequest;
-import org.dbsyncer.parser.flush.model.WriterResponse;
+import org.dbsyncer.parser.strategy.FlushStrategy;
+import org.dbsyncer.parser.model.WriterRequest;
+import org.dbsyncer.parser.model.WriterResponse;
 import org.dbsyncer.parser.model.BatchWriter;
 import org.dbsyncer.parser.model.Connector;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -41,16 +40,6 @@ public class WriterBufferActuator extends AbstractBufferActuator<WriterRequest,
 
     private final static int BATCH_SIZE = 100;
 
-    @Override
-    protected long getPeriod() {
-        return 300;
-    }
-
-    @Override
-    protected BufferResponse getValue() {
-        return new WriterResponse();
-    }
-
     @Override
     protected String getPartitionKey(WriterRequest request) {
         return new StringBuilder(request.getTableGroupId()).append("-").append(request.getEvent()).toString();
@@ -76,7 +65,7 @@ public class WriterBufferActuator extends AbstractBufferActuator<WriterRequest,
     protected void pull(WriterResponse response) {
         ConnectorMapper targetConnectorMapper = connectorFactory.connect(getConnectorConfig(response.getTargetConnectorId()));
         Result result = parserFactory.writeBatch(new BatchWriter(targetConnectorMapper, response.getCommand(), response.getTargetTableName(), response.getEvent(),
-                response.getFields(), response.getDataList(), BATCH_SIZE, true));
+                response.getFields(), response.getDataList(), BATCH_SIZE));
         flushStrategy.flushIncrementData(response.getMetaId(), result, response.getEvent());
     }
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogType.java

@@ -242,7 +242,8 @@ public interface LogType {
      */
     enum CacheLog implements LogType {
         IMPORT("70", "导入配置"),
-        EXPORT("71", "导出配置");
+        IMPORT_ERROR("71", "导入配置异常"),
+        EXPORT("72", "导出配置");
 
         private String type;
         private String message;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/AbstractWriter.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractWriter.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.model;
+package org.dbsyncer.parser.model;
 
 import org.dbsyncer.connector.model.Field;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java

@@ -8,22 +8,16 @@ import java.util.Map;
 
 public final class BatchWriter {
 
-    private ConnectorMapper     connectorMapper;
+    private ConnectorMapper connectorMapper;
     private Map<String, String> command;
-    private String              tableName;
-    private String              event;
-    private List<Field>         fields;
-    private List<Map>           dataList;
-    private int                 batchSize;
-    private boolean             isForceUpdate;
+    private String tableName;
+    private String event;
+    private List<Field> fields;
+    private List<Map> dataList;
+    private int batchSize;
 
     public BatchWriter(ConnectorMapper connectorMapper, Map<String, String> command, String tableName, String event,
                        List<Field> fields, List<Map> dataList, int batchSize) {
-        this(connectorMapper, command, tableName, event, fields, dataList, batchSize, false);
-    }
-
-    public BatchWriter(ConnectorMapper connectorMapper, Map<String, String> command, String tableName, String event,
-                       List<Field> fields, List<Map> dataList, int batchSize, boolean isForceUpdate) {
         this.connectorMapper = connectorMapper;
         this.command = command;
         this.tableName = tableName;
@@ -31,7 +25,6 @@ public final class BatchWriter {
         this.fields = fields;
         this.dataList = dataList;
         this.batchSize = batchSize;
-        this.isForceUpdate = isForceUpdate;
     }
 
     public ConnectorMapper getConnectorMapper() {
@@ -62,7 +55,4 @@ public final class BatchWriter {
         return batchSize;
     }
 
-    public boolean isForceUpdate() {
-        return isForceUpdate;
-    }
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Picker.java

@@ -3,24 +3,17 @@ package org.dbsyncer.parser.model;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.connector.model.Field;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 public class Picker {
 
     private List<Field> sourceFields;
     private List<Field> targetFields;
-    private List<Map>   targetMapList;
 
     public Picker(List<FieldMapping> fieldMapping) {
-        pickFields(fieldMapping);
-    }
-
-    public Picker(List<FieldMapping> fieldMapping, Map<String, Object> data) {
-        pickFields(fieldMapping);
-        pickData(data);
-    }
-
-    public void pickFields(List<FieldMapping> fieldMapping) {
         sourceFields = new ArrayList<>();
         targetFields = new ArrayList<>();
         if (!CollectionUtils.isEmpty(fieldMapping)) {
@@ -32,7 +25,7 @@ public class Picker {
     }
 
     public List<Map> pickData(List<Map> data) {
-        targetMapList = new ArrayList<>();
+        List<Map> targetMapList = new ArrayList<>();
         if (!CollectionUtils.isEmpty(data)) {
             final int size = data.size();
             final int sFieldSize = sourceFields.size();
@@ -46,13 +39,12 @@ public class Picker {
         return targetMapList;
     }
 
-    public void pickData(Map<String, Object> data) {
-        targetMapList = new ArrayList<>();
+    public Map pickData(Map<String, Object> data) {
+        Map targetMap = new HashMap<>();
         if (!CollectionUtils.isEmpty(data)) {
-            Map targetMap = new HashMap<>();
             exchange(sourceFields.size(), sourceFields, targetFields, data, targetMap);
-            targetMapList.add(targetMap);
         }
+        return targetMap;
     }
 
     private void exchange(int sFieldSize, List<Field> sFields, List<Field> tFields, Map<String, Object> source,
@@ -70,16 +62,8 @@ public class Picker {
         }
     }
 
-    public Map getTargetMap() {
-        return CollectionUtils.isEmpty(targetMapList) ? Collections.EMPTY_MAP : targetMapList.get(0);
-    }
-
     public List<Field> getTargetFields() {
         return targetFields;
     }
 
-    public List<Map> getTargetMapList() {
-        return targetMapList;
-    }
-
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/StorageRequest.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/StorageRequest.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.model;
+package org.dbsyncer.parser.model;
 
 import org.dbsyncer.parser.flush.BufferRequest;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/StorageResponse.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/StorageResponse.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.model;
+package org.dbsyncer.parser.model;
 
 import org.dbsyncer.parser.flush.BufferResponse;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/WriterRequest.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.model;
+package org.dbsyncer.parser.model;
 
 import org.dbsyncer.connector.model.Field;
 import org.dbsyncer.parser.flush.BufferRequest;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/model/WriterResponse.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.model;
+package org.dbsyncer.parser.model;
 
 import org.dbsyncer.parser.flush.BufferResponse;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/FlushStrategy.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/FlushStrategy.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush;
+package org.dbsyncer.parser.strategy;
 
 import org.dbsyncer.common.model.Result;
 

dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/ParserStrategy.java

@@ -0,0 +1,11 @@
+package org.dbsyncer.parser.strategy;
+
+import org.dbsyncer.common.event.RowChangedEvent;
+import org.dbsyncer.parser.model.Mapping;
+import org.dbsyncer.parser.model.TableGroup;
+
+public interface ParserStrategy {
+
+    void execute(Mapping mapping, TableGroup tableGroup, RowChangedEvent event);
+
+}

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/DisableFullFlushStrategy.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.impl;
+package org.dbsyncer.parser.strategy.impl;
 
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableWriterBufferActuatorStrategy.java

@@ -0,0 +1,67 @@
+package org.dbsyncer.parser.strategy.impl;
+
+import com.google.protobuf.ByteString;
+import org.dbsyncer.cache.CacheService;
+import org.dbsyncer.common.event.RowChangedEvent;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.constant.ConnectorConstant;
+import org.dbsyncer.parser.flush.BufferActuator;
+import org.dbsyncer.parser.model.Mapping;
+import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.parser.model.WriterRequest;
+import org.dbsyncer.parser.strategy.ParserStrategy;
+import org.dbsyncer.storage.binlog.AbstractBinlogRecorder;
+import org.dbsyncer.storage.binlog.proto.BinlogMessage;
+import org.dbsyncer.storage.binlog.proto.Data;
+import org.dbsyncer.storage.binlog.proto.EventEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Map;
+import java.util.Queue;
+
+public final class DisableWriterBufferActuatorStrategy extends AbstractBinlogRecorder<WriterRequest> implements ParserStrategy {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    @Autowired
+    private BufferActuator writerBufferActuator;
+
+    @Autowired
+    private CacheService cacheService;
+
+    @Override
+    public void execute(Mapping mapping, TableGroup tableGroup, RowChangedEvent event) {
+        try {
+            EventEnum eventEnum = EventEnum.valueOf(event.getEvent());
+            Map<String, Object> data = StringUtil.equals(ConnectorConstant.OPERTION_DELETE, eventEnum.name()) ? event.getBefore() : event.getAfter();
+            BinlogMessage.Builder builder = BinlogMessage.newBuilder()
+                    .setTableGroupId(tableGroup.getId())
+                    .setEvent(eventEnum);
+            data.forEach((k, v) -> {
+                if (null != v && v instanceof String) {
+                    builder.addData(Data.newBuilder().putRow(k, ByteString.copyFromUtf8((String) v)));
+                }
+            });
+            flush(builder.build());
+        } catch (Exception e) {
+            logger.error(e.getMessage());
+        }
+    }
+
+    @Override
+    protected String getTaskName() {
+        return "WriterBinlog";
+    }
+
+    @Override
+    protected Queue getQueue() {
+        return writerBufferActuator.getQueue();
+    }
+
+    @Override
+    protected WriterRequest deserialize(BinlogMessage message) {
+        return null;
+    }
+}

dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/EnableFlushStrategy.java → dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/EnableFlushStrategy.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.parser.flush.impl;
+package org.dbsyncer.parser.strategy.impl;
 
 import org.dbsyncer.parser.flush.AbstractFlushStrategy;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/EnableWriterBufferActuatorStrategy.java

@@ -0,0 +1,48 @@
+package org.dbsyncer.parser.strategy.impl;
+
+import org.dbsyncer.common.event.RowChangedEvent;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.constant.ConnectorConstant;
+import org.dbsyncer.parser.flush.BufferActuator;
+import org.dbsyncer.parser.model.Mapping;
+import org.dbsyncer.parser.model.Picker;
+import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.parser.model.WriterRequest;
+import org.dbsyncer.parser.strategy.ParserStrategy;
+import org.dbsyncer.parser.util.ConvertUtil;
+import org.dbsyncer.plugin.PluginFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+
+@Component
+@ConditionalOnProperty(value = "dbsyncer.parser.writer.buffer.actuator.enabled", havingValue = "true")
+public final class EnableWriterBufferActuatorStrategy implements ParserStrategy {
+
+    @Autowired
+    private PluginFactory pluginFactory;
+
+    @Autowired
+    private BufferActuator writerBufferActuator;
+
+    @Override
+    public void execute(Mapping mapping, TableGroup tableGroup, RowChangedEvent event) {
+        // 1、获取映射字段
+        final String eventName = event.getEvent();
+        Map<String, Object> data = StringUtil.equals(ConnectorConstant.OPERTION_DELETE, eventName) ? event.getBefore() : event.getAfter();
+        Picker picker = new Picker(tableGroup.getFieldMapping());
+        Map target = picker.pickData(data);
+
+        // 2、参数转换
+        ConvertUtil.convert(tableGroup.getConvert(), target);
+
+        // 3、插件转换
+        pluginFactory.convert(tableGroup.getPlugin(), eventName, data, target);
+
+        // 4、写入缓冲执行器
+        writerBufferActuator.offer(new WriterRequest(tableGroup.getId(), target, mapping.getMetaId(), mapping.getTargetConnectorId(), event.getSourceTableName(), event.getTargetTableName(), eventName, picker.getTargetFields(), tableGroup.getCommand()));
+    }
+
+}

dbsyncer-plugin/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-plugin</artifactId>

dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/enums/FileSuffixEnum.java

@@ -14,7 +14,12 @@ public enum FileSuffixEnum {
     /**
      * jar
      */
-    JAR("jar");
+    JAR("jar"),
+
+    /**
+     * JSON
+     */
+    JSON("json");
 
     private String name;
 

dbsyncer-storage/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-storage</artifactId>
@@ -32,6 +32,12 @@
             <version>${project.parent.version}</version>
         </dependency>
 
+        <!-- protobuf -->
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-log4j2</artifactId>

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogRecorder.java

@@ -0,0 +1,118 @@
+package org.dbsyncer.storage.binlog;
+
+import org.dbsyncer.common.scheduled.ScheduledTaskJob;
+import org.dbsyncer.common.scheduled.ScheduledTaskService;
+import org.dbsyncer.storage.binlog.proto.BinlogMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.PostConstruct;
+import java.io.IOException;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/8 0:53
+ */
+public abstract class AbstractBinlogRecorder<Message> implements BinlogRecorder, ScheduledTaskJob, DisposableBean {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    @Autowired
+    private ScheduledTaskService scheduledTaskService;
+
+    private static final long MAX_BATCH_COUNT = 100L;
+
+    private static final long PERIOD = 3000;
+
+    private final Lock lock = new ReentrantLock(true);
+
+    private volatile boolean running;
+
+    private BinlogContext context;
+
+    @PostConstruct
+    private void init() throws IOException {
+        // /data/binlog/WriterBinlog/
+        context = new BinlogContext(getTaskName());
+        scheduledTaskService.start(PERIOD, this);
+    }
+
+    /**
+     * 获取任务名称
+     *
+     * @return
+     */
+    protected String getTaskName() {
+        return getClass().getSimpleName();
+    }
+
+    /**
+     * 获取缓存队列
+     *
+     * @return
+     */
+    protected abstract Queue getQueue();
+
+    /**
+     * 反序列化任务
+     *
+     * @param message
+     * @return
+     */
+    protected abstract Message deserialize(BinlogMessage message);
+
+    @Override
+    public void run() {
+        if (running || !getQueue().isEmpty()) {
+            return;
+        }
+
+        final Lock binlogLock = lock;
+        boolean locked = false;
+        try {
+            locked = binlogLock.tryLock();
+            if (locked) {
+                running = true;
+                doParse();
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage());
+        } finally {
+            if (locked) {
+                running = false;
+                binlogLock.unlock();
+            }
+        }
+    }
+
+    @Override
+    public void flush(BinlogMessage message) throws IOException {
+        context.write(message);
+    }
+
+    @Override
+    public void destroy() {
+        context.close();
+    }
+
+    private void doParse() throws IOException {
+        byte[] line;
+        AtomicInteger batchCounter = new AtomicInteger();
+        while (batchCounter.get() < MAX_BATCH_COUNT && null != (line = context.readLine())) {
+            deserialize(BinlogMessage.parseFrom(line));
+            // getQueue().offer(deserialize(message));
+            batchCounter.getAndAdd(1);
+        }
+
+        if (batchCounter.get() > 0) {
+            context.flush();
+        }
+    }
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/Binlog.java

@@ -0,0 +1,29 @@
+package org.dbsyncer.storage.binlog;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/19 23:03
+ */
+public final class Binlog {
+    private String fileName;
+    private long position = 0;
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public Binlog setFileName(String fileName) {
+        this.fileName = fileName;
+        return this;
+    }
+
+    public long getPosition() {
+        return position;
+    }
+
+    public Binlog setPosition(long position) {
+        this.position = position;
+        return this;
+    }
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogContext.java

@@ -0,0 +1,178 @@
+package org.dbsyncer.storage.binlog;
+
+import org.apache.commons.io.FileUtils;
+import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.JsonUtil;
+import org.dbsyncer.common.util.NumberUtil;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.storage.binlog.proto.BinlogMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+public class BinlogContext implements Closeable {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    private static final long BINLOG_MAX_SIZE = 256 * 1024 * 1024;
+
+    private static final int BINLOG_EXPIRE_DAYS = 7;
+
+    private static final String LINE_SEPARATOR = System.lineSeparator();
+
+    private static final Charset DEFAULT_CHARSET = Charset.defaultCharset();
+
+    private static final String BINLOG = "binlog";
+
+    private static final String BINLOG_INDEX = BINLOG + ".index";
+
+    private static final String BINLOG_CONFIG = BINLOG + ".config";
+
+    private List<String> index = new LinkedList<>();
+
+    private String path;
+
+    private File configFile;
+
+    private File indexFile;
+
+    private Binlog config;
+
+    private BinlogPipeline pipeline;
+
+    public BinlogContext(String taskName) throws IOException {
+        path = new StringBuilder(System.getProperty("user.dir")).append(File.separatorChar)
+                .append("data").append(File.separatorChar)
+                .append("binlog").append(File.separatorChar)
+                .append(taskName).append(File.separatorChar)
+                .toString();
+        File dir = new File(path);
+        if (!dir.exists()) {
+            FileUtils.forceMkdir(dir);
+        }
+
+        // binlog.index
+        indexFile = new File(path + BINLOG_INDEX);
+        // binlog.config
+        configFile = new File(path + BINLOG_CONFIG);
+        if (!configFile.exists()) {
+            // binlog.000001
+            config = initBinlogConfig(createNewBinlogName(0));
+        }
+
+        // read index
+        Assert.isTrue(indexFile.exists(), String.format("The index file '%s' is not exist.", indexFile.getName()));
+        index.addAll(FileUtils.readLines(indexFile, DEFAULT_CHARSET));
+
+        // delete index file
+        deleteExpiredIndexFile();
+
+        // {"binlog":"binlog.000001","pos":0}
+        if (null == config) {
+            config = JsonUtil.jsonToObj(FileUtils.readFileToString(configFile, DEFAULT_CHARSET), Binlog.class);
+        }
+
+        // no index
+        if (CollectionUtils.isEmpty(index)) {
+            // binlog.000002
+            config = initBinlogConfig(createNewBinlogName(getBinlogIndex(config.getFileName())));
+            index.addAll(FileUtils.readLines(indexFile, DEFAULT_CHARSET));
+        }
+
+        // 配置文件已失效，取最早的索引文件
+        int indexOf = index.indexOf(config.getFileName());
+        if (-1 == indexOf) {
+            logger.warn("The binlog file '{}' is expired.", config.getFileName());
+            config = new Binlog().setFileName(index.get(0));
+            write(configFile, JsonUtil.objToJson(config), false);
+        }
+
+        pipeline = new BinlogPipeline(new File(path + config.getFileName()), config.getPosition());
+        logger.info("BinlogContext initialized with config:{}", JsonUtil.objToJson(config));
+    }
+
+    private Binlog initBinlogConfig(String binlogName) throws IOException {
+        Binlog config = new Binlog().setFileName(binlogName);
+        write(configFile, JsonUtil.objToJson(config), false);
+        write(indexFile, binlogName + LINE_SEPARATOR, false);
+        write(new File(path + binlogName), "", false);
+        return config;
+    }
+
+    private void deleteExpiredIndexFile() throws IOException {
+        if (CollectionUtils.isEmpty(index)) {
+            return;
+        }
+        Set<String> shouldDelete = new HashSet<>();
+        for (String name : index) {
+            File file = new File(path + name);
+            if (!file.exists()) {
+                shouldDelete.add(name);
+                logger.info("Delete invalid binlog file '{}'.", name);
+                continue;
+            }
+            if (isExpiredFile(file)) {
+                FileUtils.forceDelete(file);
+                shouldDelete.add(name);
+                logger.info("Delete expired binlog file '{}'.", name);
+            }
+        }
+        if (!CollectionUtils.isEmpty(shouldDelete)) {
+            index.removeAll(shouldDelete);
+            StringBuilder indexBuilder = new StringBuilder();
+            index.forEach(name -> indexBuilder.append(name).append(LINE_SEPARATOR));
+            write(indexFile, indexBuilder.toString(), false);
+        }
+    }
+
+    private boolean isExpiredFile(File file) throws IOException {
+        BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
+        Instant instant = attr.creationTime().toInstant();
+        return Timestamp.from(instant).getTime() < Timestamp.valueOf(LocalDateTime.now().minusDays(BINLOG_EXPIRE_DAYS)).getTime();
+    }
+
+    private String createNewBinlogName(int index) {
+        return String.format("%s.%06d", BINLOG, index <= 0 ? 1 : index + 1);
+    }
+
+    private int getBinlogIndex(String binlogName) {
+        return NumberUtil.toInt(StringUtil.substring(binlogName, BINLOG.length() + 1));
+    }
+
+    public void flush() throws IOException {
+        config.setFileName(pipeline.getBinlogName());
+        config.setPosition(pipeline.getOffset());
+        write(configFile, JsonUtil.objToJson(config), false);
+    }
+
+    public byte[] readLine() throws IOException {
+        return pipeline.readLine();
+    }
+
+    public void write(BinlogMessage message) throws IOException {
+        pipeline.write(message);
+    }
+
+    private void write(File file, String line, boolean append) throws IOException {
+        FileUtils.write(file, line, DEFAULT_CHARSET, append);
+    }
+
+    @Override
+    public void close() {
+        pipeline.close();
+    }
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogPipeline.java

@@ -0,0 +1,60 @@
+package org.dbsyncer.storage.binlog;
+
+import org.apache.commons.io.IOUtils;
+import org.dbsyncer.common.file.BufferedRandomAccessFile;
+import org.dbsyncer.storage.binlog.proto.BinlogMessage;
+
+import java.io.*;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/19 23:36
+ */
+public class BinlogPipeline implements Closeable {
+    private final RandomAccessFile raf;
+    private final OutputStream out;
+    private final byte[] h = new byte[1];
+    private byte[] b;
+    private File file;
+    private long offset;
+
+    public BinlogPipeline(File file, long pos) throws IOException {
+        this.file = file;
+        this.raf = new BufferedRandomAccessFile(file, "r");
+        this.out = new FileOutputStream(file, true);
+        raf.seek(pos);
+    }
+
+    public byte[] readLine() throws IOException {
+        this.offset = raf.getFilePointer();
+        if (offset >= raf.length()) {
+            return null;
+        }
+        raf.read(h);
+        b = new byte[Byte.toUnsignedInt(h[0])];
+        raf.read(b);
+        raf.seek(this.offset + (h.length + b.length));
+        return b;
+    }
+
+    public void write(BinlogMessage message) throws IOException {
+        if(null != message){
+            message.writeDelimitedTo(out);
+        }
+    }
+
+    public long getOffset() {
+        return offset;
+    }
+
+    public String getBinlogName() {
+        return file.getName();
+    }
+
+    @Override
+    public void close() {
+        IOUtils.closeQuietly(out);
+        IOUtils.closeQuietly(raf);
+    }
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogRecorder.java

@@ -0,0 +1,21 @@
+package org.dbsyncer.storage.binlog;
+
+import org.dbsyncer.storage.binlog.proto.BinlogMessage;
+
+import java.io.IOException;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/8 23:34
+ */
+public interface BinlogRecorder {
+
+    /**
+     * 将任务序列化刷入磁盘
+     *
+     * @param message
+     */
+    void flush(BinlogMessage message) throws IOException;
+
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/proto/BinlogMessage.java

@@ -0,0 +1,1081 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: BinlogMessageProto.proto
+
+package org.dbsyncer.storage.binlog.proto;
+
+/**
+ * Protobuf type {@code BinlogMessage}
+ */
+public final class BinlogMessage extends
+        com.google.protobuf.GeneratedMessageV3 implements
+        // @@protoc_insertion_point(message_implements:BinlogMessage)
+        BinlogMessageOrBuilder {
+    private static final long serialVersionUID = 0L;
+
+    // Use BinlogMessage.newBuilder() to construct.
+    private BinlogMessage(com.google.protobuf.GeneratedMessageV3.Builder<?> builder) {
+        super(builder);
+    }
+
+    private BinlogMessage() {
+        tableGroupId_ = "";
+        event_ = 0;
+        data_ = java.util.Collections.emptyList();
+    }
+
+    @java.lang.Override
+    @SuppressWarnings({"unused"})
+    protected java.lang.Object newInstance(
+            UnusedPrivateParameter unused) {
+        return new BinlogMessage();
+    }
+
+    @java.lang.Override
+    public final com.google.protobuf.UnknownFieldSet
+    getUnknownFields() {
+        return this.unknownFields;
+    }
+
+    private BinlogMessage(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        this();
+        if (extensionRegistry == null) {
+            throw new java.lang.NullPointerException();
+        }
+        int mutable_bitField0_ = 0;
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+                com.google.protobuf.UnknownFieldSet.newBuilder();
+        try {
+            boolean done = false;
+            while (!done) {
+        int tag = input.readTag();
+        switch (tag) {
+          case 0:
+            done = true;
+            break;
+          case 10: {
+            java.lang.String s = input.readStringRequireUtf8();
+
+            tableGroupId_ = s;
+            break;
+          }
+          case 16: {
+            int rawValue = input.readEnum();
+
+            event_ = rawValue;
+            break;
+          }
+          case 26: {
+            if (!((mutable_bitField0_ & 0x00000001) != 0)) {
+              data_ = new java.util.ArrayList<org.dbsyncer.storage.binlog.proto.Data>();
+              mutable_bitField0_ |= 0x00000001;
+            }
+              data_.add(
+                      input.readMessage(org.dbsyncer.storage.binlog.proto.Data.parser(), extensionRegistry));
+            break;
+          }
+          default: {
+              if (!parseUnknownField(
+                      input, unknownFields, extensionRegistry, tag)) {
+                  done = true;
+              }
+            break;
+          }
+        }
+      }
+    } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+      throw e.setUnfinishedMessage(this);
+    } catch (com.google.protobuf.UninitializedMessageException e) {
+      throw e.asInvalidProtocolBufferException().setUnfinishedMessage(this);
+    } catch (java.io.IOException e) {
+            throw new com.google.protobuf.InvalidProtocolBufferException(
+                    e).setUnfinishedMessage(this);
+        } finally {
+            if (((mutable_bitField0_ & 0x00000001) != 0)) {
+                data_ = java.util.Collections.unmodifiableList(data_);
+            }
+            this.unknownFields = unknownFields.build();
+            makeExtensionsImmutable();
+        }
+    }
+
+    public static final com.google.protobuf.Descriptors.Descriptor
+    getDescriptor() {
+        return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_BinlogMessage_descriptor;
+    }
+
+    @java.lang.Override
+    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    internalGetFieldAccessorTable() {
+        return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_BinlogMessage_fieldAccessorTable
+                .ensureFieldAccessorsInitialized(
+                        org.dbsyncer.storage.binlog.proto.BinlogMessage.class, org.dbsyncer.storage.binlog.proto.BinlogMessage.Builder.class);
+    }
+
+  public static final int TABLE_GROUP_ID_FIELD_NUMBER = 1;
+  private volatile java.lang.Object tableGroupId_;
+  /**
+   * <code>string table_group_id = 1;</code>
+   * @return The tableGroupId.
+   */
+  @java.lang.Override
+  public java.lang.String getTableGroupId() {
+    java.lang.Object ref = tableGroupId_;
+    if (ref instanceof java.lang.String) {
+      return (java.lang.String) ref;
+    } else {
+        com.google.protobuf.ByteString bs =
+                (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        tableGroupId_ = s;
+        return s;
+    }
+  }
+
+    /**
+     * <code>string table_group_id = 1;</code>
+     *
+     * @return The bytes for tableGroupId.
+     */
+    @java.lang.Override
+    public com.google.protobuf.ByteString
+    getTableGroupIdBytes() {
+        java.lang.Object ref = tableGroupId_;
+        if (ref instanceof java.lang.String) {
+            com.google.protobuf.ByteString b =
+                    com.google.protobuf.ByteString.copyFromUtf8(
+                            (java.lang.String) ref);
+            tableGroupId_ = b;
+            return b;
+        } else {
+            return (com.google.protobuf.ByteString) ref;
+        }
+  }
+
+  public static final int EVENT_FIELD_NUMBER = 2;
+  private int event_;
+
+    /**
+     * <code>.EventEnum event = 2;</code>
+     *
+     * @return The enum numeric value on the wire for event.
+     */
+    @java.lang.Override
+    public int getEventValue() {
+        return event_;
+    }
+
+    /**
+     * <code>.EventEnum event = 2;</code>
+     *
+     * @return The event.
+     */
+    @java.lang.Override
+    public org.dbsyncer.storage.binlog.proto.EventEnum getEvent() {
+        @SuppressWarnings("deprecation")
+        org.dbsyncer.storage.binlog.proto.EventEnum result = org.dbsyncer.storage.binlog.proto.EventEnum.valueOf(event_);
+        return result == null ? org.dbsyncer.storage.binlog.proto.EventEnum.UNRECOGNIZED : result;
+  }
+
+  public static final int DATA_FIELD_NUMBER = 3;
+  private java.util.List<org.dbsyncer.storage.binlog.proto.Data> data_;
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    @java.lang.Override
+    public java.util.List<org.dbsyncer.storage.binlog.proto.Data> getDataList() {
+        return data_;
+    }
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    @java.lang.Override
+    public java.util.List<? extends org.dbsyncer.storage.binlog.proto.DataOrBuilder>
+    getDataOrBuilderList() {
+        return data_;
+    }
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    @java.lang.Override
+    public int getDataCount() {
+    return data_.size();
+    }
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    @java.lang.Override
+    public org.dbsyncer.storage.binlog.proto.Data getData(int index) {
+        return data_.get(index);
+    }
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    @java.lang.Override
+    public org.dbsyncer.storage.binlog.proto.DataOrBuilder getDataOrBuilder(
+            int index) {
+    return data_.get(index);
+  }
+
+  private byte memoizedIsInitialized = -1;
+  @java.lang.Override
+  public final boolean isInitialized() {
+    byte isInitialized = memoizedIsInitialized;
+    if (isInitialized == 1) return true;
+    if (isInitialized == 0) return false;
+
+      memoizedIsInitialized = 1;
+      return true;
+  }
+
+    @java.lang.Override
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+            throws java.io.IOException {
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(tableGroupId_)) {
+            com.google.protobuf.GeneratedMessageV3.writeString(output, 1, tableGroupId_);
+        }
+        if (event_ != org.dbsyncer.storage.binlog.proto.EventEnum.UPDATE.getNumber()) {
+            output.writeEnum(2, event_);
+        }
+        for (int i = 0; i < data_.size(); i++) {
+            output.writeMessage(3, data_.get(i));
+    }
+    unknownFields.writeTo(output);
+  }
+
+  @java.lang.Override
+  public int getSerializedSize() {
+    int size = memoizedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(tableGroupId_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, tableGroupId_);
+      }
+      if (event_ != org.dbsyncer.storage.binlog.proto.EventEnum.UPDATE.getNumber()) {
+          size += com.google.protobuf.CodedOutputStream
+        .computeEnumSize(2, event_);
+      }
+      for (int i = 0; i < data_.size(); i++) {
+          size += com.google.protobuf.CodedOutputStream
+        .computeMessageSize(3, data_.get(i));
+    }
+    size += unknownFields.getSerializedSize();
+    memoizedSize = size;
+    return size;
+  }
+
+  @java.lang.Override
+  public boolean equals(final java.lang.Object obj) {
+    if (obj == this) {
+     return true;
+    }
+    if (!(obj instanceof org.dbsyncer.storage.binlog.proto.BinlogMessage)) {
+      return super.equals(obj);
+    }
+    org.dbsyncer.storage.binlog.proto.BinlogMessage other = (org.dbsyncer.storage.binlog.proto.BinlogMessage) obj;
+
+      if (!getTableGroupId()
+              .equals(other.getTableGroupId())) return false;
+      if (event_ != other.event_) return false;
+      if (!getDataList()
+              .equals(other.getDataList())) return false;
+      if (!unknownFields.equals(other.unknownFields)) return false;
+    return true;
+  }
+
+  @java.lang.Override
+  public int hashCode() {
+    if (memoizedHashCode != 0) {
+      return memoizedHashCode;
+    }
+    int hash = 41;
+    hash = (19 * hash) + getDescriptor().hashCode();
+    hash = (37 * hash) + TABLE_GROUP_ID_FIELD_NUMBER;
+    hash = (53 * hash) + getTableGroupId().hashCode();
+    hash = (37 * hash) + EVENT_FIELD_NUMBER;
+    hash = (53 * hash) + event_;
+    if (getDataCount() > 0) {
+      hash = (37 * hash) + DATA_FIELD_NUMBER;
+      hash = (53 * hash) + getDataList().hashCode();
+    }
+    hash = (29 * hash) + unknownFields.hashCode();
+      memoizedHashCode = hash;
+      return hash;
+  }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(
+            java.nio.ByteBuffer data)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(
+            java.nio.ByteBuffer data,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(
+            com.google.protobuf.ByteString data)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(
+            com.google.protobuf.ByteString data,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(byte[] data)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(
+            byte[] data,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(java.io.InputStream input)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseWithIOException(PARSER, input);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(
+            java.io.InputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseDelimitedFrom(java.io.InputStream input)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseDelimitedWithIOException(PARSER, input);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseDelimitedFrom(
+            java.io.InputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(
+            com.google.protobuf.CodedInputStream input)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseWithIOException(PARSER, input);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage parseFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    @java.lang.Override
+    public Builder newBuilderForType() {
+        return newBuilder();
+    }
+
+    public static Builder newBuilder() {
+    return DEFAULT_INSTANCE.toBuilder();
+  }
+  public static Builder newBuilder(org.dbsyncer.storage.binlog.proto.BinlogMessage prototype) {
+    return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+  }
+
+    @java.lang.Override
+    public Builder toBuilder() {
+        return this == DEFAULT_INSTANCE
+                ? new Builder() : new Builder().mergeFrom(this);
+    }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+            com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        Builder builder = new Builder(parent);
+        return builder;
+    }
+
+    /**
+     * Protobuf type {@code BinlogMessage}
+     */
+    public static final class Builder extends
+            com.google.protobuf.GeneratedMessageV3.Builder<Builder> implements
+            // @@protoc_insertion_point(builder_implements:BinlogMessage)
+            org.dbsyncer.storage.binlog.proto.BinlogMessageOrBuilder {
+        public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+            return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_BinlogMessage_descriptor;
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+            return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_BinlogMessage_fieldAccessorTable
+                    .ensureFieldAccessorsInitialized(
+                            org.dbsyncer.storage.binlog.proto.BinlogMessage.class, org.dbsyncer.storage.binlog.proto.BinlogMessage.Builder.class);
+    }
+
+    // Construct using org.dbsyncer.storage.binlog.proto.BinlogMessage.newBuilder()
+    private Builder() {
+        maybeForceBuilderInitialization();
+    }
+
+        private Builder(
+                com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+            super(parent);
+            maybeForceBuilderInitialization();
+        }
+
+        private void maybeForceBuilderInitialization() {
+            if (com.google.protobuf.GeneratedMessageV3
+                    .alwaysUseFieldBuilders) {
+        getDataFieldBuilder();
+      }
+    }
+    @java.lang.Override
+    public Builder clear() {
+      super.clear();
+      tableGroupId_ = "";
+
+      event_ = 0;
+
+      if (dataBuilder_ == null) {
+        data_ = java.util.Collections.emptyList();
+        bitField0_ = (bitField0_ & ~0x00000001);
+      } else {
+          dataBuilder_.clear();
+      }
+        return this;
+    }
+
+        @java.lang.Override
+        public com.google.protobuf.Descriptors.Descriptor
+        getDescriptorForType() {
+            return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_BinlogMessage_descriptor;
+    }
+
+    @java.lang.Override
+    public org.dbsyncer.storage.binlog.proto.BinlogMessage getDefaultInstanceForType() {
+      return org.dbsyncer.storage.binlog.proto.BinlogMessage.getDefaultInstance();
+    }
+
+    @java.lang.Override
+    public org.dbsyncer.storage.binlog.proto.BinlogMessage build() {
+      org.dbsyncer.storage.binlog.proto.BinlogMessage result = buildPartial();
+      if (!result.isInitialized()) {
+        throw newUninitializedMessageException(result);
+      }
+      return result;
+    }
+
+    @java.lang.Override
+    public org.dbsyncer.storage.binlog.proto.BinlogMessage buildPartial() {
+      org.dbsyncer.storage.binlog.proto.BinlogMessage result = new org.dbsyncer.storage.binlog.proto.BinlogMessage(this);
+      int from_bitField0_ = bitField0_;
+      result.tableGroupId_ = tableGroupId_;
+      result.event_ = event_;
+      if (dataBuilder_ == null) {
+        if (((bitField0_ & 0x00000001) != 0)) {
+          data_ = java.util.Collections.unmodifiableList(data_);
+          bitField0_ = (bitField0_ & ~0x00000001);
+        }
+        result.data_ = data_;
+      } else {
+        result.data_ = dataBuilder_.build();
+      }
+        onBuilt();
+        return result;
+    }
+
+        @java.lang.Override
+        public Builder clone() {
+            return super.clone();
+        }
+
+        @java.lang.Override
+        public Builder setField(
+                com.google.protobuf.Descriptors.FieldDescriptor field,
+                java.lang.Object value) {
+            return super.setField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder clearField(
+                com.google.protobuf.Descriptors.FieldDescriptor field) {
+            return super.clearField(field);
+        }
+
+        @java.lang.Override
+        public Builder clearOneof(
+                com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+            return super.clearOneof(oneof);
+        }
+
+        @java.lang.Override
+        public Builder setRepeatedField(
+                com.google.protobuf.Descriptors.FieldDescriptor field,
+                int index, java.lang.Object value) {
+            return super.setRepeatedField(field, index, value);
+        }
+
+        @java.lang.Override
+        public Builder addRepeatedField(
+                com.google.protobuf.Descriptors.FieldDescriptor field,
+                java.lang.Object value) {
+            return super.addRepeatedField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(com.google.protobuf.Message other) {
+            if (other instanceof org.dbsyncer.storage.binlog.proto.BinlogMessage) {
+                return mergeFrom((org.dbsyncer.storage.binlog.proto.BinlogMessage) other);
+            } else {
+        super.mergeFrom(other);
+        return this;
+      }
+    }
+
+    public Builder mergeFrom(org.dbsyncer.storage.binlog.proto.BinlogMessage other) {
+      if (other == org.dbsyncer.storage.binlog.proto.BinlogMessage.getDefaultInstance()) return this;
+      if (!other.getTableGroupId().isEmpty()) {
+        tableGroupId_ = other.tableGroupId_;
+        onChanged();
+      }
+      if (other.event_ != 0) {
+        setEventValue(other.getEventValue());
+      }
+      if (dataBuilder_ == null) {
+        if (!other.data_.isEmpty()) {
+          if (data_.isEmpty()) {
+            data_ = other.data_;
+            bitField0_ = (bitField0_ & ~0x00000001);
+          } else {
+            ensureDataIsMutable();
+            data_.addAll(other.data_);
+          }
+          onChanged();
+        }
+      } else {
+        if (!other.data_.isEmpty()) {
+            if (dataBuilder_.isEmpty()) {
+                dataBuilder_.dispose();
+                dataBuilder_ = null;
+                data_ = other.data_;
+                bitField0_ = (bitField0_ & ~0x00000001);
+                dataBuilder_ =
+                        com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ?
+                 getDataFieldBuilder() : null;
+          } else {
+            dataBuilder_.addAllMessages(other.data_);
+          }
+        }
+      }
+      this.mergeUnknownFields(other.unknownFields);
+      onChanged();
+      return this;
+    }
+
+    @java.lang.Override
+    public final boolean isInitialized() {
+        return true;
+    }
+
+        @java.lang.Override
+        public Builder mergeFrom(
+                com.google.protobuf.CodedInputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws java.io.IOException {
+            org.dbsyncer.storage.binlog.proto.BinlogMessage parsedMessage = null;
+            try {
+                parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                parsedMessage = (org.dbsyncer.storage.binlog.proto.BinlogMessage) e.getUnfinishedMessage();
+                throw e.unwrapIOException();
+            } finally {
+                if (parsedMessage != null) {
+          mergeFrom(parsedMessage);
+        }
+      }
+      return this;
+    }
+    private int bitField0_;
+
+    private java.lang.Object tableGroupId_ = "";
+    /**
+     * <code>string table_group_id = 1;</code>
+     * @return The tableGroupId.
+     */
+    public java.lang.String getTableGroupId() {
+      java.lang.Object ref = tableGroupId_;
+        if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs =
+                    (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+            tableGroupId_ = s;
+            return s;
+        } else {
+            return (java.lang.String) ref;
+        }
+    }
+
+        /**
+         * <code>string table_group_id = 1;</code>
+         *
+         * @return The bytes for tableGroupId.
+         */
+        public com.google.protobuf.ByteString
+        getTableGroupIdBytes() {
+            java.lang.Object ref = tableGroupId_;
+            if (ref instanceof String) {
+                com.google.protobuf.ByteString b =
+                        com.google.protobuf.ByteString.copyFromUtf8(
+                                (java.lang.String) ref);
+                tableGroupId_ = b;
+                return b;
+            } else {
+                return (com.google.protobuf.ByteString) ref;
+            }
+        }
+
+        /**
+         * <code>string table_group_id = 1;</code>
+         *
+         * @param value The tableGroupId to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTableGroupId(
+                java.lang.String value) {
+            if (value == null) {
+                throw new NullPointerException();
+            }
+
+            tableGroupId_ = value;
+            onChanged();
+            return this;
+    }
+    /**
+     * <code>string table_group_id = 1;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearTableGroupId() {
+
+        tableGroupId_ = getDefaultInstance().getTableGroupId();
+        onChanged();
+        return this;
+    }
+
+        /**
+         * <code>string table_group_id = 1;</code>
+         *
+         * @param value The bytes for tableGroupId to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTableGroupIdBytes(
+                com.google.protobuf.ByteString value) {
+            if (value == null) {
+                throw new NullPointerException();
+            }
+  checkByteStringIsUtf8(value);
+
+            tableGroupId_ = value;
+            onChanged();
+            return this;
+        }
+
+        private int event_ = 0;
+
+        /**
+         * <code>.EventEnum event = 2;</code>
+         *
+         * @return The enum numeric value on the wire for event.
+         */
+        @java.lang.Override
+        public int getEventValue() {
+            return event_;
+        }
+
+        /**
+         * <code>.EventEnum event = 2;</code>
+         *
+         * @param value The enum numeric value on the wire for event to set.
+         * @return This builder for chaining.
+         */
+        public Builder setEventValue(int value) {
+
+            event_ = value;
+            onChanged();
+            return this;
+        }
+
+        /**
+         * <code>.EventEnum event = 2;</code>
+         *
+         * @return The event.
+         */
+        @java.lang.Override
+        public org.dbsyncer.storage.binlog.proto.EventEnum getEvent() {
+            @SuppressWarnings("deprecation")
+            org.dbsyncer.storage.binlog.proto.EventEnum result = org.dbsyncer.storage.binlog.proto.EventEnum.valueOf(event_);
+            return result == null ? org.dbsyncer.storage.binlog.proto.EventEnum.UNRECOGNIZED : result;
+        }
+
+        /**
+         * <code>.EventEnum event = 2;</code>
+         *
+         * @param value The event to set.
+         * @return This builder for chaining.
+         */
+        public Builder setEvent(org.dbsyncer.storage.binlog.proto.EventEnum value) {
+            if (value == null) {
+                throw new NullPointerException();
+            }
+
+            event_ = value.getNumber();
+            onChanged();
+            return this;
+        }
+
+        /**
+         * <code>.EventEnum event = 2;</code>
+         * @return This builder for chaining.
+     */
+    public Builder clearEvent() {
+      
+      event_ = 0;
+      onChanged();
+      return this;
+    }
+
+    private java.util.List<org.dbsyncer.storage.binlog.proto.Data> data_ =
+      java.util.Collections.emptyList();
+
+        private void ensureDataIsMutable() {
+            if (!((bitField0_ & 0x00000001) != 0)) {
+                data_ = new java.util.ArrayList<org.dbsyncer.storage.binlog.proto.Data>(data_);
+                bitField0_ |= 0x00000001;
+       }
+    }
+
+    private com.google.protobuf.RepeatedFieldBuilderV3<
+        org.dbsyncer.storage.binlog.proto.Data, org.dbsyncer.storage.binlog.proto.Data.Builder, org.dbsyncer.storage.binlog.proto.DataOrBuilder> dataBuilder_;
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    public java.util.List<org.dbsyncer.storage.binlog.proto.Data> getDataList() {
+      if (dataBuilder_ == null) {
+        return java.util.Collections.unmodifiableList(data_);
+      } else {
+        return dataBuilder_.getMessageList();
+      }
+    }
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    public int getDataCount() {
+      if (dataBuilder_ == null) {
+          return data_.size();
+      } else {
+          return dataBuilder_.getCount();
+      }
+    }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public org.dbsyncer.storage.binlog.proto.Data getData(int index) {
+            if (dataBuilder_ == null) {
+                return data_.get(index);
+            } else {
+                return dataBuilder_.getMessage(index);
+            }
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public Builder setData(
+                int index, org.dbsyncer.storage.binlog.proto.Data value) {
+            if (dataBuilder_ == null) {
+                if (value == null) {
+                    throw new NullPointerException();
+                }
+                ensureDataIsMutable();
+                data_.set(index, value);
+                onChanged();
+            } else {
+                dataBuilder_.setMessage(index, value);
+            }
+            return this;
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public Builder setData(
+                int index, org.dbsyncer.storage.binlog.proto.Data.Builder builderForValue) {
+            if (dataBuilder_ == null) {
+        ensureDataIsMutable();
+        data_.set(index, builderForValue.build());
+        onChanged();
+      } else {
+        dataBuilder_.setMessage(index, builderForValue.build());
+      }
+      return this;
+    }
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    public Builder addData(org.dbsyncer.storage.binlog.proto.Data value) {
+        if (dataBuilder_ == null) {
+            if (value == null) {
+                throw new NullPointerException();
+            }
+            ensureDataIsMutable();
+            data_.add(value);
+            onChanged();
+        } else {
+            dataBuilder_.addMessage(value);
+        }
+        return this;
+    }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public Builder addData(
+                int index, org.dbsyncer.storage.binlog.proto.Data value) {
+            if (dataBuilder_ == null) {
+                if (value == null) {
+                    throw new NullPointerException();
+                }
+                ensureDataIsMutable();
+                data_.add(index, value);
+                onChanged();
+            } else {
+                dataBuilder_.addMessage(index, value);
+            }
+            return this;
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public Builder addData(
+                org.dbsyncer.storage.binlog.proto.Data.Builder builderForValue) {
+            if (dataBuilder_ == null) {
+                ensureDataIsMutable();
+                data_.add(builderForValue.build());
+                onChanged();
+            } else {
+                dataBuilder_.addMessage(builderForValue.build());
+            }
+            return this;
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public Builder addData(
+                int index, org.dbsyncer.storage.binlog.proto.Data.Builder builderForValue) {
+            if (dataBuilder_ == null) {
+                ensureDataIsMutable();
+                data_.add(index, builderForValue.build());
+                onChanged();
+            } else {
+                dataBuilder_.addMessage(index, builderForValue.build());
+            }
+            return this;
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public Builder addAllData(
+                java.lang.Iterable<? extends org.dbsyncer.storage.binlog.proto.Data> values) {
+            if (dataBuilder_ == null) {
+        ensureDataIsMutable();
+        com.google.protobuf.AbstractMessageLite.Builder.addAll(
+            values, data_);
+        onChanged();
+      } else {
+        dataBuilder_.addAllMessages(values);
+      }
+      return this;
+    }
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    public Builder clearData() {
+      if (dataBuilder_ == null) {
+        data_ = java.util.Collections.emptyList();
+        bitField0_ = (bitField0_ & ~0x00000001);
+          onChanged();
+      } else {
+          dataBuilder_.clear();
+      }
+        return this;
+    }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public Builder removeData(int index) {
+            if (dataBuilder_ == null) {
+                ensureDataIsMutable();
+                data_.remove(index);
+                onChanged();
+            } else {
+                dataBuilder_.remove(index);
+            }
+            return this;
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public org.dbsyncer.storage.binlog.proto.Data.Builder getDataBuilder(
+                int index) {
+            return getDataFieldBuilder().getBuilder(index);
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public org.dbsyncer.storage.binlog.proto.DataOrBuilder getDataOrBuilder(
+                int index) {
+            if (dataBuilder_ == null) {
+                return data_.get(index);
+            } else {
+                return dataBuilder_.getMessageOrBuilder(index);
+            }
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public java.util.List<? extends org.dbsyncer.storage.binlog.proto.DataOrBuilder>
+        getDataOrBuilderList() {
+            if (dataBuilder_ != null) {
+                return dataBuilder_.getMessageOrBuilderList();
+            } else {
+                return java.util.Collections.unmodifiableList(data_);
+            }
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public org.dbsyncer.storage.binlog.proto.Data.Builder addDataBuilder() {
+            return getDataFieldBuilder().addBuilder(
+                    org.dbsyncer.storage.binlog.proto.Data.getDefaultInstance());
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public org.dbsyncer.storage.binlog.proto.Data.Builder addDataBuilder(
+                int index) {
+            return getDataFieldBuilder().addBuilder(
+                    index, org.dbsyncer.storage.binlog.proto.Data.getDefaultInstance());
+        }
+
+        /**
+         * <code>repeated .Data data = 3;</code>
+         */
+        public java.util.List<org.dbsyncer.storage.binlog.proto.Data.Builder>
+        getDataBuilderList() {
+            return getDataFieldBuilder().getBuilderList();
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                org.dbsyncer.storage.binlog.proto.Data, org.dbsyncer.storage.binlog.proto.Data.Builder, org.dbsyncer.storage.binlog.proto.DataOrBuilder>
+        getDataFieldBuilder() {
+            if (dataBuilder_ == null) {
+                dataBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3<
+                        org.dbsyncer.storage.binlog.proto.Data, org.dbsyncer.storage.binlog.proto.Data.Builder, org.dbsyncer.storage.binlog.proto.DataOrBuilder>(
+                        data_,
+                        ((bitField0_ & 0x00000001) != 0),
+                        getParentForChildren(),
+                        isClean());
+                data_ = null;
+            }
+            return dataBuilder_;
+        }
+
+        @java.lang.Override
+        public final Builder setUnknownFields(
+        final com.google.protobuf.UnknownFieldSet unknownFields) {
+      return super.setUnknownFields(unknownFields);
+    }
+
+    @java.lang.Override
+    public final Builder mergeUnknownFields(
+        final com.google.protobuf.UnknownFieldSet unknownFields) {
+      return super.mergeUnknownFields(unknownFields);
+    }
+
+
+    // @@protoc_insertion_point(builder_scope:BinlogMessage)
+  }
+
+  // @@protoc_insertion_point(class_scope:BinlogMessage)
+  private static final org.dbsyncer.storage.binlog.proto.BinlogMessage DEFAULT_INSTANCE;
+
+    static {
+        DEFAULT_INSTANCE = new org.dbsyncer.storage.binlog.proto.BinlogMessage();
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.BinlogMessage getDefaultInstance() {
+        return DEFAULT_INSTANCE;
+    }
+
+    private static final com.google.protobuf.Parser<BinlogMessage>
+            PARSER = new com.google.protobuf.AbstractParser<BinlogMessage>() {
+    @java.lang.Override
+    public BinlogMessage parsePartialFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return new BinlogMessage(input, extensionRegistry);
+    }
+  };
+
+  public static com.google.protobuf.Parser<BinlogMessage> parser() {
+    return PARSER;
+  }
+
+  @java.lang.Override
+  public com.google.protobuf.Parser<BinlogMessage> getParserForType() {
+    return PARSER;
+  }
+
+  @java.lang.Override
+  public org.dbsyncer.storage.binlog.proto.BinlogMessage getDefaultInstanceForType() {
+    return DEFAULT_INSTANCE;
+  }
+
+}
+

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/proto/BinlogMessageOrBuilder.java

@@ -0,0 +1,66 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: BinlogMessageProto.proto
+
+package org.dbsyncer.storage.binlog.proto;
+
+public interface BinlogMessageOrBuilder extends
+        // @@protoc_insertion_point(interface_extends:BinlogMessage)
+        com.google.protobuf.MessageOrBuilder {
+
+    /**
+     * <code>string table_group_id = 1;</code>
+     *
+     * @return The tableGroupId.
+     */
+    java.lang.String getTableGroupId();
+
+    /**
+     * <code>string table_group_id = 1;</code>
+     *
+     * @return The bytes for tableGroupId.
+     */
+    com.google.protobuf.ByteString
+    getTableGroupIdBytes();
+
+    /**
+     * <code>.EventEnum event = 2;</code>
+     *
+     * @return The enum numeric value on the wire for event.
+     */
+    int getEventValue();
+
+    /**
+     * <code>.EventEnum event = 2;</code>
+     *
+     * @return The event.
+     */
+    org.dbsyncer.storage.binlog.proto.EventEnum getEvent();
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    java.util.List<org.dbsyncer.storage.binlog.proto.Data>
+    getDataList();
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    org.dbsyncer.storage.binlog.proto.Data getData(int index);
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    int getDataCount();
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    java.util.List<? extends org.dbsyncer.storage.binlog.proto.DataOrBuilder>
+    getDataOrBuilderList();
+
+    /**
+     * <code>repeated .Data data = 3;</code>
+     */
+    org.dbsyncer.storage.binlog.proto.DataOrBuilder getDataOrBuilder(
+            int index);
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/proto/BinlogMessageProto.java

@@ -0,0 +1,80 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: BinlogMessageProto.proto
+
+package org.dbsyncer.storage.binlog.proto;
+
+public final class BinlogMessageProto {
+    private BinlogMessageProto() {
+    }
+
+    public static void registerAllExtensions(
+            com.google.protobuf.ExtensionRegistryLite registry) {
+    }
+
+    public static void registerAllExtensions(
+            com.google.protobuf.ExtensionRegistry registry) {
+        registerAllExtensions(
+                (com.google.protobuf.ExtensionRegistryLite) registry);
+    }
+
+    static final com.google.protobuf.Descriptors.Descriptor
+            internal_static_BinlogMessage_descriptor;
+    static final
+    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internal_static_BinlogMessage_fieldAccessorTable;
+    static final com.google.protobuf.Descriptors.Descriptor
+            internal_static_Data_descriptor;
+    static final
+    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internal_static_Data_fieldAccessorTable;
+    static final com.google.protobuf.Descriptors.Descriptor
+            internal_static_Data_RowEntry_descriptor;
+    static final
+    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internal_static_Data_RowEntry_fieldAccessorTable;
+
+    public static com.google.protobuf.Descriptors.FileDescriptor
+    getDescriptor() {
+        return descriptor;
+    }
+
+    private static com.google.protobuf.Descriptors.FileDescriptor
+            descriptor;
+
+    static {
+        java.lang.String[] descriptorData = {
+                "\n\030BinlogMessageProto.proto\"W\n\rBinlogMess" +
+                        "age\022\026\n\016table_group_id\030\001 \001(\t\022\031\n\005event\030\002 \001" +
+                        "(\0162\n.EventEnum\022\023\n\004data\030\003 \003(\0132\005.Data\"O\n\004D" +
+                        "ata\022\033\n\003row\030\001 \003(\0132\016.Data.RowEntry\032*\n\010RowE" +
+                        "ntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\014:\0028\001*/\n\t" +
+                        "EventEnum\022\n\n\006UPDATE\020\000\022\n\n\006INSERT\020\001\022\n\n\006DEL" +
+                        "ETE\020\002B;\n!org.dbsyncer.storage.binlog.pro" +
+                        "toB\022BinlogMessageProtoH\001P\001b\006proto3"
+        };
+        descriptor = com.google.protobuf.Descriptors.FileDescriptor
+                .internalBuildGeneratedFileFrom(descriptorData,
+                        new com.google.protobuf.Descriptors.FileDescriptor[]{
+                        });
+        internal_static_BinlogMessage_descriptor =
+                getDescriptor().getMessageTypes().get(0);
+        internal_static_BinlogMessage_fieldAccessorTable = new
+                com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+                internal_static_BinlogMessage_descriptor,
+                new java.lang.String[]{"TableGroupId", "Event", "Data",});
+        internal_static_Data_descriptor =
+                getDescriptor().getMessageTypes().get(1);
+        internal_static_Data_fieldAccessorTable = new
+                com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+                internal_static_Data_descriptor,
+                new java.lang.String[]{"Row",});
+        internal_static_Data_RowEntry_descriptor =
+                internal_static_Data_descriptor.getNestedTypes().get(0);
+        internal_static_Data_RowEntry_fieldAccessorTable = new
+                com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+                internal_static_Data_RowEntry_descriptor,
+                new java.lang.String[]{"Key", "Value",});
+    }
+
+    // @@protoc_insertion_point(outer_class_scope)
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/proto/Data.java

@@ -0,0 +1,782 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: BinlogMessageProto.proto
+
+package org.dbsyncer.storage.binlog.proto;
+
+/**
+ * Protobuf type {@code Data}
+ */
+public final class Data extends
+        com.google.protobuf.GeneratedMessageV3 implements
+        // @@protoc_insertion_point(message_implements:Data)
+        DataOrBuilder {
+    private static final long serialVersionUID = 0L;
+
+    // Use Data.newBuilder() to construct.
+    private Data(com.google.protobuf.GeneratedMessageV3.Builder<?> builder) {
+        super(builder);
+    }
+
+    private Data() {
+    }
+
+    @java.lang.Override
+    @SuppressWarnings({"unused"})
+    protected java.lang.Object newInstance(
+            UnusedPrivateParameter unused) {
+        return new Data();
+    }
+
+    @java.lang.Override
+    public final com.google.protobuf.UnknownFieldSet
+    getUnknownFields() {
+        return this.unknownFields;
+    }
+
+    private Data(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        this();
+        if (extensionRegistry == null) {
+            throw new java.lang.NullPointerException();
+        }
+        int mutable_bitField0_ = 0;
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+                com.google.protobuf.UnknownFieldSet.newBuilder();
+        try {
+            boolean done = false;
+            while (!done) {
+                int tag = input.readTag();
+                switch (tag) {
+                    case 0:
+                        done = true;
+                        break;
+                    case 10: {
+                        if (!((mutable_bitField0_ & 0x00000001) != 0)) {
+                            row_ = com.google.protobuf.MapField.newMapField(
+                                    RowDefaultEntryHolder.defaultEntry);
+                            mutable_bitField0_ |= 0x00000001;
+                        }
+                        com.google.protobuf.MapEntry<java.lang.String, com.google.protobuf.ByteString>
+                                row__ = input.readMessage(
+                                RowDefaultEntryHolder.defaultEntry.getParserForType(), extensionRegistry);
+                        row_.getMutableMap().put(
+                                row__.getKey(), row__.getValue());
+                        break;
+                    }
+                    default: {
+                        if (!parseUnknownField(
+                                input, unknownFields, extensionRegistry, tag)) {
+                            done = true;
+                        }
+                        break;
+                    }
+                }
+            }
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+            throw e.setUnfinishedMessage(this);
+        } catch (com.google.protobuf.UninitializedMessageException e) {
+            throw e.asInvalidProtocolBufferException().setUnfinishedMessage(this);
+        } catch (java.io.IOException e) {
+            throw new com.google.protobuf.InvalidProtocolBufferException(
+                    e).setUnfinishedMessage(this);
+        } finally {
+            this.unknownFields = unknownFields.build();
+            makeExtensionsImmutable();
+        }
+    }
+
+    public static final com.google.protobuf.Descriptors.Descriptor
+    getDescriptor() {
+        return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_Data_descriptor;
+    }
+
+    @SuppressWarnings({"rawtypes"})
+    @java.lang.Override
+    protected com.google.protobuf.MapField internalGetMapField(
+            int number) {
+        switch (number) {
+            case 1:
+                return internalGetRow();
+            default:
+                throw new RuntimeException(
+                        "Invalid map field number: " + number);
+        }
+    }
+
+    @java.lang.Override
+    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+    internalGetFieldAccessorTable() {
+        return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_Data_fieldAccessorTable
+                .ensureFieldAccessorsInitialized(
+                        org.dbsyncer.storage.binlog.proto.Data.class, org.dbsyncer.storage.binlog.proto.Data.Builder.class);
+    }
+
+    public static final int ROW_FIELD_NUMBER = 1;
+
+    private static final class RowDefaultEntryHolder {
+        static final com.google.protobuf.MapEntry<
+                java.lang.String, com.google.protobuf.ByteString> defaultEntry =
+                com.google.protobuf.MapEntry
+                        .<java.lang.String, com.google.protobuf.ByteString>newDefaultInstance(
+                                org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_Data_RowEntry_descriptor,
+                                com.google.protobuf.WireFormat.FieldType.STRING,
+                                "",
+                                com.google.protobuf.WireFormat.FieldType.BYTES,
+                                com.google.protobuf.ByteString.EMPTY);
+    }
+
+    private com.google.protobuf.MapField<
+            java.lang.String, com.google.protobuf.ByteString> row_;
+
+    private com.google.protobuf.MapField<java.lang.String, com.google.protobuf.ByteString>
+    internalGetRow() {
+        if (row_ == null) {
+            return com.google.protobuf.MapField.emptyMapField(
+                    RowDefaultEntryHolder.defaultEntry);
+        }
+        return row_;
+    }
+
+    public int getRowCount() {
+        return internalGetRow().getMap().size();
+    }
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+
+    @java.lang.Override
+    public boolean containsRow(
+            java.lang.String key) {
+        if (key == null) {
+            throw new NullPointerException("map key");
+        }
+        return internalGetRow().getMap().containsKey(key);
+    }
+
+    /**
+     * Use {@link #getRowMap()} instead.
+     */
+    @java.lang.Override
+    @java.lang.Deprecated
+    public java.util.Map<java.lang.String, com.google.protobuf.ByteString> getRow() {
+        return getRowMap();
+    }
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+    @java.lang.Override
+
+    public java.util.Map<java.lang.String, com.google.protobuf.ByteString> getRowMap() {
+        return internalGetRow().getMap();
+    }
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+    @java.lang.Override
+
+    public com.google.protobuf.ByteString getRowOrDefault(
+            java.lang.String key,
+            com.google.protobuf.ByteString defaultValue) {
+        if (key == null) {
+            throw new NullPointerException("map key");
+        }
+        java.util.Map<java.lang.String, com.google.protobuf.ByteString> map =
+                internalGetRow().getMap();
+        return map.containsKey(key) ? map.get(key) : defaultValue;
+    }
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+    @java.lang.Override
+
+    public com.google.protobuf.ByteString getRowOrThrow(
+            java.lang.String key) {
+        if (key == null) {
+            throw new NullPointerException("map key");
+        }
+        java.util.Map<java.lang.String, com.google.protobuf.ByteString> map =
+                internalGetRow().getMap();
+        if (!map.containsKey(key)) {
+            throw new java.lang.IllegalArgumentException();
+        }
+        return map.get(key);
+    }
+
+    private byte memoizedIsInitialized = -1;
+
+    @java.lang.Override
+    public final boolean isInitialized() {
+        byte isInitialized = memoizedIsInitialized;
+        if (isInitialized == 1) return true;
+        if (isInitialized == 0) return false;
+
+        memoizedIsInitialized = 1;
+        return true;
+    }
+
+    @java.lang.Override
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+            throws java.io.IOException {
+        com.google.protobuf.GeneratedMessageV3
+                .serializeStringMapTo(
+                        output,
+                        internalGetRow(),
+                        RowDefaultEntryHolder.defaultEntry,
+                        1);
+        unknownFields.writeTo(output);
+    }
+
+    @java.lang.Override
+    public int getSerializedSize() {
+        int size = memoizedSize;
+        if (size != -1) return size;
+
+        size = 0;
+        for (java.util.Map.Entry<java.lang.String, com.google.protobuf.ByteString> entry
+                : internalGetRow().getMap().entrySet()) {
+            com.google.protobuf.MapEntry<java.lang.String, com.google.protobuf.ByteString>
+                    row__ = RowDefaultEntryHolder.defaultEntry.newBuilderForType()
+                    .setKey(entry.getKey())
+                    .setValue(entry.getValue())
+                    .build();
+            size += com.google.protobuf.CodedOutputStream
+                    .computeMessageSize(1, row__);
+        }
+        size += unknownFields.getSerializedSize();
+        memoizedSize = size;
+        return size;
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+        if (obj == this) {
+            return true;
+        }
+        if (!(obj instanceof org.dbsyncer.storage.binlog.proto.Data)) {
+            return super.equals(obj);
+        }
+        org.dbsyncer.storage.binlog.proto.Data other = (org.dbsyncer.storage.binlog.proto.Data) obj;
+
+        if (!internalGetRow().equals(
+                other.internalGetRow())) return false;
+        if (!unknownFields.equals(other.unknownFields)) return false;
+        return true;
+    }
+
+    @java.lang.Override
+    public int hashCode() {
+        if (memoizedHashCode != 0) {
+            return memoizedHashCode;
+        }
+        int hash = 41;
+        hash = (19 * hash) + getDescriptor().hashCode();
+        if (!internalGetRow().getMap().isEmpty()) {
+            hash = (37 * hash) + ROW_FIELD_NUMBER;
+            hash = (53 * hash) + internalGetRow().hashCode();
+        }
+        hash = (29 * hash) + unknownFields.hashCode();
+        memoizedHashCode = hash;
+        return hash;
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(
+            java.nio.ByteBuffer data)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(
+            java.nio.ByteBuffer data,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(
+            com.google.protobuf.ByteString data)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(
+            com.google.protobuf.ByteString data,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(byte[] data)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(
+            byte[] data,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(java.io.InputStream input)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseWithIOException(PARSER, input);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(
+            java.io.InputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseDelimitedFrom(java.io.InputStream input)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseDelimitedWithIOException(PARSER, input);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseDelimitedFrom(
+            java.io.InputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(
+            com.google.protobuf.CodedInputStream input)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseWithIOException(PARSER, input);
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data parseFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3
+                .parseWithIOException(PARSER, input, extensionRegistry);
+    }
+
+    @java.lang.Override
+    public Builder newBuilderForType() {
+        return newBuilder();
+    }
+
+    public static Builder newBuilder() {
+        return DEFAULT_INSTANCE.toBuilder();
+    }
+
+    public static Builder newBuilder(org.dbsyncer.storage.binlog.proto.Data prototype) {
+        return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+    }
+
+    @java.lang.Override
+    public Builder toBuilder() {
+        return this == DEFAULT_INSTANCE
+                ? new Builder() : new Builder().mergeFrom(this);
+    }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+            com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        Builder builder = new Builder(parent);
+        return builder;
+    }
+
+    /**
+     * Protobuf type {@code Data}
+     */
+    public static final class Builder extends
+            com.google.protobuf.GeneratedMessageV3.Builder<Builder> implements
+            // @@protoc_insertion_point(builder_implements:Data)
+            org.dbsyncer.storage.binlog.proto.DataOrBuilder {
+        public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+            return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_Data_descriptor;
+        }
+
+        @SuppressWarnings({"rawtypes"})
+        protected com.google.protobuf.MapField internalGetMapField(
+                int number) {
+            switch (number) {
+                case 1:
+                    return internalGetRow();
+                default:
+                    throw new RuntimeException(
+                            "Invalid map field number: " + number);
+            }
+        }
+
+        @SuppressWarnings({"rawtypes"})
+        protected com.google.protobuf.MapField internalGetMutableMapField(
+                int number) {
+            switch (number) {
+                case 1:
+                    return internalGetMutableRow();
+                default:
+                    throw new RuntimeException(
+                            "Invalid map field number: " + number);
+            }
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+            return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_Data_fieldAccessorTable
+                    .ensureFieldAccessorsInitialized(
+                            org.dbsyncer.storage.binlog.proto.Data.class, org.dbsyncer.storage.binlog.proto.Data.Builder.class);
+        }
+
+        // Construct using org.dbsyncer.storage.binlog.proto.Data.newBuilder()
+        private Builder() {
+            maybeForceBuilderInitialization();
+        }
+
+        private Builder(
+                com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+            super(parent);
+            maybeForceBuilderInitialization();
+        }
+
+        private void maybeForceBuilderInitialization() {
+            if (com.google.protobuf.GeneratedMessageV3
+                    .alwaysUseFieldBuilders) {
+            }
+        }
+
+        @java.lang.Override
+        public Builder clear() {
+            super.clear();
+            internalGetMutableRow().clear();
+            return this;
+        }
+
+        @java.lang.Override
+        public com.google.protobuf.Descriptors.Descriptor
+        getDescriptorForType() {
+            return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.internal_static_Data_descriptor;
+        }
+
+        @java.lang.Override
+        public org.dbsyncer.storage.binlog.proto.Data getDefaultInstanceForType() {
+            return org.dbsyncer.storage.binlog.proto.Data.getDefaultInstance();
+        }
+
+        @java.lang.Override
+        public org.dbsyncer.storage.binlog.proto.Data build() {
+            org.dbsyncer.storage.binlog.proto.Data result = buildPartial();
+            if (!result.isInitialized()) {
+                throw newUninitializedMessageException(result);
+            }
+            return result;
+        }
+
+        @java.lang.Override
+        public org.dbsyncer.storage.binlog.proto.Data buildPartial() {
+            org.dbsyncer.storage.binlog.proto.Data result = new org.dbsyncer.storage.binlog.proto.Data(this);
+            int from_bitField0_ = bitField0_;
+            result.row_ = internalGetRow();
+            result.row_.makeImmutable();
+            onBuilt();
+            return result;
+        }
+
+        @java.lang.Override
+        public Builder clone() {
+            return super.clone();
+        }
+
+        @java.lang.Override
+        public Builder setField(
+                com.google.protobuf.Descriptors.FieldDescriptor field,
+                java.lang.Object value) {
+            return super.setField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder clearField(
+                com.google.protobuf.Descriptors.FieldDescriptor field) {
+            return super.clearField(field);
+        }
+
+        @java.lang.Override
+        public Builder clearOneof(
+                com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+            return super.clearOneof(oneof);
+        }
+
+        @java.lang.Override
+        public Builder setRepeatedField(
+                com.google.protobuf.Descriptors.FieldDescriptor field,
+                int index, java.lang.Object value) {
+            return super.setRepeatedField(field, index, value);
+        }
+
+        @java.lang.Override
+        public Builder addRepeatedField(
+                com.google.protobuf.Descriptors.FieldDescriptor field,
+                java.lang.Object value) {
+            return super.addRepeatedField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(com.google.protobuf.Message other) {
+            if (other instanceof org.dbsyncer.storage.binlog.proto.Data) {
+                return mergeFrom((org.dbsyncer.storage.binlog.proto.Data) other);
+            } else {
+                super.mergeFrom(other);
+                return this;
+            }
+        }
+
+        public Builder mergeFrom(org.dbsyncer.storage.binlog.proto.Data other) {
+            if (other == org.dbsyncer.storage.binlog.proto.Data.getDefaultInstance()) return this;
+            internalGetMutableRow().mergeFrom(
+                    other.internalGetRow());
+            this.mergeUnknownFields(other.unknownFields);
+            onChanged();
+            return this;
+        }
+
+        @java.lang.Override
+        public final boolean isInitialized() {
+            return true;
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(
+                com.google.protobuf.CodedInputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws java.io.IOException {
+            org.dbsyncer.storage.binlog.proto.Data parsedMessage = null;
+            try {
+                parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                parsedMessage = (org.dbsyncer.storage.binlog.proto.Data) e.getUnfinishedMessage();
+                throw e.unwrapIOException();
+            } finally {
+                if (parsedMessage != null) {
+                    mergeFrom(parsedMessage);
+                }
+            }
+            return this;
+        }
+
+        private int bitField0_;
+
+        private com.google.protobuf.MapField<
+                java.lang.String, com.google.protobuf.ByteString> row_;
+
+        private com.google.protobuf.MapField<java.lang.String, com.google.protobuf.ByteString>
+        internalGetRow() {
+            if (row_ == null) {
+                return com.google.protobuf.MapField.emptyMapField(
+                        RowDefaultEntryHolder.defaultEntry);
+            }
+            return row_;
+        }
+
+        private com.google.protobuf.MapField<java.lang.String, com.google.protobuf.ByteString>
+        internalGetMutableRow() {
+            onChanged();
+            ;
+            if (row_ == null) {
+                row_ = com.google.protobuf.MapField.newMapField(
+                        RowDefaultEntryHolder.defaultEntry);
+            }
+            if (!row_.isMutable()) {
+                row_ = row_.copy();
+            }
+            return row_;
+        }
+
+        public int getRowCount() {
+            return internalGetRow().getMap().size();
+        }
+
+        /**
+         * <code>map&lt;string, bytes&gt; row = 1;</code>
+         */
+
+        @java.lang.Override
+        public boolean containsRow(
+                java.lang.String key) {
+            if (key == null) {
+                throw new NullPointerException("map key");
+            }
+            return internalGetRow().getMap().containsKey(key);
+        }
+
+        /**
+         * Use {@link #getRowMap()} instead.
+         */
+        @java.lang.Override
+        @java.lang.Deprecated
+        public java.util.Map<java.lang.String, com.google.protobuf.ByteString> getRow() {
+            return getRowMap();
+        }
+
+        /**
+         * <code>map&lt;string, bytes&gt; row = 1;</code>
+         */
+        @java.lang.Override
+
+        public java.util.Map<java.lang.String, com.google.protobuf.ByteString> getRowMap() {
+            return internalGetRow().getMap();
+        }
+
+        /**
+         * <code>map&lt;string, bytes&gt; row = 1;</code>
+         */
+        @java.lang.Override
+
+        public com.google.protobuf.ByteString getRowOrDefault(
+                java.lang.String key,
+                com.google.protobuf.ByteString defaultValue) {
+            if (key == null) {
+                throw new NullPointerException("map key");
+            }
+            java.util.Map<java.lang.String, com.google.protobuf.ByteString> map =
+                    internalGetRow().getMap();
+            return map.containsKey(key) ? map.get(key) : defaultValue;
+        }
+
+        /**
+         * <code>map&lt;string, bytes&gt; row = 1;</code>
+         */
+        @java.lang.Override
+
+        public com.google.protobuf.ByteString getRowOrThrow(
+                java.lang.String key) {
+            if (key == null) {
+                throw new NullPointerException("map key");
+            }
+            java.util.Map<java.lang.String, com.google.protobuf.ByteString> map =
+                    internalGetRow().getMap();
+            if (!map.containsKey(key)) {
+                throw new java.lang.IllegalArgumentException();
+            }
+            return map.get(key);
+        }
+
+        public Builder clearRow() {
+            internalGetMutableRow().getMutableMap()
+                    .clear();
+            return this;
+        }
+
+        /**
+         * <code>map&lt;string, bytes&gt; row = 1;</code>
+         */
+
+        public Builder removeRow(
+                java.lang.String key) {
+            if (key == null) {
+                throw new NullPointerException("map key");
+            }
+            internalGetMutableRow().getMutableMap()
+                    .remove(key);
+            return this;
+        }
+
+        /**
+         * Use alternate mutation accessors instead.
+         */
+        @java.lang.Deprecated
+        public java.util.Map<java.lang.String, com.google.protobuf.ByteString>
+        getMutableRow() {
+            return internalGetMutableRow().getMutableMap();
+        }
+
+        /**
+         * <code>map&lt;string, bytes&gt; row = 1;</code>
+         */
+        public Builder putRow(
+                java.lang.String key,
+                com.google.protobuf.ByteString value) {
+            if (key == null) {
+                throw new NullPointerException("map key");
+            }
+            if (value == null) {
+                throw new NullPointerException("map value");
+            }
+
+            internalGetMutableRow().getMutableMap()
+                    .put(key, value);
+            return this;
+        }
+
+        /**
+         * <code>map&lt;string, bytes&gt; row = 1;</code>
+         */
+
+        public Builder putAllRow(
+                java.util.Map<java.lang.String, com.google.protobuf.ByteString> values) {
+            internalGetMutableRow().getMutableMap()
+                    .putAll(values);
+            return this;
+        }
+
+        @java.lang.Override
+        public final Builder setUnknownFields(
+                final com.google.protobuf.UnknownFieldSet unknownFields) {
+            return super.setUnknownFields(unknownFields);
+        }
+
+        @java.lang.Override
+        public final Builder mergeUnknownFields(
+                final com.google.protobuf.UnknownFieldSet unknownFields) {
+            return super.mergeUnknownFields(unknownFields);
+        }
+
+
+        // @@protoc_insertion_point(builder_scope:Data)
+    }
+
+    // @@protoc_insertion_point(class_scope:Data)
+    private static final org.dbsyncer.storage.binlog.proto.Data DEFAULT_INSTANCE;
+
+    static {
+        DEFAULT_INSTANCE = new org.dbsyncer.storage.binlog.proto.Data();
+    }
+
+    public static org.dbsyncer.storage.binlog.proto.Data getDefaultInstance() {
+        return DEFAULT_INSTANCE;
+    }
+
+    private static final com.google.protobuf.Parser<Data>
+            PARSER = new com.google.protobuf.AbstractParser<Data>() {
+        @java.lang.Override
+        public Data parsePartialFrom(
+                com.google.protobuf.CodedInputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws com.google.protobuf.InvalidProtocolBufferException {
+            return new Data(input, extensionRegistry);
+        }
+    };
+
+    public static com.google.protobuf.Parser<Data> parser() {
+        return PARSER;
+    }
+
+    @java.lang.Override
+    public com.google.protobuf.Parser<Data> getParserForType() {
+        return PARSER;
+    }
+
+    @java.lang.Override
+    public org.dbsyncer.storage.binlog.proto.Data getDefaultInstanceForType() {
+        return DEFAULT_INSTANCE;
+    }
+
+}
+

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/proto/DataOrBuilder.java

@@ -0,0 +1,50 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: BinlogMessageProto.proto
+
+package org.dbsyncer.storage.binlog.proto;
+
+public interface DataOrBuilder extends
+        // @@protoc_insertion_point(interface_extends:Data)
+        com.google.protobuf.MessageOrBuilder {
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+    int getRowCount();
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+    boolean containsRow(
+            java.lang.String key);
+
+    /**
+     * Use {@link #getRowMap()} instead.
+     */
+    @java.lang.Deprecated
+    java.util.Map<java.lang.String, com.google.protobuf.ByteString>
+    getRow();
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+    java.util.Map<java.lang.String, com.google.protobuf.ByteString>
+    getRowMap();
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+
+    /* nullable */
+    com.google.protobuf.ByteString getRowOrDefault(
+            java.lang.String key,
+            /* nullable */
+            com.google.protobuf.ByteString defaultValue);
+
+    /**
+     * <code>map&lt;string, bytes&gt; row = 1;</code>
+     */
+
+    com.google.protobuf.ByteString getRowOrThrow(
+            java.lang.String key);
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/proto/EventEnum.java

@@ -0,0 +1,129 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: BinlogMessageProto.proto
+
+package org.dbsyncer.storage.binlog.proto;
+
+/**
+ * Protobuf enum {@code EventEnum}
+ */
+public enum EventEnum
+        implements com.google.protobuf.ProtocolMessageEnum {
+  /**
+   * <code>UPDATE = 0;</code>
+   */
+  UPDATE(0),
+  /**
+   * <code>INSERT = 1;</code>
+   */
+  INSERT(1),
+  /**
+   * <code>DELETE = 2;</code>
+   */
+  DELETE(2),
+  UNRECOGNIZED(-1),
+  ;
+
+  /**
+   * <code>UPDATE = 0;</code>
+   */
+  public static final int UPDATE_VALUE = 0;
+  /**
+   * <code>INSERT = 1;</code>
+   */
+  public static final int INSERT_VALUE = 1;
+  /**
+   * <code>DELETE = 2;</code>
+   */
+  public static final int DELETE_VALUE = 2;
+
+
+  public final int getNumber() {
+    if (this == UNRECOGNIZED) {
+      throw new java.lang.IllegalArgumentException(
+              "Can't get the number of an unknown enum value.");
+    }
+    return value;
+  }
+
+  /**
+   * @param value The numeric wire value of the corresponding enum entry.
+   * @return The enum associated with the given numeric wire value.
+   * @deprecated Use {@link #forNumber(int)} instead.
+   */
+  @java.lang.Deprecated
+  public static EventEnum valueOf(int value) {
+    return forNumber(value);
+  }
+
+  /**
+   * @param value The numeric wire value of the corresponding enum entry.
+   * @return The enum associated with the given numeric wire value.
+   */
+  public static EventEnum forNumber(int value) {
+    switch (value) {
+      case 0:
+        return UPDATE;
+      case 1:
+        return INSERT;
+      case 2:
+        return DELETE;
+      default:
+        return null;
+    }
+  }
+
+  public static com.google.protobuf.Internal.EnumLiteMap<EventEnum>
+  internalGetValueMap() {
+    return internalValueMap;
+  }
+
+  private static final com.google.protobuf.Internal.EnumLiteMap<
+          EventEnum> internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap<EventEnum>() {
+            public EventEnum findValueByNumber(int number) {
+              return EventEnum.forNumber(number);
+            }
+          };
+
+  public final com.google.protobuf.Descriptors.EnumValueDescriptor
+  getValueDescriptor() {
+    if (this == UNRECOGNIZED) {
+      throw new java.lang.IllegalStateException(
+              "Can't get the descriptor of an unrecognized enum value.");
+    }
+    return getDescriptor().getValues().get(ordinal());
+  }
+
+  public final com.google.protobuf.Descriptors.EnumDescriptor
+  getDescriptorForType() {
+    return getDescriptor();
+  }
+
+  public static final com.google.protobuf.Descriptors.EnumDescriptor
+  getDescriptor() {
+    return org.dbsyncer.storage.binlog.proto.BinlogMessageProto.getDescriptor().getEnumTypes().get(0);
+  }
+
+  private static final EventEnum[] VALUES = values();
+
+  public static EventEnum valueOf(
+          com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+    if (desc.getType() != getDescriptor()) {
+      throw new java.lang.IllegalArgumentException(
+              "EnumValueDescriptor is not for this type.");
+    }
+    if (desc.getIndex() == -1) {
+      return UNRECOGNIZED;
+    }
+    return VALUES[desc.getIndex()];
+  }
+
+  private final int value;
+
+  private EventEnum(int value) {
+    this.value = value;
+  }
+
+  // @@protoc_insertion_point(enum_scope:EventEnum)
+}
+

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/DiskStorageServiceImpl.java

@@ -41,8 +41,11 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
 
     private Map<String, Shard> map = new ConcurrentHashMap();
 
-    // 相对路径：./data/
-    private static final String PATH = "data" + File.separator;
+    /**
+     * 相对路径/data/
+     */
+    private static final String PATH = new StringBuilder(System.getProperty("user.dir")).append(File.separatorChar).append("data")
+            .append(File.separatorChar).toString();
 
     @PostConstruct
     private void init() {

dbsyncer-storage/src/main/proto/BinlogMessageProto.proto

@@ -0,0 +1,22 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "org.dbsyncer.storage.binlog.proto";
+option java_outer_classname = "BinlogMessageProto";
+option optimize_for = SPEED;
+
+message BinlogMessage {
+    string table_group_id = 1;
+    EventEnum event = 2;
+    repeated Data data = 3;
+}
+
+enum EventEnum {
+    UPDATE = 0;
+    INSERT = 1;
+    DELETE = 2;
+}
+
+message Data {
+    map<string, bytes> row = 1;
+}

dbsyncer-storage/src/main/test/BinlogMessageTest.java

@@ -0,0 +1,75 @@
+import com.google.protobuf.ByteString;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.dbsyncer.common.util.JsonUtil;
+import org.dbsyncer.storage.binlog.Binlog;
+import org.dbsyncer.storage.binlog.BinlogPipeline;
+import org.dbsyncer.storage.binlog.proto.BinlogMessage;
+import org.dbsyncer.storage.binlog.proto.Data;
+import org.dbsyncer.storage.binlog.proto.EventEnum;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2022/6/18 23:46
+ */
+public class BinlogMessageTest {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    private BinlogPipeline pipeline;
+
+    @Before
+    public void init() throws IOException {
+        File dir = new File(System.getProperty("user.dir")).getParentFile();
+        String path = new StringBuilder(dir.getAbsolutePath()).append(File.separatorChar)
+                .append("data").append(File.separatorChar)
+                .append("binlog").append(File.separatorChar)
+                .append("WriterBinlog").append(File.separatorChar)
+                .toString();
+        File configPath = new File(path + "binlog.config");
+        String configJson = FileUtils.readFileToString(configPath, Charset.defaultCharset());
+        Binlog binlog = JsonUtil.jsonToObj(configJson, Binlog.class);
+        pipeline = new BinlogPipeline(new File(path + binlog.getFileName()), binlog.getPosition());
+    }
+
+    @After
+    public void close() {
+        IOUtils.closeQuietly(pipeline);
+    }
+
+    @Test
+    public void testBinlogMessage() throws IOException {
+        write("123456", "abc");
+        write("000111", "xyz");
+        write("888999", "jkl");
+
+        byte[] line;
+        while (null != (line = pipeline.readLine())) {
+            BinlogMessage binlogMessage = BinlogMessage.parseFrom(line);
+            logger.info(binlogMessage.toString());
+        }
+    }
+
+    private void write(String tableGroupId, String key) throws IOException {
+        BinlogMessage build = BinlogMessage.newBuilder()
+                .setTableGroupId(tableGroupId)
+                .setEvent(EventEnum.UPDATE)
+                .addData(Data.newBuilder().putRow(key, ByteString.copyFromUtf8("hello,中国")).build())
+                .build();
+        byte[] bytes = build.toByteArray();
+        logger.info("序列化长度：{}", bytes.length);
+        logger.info("{}", bytes);
+        pipeline.write(build);
+    }
+
+}

dbsyncer-storage/src/main/test/LuceneFactoryTest.java

@@ -4,7 +4,6 @@ import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.*;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;

dbsyncer-web/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.1.8-Beta</version>
+        <version>1.1.9-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-web</artifactId>

dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java

@@ -1,5 +1,6 @@
 package org.dbsyncer.web.controller.config;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.dbsyncer.biz.ConfigService;
 import org.dbsyncer.biz.vo.RestResult;
@@ -21,6 +22,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletResponse;
+import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.Charset;
@@ -69,17 +71,20 @@ public class ConfigController {
     public RestResult upload(MultipartFile[] files) {
         try {
             if (files != null && files.length > 0) {
-                MultipartFile file = null;
                 for (int i = 0; i < files.length; i++) {
-                    file = files[i];
-                    if (file != null) {
-                        String filename = file.getOriginalFilename();
-                        // TODO checkFileSuffix(filename);
-                        logger.info(filename);
-                        String msg = String.format("导入配置文件%s。", filename);
-                        logger.info(msg);
-                        logService.log(LogType.CacheLog.IMPORT, msg);
+                    if (files[i] == null) {
+                        continue;
                     }
+                    String filename = files[i].getOriginalFilename();
+                    configService.checkFileSuffix(filename);
+                    String tmpdir = System.getProperty("java.io.tmpdir");
+                    File dest = new File(tmpdir + filename);
+                    FileUtils.deleteQuietly(dest);
+                    FileUtils.copyInputStreamToFile(files[i].getInputStream(), dest);
+                    configService.refreshConfig(dest);
+                    String msg = String.format("导入配置文件%s", filename);
+                    logger.info(msg);
+                    logService.log(LogType.CacheLog.IMPORT, msg);
                 }
             }
             return RestResult.restSuccess("ok");
@@ -101,7 +106,7 @@ public class ConfigController {
             String cache = JsonUtil.objToJson(cacheService.getAll());
             byte[] bytes = cache.getBytes(Charset.defaultCharset());
             int length = bytes.length;
-            String msg = String.format("导出配置文件%s，大小%dKB。", fileName, (length / 1024));
+            String msg = String.format("导出配置文件%s，大小%dKB", fileName, (length / 1024));
             logger.info(msg);
             logService.log(LogType.CacheLog.EXPORT, msg);
             outputStream.write(bytes, 0, length);

dbsyncer-web/src/main/resources/application.properties

@@ -13,6 +13,7 @@ server.servlet.context-path=/
 #dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false
 #dbsyncer.storage.support.mysql.config.username=root
 #dbsyncer.storage.support.mysql.config.password=123
+dbsyncer.parser.writer.buffer.actuator.enabled=true
 #dbsyncer.parser.flush.full.enabled=true
 #monitor
 management.endpoints.web.base-path=/app
@@ -20,7 +21,7 @@ management.endpoints.web.exposure.include=*
 management.endpoint.health.show-details=always
 management.health.elasticsearch.enabled=false
 info.app.name=DBSyncer
-info.app.version=1.1.8-Beta
+info.app.version=1.1.9-Beta
 info.app.copyright=&copy;2022 ${info.app.name}(${info.app.version})<footer>Designed By <a href='https://gitee.com/ghi/dbsyncer' target='_blank' >AE86</a></footer>
 
 #All < Trace < Debug < Info < Warn < Error < Fatal < OFF

dbsyncer-web/src/main/resources/public/config/config.html

@@ -47,7 +47,8 @@
 
                     <div class="form-group">
                         <div class="file-loading">
-                            <input id="fileConfig" type="file" name="files" multiple="multiple"/>
+                            <input id="fileConfig" type="file" name="files" multiple="multiple"
+                                   accept="application/json"/>
                         </div>
                     </div>
                 </form>

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>org.ghi</groupId>
     <artifactId>dbsyncer</artifactId>
-    <version>1.1.8-Beta</version>
+    <version>1.1.9-Beta</version>
     <packaging>pom</packaging>
     <name>dbsyncer</name>
     <url>https://gitee.com/ghi/dbsyncer</url>
@@ -49,6 +49,7 @@
         <kafka.version>0.9.0.0</kafka.version>
         <json.version>20090211</json.version>
         <fastjson.version>1.2.75</fastjson.version>
+        <protobuf.version>3.21.1</protobuf.version>
         <log4j2.version>2.17.1</log4j2.version>
         <junit.version>4.12</junit.version>
     </properties>
@@ -124,6 +125,13 @@
                 <version>${fastjson.version}</version>
             </dependency>
 
+            <!-- protobuf -->
+            <dependency>
+                <groupId>com.google.protobuf</groupId>
+                <artifactId>protobuf-java</artifactId>
+                <version>${protobuf.version}</version>
+            </dependency>
+
             <!-- smartcn中文分词器 -->
             <dependency>
                 <groupId>org.apache.lucene</groupId>