!207 merge

Merge pull request !207 from AE86/v_2.0

dbsyncer-biz/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.2.7_1201</version>
+        <version>2.0.0_1226-Beta</version>
     </parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>dbsyncer-biz</artifactId>
@@ -18,12 +18,5 @@
 			<version>${project.parent.version}</version>
 		</dependency>
 
-		<!-- Monitor 监控服务 -->
-		<dependency>
-			<groupId>org.ghi</groupId>
-			<artifactId>dbsyncer-monitor</artifactId>
-			<version>${project.parent.version}</version>
-		</dependency>
-
 	</dependencies>
 </project>

dbsyncer-biz/src/main/java/org/dbsyncer/biz/DataSyncService.java

@@ -32,6 +32,6 @@ public interface DataSyncService {
      * @param params
      * @return
      */
-    String sync(Map<String, String> params);
+    String sync(Map<String, String> params) throws InvalidProtocolBufferException;
 
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/MappingService.java

@@ -71,10 +71,10 @@ public interface MappingService {
     String stop(String id);
 
     /**
-     * 刷新表
+     * 刷新驱动数据源和目标源表
      *
-     * @param connector
+     * @param id
      */
-    void refreshTables(Connector connector);
+    String refreshMappingTables(String id);
 
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/MonitorService.java

@@ -3,8 +3,8 @@ package org.dbsyncer.biz;
 import org.dbsyncer.biz.vo.AppReportMetricVo;
 import org.dbsyncer.biz.vo.MetaVo;
 import org.dbsyncer.common.model.Paging;
-import org.dbsyncer.monitor.enums.MetricEnum;
-import org.dbsyncer.monitor.model.MetricResponse;
+import org.dbsyncer.biz.enums.MetricEnum;
+import org.dbsyncer.biz.model.MetricResponse;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 
 import java.util.List;

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java

@@ -1,18 +1,17 @@
 package org.dbsyncer.biz.checker;
 
 import org.dbsyncer.biz.enums.SafeInfoEnum;
-import org.dbsyncer.common.snowflake.SnowflakeIdWorker;
+import org.dbsyncer.sdk.model.Filter;
+import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.model.Filter;
-import org.dbsyncer.manager.Manager;
 import org.dbsyncer.parser.model.AbstractConfigModel;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Convert;
 import org.dbsyncer.plugin.PluginFactory;
-import org.dbsyncer.plugin.config.Plugin;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.plugin.model.Plugin;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
@@ -35,9 +34,6 @@ public abstract class AbstractChecker implements Checker {
 
     private static final String SYMBOL = "***";
 
-    @Resource
-    private Manager manager;
-
     @Resource
     private PluginFactory pluginFactory;
 
@@ -95,7 +91,7 @@ public abstract class AbstractChecker implements Checker {
         String pluginId = params.get("pluginId");
         Plugin plugin = null;
         if (StringUtil.isNotBlank(pluginId)) {
-            List<Plugin> plugins = manager.getPluginAll();
+            List<Plugin> plugins = pluginFactory.getPluginAll();
             if (!CollectionUtils.isEmpty(plugins)) {
                 for (Plugin p : plugins) {
                     if (StringUtil.equals(pluginFactory.createPluginId(p.getClassName(), p.getVersion()), pluginId)) {

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/ConnectorConfigChecker.java

@@ -1,33 +0,0 @@
-package org.dbsyncer.biz.checker;
-
-import org.dbsyncer.parser.model.Mapping;
-import org.dbsyncer.parser.model.TableGroup;
-
-import java.util.Map;
-
-/**
- * @param <C> ConnectorConfig
- * @author AE86
- * @version 1.0.0
- * @date 2020/1/7 23:17
- */
-public interface ConnectorConfigChecker<C> {
-
-    /**
-     * 处理增量同步策略
-     *
-     * @param mapping
-     * @param tableGroup
-     */
-    default void dealIncrementStrategy(Mapping mapping, TableGroup tableGroup) {}
-
-    /**
-     * 修改配置
-     *
-     * @param connectorConfig
-     * @param params
-     * @return
-     */
-    void modify(C connectorConfig, Map<String, String> params);
-
-}

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java

@@ -1,32 +1,31 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.biz.checker.impl.connector;
 
 import org.dbsyncer.biz.BizException;
 import org.dbsyncer.biz.checker.AbstractChecker;
-import org.dbsyncer.biz.checker.ConnectorConfigChecker;
-import org.dbsyncer.common.model.AbstractConnectorConfig;
-import org.dbsyncer.common.spi.ConnectorMapper;
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.enums.ConnectorEnum;
-import org.dbsyncer.connector.model.Table;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.logger.LogService;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.connector.base.ConnectorFactory;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Connector;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.connector.ConfigValidator;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.model.ConnectorConfig;
+import org.dbsyncer.sdk.spi.ConnectorService;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
 import javax.annotation.Resource;
-import java.util.List;
 import java.util.Map;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2020/1/8 15:17
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2020-01-08 15:17
  */
 @Component
 public class ConnectorChecker extends AbstractChecker {
@@ -34,13 +33,10 @@ public class ConnectorChecker extends AbstractChecker {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resource
-    private Manager manager;
-
-    @Resource
-    private LogService logService;
+    private ProfileComponent profileComponent;
 
     @Resource
-    private Map<String, ConnectorConfigChecker> map;
+    private ConnectorFactory connectorFactory;
 
     @Override
     public ConfigModel checkAddConfigModel(Map<String, String> params) {
@@ -52,17 +48,17 @@ public class ConnectorChecker extends AbstractChecker {
 
         Connector connector = new Connector();
         connector.setName(name);
-        AbstractConnectorConfig config = getConfig(connectorType);
+        ConnectorConfig config = getConfig(connectorType);
         connector.setConfig(config);
 
-        // 配置连接器配置
-        String type = StringUtil.toLowerCaseFirstOne(connectorType).concat("ConfigChecker");
-        ConnectorConfigChecker checker = map.get(type);
-        Assert.notNull(checker, "Checker can not be null.");
-        checker.modify(config, params);
+        // 连接器配置校验
+        ConfigValidator configValidator = connectorFactory.getConnectorService(connectorType).getConfigValidator();
+        Assert.notNull(configValidator, "ConfigValidator can not be null.");
+        configValidator.modify(config, params);
 
         // 获取表
-        setTable(connector);
+        ConnectorInstance connectorInstance = connectorFactory.connect(connector.getConfig());
+        connector.setTable(connectorFactory.getTable(connectorInstance));
 
         // 修改基本配置
         this.modifyConfigModel(connector, params);
@@ -75,29 +71,32 @@ public class ConnectorChecker extends AbstractChecker {
         printParams(params);
         Assert.notEmpty(params, "ConnectorChecker check params is null.");
         String id = params.get(ConfigConstant.CONFIG_MODEL_ID);
-        Connector connector = manager.getConnector(id);
+        Connector connector = profileComponent.getConnector(id);
         Assert.notNull(connector, "Can not find connector.");
+        ConnectorConfig config = connector.getConfig();
+        connectorFactory.disconnect(config);
 
         // 修改基本配置
         this.modifyConfigModel(connector, params);
 
-        // 配置连接器配置
-        AbstractConnectorConfig config = connector.getConfig();
-        String type = StringUtil.toLowerCaseFirstOne(config.getConnectorType()).concat("ConfigChecker");
-        ConnectorConfigChecker checker = map.get(type);
-        Assert.notNull(checker, "Checker can not be null.");
-        checker.modify(config, params);
+        // 连接器配置校验
+        ConfigValidator configValidator = connectorFactory.getConnectorService(config.getConnectorType()).getConfigValidator();
+        Assert.notNull(configValidator, "ConfigValidator can not be null.");
+        configValidator.modify(config, params);
 
         // 获取表
-        setTable(connector);
+        ConnectorInstance connectorInstance = connectorFactory.connect(config);
+        connector.setTable(connectorFactory.getTable(connectorInstance));
 
         return connector;
     }
 
-    private AbstractConnectorConfig getConfig(String connectorType) {
+    private ConnectorConfig getConfig(String connectorType) {
         try {
-            AbstractConnectorConfig config = ConnectorEnum.getConnectorEnum(connectorType).getConfigClass().newInstance();
-            config.setConnectorType(connectorType);
+            ConnectorService connectorService = connectorFactory.getConnectorService(connectorType);
+            Class<ConnectorConfig> configClass = connectorService.getConfigClass();
+            ConnectorConfig config = configClass.newInstance();
+            config.setConnectorType(connectorService.getConnectorType());
             return config;
         } catch (Exception e) {
             logger.error(e.getMessage());
@@ -105,16 +104,4 @@ public class ConnectorChecker extends AbstractChecker {
         }
     }
 
-    private void setTable(Connector connector) {
-        boolean isAlive = manager.refreshConnectorConfig(connector.getConfig());
-        if (!isAlive) {
-            logService.log(LogType.ConnectorLog.FAILED);
-        }
-        Assert.isTrue(isAlive, "无法连接.");
-        // 获取表信息
-        ConnectorMapper connectorMapper = manager.connect(connector.getConfig());
-        List<Table> table = manager.getTable(connectorMapper);
-        connector.setTable(table);
-    }
-
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java

@@ -1,21 +0,0 @@
-package org.dbsyncer.biz.checker.impl.connector;
-
-import org.dbsyncer.connector.config.DatabaseConfig;
-import org.springframework.stereotype.Component;
-
-import java.util.Map;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2020/1/8 15:17
- */
-@Component
-public class DqlMySQLConfigChecker extends AbstractDataBaseConfigChecker {
-
-    @Override
-    public void modify(DatabaseConfig connectorConfig, Map<String, String> params) {
-        super.modify(connectorConfig, params);
-        super.modifyDql(connectorConfig, params);
-    }
-}

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigChecker.java

@@ -1,22 +0,0 @@
-package org.dbsyncer.biz.checker.impl.connector;
-
-import org.dbsyncer.connector.config.DatabaseConfig;
-import org.springframework.stereotype.Component;
-
-import java.util.Map;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2021/5/18 20:17
- */
-@Component
-public class DqlSqlServerConfigChecker extends AbstractDataBaseConfigChecker {
-
-    @Override
-    public void modify(DatabaseConfig connectorConfig, Map<String, String> params) {
-        super.modify(connectorConfig, params);
-        super.modifyDql(connectorConfig, params);
-        super.modifySchema(connectorConfig, params);
-    }
-}

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java

@@ -1,13 +0,0 @@
-package org.dbsyncer.biz.checker.impl.connector;
-
-import org.springframework.stereotype.Component;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2020/1/8 15:17
- */
-@Component
-public class MySQLConfigChecker extends AbstractDataBaseConfigChecker {
-
-}

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java

@@ -1,172 +0,0 @@
-package org.dbsyncer.biz.checker.impl.connector;
-
-import org.dbsyncer.biz.enums.OracleIncrementEnum;
-import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.config.DatabaseConfig;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.connector.model.MetaInfo;
-import org.dbsyncer.connector.model.Table;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.model.FieldMapping;
-import org.dbsyncer.parser.model.Mapping;
-import org.dbsyncer.parser.model.TableGroup;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.util.Assert;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * <p>1、增量同步时，目标源必须有一个主键字段用于接收ROW_ID值。</p>
- * <p>2、全局可配置目标源ROW_ID字段名称，默认为ROW_ID_LABEL_NAME。 </p>
- * <p>3、如果配置了接收字段，添加字段映射关系[ROW_ID_LABEL_NAME] > [ROW_ID_LABEL_NAME]，并将ROW_ID_LABEL_NAME字段设置为目标源的唯一主键。</p>
- * <p>4、全量同步时，ROW_ID_LABEL_NAME参数非必须。</p>
- *
- * @author AE86
- * @version 1.0.0
- * @date 2020/1/8 15:17
- */
-@Component
-public class OracleConfigChecker extends AbstractDataBaseConfigChecker {
-
-    @Autowired
-    private Manager manager;
-
-    @Override
-    public void modify(DatabaseConfig connectorConfig, Map<String, String> params) {
-        super.modify(connectorConfig, params);
-
-        String schema = params.get("schema");
-        connectorConfig.setSchema(StringUtil.isBlank(schema) ? connectorConfig.getUsername().toUpperCase() : schema.toUpperCase());
-    }
-
-    @Override
-    public void dealIncrementStrategy(Mapping mapping, TableGroup tableGroup) {
-        String rowIdLabelName = OracleIncrementEnum.ROW_ID_LABEL_NAME.getName();
-        Map<String, String> params = tableGroup.getParams();
-        if (CollectionUtils.isEmpty(params) || !params.containsKey(rowIdLabelName)) {
-            revert(mapping, tableGroup);
-            return;
-        }
-        String targetRowIdName = params.get(rowIdLabelName);
-
-        // 检查目标源是否支持该自定义字段
-        List<Field> targetColumn = tableGroup.getTargetTable().getColumn();
-        Field targetField = null;
-        for (Field f : targetColumn) {
-            if (StringUtil.equals(f.getName(), targetRowIdName)) {
-                targetField = f;
-                targetField.setPk(true);
-                continue;
-            }
-            f.setPk(false);
-        }
-        Assert.isTrue(null != targetField,
-                String.format("[%s 同步到 %s]，目标源表不存在字段%s", tableGroup.getSourceTable().getName(), tableGroup.getTargetTable().getName(),
-                        targetRowIdName));
-
-        // 检查是否更新
-        List<FieldMapping> fieldMapping = tableGroup.getFieldMapping();
-        List<FieldMapping> list = new ArrayList<>();
-        for (FieldMapping m : fieldMapping) {
-            if (null != m.getSource() && OracleIncrementEnum.isRowId(m.getSource().getName())) {
-                m.getTarget().setName(targetRowIdName);
-                return;
-            }
-
-            // 排除主键关系
-            if (null != m.getTarget() && StringUtil.equals(m.getTarget().getName(), targetRowIdName)) {
-                continue;
-            }
-
-            list.add(m);
-        }
-
-        Field sourceField = new Field(OracleIncrementEnum.ROW_ID.getName(), "VARCHAR2", 12, false,
-                OracleIncrementEnum.ROW_ID_LABEL_NAME.getName(), true);
-        tableGroup.getSourceTable().getColumn().add(0, sourceField);
-
-        list.add(0, new FieldMapping(sourceField, targetField));
-        fieldMapping.clear();
-        fieldMapping.addAll(list);
-    }
-
-    /**
-     * 还原字段和映射关系
-     *
-     * @param mapping
-     * @param tableGroup
-     */
-    private void revert(Mapping mapping, TableGroup tableGroup) {
-        List<FieldMapping> fieldMapping = tableGroup.getFieldMapping();
-        if (CollectionUtils.isEmpty(fieldMapping)) {
-            return;
-        }
-
-        // 还原字段
-        Table sourceTable = tableGroup.getSourceTable();
-        List<Field> sourceColumn = sourceTable.getColumn();
-        List<Field> sourceFields = new ArrayList<>();
-        boolean existRowId = false;
-        for (Field f : sourceColumn) {
-            if (OracleIncrementEnum.isRowId(f.getName())) {
-                existRowId = true;
-                continue;
-            }
-            sourceFields.add(f);
-        }
-        sourceTable.setColumn(sourceFields);
-
-        if (!existRowId) {
-            return;
-        }
-
-        // 存在自定义主键
-        String pk = null;
-        for (FieldMapping m : tableGroup.getFieldMapping()) {
-            if (null != m.getSource() && OracleIncrementEnum.isRowId(m.getSource().getName())) {
-                continue;
-            }
-            if (null != m.getTarget() && m.getTarget().isPk()) {
-                pk = m.getTarget().getName();
-                break;
-            }
-        }
-
-        // 没有自定义主键，获取表元信息
-        if (null == pk) {
-            Table targetTable = tableGroup.getTargetTable();
-            MetaInfo metaInfo = manager.getMetaInfo(mapping.getTargetConnectorId(), targetTable.getName());
-            List<Field> targetColumn = metaInfo.getColumn();
-            targetTable.setColumn(targetColumn);
-
-            if (!CollectionUtils.isEmpty(targetColumn)) {
-                for (Field f : targetColumn) {
-                    if (f.isPk()) {
-                        pk = f.getName();
-                        break;
-                    }
-                }
-            }
-        }
-
-        // 剔除映射关系
-        List<FieldMapping> list = new ArrayList<>();
-        for (FieldMapping m : tableGroup.getFieldMapping()) {
-            if (null != m.getSource() && OracleIncrementEnum.isRowId(m.getSource().getName())) {
-                continue;
-            }
-
-            if (null != m.getTarget() && StringUtil.equals(m.getTarget().getName(), pk)) {
-                m.getTarget().setPk(true);
-            }
-            list.add(m);
-        }
-        fieldMapping.clear();
-        fieldMapping.addAll(list);
-    }
-
-}

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java

@@ -1,20 +0,0 @@
-package org.dbsyncer.biz.checker.impl.connector;
-
-import org.dbsyncer.connector.config.DatabaseConfig;
-import org.springframework.stereotype.Component;
-
-import java.util.Map;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2021/5/18 20:17
- */
-@Component
-public class SqlServerConfigChecker extends AbstractDataBaseConfigChecker {
-    @Override
-    public void modify(DatabaseConfig connectorConfig, Map<String, String> params) {
-        super.modify(connectorConfig, params);
-        super.modifySchema(connectorConfig, params);
-    }
-}

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java

@@ -3,11 +3,10 @@ package org.dbsyncer.biz.checker.impl.group;
 import org.dbsyncer.biz.checker.AbstractChecker;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.manager.Manager;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.ProjectGroup;
-import org.dbsyncer.storage.constant.ConfigConstant;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
@@ -25,7 +24,7 @@ import java.util.Map;
 public class ProjectGroupChecker extends AbstractChecker {
 
     @Resource
-    private Manager manager;
+    private ProfileComponent profileComponent;
 
     /**
      * 新增配置
@@ -56,7 +55,7 @@ public class ProjectGroupChecker extends AbstractChecker {
     @Override
     public ConfigModel checkEditConfigModel(Map<String, String> params) {
         String id = params.get(ConfigConstant.CONFIG_MODEL_ID);
-        ProjectGroup projectGroup = manager.getProjectGroup(id);
+        ProjectGroup projectGroup = profileComponent.getProjectGroup(id);
         Assert.notNull(projectGroup, "Can not find project group.");
 
         modifyProjectGroup(projectGroup, params);

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/LogConfigChecker.java

@@ -1,8 +1,8 @@
 package org.dbsyncer.biz.checker.impl.mapping;
 
 import org.dbsyncer.biz.checker.MappingConfigChecker;
-import org.dbsyncer.listener.config.ListenerConfig;
-import org.dbsyncer.listener.enums.ListenerTypeEnum;
+import org.dbsyncer.sdk.config.ListenerConfig;
+import org.dbsyncer.sdk.enums.ListenerTypeEnum;
 import org.dbsyncer.parser.model.Mapping;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java

@@ -8,15 +8,15 @@ import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.listener.config.ListenerConfig;
-import org.dbsyncer.listener.enums.ListenerTypeEnum;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.enums.ModelEnum;
+import org.dbsyncer.sdk.config.ListenerConfig;
+import org.dbsyncer.sdk.enums.ListenerTypeEnum;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.sdk.enums.ModelEnum;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
@@ -41,7 +41,7 @@ public class MappingChecker extends AbstractChecker {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resource
-    private Manager manager;
+    private ProfileComponent profileComponent;
 
     @Resource
     private TableGroupChecker tableGroupChecker;
@@ -81,7 +81,7 @@ public class MappingChecker extends AbstractChecker {
         logger.info("params:{}", params);
         Assert.notEmpty(params, "MappingChecker check params is null.");
         String id = params.get(ConfigConstant.CONFIG_MODEL_ID);
-        Mapping mapping = manager.getMapping(id);
+        Mapping mapping = profileComponent.getMapping(id);
         Assert.notNull(mapping, "Can not find mapping.");
 
         // 修改基本配置
@@ -124,7 +124,7 @@ public class MappingChecker extends AbstractChecker {
         // 修改基本配置
         this.modifyConfigModel(meta, new HashMap<>());
 
-        String id = manager.addConfigModel(meta);
+        String id = profileComponent.addConfigModel(meta);
         mapping.setMetaId(id);
     }
 
@@ -144,7 +144,7 @@ public class MappingChecker extends AbstractChecker {
      * @param params
      */
     public void batchMergeTableGroupConfig(Mapping mapping, Map<String, String> params) {
-        List<TableGroup> groupAll = manager.getTableGroupAll(mapping.getId());
+        List<TableGroup> groupAll = profileComponent.getTableGroupAll(mapping.getId());
         if (!CollectionUtils.isEmpty(groupAll)) {
             // 手动排序
             String[] sortedTableGroupIds = StringUtil.split(params.get("sortedTableGroupIds"), "|");
@@ -153,7 +153,7 @@ public class MappingChecker extends AbstractChecker {
                 groupAll.clear();
                 int size = sortedTableGroupIds.length;
                 int i = size;
-                while (i > 0){
+                while (i > 0) {
                     TableGroup g = tableGroupMap.get(sortedTableGroupIds[size - i]);
                     Assert.notNull(g, "Invalid sorted tableGroup.");
                     g.setIndex(i);
@@ -165,7 +165,7 @@ public class MappingChecker extends AbstractChecker {
             // 合并配置
             for (TableGroup g : groupAll) {
                 tableGroupChecker.mergeConfig(mapping, g);
-                manager.editConfigModel(g);
+                profileComponent.editConfigModel(g);
             }
         }
     }
@@ -194,16 +194,16 @@ public class MappingChecker extends AbstractChecker {
      * @param metaSnapshot
      */
     private void updateMeta(Mapping mapping, String metaSnapshot) {
-        Meta meta = manager.getMeta(mapping.getMetaId());
+        Meta meta = profileComponent.getMeta(mapping.getMetaId());
         Assert.notNull(meta, "驱动meta不存在.");
 
         // 清空状态
         meta.clear();
 
         // 手动配置增量点
-        if(StringUtil.isNotBlank(metaSnapshot)){
+        if (StringUtil.isNotBlank(metaSnapshot)) {
             Map snapshot = JsonUtil.jsonToObj(metaSnapshot, HashMap.class);
-            if(!CollectionUtils.isEmpty(snapshot)){
+            if (!CollectionUtils.isEmpty(snapshot)) {
                 meta.setSnapshot(snapshot);
             }
         }
@@ -211,7 +211,7 @@ public class MappingChecker extends AbstractChecker {
         getMetaTotal(meta, mapping.getModel());
 
         meta.setUpdateTime(Instant.now().toEpochMilli());
-        manager.editConfigModel(meta);
+        profileComponent.editConfigModel(meta);
     }
 
     private void getMetaTotal(Meta meta, String model) {
@@ -219,7 +219,7 @@ public class MappingChecker extends AbstractChecker {
         if (ModelEnum.isFull(model)) {
             // 统计tableGroup总条数
             AtomicLong count = new AtomicLong(0);
-            List<TableGroup> groupAll = manager.getTableGroupAll(meta.getMappingId());
+            List<TableGroup> groupAll = profileComponent.getTableGroupAll(meta.getMappingId());
             if (!CollectionUtils.isEmpty(groupAll)) {
                 for (TableGroup g : groupAll) {
                     count.getAndAdd(g.getSourceTable().getCount());

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/TimingConfigChecker.java

@@ -2,8 +2,8 @@ package org.dbsyncer.biz.checker.impl.mapping;
 
 import org.dbsyncer.biz.checker.MappingConfigChecker;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.listener.config.ListenerConfig;
-import org.dbsyncer.listener.enums.ListenerTypeEnum;
+import org.dbsyncer.sdk.config.ListenerConfig;
+import org.dbsyncer.sdk.enums.ListenerTypeEnum;
 import org.dbsyncer.parser.model.Mapping;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java

@@ -3,9 +3,9 @@ package org.dbsyncer.biz.checker.impl.system;
 import org.dbsyncer.biz.checker.AbstractChecker;
 import org.dbsyncer.common.util.BeanUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.logger.LogService;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.LogService;
+import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.SystemConfig;
 import org.slf4j.Logger;
@@ -27,7 +27,7 @@ public class SystemConfigChecker extends AbstractChecker {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resource
-    private Manager manager;
+    private ProfileComponent profileComponent;
 
     @Resource
     private LogService logService;
@@ -40,7 +40,7 @@ public class SystemConfigChecker extends AbstractChecker {
         // 修改基本配置
         this.modifyConfigModel(systemConfig, params);
 
-        manager.addConfigModel(systemConfig);
+        profileComponent.addConfigModel(systemConfig);
         return systemConfig;
     }
 
@@ -49,8 +49,11 @@ public class SystemConfigChecker extends AbstractChecker {
         logger.info("params:{}", params);
         Assert.notEmpty(params, "Config check params is null.");
         params.put("enableCDN", StringUtil.isNotBlank(params.get("enableCDN")) ? "true" : "false");
+        params.put("enableStorageWriteFull", StringUtil.isNotBlank(params.get("enableStorageWriteFull")) ? "true" : "false");
+        params.put("enableStorageWriteSuccess", StringUtil.isNotBlank(params.get("enableStorageWriteSuccess")) ? "true" : "false");
+        params.put("enableStorageWriteFail", StringUtil.isNotBlank(params.get("enableStorageWriteFail")) ? "true" : "false");
 
-        SystemConfig systemConfig = manager.getSystemConfig();
+        SystemConfig systemConfig = profileComponent.getSystemConfig();
         Assert.notNull(systemConfig, "配置文件为空.");
         BeanUtil.mapToBean(params, systemConfig);
         logService.log(LogType.SystemLog.INFO, "修改系统配置");

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java

@@ -1,23 +1,26 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.biz.checker.impl.tablegroup;
 
 import org.dbsyncer.biz.BizException;
 import org.dbsyncer.biz.checker.AbstractChecker;
-import org.dbsyncer.biz.checker.ConnectorConfigChecker;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.connector.model.MetaInfo;
-import org.dbsyncer.connector.model.Table;
-import org.dbsyncer.connector.util.PrimaryKeyUtil;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.enums.ModelEnum;
+import org.dbsyncer.parser.ParserComponent;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.FieldMapping;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.parser.util.PickerUtil;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.enums.ModelEnum;
+import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.model.MetaInfo;
+import org.dbsyncer.sdk.model.Table;
+import org.dbsyncer.sdk.util.PrimaryKeyUtil;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
@@ -35,9 +38,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2020/1/8 15:17
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2020-01-08 15:17
  */
 @Component
 public class TableGroupChecker extends AbstractChecker {
@@ -45,10 +48,10 @@ public class TableGroupChecker extends AbstractChecker {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resource
-    private Manager manager;
+    private ParserComponent parserComponent;
 
     @Resource
-    private Map<String, ConnectorConfigChecker> map;
+    private ProfileComponent profileComponent;
 
     @Override
     public ConfigModel checkAddConfigModel(Map<String, String> params) {
@@ -61,7 +64,7 @@ public class TableGroupChecker extends AbstractChecker {
         Assert.hasText(mappingId, "tableGroup mappingId is empty.");
         Assert.hasText(sourceTable, "tableGroup sourceTable is empty.");
         Assert.hasText(targetTable, "tableGroup targetTable is empty.");
-        Mapping mapping = manager.getMapping(mappingId);
+        Mapping mapping = profileComponent.getMapping(mappingId);
         Assert.notNull(mapping, "mapping can not be null.");
 
         // 检查是否存在重复映射关系
@@ -92,9 +95,9 @@ public class TableGroupChecker extends AbstractChecker {
         logger.info("params:{}", params);
         Assert.notEmpty(params, "TableGroupChecker check params is null.");
         String id = params.get(ConfigConstant.CONFIG_MODEL_ID);
-        TableGroup tableGroup = manager.getTableGroup(id);
+        TableGroup tableGroup = profileComponent.getTableGroup(id);
         Assert.notNull(tableGroup, "Can not find tableGroup.");
-        Mapping mapping = manager.getMapping(tableGroup.getMappingId());
+        Mapping mapping = profileComponent.getMapping(tableGroup.getMappingId());
         Assert.notNull(mapping, "mapping can not be null.");
         String fieldMappingJson = params.get("fieldMapping");
         Assert.hasText(fieldMappingJson, "TableGroupChecker check params fieldMapping is empty");
@@ -120,7 +123,7 @@ public class TableGroupChecker extends AbstractChecker {
      * @param tableGroup
      */
     public void refreshTableFields(TableGroup tableGroup) {
-        Mapping mapping = manager.getMapping(tableGroup.getMappingId());
+        Mapping mapping = profileComponent.getMapping(tableGroup.getMappingId());
         Assert.notNull(mapping, "mapping can not be null.");
 
         Table sourceTable = tableGroup.getSourceTable();
@@ -135,27 +138,16 @@ public class TableGroupChecker extends AbstractChecker {
         // 合并高级配置
         TableGroup group = PickerUtil.mergeTableGroupConfig(mapping, tableGroup);
 
-        // 处理策略
-        dealIncrementStrategy(mapping, group);
-
-        Map<String, String> command = manager.getCommand(mapping, group);
+        Map<String, String> command = parserComponent.getCommand(mapping, group);
         tableGroup.setCommand(command);
 
         // 获取数据源总数
-        long count = ModelEnum.isFull(mapping.getModel()) && !CollectionUtils.isEmpty(command) ? manager.getCount(mapping.getSourceConnectorId(), command) : 0;
+        long count = ModelEnum.isFull(mapping.getModel()) && !CollectionUtils.isEmpty(command) ? parserComponent.getCount(mapping.getSourceConnectorId(), command) : 0;
         tableGroup.getSourceTable().setCount(count);
     }
 
-    public void dealIncrementStrategy(Mapping mapping, TableGroup tableGroup) {
-        String connectorType = manager.getConnector(mapping.getSourceConnectorId()).getConfig().getConnectorType();
-        String type = StringUtil.toLowerCaseFirstOne(connectorType).concat("ConfigChecker");
-        ConnectorConfigChecker checker = map.get(type);
-        Assert.notNull(checker, "Checker can not be null.");
-        checker.dealIncrementStrategy(mapping, tableGroup);
-    }
-
     private Table getTable(String connectorId, String tableName, String primaryKeyStr) {
-        MetaInfo metaInfo = manager.getMetaInfo(connectorId, tableName);
+        MetaInfo metaInfo = parserComponent.getMetaInfo(connectorId, tableName);
         Assert.notNull(metaInfo, "无法获取连接器表信息:" + tableName);
         // 自定义主键
         if (StringUtil.isNotBlank(primaryKeyStr) && !CollectionUtils.isEmpty(metaInfo.getColumn())) {
@@ -173,7 +165,7 @@ public class TableGroupChecker extends AbstractChecker {
     }
 
     private void checkRepeatedTable(String mappingId, String sourceTable, String targetTable) {
-        List<TableGroup> list = manager.getTableGroupAll(mappingId);
+        List<TableGroup> list = profileComponent.getTableGroupAll(mappingId);
         if (!CollectionUtils.isEmpty(list)) {
             for (TableGroup g : list) {
                 // 数据源表和目标表都存在

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/BufferActuatorMetricEnum.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/BufferActuatorMetricEnum.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.monitor.enums;
+package org.dbsyncer.biz.enums;
 
 /**
  * 缓存执行器指标

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/DiskMetricEnum.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/DiskMetricEnum.java

@@ -1,48 +1,48 @@
-package org.dbsyncer.monitor.enums;
-
-/**
- * 硬盘指标
- *
- * @author AE86
- * @version 1.0.0
- * @date 2021/07/23 0:19
- */
-public enum DiskMetricEnum {
-
-    /**
-     * 已用
-     */
-    THRESHOLD("disk.space.threshold", "硬盘", "已用"),
-
-    /**
-     * 空闲
-     */
-    FREE("disk.space.free", "硬盘", "空闲"),
-    
-    /**
-     * 总共
-     */
-    TOTAL("disk.space.total", "硬盘", "总共"),;
-
-    private String code;
-    private String group;
-    private String metricName;
-
-    DiskMetricEnum(String code, String group, String metricName) {
-        this.code = code;
-        this.group = group;
-        this.metricName = metricName;
-    }
-
-    public String getCode() {
-        return code;
-    }
-
-    public String getGroup() {
-        return group;
-    }
-
-    public String getMetricName() {
-        return metricName;
-    }
-}
+package org.dbsyncer.biz.enums;
+
+/**
+ * 硬盘指标
+ *
+ * @author AE86
+ * @version 1.0.0
+ * @date 2021/07/23 0:19
+ */
+public enum DiskMetricEnum {
+
+    /**
+     * 已用
+     */
+    THRESHOLD("disk.space.threshold", "硬盘", "已用"),
+
+    /**
+     * 空闲
+     */
+    FREE("disk.space.free", "硬盘", "空闲"),
+    
+    /**
+     * 总共
+     */
+    TOTAL("disk.space.total", "硬盘", "总共"),;
+
+    private String code;
+    private String group;
+    private String metricName;
+
+    DiskMetricEnum(String code, String group, String metricName) {
+        this.code = code;
+        this.group = group;
+        this.metricName = metricName;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public String getGroup() {
+        return group;
+    }
+
+    public String getMetricName() {
+        return metricName;
+    }
+}

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/MetricEnum.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/MetricEnum.java

@@ -1,82 +1,81 @@
-package org.dbsyncer.monitor.enums;
-
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorException;
-import org.dbsyncer.monitor.MonitorException;
-
-/**
- * 系统指标
- *
- * @author AE86
- * @version 1.0.0
- * @date 2021/07/22 19:19
- */
-public enum MetricEnum {
-
-    /**
-     * 线程活跃数
-     */
-    THREADS_LIVE("jvm.threads.live", "应用线程", "活跃数"),
-
-    /**
-     * 线程峰值
-     */
-    THREADS_PEAK("jvm.threads.peak", "应用线程", "峰值数"),
-
-    /**
-     * 内存已用
-     */
-    MEMORY_USED("jvm.memory.used", "内存", "已用"),
-
-    /**
-     * 内存空闲
-     */
-    MEMORY_COMMITTED("jvm.memory.committed", "内存", "空闲"),
-
-    /**
-     * 内存总共
-     */
-    MEMORY_MAX("jvm.memory.max", "内存", "总共"),
-
-    /**
-     * GC
-     */
-    GC_PAUSE("jvm.gc.pause", "GC", "已用"),
-
-    /**
-     * CPU已用
-     */
-    CPU_USAGE("system.cpu.usage", "CPU", "已用");
-
-    private String code;
-    private String group;
-    private String metricName;
-
-    MetricEnum(String code, String group, String metricName) {
-        this.code = code;
-        this.group = group;
-        this.metricName = metricName;
-    }
-
-    public static MetricEnum getMetric(String code) throws ConnectorException {
-        for (MetricEnum e : MetricEnum.values()) {
-            if (StringUtil.equals(code, e.getCode())) {
-                return e;
-            }
-        }
-        throw new MonitorException(String.format("Metric code \"%s\" does not exist.", code));
-    }
-
-    public String getCode() {
-        return code;
-    }
-
-    public String getGroup() {
-        return group;
-    }
-
-    public String getMetricName() {
-        return metricName;
-    }
-
+package org.dbsyncer.biz.enums;
+
+import org.dbsyncer.biz.BizException;
+import org.dbsyncer.common.util.StringUtil;
+
+/**
+ * 系统指标
+ *
+ * @author AE86
+ * @version 1.0.0
+ * @date 2021/07/22 19:19
+ */
+public enum MetricEnum {
+
+    /**
+     * 线程活跃数
+     */
+    THREADS_LIVE("jvm.threads.live", "应用线程", "活跃数"),
+
+    /**
+     * 线程峰值
+     */
+    THREADS_PEAK("jvm.threads.peak", "应用线程", "峰值数"),
+
+    /**
+     * 内存已用
+     */
+    MEMORY_USED("jvm.memory.used", "内存", "已用"),
+
+    /**
+     * 内存空闲
+     */
+    MEMORY_COMMITTED("jvm.memory.committed", "内存", "空闲"),
+
+    /**
+     * 内存总共
+     */
+    MEMORY_MAX("jvm.memory.max", "内存", "总共"),
+
+    /**
+     * GC
+     */
+    GC_PAUSE("jvm.gc.pause", "GC", "已用"),
+
+    /**
+     * CPU已用
+     */
+    CPU_USAGE("system.cpu.usage", "CPU", "已用");
+
+    private String code;
+    private String group;
+    private String metricName;
+
+    MetricEnum(String code, String group, String metricName) {
+        this.code = code;
+        this.group = group;
+        this.metricName = metricName;
+    }
+
+    public static MetricEnum getMetric(String code) throws BizException {
+        for (MetricEnum e : MetricEnum.values()) {
+            if (StringUtil.equals(code, e.getCode())) {
+                return e;
+            }
+        }
+        throw new BizException(String.format("Metric code \"%s\" does not exist.", code));
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public String getGroup() {
+        return group;
+    }
+
+    public String getMetricName() {
+        return metricName;
+    }
+
 }

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/StatisticEnum.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/StatisticEnum.java

@@ -1,55 +1,55 @@
-package org.dbsyncer.monitor.enums;
-
-public enum StatisticEnum {
-
-    /**
-     * The sum of the amounts recorded.
-     */
-    TOTAL("total"),
-
-    /**
-     * The sum of the times recorded. Reported in the monitoring system's base unit of time
-     */
-    TOTAL_TIME("total"),
-
-    /**
-     * Rate per second for calls.
-     */
-    COUNT("count"),
-
-    /**
-     * The maximum amount recorded. When this represents a time, it is reported in the monitoring system's base unit of time.
-     */
-    MAX("max"),
-
-    /**
-     * Instantaneous value, such as those reported by gauges.
-     */
-    VALUE("value"),
-
-    /**
-     * Undetermined.
-     */
-    UNKNOWN("unknown"),
-
-    /**
-     * Number of currently active tasks for a long task timer.
-     */
-    ACTIVE_TASKS("active"),
-
-    /**
-     * Duration of a running task in a long task timer. Always reported in the monitoring system's base unit of time.
-     */
-    DURATION("duration");
-
-    private String tagValueRepresentation;
-
-    StatisticEnum(String tagValueRepresentation) {
-        this.tagValueRepresentation = tagValueRepresentation;
-    }
-
-    public String getTagValueRepresentation() {
-        return tagValueRepresentation;
-    }
-
+package org.dbsyncer.biz.enums;
+
+public enum StatisticEnum {
+
+    /**
+     * The sum of the amounts recorded.
+     */
+    TOTAL("total"),
+
+    /**
+     * The sum of the times recorded. Reported in the monitoring system's base unit of time
+     */
+    TOTAL_TIME("total"),
+
+    /**
+     * Rate per second for calls.
+     */
+    COUNT("count"),
+
+    /**
+     * The maximum amount recorded. When this represents a time, it is reported in the monitoring system's base unit of time.
+     */
+    MAX("max"),
+
+    /**
+     * Instantaneous value, such as those reported by gauges.
+     */
+    VALUE("value"),
+
+    /**
+     * Undetermined.
+     */
+    UNKNOWN("unknown"),
+
+    /**
+     * Number of currently active tasks for a long task timer.
+     */
+    ACTIVE_TASKS("active"),
+
+    /**
+     * Duration of a running task in a long task timer. Always reported in the monitoring system's base unit of time.
+     */
+    DURATION("duration");
+
+    private String tagValueRepresentation;
+
+    StatisticEnum(String tagValueRepresentation) {
+        this.tagValueRepresentation = tagValueRepresentation;
+    }
+
+    public String getTagValueRepresentation() {
+        return tagValueRepresentation;
+    }
+
 }

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/ThreadPoolMetricEnum.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/ThreadPoolMetricEnum.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.monitor.enums;
+package org.dbsyncer.biz.enums;
 
 /**
  * 线程池指标

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java

@@ -1,27 +1,28 @@
 package org.dbsyncer.biz.impl;
 
-import org.dbsyncer.manager.Manager;
+import org.dbsyncer.parser.LogService;
+import org.dbsyncer.parser.LogType;
+import org.dbsyncer.parser.MessageService;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.enums.MetaEnum;
-import org.dbsyncer.parser.enums.ModelEnum;
-import org.dbsyncer.parser.logger.LogService;
-import org.dbsyncer.parser.logger.LogType;
-import org.dbsyncer.parser.message.MessageService;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.dbsyncer.sdk.enums.ModelEnum;
 import org.springframework.util.Assert;
 
+import javax.annotation.Resource;
+
 public class BaseServiceImpl {
 
-    @Autowired
-    protected Manager manager;
+    @Resource
+    private ProfileComponent profileComponent;
 
-    @Autowired
+    @Resource
     private LogService logService;
 
-    @Autowired
+    @Resource
     private MessageService messageService;
 
     /**
@@ -30,7 +31,7 @@ public class BaseServiceImpl {
     protected final static Object LOCK = new Object();
 
     protected boolean isRunning(String metaId) {
-        Meta meta = manager.getMeta(metaId);
+        Meta meta = profileComponent.getMeta(metaId);
         if (null != meta) {
             int state = meta.getState();
             return MetaEnum.isRunning(state);
@@ -66,7 +67,7 @@ public class BaseServiceImpl {
 
     protected void log(LogType log, TableGroup tableGroup) {
         if (null != tableGroup) {
-            Mapping mapping = manager.getMapping(tableGroup.getMappingId());
+            Mapping mapping = profileComponent.getMapping(tableGroup.getMappingId());
             if (null != mapping) {
                 // 新增驱动知识库(全量)映射关系:[My_User] >> [My_User_Target]
                 String name = mapping.getName();

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java

@@ -1,32 +1,37 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.biz.impl;
 
 import org.dbsyncer.biz.ConditionService;
 import org.dbsyncer.biz.vo.ConditionVo;
-import org.dbsyncer.connector.enums.FilterEnum;
-import org.dbsyncer.connector.enums.OperationEnum;
-import org.dbsyncer.listener.enums.QuartzFilterEnum;
-import org.dbsyncer.manager.Manager;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.dbsyncer.sdk.enums.FilterEnum;
+import org.dbsyncer.sdk.enums.QuartzFilterEnum;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.sdk.enums.OperationEnum;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.Resource;
 import java.util.List;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2020/1/19 16:02
+ * 支持的条件和运算符类型
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2020-01-19 16:02
  */
 @Component
 public class ConditionServiceImpl implements ConditionService {
 
-    @Autowired
-    private Manager manager;
+    @Resource
+    private ProfileComponent profileComponent;
 
     @Override
     public ConditionVo getCondition() {
-        List<OperationEnum> operationEnumAll = manager.getOperationEnumAll();
-        List<QuartzFilterEnum> quartzFilterEnumAll = manager.getQuartzFilterEnumAll();
-        List<FilterEnum> filterEnumAll = manager.getFilterEnumAll();
+        List<OperationEnum> operationEnumAll = profileComponent.getOperationEnumAll();
+        List<QuartzFilterEnum> quartzFilterEnumAll = profileComponent.getQuartzFilterEnumAll();
+        List<FilterEnum> filterEnumAll = profileComponent.getFilterEnumAll();
         return new ConditionVo(operationEnumAll, quartzFilterEnumAll, filterEnumAll);
     }
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java

@@ -6,19 +6,23 @@ import org.dbsyncer.biz.checker.Checker;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.connector.base.ConnectorFactory;
+import org.dbsyncer.parser.LogService;
+import org.dbsyncer.parser.LogType;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Connector;
 import org.dbsyncer.parser.model.Mapping;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.model.ConnectorConfig;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
+import javax.annotation.Resource;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -39,10 +43,16 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe
 
     private Map<String, Boolean> health = new LinkedHashMap<>();
 
-    @Autowired
-    private Manager manager;
+    @Resource
+    private ProfileComponent profileComponent;
 
-    @Autowired
+    @Resource
+    private ConnectorFactory connectorFactory;
+
+    @Resource
+    private LogService logService;
+
+    @Resource
     private Checker connectorChecker;
 
     @Override
@@ -50,7 +60,7 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe
         ConfigModel model = connectorChecker.checkAddConfigModel(params);
         log(LogType.ConnectorLog.INSERT, model);
 
-        return manager.addConfigModel(model);
+        return profileComponent.addConfigModel(model);
     }
 
     @Override
@@ -62,7 +72,7 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe
         params.put(ConfigConstant.CONFIG_MODEL_NAME, connector.getName() + "(复制)");
         ConfigModel model = connectorChecker.checkAddConfigModel(params);
         log(LogType.ConnectorLog.COPY, model);
-        manager.addConfigModel(model);
+        profileComponent.addConfigModel(model);
 
         return String.format("复制成功[%s]", model.getName());
     }
@@ -72,12 +82,12 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe
         ConfigModel model = connectorChecker.checkEditConfigModel(params);
         log(LogType.ConnectorLog.UPDATE, model);
 
-        return manager.editConfigModel(model);
+        return profileComponent.editConfigModel(model);
     }
 
     @Override
     public String remove(String id) {
-        List<Mapping> mappingAll = manager.getMappingAll();
+        List<Mapping> mappingAll = profileComponent.getMappingAll();
         if (!CollectionUtils.isEmpty(mappingAll)) {
             mappingAll.forEach(mapping -> {
                 if (StringUtil.equals(mapping.getSourceConnectorId(), id) || StringUtil.equals(mapping.getTargetConnectorId(), id)) {
@@ -88,21 +98,23 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe
             });
         }
 
-        Connector connector = manager.getConnector(id);
-        log(LogType.ConnectorLog.DELETE, connector);
-
-        manager.removeConfigModel(id);
+        Connector connector = profileComponent.getConnector(id);
+        if (connector != null) {
+            connectorFactory.disconnect(connector.getConfig());
+            log(LogType.ConnectorLog.DELETE, connector);
+            profileComponent.removeConfigModel(id);
+        }
         return "删除连接器成功!";
     }
 
     @Override
     public Connector getConnector(String id) {
-        return StringUtil.isNotBlank(id) ? manager.getConnector(id) : null;
+        return StringUtil.isNotBlank(id) ? profileComponent.getConnector(id) : null;
     }
 
     @Override
     public List<Connector> getConnectorAll() {
-        List<Connector> list = manager.getConnectorAll()
+        List<Connector> list = profileComponent.getConnectorAll()
                 .stream()
                 .sorted(Comparator.comparing(Connector::getUpdateTime).reversed())
                 .collect(Collectors.toList());
@@ -111,14 +123,14 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe
 
     @Override
     public List<String> getConnectorTypeAll() {
-        List<String> list = new ArrayList<>();
-        manager.getConnectorEnumAll().forEach(c -> list.add(c.getType()));
-        return list;
+        ArrayList<String> connectorTypes = new ArrayList<>(connectorFactory.getConnectorTypeAll());
+        Collections.sort(connectorTypes, Comparator.comparing(String::toString));
+        return connectorTypes;
     }
 
     @Override
     public void refreshHealth() {
-        List<Connector> list = manager.getConnectorAll();
+        List<Connector> list = profileComponent.getConnectorAll();
         if (CollectionUtils.isEmpty(list)) {
             if (!CollectionUtils.isEmpty(health)) {
                 health.clear();
@@ -129,7 +141,7 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe
         // 更新连接器状态
         Set<String> exist = new HashSet<>();
         list.forEach(c -> {
-            health.put(c.getId(), manager.isAliveConnectorConfig(c.getConfig()));
+            health.put(c.getId(), isAlive(c.getConfig()));
             exist.add(c.getId());
         });
 
@@ -150,4 +162,15 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe
     public boolean isAlive(String id) {
         return health.containsKey(id) && health.get(id);
     }
+
+    private boolean isAlive(ConnectorConfig config) {
+        try {
+            return connectorFactory.isAlive(config);
+        } catch (Exception e) {
+            LogType.ConnectorLog logType = LogType.ConnectorLog.FAILED;
+            logService.log(logType, "%s%s", logType.getName(), e.getMessage());
+            return false;
+        }
+    }
+
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConvertServiceImpl.java

@@ -1,11 +1,11 @@
 package org.dbsyncer.biz.impl;
 
 import org.dbsyncer.biz.ConvertService;
-import org.dbsyncer.manager.Manager;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.enums.ConvertEnum;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.Resource;
 import java.util.List;
 
 /**
@@ -16,12 +16,12 @@ import java.util.List;
 @Component
 public class ConvertServiceImpl implements ConvertService {
 
-    @Autowired
-    private Manager manager;
+    @Resource
+    private ProfileComponent profileComponent;
 
     @Override
     public List<ConvertEnum> getConvertEnumAll() {
-        return manager.getConvertEnumAll();
+        return profileComponent.getConvertEnumAll();
     }
 
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java

@@ -1,24 +1,29 @@
 package org.dbsyncer.biz.impl;
 
 import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.lucene.index.IndexableField;
 import org.dbsyncer.biz.DataSyncService;
 import org.dbsyncer.biz.vo.BinlogColumnVo;
 import org.dbsyncer.biz.vo.MessageVo;
-import org.dbsyncer.common.event.RowChangedEvent;
+import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.DateFormatUtil;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.monitor.Monitor;
-import org.dbsyncer.parser.Parser;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.flush.impl.BufferActuatorRouter;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.Picker;
 import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.listener.event.RowChangedEvent;
+import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.filter.FieldResolver;
+import org.dbsyncer.sdk.filter.Query;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.dbsyncer.storage.binlog.proto.BinlogMap;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.util.BinlogMessageUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,6 +36,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -48,13 +54,13 @@ public class DataSyncServiceImpl implements DataSyncService {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resource
-    private Monitor monitor;
+    private BufferActuatorRouter bufferActuatorRouter;
 
     @Resource
-    private Manager manager;
+    private ProfileComponent profileComponent;
 
     @Resource
-    private Parser parser;
+    private StorageService storageService;
 
     @Override
     public MessageVo getMessageVo(String metaId, String messageId) {
@@ -63,10 +69,10 @@ public class DataSyncServiceImpl implements DataSyncService {
 
         MessageVo messageVo = new MessageVo();
         try {
-            Map row = monitor.getData(metaId, messageId);
+            Map row = getData(metaId, messageId);
             Map binlogData = getBinlogData(row, true);
             String tableGroupId = (String) row.get(ConfigConstant.DATA_TABLE_GROUP_ID);
-            TableGroup tableGroup = monitor.getTableGroup(tableGroupId);
+            TableGroup tableGroup = profileComponent.getTableGroup(tableGroupId);
             messageVo.setSourceTableName(tableGroup.getSourceTable().getName());
             messageVo.setTargetTableName(tableGroup.getTargetTable().getName());
             messageVo.setId(messageId);
@@ -87,7 +93,7 @@ public class DataSyncServiceImpl implements DataSyncService {
     public Map getBinlogData(Map row, boolean prettyBytes) throws InvalidProtocolBufferException {
         String tableGroupId = (String) row.get(ConfigConstant.DATA_TABLE_GROUP_ID);
         // 1、获取配置信息
-        final TableGroup tableGroup = manager.getTableGroup(tableGroupId);
+        final TableGroup tableGroup = profileComponent.getTableGroup(tableGroupId);
         if (tableGroup == null) {
             return Collections.EMPTY_MAP;
         }
@@ -134,45 +140,58 @@ public class DataSyncServiceImpl implements DataSyncService {
     }
 
     @Override
-    public String sync(Map<String, String> params) {
+    public String sync(Map<String, String> params) throws InvalidProtocolBufferException {
         String metaId = params.get("metaId");
         String messageId = params.get("messageId");
         Assert.hasText(metaId, "The metaId is null.");
         Assert.hasText(messageId, "The messageId is null.");
 
-        try {
-            Map row = monitor.getData(metaId, messageId);
-            Map binlogData = getBinlogData(row, false);
-            if (CollectionUtils.isEmpty(binlogData)) {
-                return messageId;
-            }
-            String tableGroupId = (String) row.get(ConfigConstant.DATA_TABLE_GROUP_ID);
-            String event = (String) row.get(ConfigConstant.DATA_EVENT);
-            // 有修改同步值
-            String retryDataParams = params.get("retryDataParams");
-            if (StringUtil.isNotBlank(retryDataParams)) {
-                JsonUtil.parseMap(retryDataParams).forEach((k, v) -> binlogData.put(k, convertValue(binlogData.get(k), (String) v)));
-            }
-            TableGroup tableGroup = manager.getTableGroup(tableGroupId);
-            String sourceTableName = tableGroup.getSourceTable().getName();
-            RowChangedEvent changedEvent = new RowChangedEvent(sourceTableName, event, Collections.EMPTY_LIST);
-            // 转换为源字段
-            final Picker picker = new Picker(tableGroup.getFieldMapping());
-            changedEvent.setChangedRow(picker.pickSourceData(binlogData));
-            parser.execute(tableGroupId, changedEvent);
-            monitor.removeData(metaId, messageId);
-            // 更新失败数
-            Meta meta = manager.getMeta(metaId);
-            Assert.notNull(meta, "Meta can not be null.");
-            meta.getFail().decrementAndGet();
-            meta.setUpdateTime(Instant.now().toEpochMilli());
-            manager.editConfigModel(meta);
-        } catch (Exception e) {
-            logger.error(e.getLocalizedMessage());
+        Map row = getData(metaId, messageId);
+        Map binlogData = getBinlogData(row, false);
+        if (CollectionUtils.isEmpty(binlogData)) {
+            return messageId;
         }
+        String tableGroupId = (String) row.get(ConfigConstant.DATA_TABLE_GROUP_ID);
+        String event = (String) row.get(ConfigConstant.DATA_EVENT);
+        // 有修改同步值
+        String retryDataParams = params.get("retryDataParams");
+        if (StringUtil.isNotBlank(retryDataParams)) {
+            JsonUtil.parseMap(retryDataParams).forEach((k, v) -> binlogData.put(k, convertValue(binlogData.get(k), (String) v)));
+        }
+        TableGroup tableGroup = profileComponent.getTableGroup(tableGroupId);
+        String sourceTableName = tableGroup.getSourceTable().getName();
+        RowChangedEvent changedEvent = new RowChangedEvent(sourceTableName, event, Collections.EMPTY_LIST);
+        // 转换为源字段
+        final Picker picker = new Picker(tableGroup.getFieldMapping());
+        changedEvent.setChangedRow(picker.pickSourceData(binlogData));
+        // 执行同步是否成功
+        bufferActuatorRouter.execute(metaId, tableGroupId, changedEvent);
+        storageService.remove(StorageEnum.DATA, metaId, messageId);
+        // 更新失败数
+        Meta meta = profileComponent.getMeta(metaId);
+        Assert.notNull(meta, "Meta can not be null.");
+        meta.getFail().decrementAndGet();
+        meta.setUpdateTime(Instant.now().toEpochMilli());
+        profileComponent.editConfigModel(meta);
         return messageId;
     }
 
+    private Map getData(String metaId, String messageId) {
+        Query query = new Query(1, 1);
+        Map<String, FieldResolver> fieldResolvers = new LinkedHashMap<>();
+        fieldResolvers.put(ConfigConstant.BINLOG_DATA, (FieldResolver<IndexableField>) field -> field.binaryValue().bytes);
+        query.setFieldResolverMap(fieldResolvers);
+        query.addFilter(ConfigConstant.CONFIG_MODEL_ID, messageId);
+        query.setMetaId(metaId);
+        query.setType(StorageEnum.DATA);
+        Paging paging = storageService.query(query);
+        if (!CollectionUtils.isEmpty(paging.getData())) {
+            List<Map> data = (List<Map>) paging.getData();
+            return data.get(0);
+        }
+        return Collections.EMPTY_MAP;
+    }
+
     private Object convertValue(Object oldValue, String newValue) {
         if (oldValue == null) {
             return newValue;

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java

@@ -3,30 +3,34 @@ package org.dbsyncer.biz.impl;
 import org.dbsyncer.biz.BizException;
 import org.dbsyncer.biz.ConnectorService;
 import org.dbsyncer.biz.MappingService;
+import org.dbsyncer.biz.MonitorService;
 import org.dbsyncer.biz.TableGroupService;
 import org.dbsyncer.biz.checker.impl.mapping.MappingChecker;
 import org.dbsyncer.biz.vo.ConnectorVo;
 import org.dbsyncer.biz.vo.MappingVo;
 import org.dbsyncer.biz.vo.MetaVo;
-import org.dbsyncer.common.snowflake.SnowflakeIdWorker;
-import org.dbsyncer.common.spi.ConnectorMapper;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.model.Table;
-import org.dbsyncer.parser.enums.ModelEnum;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.connector.base.ConnectorFactory;
+import org.dbsyncer.manager.ManagerFactory;
+import org.dbsyncer.parser.LogType;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Connector;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.enums.ModelEnum;
+import org.dbsyncer.sdk.model.Table;
+import org.dbsyncer.sdk.constant.ConfigConstant;
+import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
+import javax.annotation.Resource;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.Comparator;
@@ -43,24 +47,36 @@ import java.util.stream.Collectors;
 @Service
 public class MappingServiceImpl extends BaseServiceImpl implements MappingService {
 
-    @Autowired
+    @Resource
     private MappingChecker mappingChecker;
 
-    @Autowired
+    @Resource
     private TableGroupService tableGroupService;
 
-    @Autowired
+    @Resource
     private ConnectorService connectorService;
 
-    @Autowired
+    @Resource
     private SnowflakeIdWorker snowflakeIdWorker;
 
+    @Resource
+    private ProfileComponent profileComponent;
+
+    @Resource
+    private MonitorService monitorService;
+
+    @Resource
+    private ManagerFactory managerFactory;
+
+    @Resource
+    private ConnectorFactory connectorFactory;
+
     @Override
     public String add(Map<String, String> params) {
         ConfigModel model = mappingChecker.checkAddConfigModel(params);
         log(LogType.MappingLog.INSERT, (Mapping) model);
 
-        String id = manager.addConfigModel(model);
+        String id = profileComponent.addConfigModel(model);
 
         // 匹配相似表 on
         String autoMatchTable = params.get("autoMatchTable");
@@ -73,7 +89,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
 
     @Override
     public String copy(String id) {
-        Mapping mapping = manager.getMapping(id);
+        Mapping mapping = profileComponent.getMapping(id);
         Assert.notNull(mapping, "The mapping id is invalid.");
 
         String json = JsonUtil.objToJson(mapping);
@@ -83,18 +99,18 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
         newMapping.setUpdateTime(Instant.now().toEpochMilli());
         mappingChecker.addMeta(newMapping);
 
-        manager.addConfigModel(newMapping);
+        profileComponent.addConfigModel(newMapping);
         log(LogType.MappingLog.COPY, newMapping);
 
         // 复制映射表关系
-        List<TableGroup> groupList = manager.getTableGroupAll(mapping.getId());
-        if(!CollectionUtils.isEmpty(groupList)){
+        List<TableGroup> groupList = profileComponent.getTableGroupAll(mapping.getId());
+        if (!CollectionUtils.isEmpty(groupList)) {
             groupList.forEach(tableGroup -> {
                 String tableGroupJson = JsonUtil.objToJson(tableGroup);
                 TableGroup newTableGroup = JsonUtil.jsonToObj(tableGroupJson, TableGroup.class);
                 newTableGroup.setId(String.valueOf(snowflakeIdWorker.nextId()));
                 newTableGroup.setMappingId(newMapping.getId());
-                manager.addTableGroup(newTableGroup);
+                profileComponent.addTableGroup(newTableGroup);
                 log(LogType.TableGroupLog.COPY, newTableGroup);
             });
         }
@@ -111,7 +127,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
             log(LogType.MappingLog.UPDATE, model);
 
             mappingChecker.batchMergeTableGroupConfig(model, params);
-            return manager.editConfigModel(model);
+            return profileComponent.editConfigModel(model);
         }
     }
 
@@ -119,26 +135,26 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
     public String remove(String id) {
         Mapping mapping = assertMappingExist(id);
         String metaId = mapping.getMetaId();
-        Meta meta = manager.getMeta(metaId);
+        Meta meta = profileComponent.getMeta(metaId);
         synchronized (LOCK) {
             assertRunning(metaId);
 
             // 删除数据
-            manager.clearData(metaId);
+            monitorService.clearData(metaId);
             log(LogType.MetaLog.CLEAR, meta);
 
             // 删除meta
-            manager.removeConfigModel(metaId);
+            profileComponent.removeConfigModel(metaId);
             log(LogType.MetaLog.DELETE, meta);
 
             // 删除tableGroup
-            List<TableGroup> groupList = manager.getTableGroupAll(id);
+            List<TableGroup> groupList = profileComponent.getTableGroupAll(id);
             if (!CollectionUtils.isEmpty(groupList)) {
-                groupList.forEach(t -> manager.removeTableGroup(t.getId()));
+                groupList.forEach(t -> profileComponent.removeTableGroup(t.getId()));
             }
 
             // 删除驱动
-            manager.removeConfigModel(id);
+            profileComponent.removeConfigModel(id);
             log(LogType.MappingLog.DELETE, mapping);
         }
         return "驱动删除成功";
@@ -146,13 +162,13 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
 
     @Override
     public MappingVo getMapping(String id) {
-        Mapping mapping = manager.getMapping(id);
+        Mapping mapping = profileComponent.getMapping(id);
         return convertMapping2Vo(mapping);
     }
 
     @Override
     public List<MappingVo> getMappingAll() {
-        List<MappingVo> list = manager.getMappingAll()
+        List<MappingVo> list = profileComponent.getMappingAll()
                 .stream()
                 .map(m -> convertMapping2Vo(m))
                 .sorted(Comparator.comparing(MappingVo::getUpdateTime).reversed())
@@ -171,7 +187,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
             assertRunning(metaId);
 
             // 启动
-            manager.start(mapping);
+            managerFactory.start(mapping);
 
             log(LogType.MappingLog.RUNNING, mapping);
         }
@@ -185,7 +201,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
             if (!isRunning(mapping.getMetaId())) {
                 throw new BizException("驱动已停止.");
             }
-            manager.close(mapping);
+            managerFactory.close(mapping);
 
             log(LogType.MappingLog.STOP, mapping);
 
@@ -196,11 +212,29 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
         return "驱动停止成功";
     }
 
+    @Override
+    public String refreshMappingTables(String id) {
+        Mapping mapping = profileComponent.getMapping(id);
+        Assert.notNull(mapping, "The mapping id is invalid.");
+        updateConnectorTables(mapping.getSourceConnectorId());
+        updateConnectorTables(mapping.getTargetConnectorId());
+        return "刷新驱动表成功";
+    }
+
+    private void updateConnectorTables(String connectorId) {
+        Connector connector = profileComponent.getConnector(connectorId);
+        Assert.notNull(connector, "The connector id is invalid.");
+        // 刷新数据表
+        ConnectorInstance connectorInstance = connectorFactory.connect(connector.getConfig());
+        connector.setTable(connectorFactory.getTable(connectorInstance));
+        profileComponent.editConfigModel(connector);
+    }
+
     private MappingVo convertMapping2Vo(Mapping mapping) {
         String model = mapping.getModel();
         Assert.notNull(mapping, "Mapping can not be null.");
-        Connector s = manager.getConnector(mapping.getSourceConnectorId());
-        Connector t = manager.getConnector(mapping.getTargetConnectorId());
+        Connector s = profileComponent.getConnector(mapping.getSourceConnectorId());
+        Connector t = profileComponent.getConnector(mapping.getTargetConnectorId());
         ConnectorVo sConn = new ConnectorVo(connectorService.isAlive(s.getId()));
         BeanUtils.copyProperties(s, sConn);
         ConnectorVo tConn = new ConnectorVo(connectorService.isAlive(t.getId()));
@@ -211,7 +245,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
         Collections.sort(tConn.getTable(), Comparator.comparing(Table::getName));
 
         // 元信息
-        Meta meta = manager.getMeta(mapping.getMetaId());
+        Meta meta = profileComponent.getMeta(mapping.getMetaId());
         Assert.notNull(meta, "Meta can not be null.");
         MetaVo metaVo = new MetaVo(ModelEnum.getModelEnum(model).getName(), mapping.getName());
         BeanUtils.copyProperties(meta, metaVo);
@@ -228,7 +262,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
      * @return
      */
     private Mapping assertMappingExist(String mappingId) {
-        Mapping mapping = manager.getMapping(mappingId);
+        Mapping mapping = profileComponent.getMapping(mappingId);
         Assert.notNull(mapping, "驱动不存在.");
         return mapping;
     }
@@ -240,8 +274,8 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
      */
     private void matchSimilarTable(ConfigModel model) {
         Mapping mapping = (Mapping) model;
-        Connector s = manager.getConnector(mapping.getSourceConnectorId());
-        Connector t = manager.getConnector(mapping.getTargetConnectorId());
+        Connector s = profileComponent.getConnector(mapping.getSourceConnectorId());
+        Connector t = profileComponent.getConnector(mapping.getTargetConnectorId());
         if (CollectionUtils.isEmpty(s.getTable()) || CollectionUtils.isEmpty(t.getTable())) {
             return;
         }
@@ -263,23 +297,14 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
     }
 
     private void clearMetaIfFinished(String metaId) {
-        Meta meta = manager.getMeta(metaId);
+        Meta meta = profileComponent.getMeta(metaId);
         Assert.notNull(meta, "Mapping meta can not be null.");
         // 完成任务则重置状态
         if (meta.getTotal().get() <= (meta.getSuccess().get() + meta.getFail().get())) {
             meta.getFail().set(0);
             meta.getSuccess().set(0);
-            manager.editConfigModel(meta);
+            profileComponent.editConfigModel(meta);
         }
     }
 
-    @Override
-    public void refreshTables(Connector connector) {
-        // 刷新数据表
-        ConnectorMapper connectorMapper = manager.connect(connector.getConfig());
-        List<Table> table = manager.getTable(connectorMapper);
-        connector.setTable(table);
-        manager.editConfigModel(connector);
-    }
-
 }

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java

@@ -1,338 +1,246 @@
-package org.dbsyncer.monitor;
-
-import org.dbsyncer.common.model.Paging;
-import org.dbsyncer.common.scheduled.ScheduledTaskJob;
-import org.dbsyncer.common.scheduled.ScheduledTaskService;
-import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.common.util.NumberUtil;
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.constant.ConnectorConstant;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.manager.puller.BufferActuatorRouter;
-import org.dbsyncer.monitor.enums.BufferActuatorMetricEnum;
-import org.dbsyncer.monitor.enums.MetricEnum;
-import org.dbsyncer.monitor.enums.StatisticEnum;
-import org.dbsyncer.monitor.enums.ThreadPoolMetricEnum;
-import org.dbsyncer.monitor.model.AppReportMetric;
-import org.dbsyncer.monitor.model.MappingReportMetric;
-import org.dbsyncer.monitor.model.MetricResponse;
-import org.dbsyncer.monitor.model.MetricResponseInfo;
-import org.dbsyncer.monitor.model.Sample;
-import org.dbsyncer.parser.flush.BufferActuator;
-import org.dbsyncer.parser.flush.impl.TableGroupBufferActuator;
-import org.dbsyncer.parser.model.Mapping;
-import org.dbsyncer.parser.model.Meta;
-import org.dbsyncer.parser.model.TableGroup;
-import org.dbsyncer.storage.constant.ConfigConstant;
-import org.dbsyncer.storage.enums.IndexFieldResolverEnum;
-import org.dbsyncer.storage.enums.StorageDataStatusEnum;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.Query;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
-import java.time.LocalDateTime;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2020/04/23 11:30
- */
-@Component
-public class MonitorFactory implements Monitor, ScheduledTaskJob {
-
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
-    @Resource
-    private Manager manager;
-
-    @Resource
-    private BufferActuator generalBufferActuator;
-
-    @Resource
-    private BufferActuator storageBufferActuator;
-
-    @Resource
-    private BufferActuatorRouter bufferActuatorRouter;
-
-    @Resource
-    private ScheduledTaskService scheduledTaskService;
-
-    private volatile boolean running;
-
-    private LocalDateTime queryTime;
-
-    private final MappingReportMetric mappingReportMetric = new MappingReportMetric();
-
-    private final int SHOW_BUFFER_ACTUATOR_SIZE = 6;
-
-    @PostConstruct
-    private void init() {
-        scheduledTaskService.start(5000, this);
-    }
-
-    @Override
-    public Mapping getMapping(String mappingId) {
-        return manager.getMapping(mappingId);
-    }
-
-    @Override
-    public TableGroup getTableGroup(String tableGroupId) {
-        return manager.getTableGroup(tableGroupId);
-    }
-
-    @Override
-    public List<Meta> getMetaAll() {
-        return manager.getMetaAll();
-    }
-
-    @Override
-    public Meta getMeta(String metaId) {
-        return manager.getMeta(metaId);
-    }
-
-    @Override
-    public Paging queryData(String metaId, int pageNum, int pageSize, String error, String success) {
-        // 没有驱动
-        if (StringUtil.isBlank(metaId)) {
-            return new Paging(pageNum, pageSize);
-        }
-        Query query = new Query(pageNum, pageSize);
-        Map<String, IndexFieldResolverEnum> fieldResolvers = new LinkedHashMap<>();
-        fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY);
-        query.setIndexFieldResolverMap(fieldResolvers);
-
-        // 查询异常信息
-        if (StringUtil.isNotBlank(error)) {
-            query.addFilter(ConfigConstant.DATA_ERROR, error, true);
-        }
-        // 查询是否成功, 默认查询失败
-        query.addFilter(ConfigConstant.DATA_SUCCESS, StringUtil.isNotBlank(success) ? NumberUtil.toInt(success) : StorageDataStatusEnum.FAIL.getValue());
-        query.setMetaId(metaId);
-        return manager.queryData(query);
-    }
-
-    @Override
-    public Map getData(String metaId, String messageId) {
-        Query query = new Query(1, 1);
-        Map<String, IndexFieldResolverEnum> fieldResolvers = new LinkedHashMap<>();
-        fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY);
-        query.setIndexFieldResolverMap(fieldResolvers);
-        query.addFilter(ConfigConstant.CONFIG_MODEL_ID, messageId);
-        query.setMetaId(metaId);
-        Paging paging = manager.queryData(query);
-        if (!CollectionUtils.isEmpty(paging.getData())) {
-            List<Map> data = (List<Map>) paging.getData();
-            return data.get(0);
-        }
-        return Collections.EMPTY_MAP;
-    }
-
-    @Override
-    public void removeData(String metaId, String messageId) {
-        manager.removeData(metaId, messageId);
-    }
-
-    @Override
-    public void clearData(String metaId) {
-        manager.clearData(metaId);
-    }
-
-    @Override
-    public Paging queryLog(int pageNum, int pageSize, String json) {
-        Query query = new Query(pageNum, pageSize);
-        if (StringUtil.isNotBlank(json)) {
-            query.addFilter(ConfigConstant.CONFIG_MODEL_JSON, json, true);
-        }
-        return manager.queryLog(query);
-    }
-
-    @Override
-    public void clearLog() {
-        manager.clearLog();
-    }
-
-    @Override
-    public List<StorageDataStatusEnum> getStorageDataStatusEnumAll() {
-        return manager.getStorageDataStatusEnumAll();
-    }
-
-    @Override
-    public List<MetricEnum> getMetricEnumAll() {
-        return Arrays.asList(MetricEnum.values());
-    }
-
-    @Override
-    public List<MetricResponse> getMetricInfo() {
-        List<MetricResponseInfo> list = new ArrayList<>();
-        BufferActuatorMetricEnum general = BufferActuatorMetricEnum.GENERAL;
-        BufferActuatorMetricEnum storage = BufferActuatorMetricEnum.STORAGE;
-        list.add(collect(generalBufferActuator, general.getCode(), general.getGroup(), general.getMetricName()));
-        list.add(collect(storageBufferActuator, storage.getCode(), storage.getGroup(), storage.getMetricName()));
-        if (!CollectionUtils.isEmpty(bufferActuatorRouter.getRouter())) {
-            List<MetricResponseInfo> tableList = new ArrayList<>();
-            String tableGroupCode = BufferActuatorMetricEnum.TABLE_GROUP.getCode();
-            bufferActuatorRouter.getRouter().forEach((metaId, group) -> {
-                Meta meta = manager.getMeta(metaId);
-                Mapping mapping = manager.getMapping(meta.getMappingId());
-                group.forEach((k, bufferActuator) -> {
-                    if (bufferActuator instanceof TableGroupBufferActuator) {
-                        TableGroupBufferActuator actuator = bufferActuator;
-                        TableGroup tableGroup = manager.getTableGroup(actuator.getTableGroupId());
-                        String metricName = new StringBuilder()
-                                .append(tableGroup.getSourceTable().getName())
-                                .append(" > ")
-                                .append(tableGroup.getTargetTable().getName()).toString();
-                        tableList.add(collect(bufferActuator, tableGroupCode, mapping.getName(), metricName));
-                    }
-                });
-            });
-            List<MetricResponseInfo> sortList = tableList.stream()
-                    .sorted(Comparator.comparing(MetricResponseInfo::getQueueUp).reversed())
-                    .collect(Collectors.toList());
-            list.addAll(sortList.size() <= SHOW_BUFFER_ACTUATOR_SIZE ? sortList : sortList.subList(0, SHOW_BUFFER_ACTUATOR_SIZE));
-        }
-        return list.stream().map(info -> info.getResponse()).collect(Collectors.toList());
-    }
-
-    @Override
-    public AppReportMetric getAppReportMetric() {
-        queryTime = LocalDateTime.now();
-        AppReportMetric report = new AppReportMetric();
-        report.setSuccess(mappingReportMetric.getSuccess());
-        report.setFail(mappingReportMetric.getFail());
-        report.setInsert(mappingReportMetric.getInsert());
-        report.setUpdate(mappingReportMetric.getUpdate());
-        report.setDelete(mappingReportMetric.getDelete());
-        // 堆积任务(通用执行器 + 表执行器)
-        report.setQueueUp(bufferActuatorRouter.getQueueSize().addAndGet(generalBufferActuator.getQueue().size()));
-        report.setQueueCapacity(bufferActuatorRouter.getQueueCapacity().addAndGet(generalBufferActuator.getQueueCapacity()));
-        // 持久化任务
-        report.setStorageQueueUp(storageBufferActuator.getQueue().size());
-        report.setStorageQueueCapacity(storageBufferActuator.getQueueCapacity());
-        return report;
-    }
-
-    @Override
-    public void run() {
-        if (running || null == queryTime) {
-            return;
-        }
-        // 非活动时间范围(30s内)
-        if (LocalDateTime.now().minusSeconds(30).isAfter(queryTime)) {
-            return;
-        }
-
-        // 刷新报表
-        try {
-            running = true;
-            final List<Meta> metaAll = manager.getMetaAll();
-            mappingReportMetric.setSuccess(getMappingSuccess(metaAll));
-            mappingReportMetric.setFail(getMappingFail(metaAll));
-            mappingReportMetric.setInsert(getMappingInsert(metaAll));
-            mappingReportMetric.setUpdate(getMappingUpdate(metaAll));
-            mappingReportMetric.setDelete(getMappingDelete(metaAll));
-        } catch (Exception e) {
-            logger.error(e.getMessage());
-        } finally {
-            running = false;
-        }
-    }
-
-    /**
-     * 获取所有驱动成功数
-     *
-     * @param metaAll
-     * @return
-     */
-    private long getMappingSuccess(List<Meta> metaAll) {
-        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.SUCCESS.getValue()));
-    }
-
-    /**
-     * 获取所有驱动失败数
-     *
-     * @param metaAll
-     * @return
-     */
-    private long getMappingFail(List<Meta> metaAll) {
-        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.FAIL.getValue()));
-    }
-
-    /**
-     * 获取所有驱动事件插入数
-     *
-     * @param metaAll
-     * @return
-     */
-    private long getMappingInsert(List<Meta> metaAll) {
-        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_INSERT));
-    }
-
-    /**
-     * 获取所有驱动事件更新数
-     *
-     * @param metaAll
-     * @return
-     */
-    private long getMappingUpdate(List<Meta> metaAll) {
-        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_UPDATE));
-    }
-
-    /**
-     * 获取所有驱动事件删除数
-     *
-     * @param metaAll
-     * @return
-     */
-    private long getMappingDelete(List<Meta> metaAll) {
-        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_DELETE));
-    }
-
-    private long queryMappingMetricCount(List<Meta> metaAll, Consumer<Query> operation) {
-        AtomicLong total = new AtomicLong(0);
-        if (!CollectionUtils.isEmpty(metaAll)) {
-            Query query = new Query(1, 1);
-            query.setQueryTotal(true);
-            query.setType(StorageEnum.DATA);
-            operation.accept(query);
-            metaAll.forEach(meta -> {
-                query.setMetaId(meta.getId());
-                Paging paging = manager.queryData(query);
-                total.getAndAdd(paging.getTotal());
-            });
-        }
-        return total.get();
-    }
-
-    private MetricResponseInfo collect(BufferActuator bufferActuator, String code, String group, String metricName) {
-        MetricResponseInfo info = new MetricResponseInfo();
-        ThreadPoolTaskExecutor threadTask = (ThreadPoolTaskExecutor) bufferActuator.getExecutor();
-        ThreadPoolExecutor pool = threadTask.getThreadPoolExecutor();
-        info.setQueueUp(bufferActuator.getQueue().size());
-        StringBuilder msg = new StringBuilder();
-        msg.append("堆积").append(StringUtil.COLON).append(info.getQueueUp());
-        msg.append(StringUtil.FORWARD_SLASH).append(bufferActuator.getQueueCapacity()).append(StringUtil.SPACE);
-        msg.append(ThreadPoolMetricEnum.CORE_SIZE.getMetricName()).append(StringUtil.COLON).append(pool.getActiveCount());
-        msg.append(StringUtil.FORWARD_SLASH).append(pool.getCorePoolSize()).append(StringUtil.SPACE);
-        msg.append(ThreadPoolMetricEnum.COMPLETED.getMetricName()).append(StringUtil.COLON).append(pool.getCompletedTaskCount());
-        info.setResponse(new MetricResponse(code, group, metricName, Arrays.asList(new Sample(StatisticEnum.COUNT.getTagValueRepresentation(), msg.toString()))));
-        return info;
-    }
-
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.biz.impl;
+
+import org.dbsyncer.biz.enums.BufferActuatorMetricEnum;
+import org.dbsyncer.biz.enums.StatisticEnum;
+import org.dbsyncer.biz.enums.ThreadPoolMetricEnum;
+import org.dbsyncer.biz.model.AppReportMetric;
+import org.dbsyncer.biz.model.MappingReportMetric;
+import org.dbsyncer.biz.model.MetricResponse;
+import org.dbsyncer.biz.model.MetricResponseInfo;
+import org.dbsyncer.biz.model.Sample;
+import org.dbsyncer.common.model.Paging;
+import org.dbsyncer.common.scheduled.ScheduledTaskJob;
+import org.dbsyncer.common.scheduled.ScheduledTaskService;
+import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.flush.BufferActuator;
+import org.dbsyncer.parser.flush.impl.BufferActuatorRouter;
+import org.dbsyncer.parser.flush.impl.TableGroupBufferActuator;
+import org.dbsyncer.parser.model.Mapping;
+import org.dbsyncer.parser.model.Meta;
+import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.sdk.constant.ConnectorConstant;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.filter.Query;
+import org.dbsyncer.sdk.storage.StorageService;
+import org.dbsyncer.sdk.constant.ConfigConstant;
+import org.dbsyncer.storage.enums.StorageDataStatusEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2020-04-23 11:30
+ */
+@Component
+public class MetricReporter implements ScheduledTaskJob {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    @Resource
+    private ProfileComponent profileComponent;
+
+    @Resource
+    private BufferActuator generalBufferActuator;
+
+    @Resource
+    private BufferActuator storageBufferActuator;
+
+    @Resource
+    private BufferActuatorRouter bufferActuatorRouter;
+
+    @Resource
+    private ScheduledTaskService scheduledTaskService;
+
+    @Resource
+    private StorageService storageService;
+
+    private volatile boolean running;
+
+    private LocalDateTime queryTime;
+
+    private final MappingReportMetric mappingReportMetric = new MappingReportMetric();
+
+    private final int SHOW_BUFFER_ACTUATOR_SIZE = 6;
+
+    @PostConstruct
+    private void init() {
+        scheduledTaskService.start(5000, this);
+    }
+
+    public List<MetricResponse> getMetricInfo() {
+        List<MetricResponseInfo> list = new ArrayList<>();
+        BufferActuatorMetricEnum general = BufferActuatorMetricEnum.GENERAL;
+        BufferActuatorMetricEnum storage = BufferActuatorMetricEnum.STORAGE;
+        list.add(collect(generalBufferActuator, general.getCode(), general.getGroup(), general.getMetricName()));
+        list.add(collect(storageBufferActuator, storage.getCode(), storage.getGroup(), storage.getMetricName()));
+        if (!CollectionUtils.isEmpty(bufferActuatorRouter.getRouter())) {
+            List<MetricResponseInfo> tableList = new ArrayList<>();
+            String tableGroupCode = BufferActuatorMetricEnum.TABLE_GROUP.getCode();
+            bufferActuatorRouter.getRouter().forEach((metaId, group) -> {
+                Meta meta = profileComponent.getMeta(metaId);
+                Mapping mapping = profileComponent.getMapping(meta.getMappingId());
+                group.forEach((k, bufferActuator) -> {
+                    if (bufferActuator instanceof TableGroupBufferActuator) {
+                        TableGroupBufferActuator actuator = bufferActuator;
+                        TableGroup tableGroup = profileComponent.getTableGroup(actuator.getTableGroupId());
+                        String metricName = new StringBuilder()
+                                .append(tableGroup.getSourceTable().getName())
+                                .append(" > ")
+                                .append(tableGroup.getTargetTable().getName()).toString();
+                        tableList.add(collect(bufferActuator, tableGroupCode, mapping.getName(), metricName));
+                    }
+                });
+            });
+            List<MetricResponseInfo> sortList = tableList.stream()
+                    .sorted(Comparator.comparing(MetricResponseInfo::getQueueUp).reversed())
+                    .collect(Collectors.toList());
+            list.addAll(sortList.size() <= SHOW_BUFFER_ACTUATOR_SIZE ? sortList : sortList.subList(0, SHOW_BUFFER_ACTUATOR_SIZE));
+        }
+        return list.stream().map(info -> info.getResponse()).collect(Collectors.toList());
+    }
+
+    public AppReportMetric getAppReportMetric() {
+        queryTime = LocalDateTime.now();
+        AppReportMetric report = new AppReportMetric();
+        report.setSuccess(mappingReportMetric.getSuccess());
+        report.setFail(mappingReportMetric.getFail());
+        report.setInsert(mappingReportMetric.getInsert());
+        report.setUpdate(mappingReportMetric.getUpdate());
+        report.setDelete(mappingReportMetric.getDelete());
+        // 堆积任务(通用执行器 + 表执行器)
+        report.setQueueUp(bufferActuatorRouter.getQueueSize().addAndGet(generalBufferActuator.getQueue().size()));
+        report.setQueueCapacity(bufferActuatorRouter.getQueueCapacity().addAndGet(generalBufferActuator.getQueueCapacity()));
+        // 持久化任务
+        report.setStorageQueueUp(storageBufferActuator.getQueue().size());
+        report.setStorageQueueCapacity(storageBufferActuator.getQueueCapacity());
+        return report;
+    }
+
+    @Override
+    public void run() {
+        if (running || null == queryTime) {
+            return;
+        }
+        // 非活动时间范围(30s内)
+        if (LocalDateTime.now().minusSeconds(30).isAfter(queryTime)) {
+            return;
+        }
+
+        // 刷新报表
+        try {
+            running = true;
+            final List<Meta> metaAll = profileComponent.getMetaAll();
+            mappingReportMetric.setSuccess(getMappingSuccess(metaAll));
+            mappingReportMetric.setFail(getMappingFail(metaAll));
+            mappingReportMetric.setInsert(getMappingInsert(metaAll));
+            mappingReportMetric.setUpdate(getMappingUpdate(metaAll));
+            mappingReportMetric.setDelete(getMappingDelete(metaAll));
+        } catch (Exception e) {
+            logger.error(e.getMessage());
+        } finally {
+            running = false;
+        }
+    }
+
+    /**
+     * 获取所有驱动成功数
+     *
+     * @param metaAll
+     * @return
+     */
+    private long getMappingSuccess(List<Meta> metaAll) {
+        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.SUCCESS.getValue()));
+    }
+
+    /**
+     * 获取所有驱动失败数
+     *
+     * @param metaAll
+     * @return
+     */
+    private long getMappingFail(List<Meta> metaAll) {
+        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.FAIL.getValue()));
+    }
+
+    /**
+     * 获取所有驱动事件插入数
+     *
+     * @param metaAll
+     * @return
+     */
+    private long getMappingInsert(List<Meta> metaAll) {
+        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_INSERT));
+    }
+
+    /**
+     * 获取所有驱动事件更新数
+     *
+     * @param metaAll
+     * @return
+     */
+    private long getMappingUpdate(List<Meta> metaAll) {
+        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_UPDATE));
+    }
+
+    /**
+     * 获取所有驱动事件删除数
+     *
+     * @param metaAll
+     * @return
+     */
+    private long getMappingDelete(List<Meta> metaAll) {
+        return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_DELETE));
+    }
+
+    private long queryMappingMetricCount(List<Meta> metaAll, Consumer<Query> operation) {
+        AtomicLong total = new AtomicLong(0);
+        if (!CollectionUtils.isEmpty(metaAll)) {
+            Query query = new Query(1, 1);
+            query.setQueryTotal(true);
+            query.setType(StorageEnum.DATA);
+            operation.accept(query);
+            metaAll.forEach(meta -> {
+                query.setMetaId(meta.getId());
+                Paging paging = storageService.query(query);
+                total.getAndAdd(paging.getTotal());
+            });
+        }
+        return total.get();
+    }
+
+    private MetricResponseInfo collect(BufferActuator bufferActuator, String code, String group, String metricName) {
+        MetricResponseInfo info = new MetricResponseInfo();
+        ThreadPoolTaskExecutor threadTask = (ThreadPoolTaskExecutor) bufferActuator.getExecutor();
+        ThreadPoolExecutor pool = threadTask.getThreadPoolExecutor();
+        info.setQueueUp(bufferActuator.getQueue().size());
+        StringBuilder msg = new StringBuilder();
+        msg.append("堆积").append(StringUtil.COLON).append(info.getQueueUp());
+        msg.append(StringUtil.FORWARD_SLASH).append(bufferActuator.getQueueCapacity()).append(StringUtil.SPACE);
+        msg.append(ThreadPoolMetricEnum.CORE_SIZE.getMetricName()).append(StringUtil.COLON).append(pool.getActiveCount());
+        msg.append(StringUtil.FORWARD_SLASH).append(pool.getMaximumPoolSize()).append(StringUtil.SPACE);
+        msg.append(ThreadPoolMetricEnum.COMPLETED.getMetricName()).append(StringUtil.COLON).append(pool.getCompletedTaskCount());
+        info.setResponse(new MetricResponse(code, group, metricName, Arrays.asList(new Sample(StatisticEnum.COUNT.getTagValueRepresentation(), msg.toString()))));
+        return info;
+    }
+
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java

@@ -1,8 +1,15 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.biz.impl;
 
+import org.apache.lucene.index.IndexableField;
 import org.dbsyncer.biz.DataSyncService;
 import org.dbsyncer.biz.MonitorService;
 import org.dbsyncer.biz.SystemConfigService;
+import org.dbsyncer.biz.enums.BufferActuatorMetricEnum;
+import org.dbsyncer.biz.enums.DiskMetricEnum;
+import org.dbsyncer.biz.enums.MetricEnum;
 import org.dbsyncer.biz.metric.MetricDetailFormatter;
 import org.dbsyncer.biz.metric.impl.CpuMetricDetailFormatter;
 import org.dbsyncer.biz.metric.impl.DiskMetricDetailFormatter;
@@ -10,12 +17,13 @@ import org.dbsyncer.biz.metric.impl.DoubleRoundMetricDetailFormatter;
 import org.dbsyncer.biz.metric.impl.GCMetricDetailFormatter;
 import org.dbsyncer.biz.metric.impl.MemoryMetricDetailFormatter;
 import org.dbsyncer.biz.metric.impl.ValueMetricDetailFormatter;
+import org.dbsyncer.biz.model.AppReportMetric;
+import org.dbsyncer.biz.model.MetricResponse;
 import org.dbsyncer.biz.vo.AppReportMetricVo;
 import org.dbsyncer.biz.vo.DataVo;
 import org.dbsyncer.biz.vo.LogVo;
 import org.dbsyncer.biz.vo.MetaVo;
 import org.dbsyncer.biz.vo.MetricResponseVo;
-import org.dbsyncer.common.event.PreloadCompletedEvent;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.scheduled.ScheduledTaskJob;
 import org.dbsyncer.common.scheduled.ScheduledTaskService;
@@ -23,28 +31,25 @@ import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.enums.FilterEnum;
-import org.dbsyncer.monitor.Monitor;
-import org.dbsyncer.monitor.enums.BufferActuatorMetricEnum;
-import org.dbsyncer.monitor.enums.DiskMetricEnum;
-import org.dbsyncer.monitor.enums.MetricEnum;
-import org.dbsyncer.monitor.model.AppReportMetric;
-import org.dbsyncer.monitor.model.MetricResponse;
+import org.dbsyncer.parser.LogService;
+import org.dbsyncer.parser.LogType;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.enums.MetaEnum;
-import org.dbsyncer.parser.enums.ModelEnum;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
-import org.dbsyncer.storage.StorageService;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.enums.FilterEnum;
+import org.dbsyncer.sdk.enums.ModelEnum;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.filter.BooleanFilter;
+import org.dbsyncer.sdk.filter.FieldResolver;
+import org.dbsyncer.sdk.filter.Query;
+import org.dbsyncer.sdk.filter.impl.LongFilter;
+import org.dbsyncer.sdk.storage.StorageService;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.BooleanFilter;
-import org.dbsyncer.storage.query.Query;
-import org.dbsyncer.storage.query.filter.LongFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
-import org.springframework.context.ApplicationListener;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
@@ -53,6 +58,7 @@ import javax.annotation.Resource;
 import java.sql.Timestamp;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -60,17 +66,20 @@ import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2020/04/27 10:20
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2020-04-27 10:20
  */
 @Service
-public class MonitorServiceImpl extends BaseServiceImpl implements MonitorService, ScheduledTaskJob, ApplicationListener<PreloadCompletedEvent> {
+public class MonitorServiceImpl extends BaseServiceImpl implements MonitorService, ScheduledTaskJob {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resource
-    private Monitor monitor;
+    private MetricReporter metricReporter;
+
+    @Resource
+    private ProfileComponent profileComponent;
 
     @Resource
     private DataSyncService dataSyncService;
@@ -82,9 +91,10 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
     private StorageService storageService;
 
     @Resource
-    private SystemConfigService systemConfigService;
+    private LogService logService;
 
-    private boolean preloadCompleted;
+    @Resource
+    private SystemConfigService systemConfigService;
 
     private Map<String, MetricDetailFormatter> metricDetailFormatterMap = new LinkedHashMap<>();
 
@@ -110,7 +120,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
 
     @Override
     public List<MetaVo> getMetaAll() {
-        List<MetaVo> list = monitor.getMetaAll()
+        List<MetaVo> list = profileComponent.getMetaAll()
                 .stream()
                 .map(m -> convertMeta2Vo(m))
                 .sorted(Comparator.comparing(MetaVo::getUpdateTime).reversed())
@@ -120,7 +130,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
 
     @Override
     public MetaVo getMetaVo(String metaId) {
-        Meta meta = monitor.getMeta(metaId);
+        Meta meta = profileComponent.getMeta(metaId);
         Assert.notNull(meta, "The meta is null.");
 
         return convertMeta2Vo(meta);
@@ -140,7 +150,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
         String error = params.get(ConfigConstant.DATA_ERROR);
         String success = params.get(ConfigConstant.DATA_SUCCESS);
 
-        Paging paging = monitor.queryData(getDefaultMetaId(id), pageNum, pageSize, error, success);
+        Paging paging = queryData(getDefaultMetaId(id), pageNum, pageSize, error, success);
         List<Map> data = (List<Map>) paging.getData();
         List<DataVo> list = new ArrayList<>();
         for (Map row : data) {
@@ -160,7 +170,12 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
     @Override
     public String clearData(String id) {
         Assert.hasText(id, "驱动不存在.");
-        monitor.clearData(id);
+        Meta meta = profileComponent.getMeta(id);
+        Mapping mapping = profileComponent.getMapping(meta.getMappingId());
+        String model = ModelEnum.getModelEnum(mapping.getModel()).getName();
+        LogType.MappingLog log = LogType.MappingLog.CLEAR_DATA;
+        logService.log(log, "%s:%s(%s)", log.getMessage(), mapping.getName(), model);
+        storageService.clear(StorageEnum.DATA, id);
         return "清空同步数据成功";
     }
 
@@ -169,7 +184,12 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
         int pageNum = NumberUtil.toInt(params.get("pageNum"), 1);
         int pageSize = NumberUtil.toInt(params.get("pageSize"), 10);
         String json = params.get(ConfigConstant.CONFIG_MODEL_JSON);
-        Paging paging = monitor.queryLog(pageNum, pageSize, json);
+        Query query = new Query(pageNum, pageSize);
+        if (StringUtil.isNotBlank(json)) {
+            query.addFilter(ConfigConstant.CONFIG_MODEL_JSON, json, true);
+        }
+        query.setType(StorageEnum.LOG);
+        Paging paging = storageService.query(query);
         List<Map> data = (List<Map>) paging.getData();
         paging.setData(data.stream()
                 .map(m -> convert2Vo(m, LogVo.class))
@@ -179,31 +199,29 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
 
     @Override
     public String clearLog() {
-        monitor.clearLog();
+        storageService.clear(StorageEnum.LOG, null);
         return "清空日志成功";
     }
 
     @Override
     public void deleteExpiredDataAndLog() {
-        if (preloadCompleted) {
-            deleteExpiredData();
-            deleteExpiredLog();
-        }
+        deleteExpiredData();
+        deleteExpiredLog();
     }
 
     @Override
     public List<StorageDataStatusEnum> getStorageDataStatusEnumAll() {
-        return monitor.getStorageDataStatusEnumAll();
+        return profileComponent.getStorageDataStatusEnumAll();
     }
 
     @Override
     public List<MetricEnum> getMetricEnumAll() {
-        return monitor.getMetricEnumAll();
+        return Arrays.asList(MetricEnum.values());
     }
 
     @Override
     public AppReportMetricVo queryAppReportMetric(List<MetricResponse> metrics) {
-        AppReportMetric appReportMetric = monitor.getAppReportMetric();
+        AppReportMetric appReportMetric = metricReporter.getAppReportMetric();
         AppReportMetricVo vo = new AppReportMetricVo();
         BeanUtils.copyProperties(appReportMetric, vo);
         vo.setMetrics(getMetrics(metrics));
@@ -213,7 +231,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
     @Override
     public void run() {
         // 预警：驱动出现失败记录，发送通知消息
-        List<Meta> metaAll = monitor.getMetaAll();
+        List<Meta> metaAll = profileComponent.getMetaAll();
         if (CollectionUtils.isEmpty(metaAll)) {
             return;
         }
@@ -222,7 +240,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
         metaAll.forEach(meta -> {
             // 有失败记录
             if (MetaEnum.isRunning(meta.getState()) && meta.getFail().get() > 0) {
-                Mapping mapping = monitor.getMapping(meta.getMappingId());
+                Mapping mapping = profileComponent.getMapping(meta.getMappingId());
                 if (null != mapping) {
                     ModelEnum modelEnum = ModelEnum.getModelEnum(mapping.getModel());
                     content.append("<p>");
@@ -241,6 +259,27 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
         }
     }
 
+    private Paging queryData(String metaId, int pageNum, int pageSize, String error, String success) {
+        // 没有驱动
+        if (StringUtil.isBlank(metaId)) {
+            return new Paging(pageNum, pageSize);
+        }
+        Query query = new Query(pageNum, pageSize);
+        Map<String, FieldResolver> fieldResolvers = new LinkedHashMap<>();
+        fieldResolvers.put(ConfigConstant.BINLOG_DATA, (FieldResolver<IndexableField>) field -> field.binaryValue().bytes);
+        query.setFieldResolverMap(fieldResolvers);
+
+        // 查询异常信息
+        if (StringUtil.isNotBlank(error)) {
+            query.addFilter(ConfigConstant.DATA_ERROR, error, true);
+        }
+        // 查询是否成功, 默认查询失败
+        query.addFilter(ConfigConstant.DATA_SUCCESS, StringUtil.isNotBlank(success) ? NumberUtil.toInt(success) : StorageDataStatusEnum.FAIL.getValue());
+        query.setMetaId(metaId);
+        query.setType(StorageEnum.DATA);
+        return storageService.query(query);
+    }
+
     private void deleteExpiredData() {
         List<MetaVo> metaAll = getMetaAll();
         if (!CollectionUtils.isEmpty(metaAll)) {
@@ -268,7 +307,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
     }
 
     private MetaVo convertMeta2Vo(Meta meta) {
-        Mapping mapping = monitor.getMapping(meta.getMappingId());
+        Mapping mapping = profileComponent.getMapping(meta.getMappingId());
         Assert.notNull(mapping, "驱动不存在.");
         ModelEnum modelEnum = ModelEnum.getModelEnum(mapping.getModel());
         MetaVo metaVo = new MetaVo(modelEnum.getName(), mapping.getName());
@@ -293,7 +332,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
 
     private List<MetricResponseVo> getMetrics(List<MetricResponse> metrics) {
         // 系统指标
-        List<MetricResponse> metricList = monitor.getMetricInfo();
+        List<MetricResponse> metricList = metricReporter.getMetricInfo();
         // 线程池状态
         metrics.addAll(metricList);
 
@@ -309,8 +348,4 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
         }).collect(Collectors.toList());
     }
 
-    @Override
-    public void onApplicationEvent(PreloadCompletedEvent event) {
-        preloadCompleted = true;
-    }
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java

@@ -5,18 +5,19 @@ import org.dbsyncer.biz.PluginService;
 import org.dbsyncer.biz.vo.PluginVo;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.logger.LogService;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.LogService;
+import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
-import org.dbsyncer.plugin.config.Plugin;
-import org.dbsyncer.plugin.enums.FileSuffixEnum;
+import org.dbsyncer.plugin.PluginFactory;
+import org.dbsyncer.plugin.model.Plugin;
+import org.dbsyncer.common.enums.FileSuffixEnum;
 import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
+import javax.annotation.Resource;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -31,15 +32,18 @@ import java.util.stream.Collectors;
 @Component
 public class PluginServiceImpl implements PluginService {
 
-    @Autowired
-    private Manager manager;
+    @Resource
+    private PluginFactory pluginFactory;
 
-    @Autowired
+    @Resource
+    private ProfileComponent profileComponent;
+
+    @Resource
     private LogService logService;
 
     @Override
     public List<PluginVo> getPluginAll() {
-        List<Plugin> pluginAll = manager.getPluginAll();
+        List<Plugin> pluginAll = pluginFactory.getPluginAll();
         List<PluginVo> vos = new ArrayList<>();
         if (!CollectionUtils.isEmpty(pluginAll)) {
             Map<String, List<String>> pluginClassNameMap = getPluginClassNameMap();
@@ -55,17 +59,17 @@ public class PluginServiceImpl implements PluginService {
 
     @Override
     public String getPluginPath() {
-        return manager.getPluginPath();
+        return pluginFactory.getPluginPath();
     }
 
     @Override
     public String getLibraryPath() {
-        return manager.getLibraryPath();
+        return pluginFactory.getLibraryPath();
     }
 
     @Override
     public void loadPlugins() {
-        manager.loadPlugins();
+        pluginFactory.loadPlugins();
         logService.log(LogType.PluginLog.UPDATE);
     }
 
@@ -84,7 +88,7 @@ public class PluginServiceImpl implements PluginService {
 
     private Map<String, List<String>> getPluginClassNameMap() {
         Map<String, List<String>> map = new HashMap<>();
-        List<Mapping> mappingAll = manager.getMappingAll();
+        List<Mapping> mappingAll = profileComponent.getMappingAll();
         if (CollectionUtils.isEmpty(mappingAll)) {
             return map;
         }
@@ -97,7 +101,7 @@ public class PluginServiceImpl implements PluginService {
                 continue;
             }
 
-            List<TableGroup> tableGroupAll = manager.getTableGroupAll(m.getId());
+            List<TableGroup> tableGroupAll = profileComponent.getTableGroupAll(m.getId());
             if (CollectionUtils.isEmpty(tableGroupAll)) {
                 continue;
             }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java

@@ -8,17 +8,21 @@ import org.dbsyncer.biz.vo.MappingVo;
 import org.dbsyncer.biz.vo.ProjectGroupVo;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Connector;
 import org.dbsyncer.parser.model.ProjectGroup;
 import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
-import java.util.*;
+import javax.annotation.Resource;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
@@ -31,16 +35,16 @@ import java.util.stream.Collectors;
 @Service
 public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectGroupService {
 
-    @Autowired
+    @Resource
     private ConnectorService connectorService;
 
-    @Autowired
+    @Resource
     private MappingService mappingService;
 
-    @Autowired
-    private Manager manager;
+    @Resource
+    private ProfileComponent profileComponent;
 
-    @Autowired
+    @Resource
     private Checker projectGroupChecker;
 
     @Override
@@ -48,7 +52,7 @@ public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectG
         ConfigModel model = projectGroupChecker.checkAddConfigModel(params);
         log(LogType.ConnectorLog.INSERT, model);
 
-        return manager.addConfigModel(model);
+        return profileComponent.addConfigModel(model);
     }
 
     @Override
@@ -56,15 +60,15 @@ public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectG
         ConfigModel model = projectGroupChecker.checkEditConfigModel(params);
         log(LogType.ConnectorLog.UPDATE, model);
 
-        return manager.editConfigModel(model);
+        return profileComponent.editConfigModel(model);
     }
 
     @Override
     public String remove(String id) {
-        ProjectGroup projectGroup = manager.getProjectGroup(id);
+        ProjectGroup projectGroup = profileComponent.getProjectGroup(id);
         log(LogType.ConnectorLog.DELETE, projectGroup);
         Assert.notNull(projectGroup, "该分组已被删除");
-        manager.removeConfigModel(id);
+        profileComponent.removeConfigModel(id);
         return "删除分组成功!";
     }
 
@@ -80,7 +84,7 @@ public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectG
             return vo;
         }
 
-        ProjectGroup projectGroup = manager.getProjectGroup(id);
+        ProjectGroup projectGroup = profileComponent.getProjectGroup(id);
         Assert.notNull(projectGroup, "该分组已被删除");
         BeanUtils.copyProperties(projectGroup, vo);
         vo.setConnectors(Collections.EMPTY_LIST);
@@ -115,7 +119,7 @@ public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectG
 
     @Override
     public List<ProjectGroup> getProjectGroupAll() {
-        return manager.getProjectGroupAll();
+        return profileComponent.getProjectGroupAll();
     }
 
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java

@@ -6,13 +6,13 @@ import org.dbsyncer.biz.UserConfigService;
 import org.dbsyncer.biz.checker.Checker;
 import org.dbsyncer.biz.vo.SystemConfigVo;
 import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.manager.template.PreloadTemplate;
-import org.dbsyncer.parser.logger.LogService;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.LogService;
+import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.SystemConfig;
-import org.dbsyncer.plugin.enums.FileSuffixEnum;
+import org.dbsyncer.manager.impl.PreloadTemplate;
+import org.dbsyncer.common.enums.FileSuffixEnum;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
@@ -35,13 +35,13 @@ import java.util.Map;
 public class SystemConfigServiceImpl implements SystemConfigService {
 
     @Resource
-    private Manager manager;
+    private ProfileComponent profileComponent;
 
     @Resource
-    private Checker systemConfigChecker;
+    private PreloadTemplate preloadTemplate;
 
     @Resource
-    private PreloadTemplate preloadTemplate;
+    private Checker systemConfigChecker;
 
     @Resource
     private LogService logService;
@@ -52,7 +52,7 @@ public class SystemConfigServiceImpl implements SystemConfigService {
     @Override
     public String edit(Map<String, String> params) {
         ConfigModel model = systemConfigChecker.checkEditConfigModel(params);
-        manager.editConfigModel(model);
+        profileComponent.editConfigModel(model);
         return "修改成功.";
     }
 
@@ -66,9 +66,9 @@ public class SystemConfigServiceImpl implements SystemConfigService {
         List<ConfigModel> list = new ArrayList<>();
         list.add(getSystemConfig());
         list.add(userConfigService.getUserConfig());
-        manager.getConnectorAll().forEach(config -> list.add(config));
-        manager.getMappingAll().forEach(config -> list.add(config));
-        manager.getMetaAll().forEach(config -> list.add(config));
+        profileComponent.getConnectorAll().forEach(config -> list.add(config));
+        profileComponent.getMappingAll().forEach(config -> list.add(config));
+        profileComponent.getMetaAll().forEach(config -> list.add(config));
         return list;
     }
 
@@ -104,13 +104,13 @@ public class SystemConfigServiceImpl implements SystemConfigService {
     }
 
     private SystemConfig getSystemConfig() {
-        SystemConfig config = manager.getSystemConfig();
+        SystemConfig config = profileComponent.getSystemConfig();
         if (null != config) {
             return config;
         }
 
         synchronized (this) {
-            config = manager.getSystemConfig();
+            config = profileComponent.getSystemConfig();
             if (null == config) {
                 config = (SystemConfig) systemConfigChecker.checkAddConfigModel(new HashMap<>());
             }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java

@@ -5,15 +5,16 @@ import org.dbsyncer.biz.checker.Checker;
 import org.dbsyncer.biz.checker.impl.tablegroup.TableGroupChecker;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.parser.LogType;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
-import org.dbsyncer.storage.constant.ConfigConstant;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
+import javax.annotation.Resource;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -29,15 +30,18 @@ import java.util.stream.Stream;
 @Service
 public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroupService {
 
-    @Autowired
+    @Resource
     private Checker tableGroupChecker;
 
+    @Resource
+    private ProfileComponent profileComponent;
+
     @Override
     public String add(Map<String, String> params) {
         String mappingId = params.get("mappingId");
-        assertRunning(manager.getMapping(mappingId));
+        assertRunning(profileComponent.getMapping(mappingId));
 
-        synchronized (LOCK){
+        synchronized (LOCK) {
             // table1, table2
             String[] sourceTableArray = StringUtil.split(params.get("sourceTable"), "|");
             String[] targetTableArray = StringUtil.split(params.get("targetTable"), "|");
@@ -50,9 +54,9 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
                 params.put("targetTable", targetTableArray[i]);
                 TableGroup model = (TableGroup) tableGroupChecker.checkAddConfigModel(params);
                 log(LogType.TableGroupLog.INSERT, model);
-                int tableGroupCount = manager.getTableGroupCount(mappingId);
+                int tableGroupCount = profileComponent.getTableGroupCount(mappingId);
                 model.setIndex(tableGroupCount + 1);
-                id = manager.addTableGroup(model);
+                id = profileComponent.addTableGroup(model);
             }
 
             // 合并驱动公共字段
@@ -64,39 +68,39 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
     @Override
     public String edit(Map<String, String> params) {
         String id = params.get(ConfigConstant.CONFIG_MODEL_ID);
-        TableGroup tableGroup = manager.getTableGroup(id);
+        TableGroup tableGroup = profileComponent.getTableGroup(id);
         Assert.notNull(tableGroup, "Can not find tableGroup.");
-        assertRunning(manager.getMapping(tableGroup.getMappingId()));
+        assertRunning(profileComponent.getMapping(tableGroup.getMappingId()));
 
         TableGroup model = (TableGroup) tableGroupChecker.checkEditConfigModel(params);
         log(LogType.TableGroupLog.UPDATE, model);
 
-        return manager.editTableGroup(model);
+        return profileComponent.editTableGroup(model);
     }
 
     @Override
     public String refreshFields(String id) {
-        TableGroup tableGroup = manager.getTableGroup(id);
+        TableGroup tableGroup = profileComponent.getTableGroup(id);
         Assert.notNull(tableGroup, "Can not find tableGroup.");
-        assertRunning(manager.getMapping(tableGroup.getMappingId()));
+        assertRunning(profileComponent.getMapping(tableGroup.getMappingId()));
 
         TableGroupChecker checker = (TableGroupChecker) tableGroupChecker;
         checker.refreshTableFields(tableGroup);
 
-        return manager.editTableGroup(tableGroup);
+        return profileComponent.editTableGroup(tableGroup);
     }
 
     @Override
     public boolean remove(String mappingId, String ids) {
         Assert.hasText(mappingId, "Mapping id can not be null");
         Assert.hasText(ids, "TableGroup ids can not be null");
-        assertRunning(manager.getMapping(mappingId));
+        assertRunning(profileComponent.getMapping(mappingId));
 
         // 批量删除表
         Stream.of(StringUtil.split(ids, ",")).parallel().forEach(id -> {
-            TableGroup model = manager.getTableGroup(id);
+            TableGroup model = profileComponent.getTableGroup(id);
             log(LogType.TableGroupLog.DELETE, model);
-            manager.removeTableGroup(id);
+            profileComponent.removeTableGroup(id);
         });
 
         // 合并驱动公共字段
@@ -109,34 +113,34 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
 
     @Override
     public TableGroup getTableGroup(String id) {
-        TableGroup tableGroup = manager.getTableGroup(id);
+        TableGroup tableGroup = profileComponent.getTableGroup(id);
         Assert.notNull(tableGroup, "TableGroup can not be null");
         return tableGroup;
     }
 
     @Override
     public List<TableGroup> getTableGroupAll(String mappingId) {
-        return manager.getSortedTableGroupAll(mappingId);
+        return profileComponent.getSortedTableGroupAll(mappingId);
     }
 
     private void resetTableGroupAllIndex(String mappingId) {
         synchronized (LOCK) {
-            List<TableGroup> list = manager.getSortedTableGroupAll(mappingId);
+            List<TableGroup> list = profileComponent.getSortedTableGroupAll(mappingId);
             int size = list.size();
             int i = size;
             while (i > 0) {
                 TableGroup g = list.get(size - i);
                 g.setIndex(i);
-                manager.editConfigModel(g);
+                profileComponent.editConfigModel(g);
                 i--;
             }
         }
     }
 
     private void mergeMappingColumn(String mappingId) {
-        List<TableGroup> groups = manager.getTableGroupAll(mappingId);
+        List<TableGroup> groups = profileComponent.getTableGroupAll(mappingId);
 
-        Mapping mapping = manager.getMapping(mappingId);
+        Mapping mapping = profileComponent.getMapping(mappingId);
         Assert.notNull(mapping, "mapping not exist.");
 
         List<Field> sourceColumn = null;
@@ -148,7 +152,7 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
 
         mapping.setSourceColumn(sourceColumn);
         mapping.setTargetColumn(targetColumn);
-        manager.editConfigModel(mapping);
+        profileComponent.editConfigModel(mapping);
     }
 
     private List<Field> pickCommonFields(List<Field> column, List<Field> target) {

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java

@@ -7,16 +7,16 @@ import org.dbsyncer.biz.enums.UserRoleEnum;
 import org.dbsyncer.biz.vo.UserInfoVo;
 import org.dbsyncer.common.util.SHA1Util;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.manager.Manager;
-import org.dbsyncer.parser.logger.LogService;
-import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.LogService;
+import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.model.UserConfig;
 import org.dbsyncer.parser.model.UserInfo;
 import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
+import javax.annotation.Resource;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -35,13 +35,13 @@ public class UserConfigServiceImpl implements UserConfigService {
 
     private static final String DEFAULT_PASSWORD = "0DPiKuNIrrVmD8IUCuw1hQxNqZc=";
 
-    @Autowired
-    private Manager manager;
+    @Resource
+    private ProfileComponent profileComponent;
 
-    @Autowired
+    @Resource
     private UserConfigChecker userConfigChecker;
 
-    @Autowired
+    @Resource
     private LogService logService;
 
     @Override
@@ -64,7 +64,7 @@ public class UserConfigServiceImpl implements UserConfigService {
         userConfig.getUserInfoList().add(new UserInfo(username, nickname, SHA1Util.b64_sha1(password), UserRoleEnum.USER.getCode(), mail));
 
         logService.log(LogType.UserLog.INSERT, String.format("[%s]添加[%s]账号成功", currentUser.getUsername(), username));
-        return manager.editConfigModel(userConfig);
+        return profileComponent.editConfigModel(userConfig);
     }
 
     @Override
@@ -107,7 +107,7 @@ public class UserConfigServiceImpl implements UserConfigService {
             logService.log(LogType.UserLog.UPDATE, String.format("[%s]修改[%s]账号密码成功", currentUser.getUsername(), username));
         }
 
-        return manager.editConfigModel(userConfig);
+        return profileComponent.editConfigModel(userConfig);
     }
 
     @Override
@@ -127,7 +127,7 @@ public class UserConfigServiceImpl implements UserConfigService {
         UserInfo deleteUser = userConfig.getUserInfo(username);
         Assert.notNull(deleteUser, "用户已删除.");
         userConfig.removeUserInfo(username);
-        manager.editConfigModel(userConfig);
+        profileComponent.editConfigModel(userConfig);
         logService.log(LogType.UserLog.DELETE, String.format("[%s]删除[%s]账号成功", currentUser.getUsername(), username));
         return "删除用户成功!";
     }
@@ -168,18 +168,18 @@ public class UserConfigServiceImpl implements UserConfigService {
 
     @Override
     public UserConfig getUserConfig() {
-        UserConfig config = manager.getUserConfig();
+        UserConfig config = profileComponent.getUserConfig();
         if (null != config) {
             return config;
         }
 
         synchronized (this) {
-            config = manager.getUserConfig();
+            config = profileComponent.getUserConfig();
             if (null == config) {
                 config = (UserConfig) userConfigChecker.checkAddConfigModel(new HashMap<>());
                 UserRoleEnum admin = UserRoleEnum.ADMIN;
                 config.getUserInfoList().add(new UserInfo(DEFAULT_USERNAME, DEFAULT_USERNAME, DEFAULT_PASSWORD, admin.getCode(), ""));
-                manager.addConfigModel(config);
+                profileComponent.addConfigModel(config);
             }
             return config;
         }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/CpuMetricDetailFormatter.java

@@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl;
 
 import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter;
 import org.dbsyncer.biz.vo.MetricResponseVo;
-import org.dbsyncer.monitor.model.Sample;
+import org.dbsyncer.biz.model.Sample;
 
 public final class CpuMetricDetailFormatter extends AbstractMetricDetailFormatter {
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DiskMetricDetailFormatter.java

@@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl;
 
 import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter;
 import org.dbsyncer.biz.vo.MetricResponseVo;
-import org.dbsyncer.monitor.model.Sample;
+import org.dbsyncer.biz.model.Sample;
 
 import java.math.BigDecimal;
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DoubleRoundMetricDetailFormatter.java

@@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl;
 
 import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter;
 import org.dbsyncer.biz.vo.MetricResponseVo;
-import org.dbsyncer.monitor.model.Sample;
+import org.dbsyncer.biz.model.Sample;
 
 public final class DoubleRoundMetricDetailFormatter extends AbstractMetricDetailFormatter {
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/GCMetricDetailFormatter.java

@@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl;
 
 import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter;
 import org.dbsyncer.biz.vo.MetricResponseVo;
-import org.dbsyncer.monitor.model.Sample;
+import org.dbsyncer.biz.model.Sample;
 
 import java.util.List;
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/MemoryMetricDetailFormatter.java

@@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl;
 
 import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter;
 import org.dbsyncer.biz.vo.MetricResponseVo;
-import org.dbsyncer.monitor.model.Sample;
+import org.dbsyncer.biz.model.Sample;
 
 public final class MemoryMetricDetailFormatter extends AbstractMetricDetailFormatter {
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/ValueMetricDetailFormatter.java

@@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl;
 
 import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter;
 import org.dbsyncer.biz.vo.MetricResponseVo;
-import org.dbsyncer.monitor.model.Sample;
+import org.dbsyncer.biz.model.Sample;
 
 public final class ValueMetricDetailFormatter extends AbstractMetricDetailFormatter {
 

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/AppReportMetric.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/AppReportMetric.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.monitor.model;
+package org.dbsyncer.biz.model;
 
 public class AppReportMetric extends MappingReportMetric{
 

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MappingReportMetric.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MappingReportMetric.java

@@ -1,70 +1,70 @@
-package org.dbsyncer.monitor.model;
-
-public class MappingReportMetric {
-
-    /**
-     * 已处理成功数
-     */
-    private long success;
-
-    /**
-     * 已处理失败数
-     */
-    private long fail;
-
-    /**
-     * 插入事件
-     */
-    private long insert;
-
-    /**
-     * 更新事件
-     */
-    private long update;
-
-    /**
-     * 删除事件
-     */
-    private long delete;
-
-    public long getSuccess() {
-        return success;
-    }
-
-    public void setSuccess(long success) {
-        this.success = success;
-    }
-
-    public long getFail() {
-        return fail;
-    }
-
-    public void setFail(long fail) {
-        this.fail = fail;
-    }
-
-    public long getInsert() {
-        return insert;
-    }
-
-    public void setInsert(long insert) {
-        this.insert = insert;
-    }
-
-    public long getUpdate() {
-        return update;
-    }
-
-    public void setUpdate(long update) {
-        this.update = update;
-    }
-
-    public long getDelete() {
-        return delete;
-    }
-
-    public void setDelete(long delete) {
-        this.delete = delete;
-    }
-
+package org.dbsyncer.biz.model;
+
+public class MappingReportMetric {
+
+    /**
+     * 已处理成功数
+     */
+    private long success;
+
+    /**
+     * 已处理失败数
+     */
+    private long fail;
+
+    /**
+     * 插入事件
+     */
+    private long insert;
+
+    /**
+     * 更新事件
+     */
+    private long update;
+
+    /**
+     * 删除事件
+     */
+    private long delete;
+
+    public long getSuccess() {
+        return success;
+    }
+
+    public void setSuccess(long success) {
+        this.success = success;
+    }
+
+    public long getFail() {
+        return fail;
+    }
+
+    public void setFail(long fail) {
+        this.fail = fail;
+    }
+
+    public long getInsert() {
+        return insert;
+    }
+
+    public void setInsert(long insert) {
+        this.insert = insert;
+    }
+
+    public long getUpdate() {
+        return update;
+    }
+
+    public void setUpdate(long update) {
+        this.update = update;
+    }
+
+    public long getDelete() {
+        return delete;
+    }
+
+    public void setDelete(long delete) {
+        this.delete = delete;
+    }
+
 }

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MetricResponse.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MetricResponse.java

@@ -1,56 +1,56 @@
-package org.dbsyncer.monitor.model;
-
-import java.util.List;
-
-public class MetricResponse {
-
-    private String code;
-
-    private String group;
-
-    private String metricName;
-
-    private List<Sample> measurements;
-
-    public MetricResponse() {
-    }
-
-    public MetricResponse(String code, String group, String metricName, List<Sample> measurements) {
-        this.code = code;
-        this.group = group;
-        this.metricName = metricName;
-        this.measurements = measurements;
-    }
-
-    public String getCode() {
-        return code;
-    }
-
-    public void setCode(String code) {
-        this.code = code;
-    }
-
-    public String getGroup() {
-        return group;
-    }
-
-    public void setGroup(String group) {
-        this.group = group;
-    }
-
-    public String getMetricName() {
-        return metricName;
-    }
-
-    public void setMetricName(String metricName) {
-        this.metricName = metricName;
-    }
-
-    public List<Sample> getMeasurements() {
-        return measurements;
-    }
-
-    public void setMeasurements(List<Sample> measurements) {
-        this.measurements = measurements;
-    }
+package org.dbsyncer.biz.model;
+
+import java.util.List;
+
+public class MetricResponse {
+
+    private String code;
+
+    private String group;
+
+    private String metricName;
+
+    private List<Sample> measurements;
+
+    public MetricResponse() {
+    }
+
+    public MetricResponse(String code, String group, String metricName, List<Sample> measurements) {
+        this.code = code;
+        this.group = group;
+        this.metricName = metricName;
+        this.measurements = measurements;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public void setCode(String code) {
+        this.code = code;
+    }
+
+    public String getGroup() {
+        return group;
+    }
+
+    public void setGroup(String group) {
+        this.group = group;
+    }
+
+    public String getMetricName() {
+        return metricName;
+    }
+
+    public void setMetricName(String metricName) {
+        this.metricName = metricName;
+    }
+
+    public List<Sample> getMeasurements() {
+        return measurements;
+    }
+
+    public void setMeasurements(List<Sample> measurements) {
+        this.measurements = measurements;
+    }
 }

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MetricResponseInfo.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MetricResponseInfo.java

@@ -1,24 +1,24 @@
-package org.dbsyncer.monitor.model;
-
-public class MetricResponseInfo {
-
-    private MetricResponse response;
-
-    private long queueUp;
-
-    public MetricResponse getResponse() {
-        return response;
-    }
-
-    public void setResponse(MetricResponse response) {
-        this.response = response;
-    }
-
-    public long getQueueUp() {
-        return queueUp;
-    }
-
-    public void setQueueUp(long queueUp) {
-        this.queueUp = queueUp;
-    }
+package org.dbsyncer.biz.model;
+
+public class MetricResponseInfo {
+
+    private MetricResponse response;
+
+    private long queueUp;
+
+    public MetricResponse getResponse() {
+        return response;
+    }
+
+    public void setResponse(MetricResponse response) {
+        this.response = response;
+    }
+
+    public long getQueueUp() {
+        return queueUp;
+    }
+
+    public void setQueueUp(long queueUp) {
+        this.queueUp = queueUp;
+    }
 }

dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/Sample.java → dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/Sample.java

@@ -1,29 +1,29 @@
-package org.dbsyncer.monitor.model;
-
-public final class Sample {
-
-    private String statistic;
-
-    private Object value;
-
-    public Sample(String statistic, Object value) {
-        this.statistic = statistic;
-        this.value = value;
-    }
-
-    public String getStatistic() {
-        return statistic;
-    }
-
-    public void setStatistic(String statistic) {
-        this.statistic = statistic;
-    }
-
-    public Object getValue() {
-        return value;
-    }
-
-    public void setValue(Object value) {
-        this.value = value;
-    }
+package org.dbsyncer.biz.model;
+
+public final class Sample {
+
+    private String statistic;
+
+    private Object value;
+
+    public Sample(String statistic, Object value) {
+        this.statistic = statistic;
+        this.value = value;
+    }
+
+    public String getStatistic() {
+        return statistic;
+    }
+
+    public void setStatistic(String statistic) {
+        this.statistic = statistic;
+    }
+
+    public Object getValue() {
+        return value;
+    }
+
+    public void setValue(Object value) {
+        this.value = value;
+    }
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/AppReportMetricVo.java

@@ -1,6 +1,6 @@
 package org.dbsyncer.biz.vo;
 
-import org.dbsyncer.monitor.model.AppReportMetric;
+import org.dbsyncer.biz.model.AppReportMetric;
 
 import java.util.List;
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java

@@ -1,15 +1,18 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.biz.vo;
 
-import org.dbsyncer.connector.enums.FilterEnum;
-import org.dbsyncer.connector.enums.OperationEnum;
-import org.dbsyncer.listener.enums.QuartzFilterEnum;
+import org.dbsyncer.sdk.enums.FilterEnum;
+import org.dbsyncer.sdk.enums.QuartzFilterEnum;
+import org.dbsyncer.sdk.enums.OperationEnum;
 
 import java.util.List;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2020/01/19 17:20
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2020-01-19 17:20
  */
 public class ConditionVo {
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/MetricResponseVo.java

@@ -1,6 +1,6 @@
 package org.dbsyncer.biz.vo;
 
-import org.dbsyncer.monitor.model.MetricResponse;
+import org.dbsyncer.biz.model.MetricResponse;
 
 public class MetricResponseVo extends MetricResponse {
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/PluginVo.java

@@ -1,6 +1,6 @@
 package org.dbsyncer.biz.vo;
 
-import org.dbsyncer.plugin.config.Plugin;
+import org.dbsyncer.plugin.model.Plugin;
 
 public class PluginVo extends Plugin {
 

dbsyncer-cache/pom.xml

@@ -1,21 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<parent>
-        <artifactId>dbsyncer</artifactId>
-        <groupId>org.ghi</groupId>
-        <version>1.2.7_1201</version>
-    </parent>
-	<modelVersion>4.0.0</modelVersion>
-	<artifactId>dbsyncer-cache</artifactId>
-
-	<dependencies>
-		<!-- Common 公共层 -->
-		<dependency>
-			<groupId>org.ghi</groupId>
-			<artifactId>dbsyncer-common</artifactId>
-			<version>${project.parent.version}</version>
-		</dependency>
-	</dependencies>
-
-</project>

dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheException.java

@@ -1,27 +0,0 @@
-package org.dbsyncer.cache;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2020/1/8 15:17
- */
-public class CacheException extends RuntimeException {
-
-    private static final long serialVersionUID = 1L;
-
-    public CacheException(String message) {
-        super(message);
-    }
-
-    public CacheException(String message, Throwable cause) {
-        super(message, cause);
-    }
-
-    public CacheException(Throwable cause) {
-        super(cause);
-    }
-
-    protected CacheException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
-        super(message, cause, enableSuppression, writableStackTrace);
-    }
-}

dbsyncer-cluster/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.2.7_1201</version>
+        <version>2.0.0_1226-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-cluster</artifactId>

dbsyncer-common/pom.xml

@@ -5,13 +5,12 @@
     <parent>
         <artifactId>dbsyncer</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.2.7_1201</version>
+        <version>2.0.0_1226-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dbsyncer-common</artifactId>
 
     <dependencies>
-
         <!-- Log4j2框架的性能是最强的 -->
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -57,11 +56,6 @@
             <artifactId>fastjson2</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>com.github.jsqlparser</groupId>
-            <artifactId>jsqlparser</artifactId>
-        </dependency>
-
     </dependencies>
 
 </project>

dbsyncer-common/src/main/java/org/dbsyncer/common/QueueOverflowException.java

@@ -0,0 +1,27 @@
+package org.dbsyncer.common;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2023/12/04 22:39
+ */
+public class QueueOverflowException extends RuntimeException {
+
+	private static final long serialVersionUID = 1L;
+
+	public QueueOverflowException(String message) {
+        super(message);
+    }
+
+    public QueueOverflowException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public QueueOverflowException(Throwable cause) {
+        super(cause);
+    }
+
+    protected QueueOverflowException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

dbsyncer-common/src/main/java/org/dbsyncer/common/column/AbstractColumnValue.java

@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.common.column;
 
 public abstract class AbstractColumnValue<T> implements ColumnValue {

dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java

@@ -1,14 +1,18 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.common.column;
 
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/4/22 22:39
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-04-22 22:39
  */
 public interface ColumnValue {
 
@@ -37,4 +41,8 @@ public interface ColumnValue {
     Timestamp asTimestamp();
 
     Time asTime();
+
+    default BigInteger asBigInteger(){
+        return null;
+    }
 }

dbsyncer-common/src/main/java/org/dbsyncer/common/config/GeneralBufferConfig.java

@@ -21,6 +21,11 @@ public class GeneralBufferConfig extends BufferActuatorConfig {
      */
     private int threadCoreSize = Runtime.getRuntime().availableProcessors() * 2;
 
+    /**
+     * 最大工作线程数
+     */
+    private int maxThreadSize = 16;
+
     /**
      * 工作线任务队列
      */
@@ -28,7 +33,7 @@ public class GeneralBufferConfig extends BufferActuatorConfig {
 
     @Bean(name = "generalExecutor", destroyMethod = "shutdown")
     public ThreadPoolTaskExecutor generalExecutor() {
-        return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, threadCoreSize, threadQueueCapacity, 30, "GeneralExecutor-");
+        return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, maxThreadSize, threadQueueCapacity, 30, "GeneralExecutor-");
     }
 
     public int getThreadCoreSize() {
@@ -39,6 +44,14 @@ public class GeneralBufferConfig extends BufferActuatorConfig {
         this.threadCoreSize = threadCoreSize;
     }
 
+    public int getMaxThreadSize() {
+        return maxThreadSize;
+    }
+
+    public void setMaxThreadSize(int maxThreadSize) {
+        this.maxThreadSize = maxThreadSize;
+    }
+
     public int getThreadQueueCapacity() {
         return threadQueueCapacity;
     }

dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java

@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.common.config;
 
 import org.dbsyncer.common.util.ThreadPoolUtil;
@@ -23,28 +26,18 @@ public class StorageConfig extends BufferActuatorConfig {
     private int threadCoreSize = Runtime.getRuntime().availableProcessors();
 
     /**
-     * 工作线任务队列
-     */
-    private int threadQueueCapacity = 500;
-
-    /**
-     * 是否记录同步成功数据
+     * 最大工作线程数
      */
-    private boolean writeSuccess;
+    private int maxThreadSize = 16;
 
     /**
-     * 是否记录同步失败数据
-     */
-    private boolean writeFail;
-
-    /**
-     * 最大记录异常信息长度
+     * 工作线任务队列
      */
-    private int maxErrorLength;
+    private int threadQueueCapacity = 500;
 
     @Bean(name = "storageExecutor", destroyMethod = "shutdown")
     public ThreadPoolTaskExecutor storageExecutor() {
-        return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, threadCoreSize, threadQueueCapacity, 30, "StorageExecutor-");
+        return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, maxThreadSize, threadQueueCapacity, 30, "StorageExecutor-");
     }
 
     public int getThreadCoreSize() {
@@ -55,35 +48,20 @@ public class StorageConfig extends BufferActuatorConfig {
         this.threadCoreSize = threadCoreSize;
     }
 
-    public int getThreadQueueCapacity() {
-        return threadQueueCapacity;
+    public int getMaxThreadSize() {
+        return maxThreadSize;
     }
 
-    public void setThreadQueueCapacity(int threadQueueCapacity) {
-        this.threadQueueCapacity = threadQueueCapacity;
-    }
-
-    public boolean isWriteSuccess() {
-        return writeSuccess;
+    public void setMaxThreadSize(int maxThreadSize) {
+        this.maxThreadSize = maxThreadSize;
     }
 
-    public void setWriteSuccess(boolean writeSuccess) {
-        this.writeSuccess = writeSuccess;
-    }
-
-    public boolean isWriteFail() {
-        return writeFail;
-    }
-
-    public void setWriteFail(boolean writeFail) {
-        this.writeFail = writeFail;
+    public int getThreadQueueCapacity() {
+        return threadQueueCapacity;
     }
 
-    public int getMaxErrorLength() {
-        return maxErrorLength;
+    public void setThreadQueueCapacity(int threadQueueCapacity) {
+        this.threadQueueCapacity = threadQueueCapacity;
     }
 
-    public void setMaxErrorLength(int maxErrorLength) {
-        this.maxErrorLength = maxErrorLength;
-    }
 }

dbsyncer-common/src/main/java/org/dbsyncer/common/config/TableGroupBufferConfig.java

@@ -23,6 +23,11 @@ public class TableGroupBufferConfig extends BufferActuatorConfig {
      */
     private int threadCoreSize = Runtime.getRuntime().availableProcessors() * 2;
 
+    /**
+     * 最大工作线程数
+     */
+    private int maxThreadSize = 10;
+
     /**
      * 工作线任务队列
      */
@@ -44,6 +49,14 @@ public class TableGroupBufferConfig extends BufferActuatorConfig {
         this.threadCoreSize = threadCoreSize;
     }
 
+    public int getMaxThreadSize() {
+        return maxThreadSize;
+    }
+
+    public void setMaxThreadSize(int maxThreadSize) {
+        this.maxThreadSize = maxThreadSize;
+    }
+
     public int getThreadQueueCapacity() {
         return threadQueueCapacity;
     }

dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/enums/FileSuffixEnum.java → dbsyncer-common/src/main/java/org/dbsyncer/common/enums/FileSuffixEnum.java

@@ -1,48 +1,48 @@
-package org.dbsyncer.plugin.enums;
-
-import org.dbsyncer.common.util.StringUtil;
-
-/**
- * 文件格式
- *
- * @author AE86
- * @version 1.0.0
- * @date 2021/2/23 23:21
- */
-public enum FileSuffixEnum {
-
-    /**
-     * jar
-     */
-    JAR("jar"),
-
-    /**
-     * JSON
-     */
-    JSON("json");
-
-    private String name;
-
-    FileSuffixEnum(String name) {
-        this.name = name;
-    }
-
-    /**
-     * 获取文件类型
-     *
-     * @param suffix
-     * @return
-     */
-    public static FileSuffixEnum getFileSuffix(String suffix) {
-        for (FileSuffixEnum e : FileSuffixEnum.values()) {
-            if (StringUtil.equals(suffix, e.getName())) {
-                return e;
-            }
-        }
-        return null;
-    }
-
-    public String getName() {
-        return name;
-    }
+package org.dbsyncer.common.enums;
+
+import org.dbsyncer.common.util.StringUtil;
+
+/**
+ * 文件格式
+ *
+ * @author AE86
+ * @version 1.0.0
+ * @date 2021/2/23 23:21
+ */
+public enum FileSuffixEnum {
+
+    /**
+     * jar
+     */
+    JAR("jar"),
+
+    /**
+     * JSON
+     */
+    JSON("json");
+
+    private String name;
+
+    FileSuffixEnum(String name) {
+        this.name = name;
+    }
+
+    /**
+     * 获取文件类型
+     *
+     * @param suffix
+     * @return
+     */
+    public static FileSuffixEnum getFileSuffix(String suffix) {
+        for (FileSuffixEnum e : FileSuffixEnum.values()) {
+            if (StringUtil.equals(suffix, e.getName())) {
+                return e;
+            }
+        }
+        return null;
+    }
+
+    public String getName() {
+        return name;
+    }
 }

dbsyncer-common/src/main/java/org/dbsyncer/common/event/PreloadCompletedEvent.java

@@ -1,24 +0,0 @@
-package org.dbsyncer.common.event;
-
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.event.ApplicationContextEvent;
-
-/**
- * 预加载完成事件
- *
- * @version 1.0.0
- * @Author AE86
- * @Date 2020-08-26 22:45
- */
-public final class PreloadCompletedEvent extends ApplicationContextEvent {
-
-    /**
-     * Create a new ContextStartedEvent.
-     *
-     * @param source the {@code ApplicationContext} that the event is raised for (must not be {@code null})
-     */
-    public PreloadCompletedEvent(ApplicationContext source) {
-        super(source);
-    }
-
-}

dbsyncer-common/src/main/java/org/dbsyncer/common/model/FullConvertContext.java

@@ -1,15 +0,0 @@
-package org.dbsyncer.common.model;
-
-import org.dbsyncer.common.spi.ConnectorMapper;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2022/6/30 16:04
- */
-public final class FullConvertContext extends AbstractConvertContext {
-
-    public FullConvertContext(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event) {
-        super.init(sourceConnectorMapper, targetConnectorMapper, sourceTableName, targetTableName, event, null, null);
-    }
-}

dbsyncer-common/src/main/java/org/dbsyncer/common/model/IncrementConvertContext.java

@@ -1,18 +0,0 @@
-package org.dbsyncer.common.model;
-
-import org.dbsyncer.common.spi.ConnectorMapper;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2022/6/30 16:06
- */
-public final class IncrementConvertContext extends AbstractConvertContext {
-
-    public IncrementConvertContext(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event, List<Map> sourceList, List<Map> targetList) {
-        super.init(sourceConnectorMapper, targetConnectorMapper, sourceTableName, targetTableName, event, sourceList, targetList);
-    }
-}

dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskServiceImpl.java → dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java

@@ -1,6 +1,8 @@
-package org.dbsyncer.common.scheduled;
+package org.dbsyncer.common.scheduled.impl;
 
 import org.dbsyncer.common.CommonException;
+import org.dbsyncer.common.scheduled.ScheduledTaskJob;
+import org.dbsyncer.common.scheduled.ScheduledTaskService;
 import org.dbsyncer.common.util.UUIDUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ProxyApplicationContext.java

@@ -1,13 +0,0 @@
-package org.dbsyncer.common.spi;
-
-import org.springframework.context.ApplicationContext;
-
-/**
- * Spring上下文代理对象
- *
- * @author AE86
- * @version 1.0.0
- * @date 2022/6/30 15:13
- */
-public interface ProxyApplicationContext extends ApplicationContext {
-}

dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java

@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.common.util;
 
 import org.apache.commons.lang3.StringUtils;
@@ -8,14 +11,22 @@ public abstract class StringUtil {
 
     public static final String SYMBOL = "-";
 
+    public static final String UNDERLINE = "_";
+
     public static final String COLON = ":";
 
     public static final String SPACE = " ";
 
     public static final String SINGLE_QUOTATION = "'";
 
+    public static final String DOUBLE_QUOTATION = "\"";
+
     public static final String FORWARD_SLASH = "/";
 
+    public static final String BACK_QUOTE = "`";
+
+    public static final String POINT = ".";
+
     public static boolean equals(CharSequence cs1, CharSequence cs2) {
         return StringUtils.equals(cs1, cs2);
     }

dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/UnderlineToCamelUtils.java → dbsyncer-common/src/main/java/org/dbsyncer/common/util/UnderlineToCamelUtils.java

@@ -1,56 +1,56 @@
-package org.dbsyncer.storage.util;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public abstract class UnderlineToCamelUtils {
-
-    /**
-     * 下划线转驼峰法
-     *
-     * @param line       源字符串
-     * @param smallCamel 大小驼峰,是否为小驼峰
-     * @return 转换后的字符串
-     */
-    public static String underlineToCamel(String line, boolean smallCamel) {
-        if (line == null || "".equals(line)) {
-            return "";
-        }
-        StringBuffer sb = new StringBuffer();
-        Pattern pattern = Pattern.compile("([A-Za-z\\d]+)(_)?");
-        Matcher matcher = pattern.matcher(line);
-        while (matcher.find()) {
-            String word = matcher.group();
-            sb.append(smallCamel && matcher.start() == 0 ? Character.toLowerCase(word.charAt(0)) : Character.toUpperCase(word.charAt(0)));
-            int index = word.lastIndexOf('_');
-            if (index > 0) {
-                sb.append(word.substring(1, index).toLowerCase());
-            } else {
-                sb.append(word.substring(1).toLowerCase());
-            }
-        }
-        return sb.toString();
-    }
-
-    /**
-     * 驼峰法转下划线
-     *
-     * @param line 源字符串
-     * @return 转换后的字符串
-     */
-    public static String camelToUnderline(String line) {
-        if (line == null || "".equals(line)) {
-            return "";
-        }
-        line = String.valueOf(line.charAt(0)).toUpperCase().concat(line.substring(1));
-        StringBuffer sb = new StringBuffer();
-        Pattern pattern = Pattern.compile("[A-Z]([a-z\\d]+)?");
-        Matcher matcher = pattern.matcher(line);
-        while (matcher.find()) {
-            String word = matcher.group();
-            sb.append(word.toUpperCase());
-            sb.append(matcher.end() == line.length() ? "" : "_");
-        }
-        return sb.toString();
-    }
+package org.dbsyncer.common.util;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public abstract class UnderlineToCamelUtils {
+
+    /**
+     * 下划线转驼峰法
+     *
+     * @param line       源字符串
+     * @param smallCamel 大小驼峰,是否为小驼峰
+     * @return 转换后的字符串
+     */
+    public static String underlineToCamel(String line, boolean smallCamel) {
+        if (line == null || "".equals(line)) {
+            return "";
+        }
+        StringBuffer sb = new StringBuffer();
+        Pattern pattern = Pattern.compile("([A-Za-z\\d]+)(_)?");
+        Matcher matcher = pattern.matcher(line);
+        while (matcher.find()) {
+            String word = matcher.group();
+            sb.append(smallCamel && matcher.start() == 0 ? Character.toLowerCase(word.charAt(0)) : Character.toUpperCase(word.charAt(0)));
+            int index = word.lastIndexOf('_');
+            if (index > 0) {
+                sb.append(word.substring(1, index).toLowerCase());
+            } else {
+                sb.append(word.substring(1).toLowerCase());
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     * 驼峰法转下划线
+     *
+     * @param line 源字符串
+     * @return 转换后的字符串
+     */
+    public static String camelToUnderline(String line) {
+        if (line == null || "".equals(line)) {
+            return "";
+        }
+        line = String.valueOf(line.charAt(0)).toUpperCase().concat(line.substring(1));
+        StringBuffer sb = new StringBuffer();
+        Pattern pattern = Pattern.compile("[A-Z]([a-z\\d]+)?");
+        Matcher matcher = pattern.matcher(line);
+        while (matcher.find()) {
+            String word = matcher.group();
+            sb.append(word.toUpperCase());
+            sb.append(matcher.end() == line.length() ? "" : "_");
+        }
+        return sb.toString();
+    }
 }

dbsyncer-connector/dbsyncer-connector-base/pom.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>dbsyncer-connector</artifactId>
+        <groupId>org.ghi</groupId>
+        <version>2.0.0_1226-Beta</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>dbsyncer-connector-base</artifactId>
+
+    <dependencies>
+        <!-- sdk -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-sdk</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <!-- dbsyncer-connector-elasticsearch -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-elasticsearch</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <!-- dbsyncer-connector-file -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-file</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <!-- dbsyncer-connector-kafka -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-kafka</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <!-- dbsyncer-connector-mysql -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-mysql</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <!-- dbsyncer-connector-oracle -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-oracle</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <!-- dbsyncer-connector-postgresql -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-postgresql</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <!-- dbsyncer-connector-sqlite -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-sqlite</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+
+        <!-- dbsyncer-connector-sqlserver -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-sqlserver</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+</project>

dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorException.java → dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/base/ConnectorException.java

@@ -1,4 +1,4 @@
-package org.dbsyncer.connector;
+package org.dbsyncer.connector.base;
 
 /**
  * @author AE86

dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/base/ConnectorFactory.java

@@ -0,0 +1,253 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.base;
+
+import org.dbsyncer.common.model.Result;
+import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.sdk.config.CommandConfig;
+import org.dbsyncer.sdk.config.DDLConfig;
+import org.dbsyncer.sdk.config.ReaderConfig;
+import org.dbsyncer.sdk.config.WriterBatchConfig;
+import org.dbsyncer.sdk.connector.AbstractConnector;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.listener.Listener;
+import org.dbsyncer.sdk.model.ConnectorConfig;
+import org.dbsyncer.sdk.model.MetaInfo;
+import org.dbsyncer.sdk.model.Table;
+import org.dbsyncer.sdk.spi.ConnectorService;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
+import org.springframework.beans.factory.support.BeanDefinitionRegistry;
+import org.springframework.beans.factory.support.BeanDefinitionRegistryPostProcessor;
+import org.springframework.stereotype.Component;
+import org.springframework.util.Assert;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * 连接器工厂
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-09-18 23:30
+ */
+@Component
+public class ConnectorFactory implements BeanDefinitionRegistryPostProcessor, DisposableBean {
+
+    private final Map<String, ConnectorInstance> pool = new ConcurrentHashMap<>();
+
+    private final Map<String, ConnectorService> service = new ConcurrentHashMap<>();
+
+    private final Set<String> connectorTypes = new HashSet<>();
+
+    @Override
+    public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry beanDefinitionRegistry) throws BeansException {
+        ServiceLoader<ConnectorService> services = ServiceLoader.load(ConnectorService.class, Thread.currentThread().getContextClassLoader());
+        for (ConnectorService s : services) {
+            service.putIfAbsent(s.getConnectorType(), s);
+            connectorTypes.add(s.getConnectorType());
+        }
+    }
+
+    @Override
+    public void postProcessBeanFactory(ConfigurableListableBeanFactory configurableListableBeanFactory) throws BeansException {
+
+    }
+
+    @Override
+    public void destroy() {
+        pool.values().forEach(this::disconnect);
+        pool.clear();
+    }
+
+    /**
+     * 建立连接，返回缓存连接对象
+     *
+     * @param config
+     */
+    public ConnectorInstance connect(ConnectorConfig config) {
+        Assert.notNull(config, "ConnectorConfig can not be null.");
+        ConnectorService connectorService = getConnectorService(config);
+        String cacheKey = connectorService.getConnectorInstanceCacheKey(config);
+        if (!pool.containsKey(cacheKey)) {
+            synchronized (pool) {
+                if (!pool.containsKey(cacheKey)) {
+                    ConnectorInstance instance = connectorService.connect(config);
+                    Assert.isTrue(connectorService.isAlive(instance), "连接配置异常");
+                    pool.putIfAbsent(cacheKey, instance);
+                }
+            }
+        }
+        try {
+            ConnectorInstance connectorInstance = pool.get(cacheKey);
+            ConnectorInstance clone = (ConnectorInstance) connectorInstance.clone();
+            clone.setConfig(config);
+            return clone;
+        } catch (CloneNotSupportedException e) {
+            throw new ConnectorException(e);
+        }
+    }
+
+    /**
+     * 获取监听器
+     *
+     * @param connectorType
+     * @param listenerType
+     * @return
+     */
+    public Listener getListener(String connectorType, String listenerType) {
+        return getConnectorService(connectorType).getListener(listenerType);
+    }
+
+    /**
+     * 检查连接配置是否可用
+     *
+     * @param config
+     * @return
+     */
+    public boolean isAlive(ConnectorConfig config) {
+        Assert.notNull(config, "ConnectorConfig can not be null.");
+        ConnectorService connectorService = getConnectorService(config);
+        String cacheKey = connectorService.getConnectorInstanceCacheKey(config);
+        if (pool.containsKey(cacheKey)) {
+            return connectorService.isAlive(pool.get(cacheKey));
+        }
+        return false;
+    }
+
+    /**
+     * 获取配置表
+     *
+     * @param connectorInstance
+     * @return
+     */
+    public List<Table> getTable(ConnectorInstance connectorInstance) {
+        Assert.notNull(connectorInstance, "ConnectorInstance can not be null.");
+        return getConnectorService(connectorInstance.getConfig()).getTable(connectorInstance);
+    }
+
+    /**
+     * 获取配置表元信息
+     *
+     * @param connectorInstance
+     * @param tableName
+     * @return
+     */
+    public MetaInfo getMetaInfo(ConnectorInstance connectorInstance, String tableName) {
+        Assert.notNull(connectorInstance, "ConnectorInstance can not be null.");
+        Assert.hasText(tableName, "tableName can not be empty.");
+        return getConnectorService(connectorInstance.getConfig()).getMetaInfo(connectorInstance, tableName);
+    }
+
+    /**
+     * 获取连接器同步参数
+     *
+     * @param sourceCommandConfig
+     * @param targetCommandConfig
+     * @return
+     */
+    public Map<String, String> getCommand(CommandConfig sourceCommandConfig, CommandConfig targetCommandConfig) {
+        Assert.notNull(sourceCommandConfig, "SourceCommandConfig can not be null.");
+        Assert.notNull(targetCommandConfig, "TargetCommandConfig can not be null.");
+        Map<String, String> map = new HashMap<>();
+        Map sCmd = getConnectorService(sourceCommandConfig.getConnectorType()).getSourceCommand(sourceCommandConfig);
+        if (!CollectionUtils.isEmpty(sCmd)) {
+            map.putAll(sCmd);
+        }
+
+        Map tCmd = getConnectorService(targetCommandConfig.getConnectorType()).getTargetCommand(targetCommandConfig);
+        if (!CollectionUtils.isEmpty(tCmd)) {
+            map.putAll(tCmd);
+        }
+        return map;
+    }
+
+    public long getCount(ConnectorInstance connectorInstance, Map<String, String> command) {
+        Assert.notNull(connectorInstance, "ConnectorInstance can not null");
+        Assert.notNull(command, "command can not null");
+        return getConnectorService(connectorInstance.getConfig()).getCount(connectorInstance, command);
+    }
+
+    public Result reader(ConnectorInstance connectorInstance, ReaderConfig config) {
+        Assert.notNull(connectorInstance, "ConnectorInstance can not null");
+        Assert.notNull(config, "ReaderConfig can not null");
+        Result result = getConnectorService(connectorInstance.getConfig()).reader(connectorInstance, config);
+        Assert.notNull(result, "Connector reader result can not null");
+        return result;
+    }
+
+    public Result writer(ConnectorInstance connectorInstance, WriterBatchConfig config) {
+        Assert.notNull(connectorInstance, "ConnectorInstance can not null");
+        Assert.notNull(config, "WriterBatchConfig can not null");
+        ConnectorService connector = getConnectorService(connectorInstance.getConfig());
+        if (connector instanceof AbstractConnector) {
+            AbstractConnector conn = (AbstractConnector) connector;
+            try {
+                conn.convertProcessBeforeWriter(connectorInstance, config);
+            } catch (Exception e) {
+                Result result = new Result();
+                result.getError().append(e.getMessage());
+                result.addFailData(config.getData());
+                return result;
+            }
+        }
+
+        Result result = connector.writer(connectorInstance, config);
+        Assert.notNull(result, "Connector writer batch result can not null");
+        return result;
+    }
+
+    public Result writerDDL(ConnectorInstance connectorInstance, DDLConfig ddlConfig) {
+        Assert.notNull(connectorInstance, "ConnectorInstance can not null");
+        Result result = getConnectorService(connectorInstance.getConfig()).writerDDL(connectorInstance, ddlConfig);
+        Assert.notNull(result, "Connector writer batch result can not null");
+        return result;
+    }
+
+    public ConnectorService getConnectorService(ConnectorConfig connectorConfig) {
+        Assert.notNull(connectorConfig, "ConnectorConfig can not null");
+        return getConnectorService(connectorConfig.getConnectorType());
+    }
+
+    public ConnectorService getConnectorService(String connectorType) {
+        ConnectorService connectorService = service.get(connectorType);
+        if (connectorService == null) {
+            Assert.isTrue(false, "Unsupported connector type:" + connectorType);
+        }
+        return connectorService;
+    }
+
+    public Set<String> getConnectorTypeAll() {
+        return connectorTypes;
+    }
+
+    /**
+     * 断开连接
+     *
+     * @param config
+     * @return
+     */
+    public void disconnect(ConnectorConfig config) {
+        Assert.notNull(config, "ConnectorConfig can not be null.");
+        String cacheKey = getConnectorService(config).getConnectorInstanceCacheKey(config);
+        ConnectorInstance connectorInstance = pool.get(cacheKey);
+        if (connectorInstance != null) {
+            disconnect(connectorInstance);
+            pool.remove(cacheKey);
+        }
+    }
+
+    private void disconnect(ConnectorInstance connectorInstance) {
+        Assert.notNull(connectorInstance, "ConnectorInstance can not be null.");
+        getConnectorService(connectorInstance.getConfig()).disconnect(connectorInstance);
+    }
+
+}

dbsyncer-connector/src/main/test/ConnectionTest.java → dbsyncer-connector/dbsyncer-connector-base/src/test/java/ConnectionTest.java

@@ -1,13 +1,16 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 import oracle.jdbc.OracleConnection;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.RandomUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.config.DatabaseConfig;
-import org.dbsyncer.connector.database.DatabaseConnectorMapper;
-import org.dbsyncer.connector.database.ds.SimpleConnection;
-import org.dbsyncer.connector.enums.TableTypeEnum;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.connector.model.Table;
+import org.dbsyncer.sdk.config.DatabaseConfig;
+import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
+import org.dbsyncer.sdk.connector.database.ds.SimpleConnection;
+import org.dbsyncer.sdk.enums.TableTypeEnum;
+import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.model.Table;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -22,9 +25,9 @@ import java.util.List;
 import java.util.concurrent.*;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/4/11 20:19
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-04-11 20:19
  */
 public class ConnectionTest {
 
@@ -32,10 +35,10 @@ public class ConnectionTest {
 
     @Test
     public void testByte() {
-        final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createOracleConfig());
+        final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createOracleConfig());
 
         String executeSql = "UPDATE \"my_user\" SET \"name\"=?,\"clo\"=? WHERE \"id\"=?";
-        int[] execute = connectorMapper.execute(databaseTemplate ->
+        int[] execute = connectorInstance.execute(databaseTemplate ->
                 databaseTemplate.batchUpdate(executeSql, new BatchPreparedStatementSetter() {
                     @Override
                     public void setValues(PreparedStatement ps, int i) {
@@ -64,7 +67,7 @@ public class ConnectionTest {
 
     @Test
     public void testConnection() throws InterruptedException {
-        final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createSqlServerConfig());
+        final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createSqlServerConfig());
 
         // 模拟并发
         final int threadSize = 100;
@@ -80,7 +83,7 @@ public class ConnectionTest {
                     // 模拟操作
                     System.out.println(String.format("%s %s:%s", LocalDateTime.now(), Thread.currentThread().getName(), k));
 
-                    Object execute = connectorMapper.execute(tem -> tem.queryForObject("select 1", Integer.class));
+                    Object execute = connectorInstance.execute(tem -> tem.queryForObject("select 1", Integer.class));
                     System.out.println(String.format("%s %s:%s execute=>%s", LocalDateTime.now(), Thread.currentThread().getName(), k, execute));
 
                 } catch (InterruptedException e) {
@@ -109,7 +112,7 @@ public class ConnectionTest {
 
     @Test
     public void testBatchInsert() {
-        final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createMysqlConfig());
+        final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig());
 
         long begin = Instant.now().toEpochMilli();
         final int threadSize = 10;
@@ -131,14 +134,14 @@ public class ConnectionTest {
 
             if (i % 10000 == 0) {
                 System.out.println(i + "-----------------正在处理");
-                batchUpdate(connectorMapper, pool, sql, dataList, 1000);
+                batchUpdate(connectorInstance, pool, sql, dataList, 1000);
                 dataList.clear();
             }
         }
 
         if(!CollectionUtils.isEmpty(dataList)){
             System.out.println("-----------------正在处理剩余数据");
-            batchUpdate(connectorMapper, pool, sql, dataList, 1000);
+            batchUpdate(connectorInstance, pool, sql, dataList, 1000);
         }
 
         pool.shutdown();
@@ -147,7 +150,7 @@ public class ConnectionTest {
 
     @Test
     public void testBatchUpdate() {
-        final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createMysqlConfig());
+        final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig());
 
         long begin = Instant.now().toEpochMilli();
         final int threadSize = 10;
@@ -167,14 +170,14 @@ public class ConnectionTest {
 
                 if (i % 10000 == 0) {
                     System.out.println(i + "-----------------正在处理");
-                    batchUpdate(connectorMapper, pool, sql, dataList, 1000);
+                    batchUpdate(connectorInstance, pool, sql, dataList, 1000);
                     dataList.clear();
                 }
             }
 
             if (!CollectionUtils.isEmpty(dataList)) {
                 System.out.println("-----------------正在处理剩余数据");
-                batchUpdate(connectorMapper, pool, sql, dataList, 1000);
+                batchUpdate(connectorInstance, pool, sql, dataList, 1000);
             }
             k--;
         }
@@ -185,7 +188,7 @@ public class ConnectionTest {
 
     @Test
     public void testBatchDelete() {
-        final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createMysqlConfig());
+        final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig());
 
         long begin = Instant.now().toEpochMilli();
         final int threadSize = 10;
@@ -201,14 +204,14 @@ public class ConnectionTest {
 
             if (i % 10000 == 0) {
                 System.out.println(i + "-----------------正在处理");
-                batchUpdate(connectorMapper, pool, sql, dataList, 1000);
+                batchUpdate(connectorInstance, pool, sql, dataList, 1000);
                 dataList.clear();
             }
         }
 
         if (!CollectionUtils.isEmpty(dataList)) {
             System.out.println("-----------------正在处理剩余数据");
-            batchUpdate(connectorMapper, pool, sql, dataList, 1000);
+            batchUpdate(connectorInstance, pool, sql, dataList, 1000);
         }
 
         pool.shutdown();
@@ -226,7 +229,7 @@ public class ConnectionTest {
         return s.toString();
     }
 
-    private void batchUpdate(DatabaseConnectorMapper connectorMapper, ExecutorService pool, String sql, List<Object[]> dataList, int batchSize) {
+    private void batchUpdate(DatabaseConnectorInstance connectorInstance, ExecutorService pool, String sql, List<Object[]> dataList, int batchSize) {
         int total = dataList.size();
         int taskSize = total % batchSize == 0 ? total / batchSize : total / batchSize + 1;
         final CountDownLatch latch = new CountDownLatch(taskSize);
@@ -245,7 +248,7 @@ public class ConnectionTest {
 
             pool.submit(() -> {
                 try {
-                    connectorMapper.execute(databaseTemplate -> databaseTemplate.batchUpdate(sql, data));
+                    connectorInstance.execute(databaseTemplate -> databaseTemplate.batchUpdate(sql, data));
                 } catch (Exception e) {
                     logger.error(e.getMessage());
                 } finally {
@@ -279,8 +282,8 @@ public class ConnectionTest {
     public void testGetColumnsDetails() {
         final String schema = "root";
         final String tableNamePattern = "sw_test";
-        final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createMysqlConfig());
-        connectorMapper.execute(databaseTemplate -> {
+        final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig());
+        connectorInstance.execute(databaseTemplate -> {
             SimpleConnection connection = databaseTemplate.getSimpleConnection();
             Connection conn = connection.getConnection();
             String databaseCatalog = conn.getCatalog();
@@ -299,9 +302,9 @@ public class ConnectionTest {
     }
 
     private List<Table> getTables(DatabaseConfig config, final String catalog, final String schema, final String tableNamePattern) {
-        final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(config);
+        final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(config);
         List<Table> tables = new ArrayList<>();
-        connectorMapper.execute(databaseTemplate -> {
+        connectorInstance.execute(databaseTemplate -> {
             SimpleConnection connection = databaseTemplate.getSimpleConnection();
             Connection conn = connection.getConnection();
             String databaseCatalog = null == catalog ? conn.getCatalog() : catalog;

dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>dbsyncer-connector</artifactId>
+        <groupId>org.ghi</groupId>
+        <version>2.0.0_1226-Beta</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>dbsyncer-connector-elasticsearch</artifactId>
+
+    <dependencies>
+        <!-- sdk -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-sdk</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- elasticsearch -->
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-high-level-client</artifactId>
+        </dependency>
+
+        <!-- postgresql -->
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-log4j2</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+    </dependencies>
+</project>

dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorMapper.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ESConnectorInstance.java

@@ -1,54 +1,70 @@
-package org.dbsyncer.connector.es;
-
-import org.dbsyncer.common.spi.ConnectorMapper;
-import org.dbsyncer.connector.ConnectorException;
-import org.dbsyncer.connector.config.ESConfig;
-import org.dbsyncer.connector.util.ESUtil;
-import org.elasticsearch.Version;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.core.MainResponse;
-
-public final class ESConnectorMapper implements ConnectorMapper<ESConfig, EasyRestHighLevelClient> {
-    private ESConfig config;
-    private EasyRestHighLevelClient client;
-
-    public ESConnectorMapper(ESConfig config) {
-        this.config = config;
-        this.client = ESUtil.getConnection(config);
-        try {
-            MainResponse info = client.info(RequestOptions.DEFAULT);
-            client.setVersion(Version.fromString(info.getVersion().getNumber()));
-        } catch (Exception e) {
-            throw new ConnectorException(String.format("获取ES版本信息异常 %s, %s", config.getUrl(), e.getMessage()));
-        }
-    }
-
-    @Override
-    public ESConfig getConfig() {
-        return config;
-    }
-
-    @Override
-    public void setConfig(ESConfig config) {
-        this.config = config;
-    }
-
-    @Override
-    public EasyRestHighLevelClient getConnection() {
-        return client;
-    }
-
-    public Version getVersion() {
-        return client.getVersion();
-    }
-
-    @Override
-    public void close() {
-        ESUtil.close(client);
-    }
-
-    @Override
-    public Object clone() throws CloneNotSupportedException {
-        return super.clone();
-    }
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch;
+
+import org.dbsyncer.connector.elasticsearch.api.EasyRestHighLevelClient;
+import org.dbsyncer.connector.elasticsearch.config.ESConfig;
+import org.dbsyncer.connector.elasticsearch.util.ESUtil;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.core.MainResponse;
+
+/**
+ * ES连接器实例
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
+ */
+public final class ESConnectorInstance implements ConnectorInstance<ESConfig, EasyRestHighLevelClient> {
+    private ESConfig config;
+    private EasyRestHighLevelClient client;
+
+    public ESConnectorInstance(ESConfig config) {
+        this.config = config;
+        this.client = ESUtil.getConnection(config);
+        try {
+            MainResponse info = client.info(RequestOptions.DEFAULT);
+            client.setVersion(Version.fromString(info.getVersion().getNumber()));
+        } catch (Exception e) {
+            throw new ElasticsearchException(String.format("获取ES版本信息异常 %s, %s", config.getUrl(), e.getMessage()));
+        }
+    }
+
+    @Override
+    public String getServiceUrl() {
+        return config.getUrl();
+    }
+
+    @Override
+    public ESConfig getConfig() {
+        return config;
+    }
+
+    @Override
+    public void setConfig(ESConfig config) {
+        this.config = config;
+    }
+
+    @Override
+    public EasyRestHighLevelClient getConnection() {
+        return client;
+    }
+
+    public Version getVersion() {
+        return client.getVersion();
+    }
+
+    @Override
+    public void close() {
+        ESUtil.close(client);
+    }
+
+    @Override
+    public Object clone() throws CloneNotSupportedException {
+        return super.clone();
+    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ElasticsearchConnector.java

@@ -1,27 +1,37 @@
-package org.dbsyncer.connector.es;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch;
 
 import org.dbsyncer.common.model.Result;
-import org.dbsyncer.common.spi.ConnectorMapper;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.AbstractConnector;
-import org.dbsyncer.connector.Connector;
-import org.dbsyncer.connector.ConnectorException;
-import org.dbsyncer.connector.config.CommandConfig;
-import org.dbsyncer.connector.config.ESConfig;
-import org.dbsyncer.connector.config.ReaderConfig;
-import org.dbsyncer.connector.config.WriterBatchConfig;
-import org.dbsyncer.connector.constant.ConnectorConstant;
-import org.dbsyncer.connector.enums.ESFieldTypeEnum;
-import org.dbsyncer.connector.enums.FilterEnum;
-import org.dbsyncer.connector.enums.OperationEnum;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.connector.model.Filter;
-import org.dbsyncer.connector.model.MetaInfo;
-import org.dbsyncer.connector.model.Table;
-import org.dbsyncer.connector.util.ESUtil;
-import org.dbsyncer.connector.util.PrimaryKeyUtil;
+import org.dbsyncer.connector.elasticsearch.cdc.ESQuartzListener;
+import org.dbsyncer.connector.elasticsearch.config.ESConfig;
+import org.dbsyncer.connector.elasticsearch.enums.ESFieldTypeEnum;
+import org.dbsyncer.connector.elasticsearch.schema.ESDateValueMapper;
+import org.dbsyncer.connector.elasticsearch.schema.ESOtherValueMapper;
+import org.dbsyncer.connector.elasticsearch.util.ESUtil;
+import org.dbsyncer.connector.elasticsearch.validator.ESConfigValidator;
+import org.dbsyncer.sdk.config.CommandConfig;
+import org.dbsyncer.sdk.config.ReaderConfig;
+import org.dbsyncer.sdk.config.WriterBatchConfig;
+import org.dbsyncer.sdk.connector.AbstractConnector;
+import org.dbsyncer.sdk.connector.ConfigValidator;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.constant.ConnectorConstant;
+import org.dbsyncer.sdk.enums.FilterEnum;
+import org.dbsyncer.sdk.enums.ListenerTypeEnum;
+import org.dbsyncer.sdk.enums.OperationEnum;
+import org.dbsyncer.sdk.listener.Listener;
+import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.model.Filter;
+import org.dbsyncer.sdk.model.MetaInfo;
+import org.dbsyncer.sdk.model.Table;
+import org.dbsyncer.sdk.spi.ConnectorService;
+import org.dbsyncer.sdk.util.PrimaryKeyUtil;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -58,13 +68,27 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-public final class ESConnector extends AbstractConnector implements Connector<ESConnectorMapper, ESConfig> {
+/**
+ * ES连接器实现
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
+ */
+public final class ElasticsearchConnector extends AbstractConnector implements ConnectorService<ESConnectorInstance, ESConfig> {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    private static Map<String, FilterMapper> filters = new LinkedHashMap<>();
+    private final String TYPE = "Elasticsearch";
+
+    private final Map<String, FilterMapper> filters = new LinkedHashMap<>();
+
+    private final ESConfigValidator configValidator = new ESConfigValidator();
+
+    public ElasticsearchConnector() {
+        VALUE_MAPPERS.put(Types.DATE, new ESDateValueMapper());
+        VALUE_MAPPERS.put(Types.OTHER, new ESOtherValueMapper());
 
-    static {
         filters.putIfAbsent(FilterEnum.EQUAL.getName(), (builder, k, v) -> builder.must(QueryBuilders.matchQuery(k, v)));
         filters.putIfAbsent(FilterEnum.NOT_EQUAL.getName(), (builder, k, v) -> builder.mustNot(QueryBuilders.matchQuery(k, v)));
         filters.putIfAbsent(FilterEnum.GT.getName(), (builder, k, v) -> builder.filter(QueryBuilders.rangeQuery(k).gt(v)));
@@ -74,47 +98,67 @@ public final class ESConnector extends AbstractConnector implements Connector<ES
         filters.putIfAbsent(FilterEnum.LIKE.getName(), (builder, k, v) -> builder.filter(QueryBuilders.wildcardQuery(k, v)));
     }
 
-    public ESConnector() {
-        VALUE_MAPPERS.put(Types.DATE, new ESDateValueMapper());
-        VALUE_MAPPERS.put(Types.OTHER, new ESOtherValueMapper());
+    @Override
+    public String getConnectorType() {
+        return TYPE;
+    }
+
+    @Override
+    public boolean isSupportedTiming() {
+        return true;
+    }
+
+    @Override
+    public boolean isSupportedLog() {
+        return false;
+    }
+
+    @Override
+    public Class<ESConfig> getConfigClass() {
+        return ESConfig.class;
+    }
+
+    @Override
+    public ConnectorInstance connect(ESConfig config) {
+        return new ESConnectorInstance(config);
     }
 
     @Override
-    public ConnectorMapper connect(ESConfig config) {
-        return new ESConnectorMapper(config);
+    public ConfigValidator getConfigValidator() {
+        return configValidator;
     }
 
     @Override
-    public void disconnect(ESConnectorMapper connectorMapper) {
-        connectorMapper.close();
+    public void disconnect(ESConnectorInstance connectorInstance) {
+        connectorInstance.close();
     }
 
     @Override
-    public boolean isAlive(ESConnectorMapper connectorMapper) {
+    public boolean isAlive(ESConnectorInstance connectorInstance) {
         try {
-            RestHighLevelClient client = connectorMapper.getConnection();
+            RestHighLevelClient client = connectorInstance.getConnection();
             return client.ping(RequestOptions.DEFAULT);
         } catch (IOException e) {
             logger.error(e.getMessage());
-            throw new ConnectorException(e.getMessage());
+            throw new ElasticsearchException(e.getMessage());
         }
     }
 
     @Override
-    public String getConnectorMapperCacheKey(ESConfig config) {
+    public String getConnectorInstanceCacheKey(ESConfig config) {
         return String.format("%s-%s-%s-%s", config.getConnectorType(), config.getUrl(), config.getIndex(), config.getUsername());
     }
 
     @Override
-    public List<Table> getTable(ESConnectorMapper connectorMapper) {
+    public List<Table> getTable(ESConnectorInstance connectorInstance) {
         try {
-            ESConfig config = connectorMapper.getConfig();
+            ESConfig config = connectorInstance.getConfig();
             GetIndexRequest request = new GetIndexRequest(config.getIndex());
-            GetIndexResponse indexResponse = connectorMapper.getConnection().indices().get(request, RequestOptions.DEFAULT);
+            GetIndexResponse indexResponse = connectorInstance.getConnection().indices().get(request, RequestOptions.DEFAULT);
             MappingMetadata mappingMetaData = indexResponse.getMappings().get(config.getIndex());
             List<Table> tables = new ArrayList<>();
             // 6.x 版本
-            if (Version.V_7_0_0.after(connectorMapper.getVersion())) {
+            if (Version.V_7_0_0.after(connectorInstance.getVersion())) {
                 Map<String, Object> sourceMap = XContentHelper.convertToMap(mappingMetaData.source().compressedReference(), true, null).v2();
                 sourceMap.keySet().forEach(tableName -> tables.add(new Table(tableName)));
                 return tables;
@@ -125,20 +169,20 @@ public final class ESConnector extends AbstractConnector implements Connector<ES
             return tables;
         } catch (IOException e) {
             logger.error(e.getMessage());
-            throw new ConnectorException(e);
+            throw new ElasticsearchException(e);
         }
     }
 
     @Override
-    public MetaInfo getMetaInfo(ESConnectorMapper connectorMapper, String tableName) {
+    public MetaInfo getMetaInfo(ESConnectorInstance connectorInstance, String tableName) {
         List<Field> fields = new ArrayList<>();
         try {
-            ESConfig config = connectorMapper.getConfig();
+            ESConfig config = connectorInstance.getConfig();
             GetIndexRequest request = new GetIndexRequest(config.getIndex());
-            GetIndexResponse indexResponse = connectorMapper.getConnection().indices().get(request, RequestOptions.DEFAULT);
+            GetIndexResponse indexResponse = connectorInstance.getConnection().indices().get(request, RequestOptions.DEFAULT);
             MappingMetadata mappingMetaData = indexResponse.getMappings().get(config.getIndex());
             // 6.x 版本
-            if (Version.V_7_0_0.after(connectorMapper.getVersion())) {
+            if (Version.V_7_0_0.after(connectorInstance.getVersion())) {
                 Map<String, Object> sourceMap = XContentHelper.convertToMap(mappingMetaData.source().compressedReference(), true, null).v2();
                 parseProperties(fields, (Map) sourceMap.get(tableName));
                 return new MetaInfo().setColumn(fields);
@@ -149,34 +193,34 @@ public final class ESConnector extends AbstractConnector implements Connector<ES
             return new MetaInfo().setColumn(fields);
         } catch (IOException e) {
             logger.error(e.getMessage());
-            throw new ConnectorException(e);
+            throw new ElasticsearchException(e);
         }
     }
 
     @Override
-    public long getCount(ESConnectorMapper connectorMapper, Map<String, String> command) {
+    public long getCount(ESConnectorInstance connectorInstance, Map<String, String> command) {
         try {
-            ESConfig config = connectorMapper.getConfig();
+            ESConfig config = connectorInstance.getConfig();
             SearchSourceBuilder builder = new SearchSourceBuilder();
             genSearchSourceBuilder(builder, command);
             // 7.x 版本以上
-            if (Version.V_7_0_0.onOrBefore(connectorMapper.getVersion())) {
+            if (Version.V_7_0_0.onOrBefore(connectorInstance.getVersion())) {
                 builder.trackTotalHits(true);
             }
             builder.from(0);
             builder.size(0);
-            SearchRequest request = new SearchRequest(new String[] {config.getIndex()}, builder);
-            SearchResponse response = connectorMapper.getConnection().searchWithVersion(request, RequestOptions.DEFAULT);
+            SearchRequest request = new SearchRequest(new String[]{config.getIndex()}, builder);
+            SearchResponse response = connectorInstance.getConnection().searchWithVersion(request, RequestOptions.DEFAULT);
             return response.getHits().getTotalHits().value;
         } catch (IOException e) {
             logger.error(e.getMessage());
-            throw new ConnectorException(e);
+            throw new ElasticsearchException(e);
         }
     }
 
     @Override
-    public Result reader(ESConnectorMapper connectorMapper, ReaderConfig config) {
-        ESConfig cfg = connectorMapper.getConfig();
+    public Result reader(ESConnectorInstance connectorInstance, ReaderConfig config) {
+        ESConfig cfg = connectorInstance.getConfig();
         SearchSourceBuilder builder = new SearchSourceBuilder();
         genSearchSourceBuilder(builder, config.getCommand());
         builder.from((config.getPageIndex() - 1) * config.getPageSize());
@@ -193,8 +237,8 @@ public final class ESConnector extends AbstractConnector implements Connector<ES
         }
 
         try {
-            SearchRequest rq = new SearchRequest(new String[] {cfg.getIndex()}, builder);
-            SearchResponse searchResponse = connectorMapper.getConnection().searchWithVersion(rq, RequestOptions.DEFAULT);
+            SearchRequest rq = new SearchRequest(new String[]{cfg.getIndex()}, builder);
+            SearchResponse searchResponse = connectorInstance.getConnection().searchWithVersion(rq, RequestOptions.DEFAULT);
             SearchHits hits = searchResponse.getHits();
             SearchHit[] searchHits = hits.getHits();
             List<Map<String, Object>> list = new ArrayList<>();
@@ -204,20 +248,20 @@ public final class ESConnector extends AbstractConnector implements Connector<ES
             return new Result(list);
         } catch (IOException e) {
             logger.error(e.getMessage());
-            throw new ConnectorException(e.getMessage());
+            throw new ElasticsearchException(e.getMessage());
         }
     }
 
     @Override
-    public Result writer(ESConnectorMapper connectorMapper, WriterBatchConfig config) {
+    public Result writer(ESConnectorInstance connectorInstance, WriterBatchConfig config) {
         List<Map> data = config.getData();
         if (CollectionUtils.isEmpty(data) || CollectionUtils.isEmpty(config.getFields())) {
             logger.error("writer data can not be empty.");
-            throw new ConnectorException("writer data can not be empty.");
+            throw new ElasticsearchException("writer data can not be empty.");
         }
 
         final Result result = new Result();
-        final ESConfig cfg = connectorMapper.getConfig();
+        final ESConfig cfg = connectorInstance.getConfig();
         final List<Field> pkFields = PrimaryKeyUtil.findConfigPrimaryKeyFields(config);
         try {
             BulkRequest request = new BulkRequest();
@@ -225,10 +269,10 @@ public final class ESConnector extends AbstractConnector implements Connector<ES
             final String pk = pkFields.get(0).getName();
             data.forEach(row -> addRequest(request, cfg.getIndex(), config.getTableName(), config.getEvent(), String.valueOf(row.get(pk)), row));
 
-            BulkResponse response = connectorMapper.getConnection().bulkWithVersion(request, RequestOptions.DEFAULT);
+            BulkResponse response = connectorInstance.getConnection().bulkWithVersion(request, RequestOptions.DEFAULT);
             RestStatus restStatus = response.status();
             if (restStatus.getStatus() != RestStatus.OK.getStatus()) {
-                throw new ConnectorException(String.format("error code:%s", restStatus.getStatus()));
+                throw new ElasticsearchException(String.format("error code:%s", restStatus.getStatus()));
             }
             result.addSuccessData(data);
         } catch (Exception e) {
@@ -265,10 +309,18 @@ public final class ESConnector extends AbstractConnector implements Connector<ES
         return Collections.EMPTY_MAP;
     }
 
+    @Override
+    public Listener getListener(String listenerType) {
+        if (ListenerTypeEnum.isTiming(listenerType)) {
+            return new ESQuartzListener();
+        }
+        return null;
+    }
+
     private void parseProperties(List<Field> fields, Map<String, Object> sourceMap) {
         Map<String, Object> properties = (Map<String, Object>) sourceMap.get(ESUtil.PROPERTIES);
         if (CollectionUtils.isEmpty(properties)) {
-            throw new ConnectorException("查询字段不能为空.");
+            throw new ElasticsearchException("查询字段不能为空.");
         }
         properties.forEach((fieldName, c) -> {
             Map fieldDesc = (Map) c;

dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/EasyRestHighLevelClient.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/EasyRestHighLevelClient.java

@@ -1,4 +1,7 @@
-package org.dbsyncer.connector.es;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.api;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -22,7 +25,14 @@ import java.util.List;
 
 import static java.util.Collections.emptySet;
 
-public class EasyRestHighLevelClient extends RestHighLevelClient {
+/**
+ * ES客户端（同时兼容6-7版本API）
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
+ */
+public final class EasyRestHighLevelClient extends RestHighLevelClient {
 
     private Version version;
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/RequestConverters.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/RequestConverters.java

@@ -1,4 +1,7 @@
-package org.dbsyncer.connector.es;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.api;
 
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.HttpDelete;
@@ -81,6 +84,13 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.StringJoiner;
 
+/**
+ * ES请求（同时兼容6-7版本API）
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
+ */
 final class RequestConverters {
     static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON;
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/RethrottleRequest.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/RethrottleRequest.java

@@ -1,4 +1,7 @@
-package org.dbsyncer.connector.es;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.api;
 
 import org.elasticsearch.client.Validatable;
 import org.elasticsearch.tasks.TaskId;
@@ -7,6 +10,10 @@ import java.util.Objects;
 
 /**
  * A request changing throttling of a task.
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
  */
 public class RethrottleRequest implements Validatable {
 

dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/ESQuartzExtractor.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc/ESQuartzListener.java

@@ -1,23 +1,33 @@
-package org.dbsyncer.listener.quartz;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.cdc;
 
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.model.Filter;
-import org.dbsyncer.connector.constant.ConnectorConstant;
-import org.dbsyncer.listener.ListenerException;
-import org.dbsyncer.listener.enums.QuartzFilterEnum;
+import org.dbsyncer.sdk.constant.ConnectorConstant;
+import org.dbsyncer.sdk.enums.QuartzFilterEnum;
+import org.dbsyncer.sdk.listener.AbstractQuartzListener;
+import org.dbsyncer.sdk.listener.QuartzFilter;
+import org.dbsyncer.sdk.model.Filter;
+import org.dbsyncer.sdk.model.Point;
+import org.elasticsearch.ElasticsearchException;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
- * ES定时抽取
+ * CDC-ES定时监听器
  *
- * @version 1.0.0
  * @Author AE86
+ * @Version 1.0.0
  * @Date 2021-09-01 20:35
  */
-public final class ESQuartzExtractor extends AbstractQuartzExtractor {
+public final class ESQuartzListener extends AbstractQuartzListener {
 
     @Override
     protected Point checkLastPoint(Map<String, String> command, int index) {
@@ -36,7 +46,7 @@ public final class ESQuartzExtractor extends AbstractQuartzExtractor {
         Set<String> set = new HashSet<>();
         for (Filter f : filters) {
             if (set.contains(f.getValue())) {
-                throw new ListenerException(String.format("系统参数%s存在多个.", f.getValue()));
+                throw new ElasticsearchException(String.format("系统参数%s存在多个.", f.getValue()));
             }
             QuartzFilterEnum filterEnum = QuartzFilterEnum.getQuartzFilterEnum(f.getValue());
             if (null != filterEnum) {
@@ -57,7 +67,7 @@ public final class ESQuartzExtractor extends AbstractQuartzExtractor {
                     }
 
                     // 读取历史增量点
-                    f.setValue(snapshot.get(key));
+                    f.setValue((String) snapshot.get(key));
                     point.setBeginKey(key);
                     point.setBeginValue(quartzFilter.toString(quartzFilter.getObject()));
                     continue;

dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ESConfig.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/config/ESConfig.java

@@ -1,65 +1,69 @@
-package org.dbsyncer.connector.config;
-
-import org.dbsyncer.common.model.AbstractConnectorConfig;
-
-/**
- * @author AE86
- * @ClassName: ESConfig
- * @Description: ES连接配置
- * @date: 2021年8月23日 下午8:00:00
- */
-public class ESConfig extends AbstractConnectorConfig {
-
-    /**
-     * 集群地址, http(s)-9200, tcp-9300 http://192.168.1.100:9200,http://192.168.1.200:9200
-     */
-    private String url;
-
-    /**
-     * 帐号
-     */
-    private String username;
-
-    /**
-     * 密码
-     */
-    private String password;
-
-    /**
-     * 索引(相当于数据库)
-     */
-    private String index;
-
-    public String getUrl() {
-        return url;
-    }
-
-    public void setUrl(String url) {
-        this.url = url;
-    }
-
-    public String getUsername() {
-        return username;
-    }
-
-    public void setUsername(String username) {
-        this.username = username;
-    }
-
-    public String getPassword() {
-        return password;
-    }
-
-    public void setPassword(String password) {
-        this.password = password;
-    }
-
-    public String getIndex() {
-        return index;
-    }
-
-    public void setIndex(String index) {
-        this.index = index;
-    }
-
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.config;
+
+import org.dbsyncer.sdk.model.ConnectorConfig;
+
+/**
+ * ES连接配置
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2021-08-23 20:00
+ */
+public class ESConfig extends ConnectorConfig {
+
+    /**
+     * 集群地址, http(s)-9200, tcp-9300 http://192.168.1.100:9200,http://192.168.1.200:9200
+     */
+    private String url;
+
+    /**
+     * 帐号
+     */
+    private String username;
+
+    /**
+     * 密码
+     */
+    private String password;
+
+    /**
+     * 索引(相当于数据库)
+     */
+    private String index;
+
+    public String getUrl() {
+        return url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getIndex() {
+        return index;
+    }
+
+    public void setIndex(String index) {
+        this.index = index;
+    }
+
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ESFieldTypeEnum.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/enums/ESFieldTypeEnum.java

@@ -1,7 +1,10 @@
-package org.dbsyncer.connector.enums;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.enums;
 
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorException;
+import org.elasticsearch.ElasticsearchException;
 
 import java.sql.Types;
 
@@ -10,7 +13,7 @@ import java.sql.Types;
  *
  * @author AE86
  * @version 1.0.0
- * @date 2021/08/26 21:13
+ * @Date 2023-08-26 21:13
  */
 public enum ESFieldTypeEnum {
 
@@ -80,7 +83,7 @@ public enum ESFieldTypeEnum {
         this.type = type;
     }
 
-    public static int getType(String code) throws ConnectorException {
+    public static int getType(String code) throws ElasticsearchException {
         for (ESFieldTypeEnum e : ESFieldTypeEnum.values()) {
             if (StringUtil.equals(e.getCode(), code)) {
                 return e.getType();

dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema/ESDateValueMapper.java

@@ -0,0 +1,36 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.schema;
+
+import org.dbsyncer.sdk.connector.AbstractValueMapper;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.elasticsearch.ElasticsearchException;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+
+/**
+ * ES日期字段值转换
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
+ */
+public class ESDateValueMapper extends AbstractValueMapper<java.util.Date> {
+
+    @Override
+    protected java.util.Date convert(ConnectorInstance connectorInstance, Object val) {
+        if (val instanceof Timestamp) {
+            Timestamp timestamp = (Timestamp) val;
+            return new java.util.Date(timestamp.getTime());
+        }
+
+        if (val instanceof Date) {
+            Date date = (Date) val;
+            return new java.util.Date(date.getTime());
+        }
+
+        throw new ElasticsearchException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val));
+    }
+}

dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema/ESOtherValueMapper.java

@@ -1,35 +1,40 @@
-package org.dbsyncer.connector.es;
-
-import org.dbsyncer.common.spi.ConnectorMapper;
-import org.dbsyncer.common.util.JsonUtil;
-import org.dbsyncer.connector.AbstractValueMapper;
-import org.dbsyncer.connector.ConnectorException;
-import org.postgresql.util.PGobject;
-
-import java.util.Map;
-
-/**
- * @author moyu
- * @version 1.0.0
- * @date 2023/10/12 0:07
- */
-public class ESOtherValueMapper extends AbstractValueMapper<Map> {
-
-    @Override
-    protected Map convert(ConnectorMapper connectorMapper, Object val) {
-        if (val instanceof String) {
-            return JsonUtil.jsonToObj((String) val, Map.class);
-        }
-
-        if (val instanceof byte[]) {
-            return JsonUtil.jsonToObj(new String((byte[]) val), Map.class);
-        }
-
-        if (val instanceof PGobject) {
-            PGobject pgObject = (PGobject) val;
-            return JsonUtil.jsonToObj(pgObject.getValue(), Map.class);
-        }
-
-        throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val));
-    }
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.schema;
+
+import org.dbsyncer.common.util.JsonUtil;
+import org.dbsyncer.sdk.connector.AbstractValueMapper;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.elasticsearch.ElasticsearchException;
+import org.postgresql.util.PGobject;
+
+import java.util.Map;
+
+/**
+ * ES异构字段值转换
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
+ */
+public class ESOtherValueMapper extends AbstractValueMapper<Map> {
+
+    @Override
+    protected Map convert(ConnectorInstance connectorInstance, Object val) {
+        if (val instanceof String) {
+            return JsonUtil.jsonToObj((String) val, Map.class);
+        }
+
+        if (val instanceof byte[]) {
+            return JsonUtil.jsonToObj(new String((byte[]) val), Map.class);
+        }
+
+        if (val instanceof PGobject) {
+            PGobject pgObject = (PGobject) val;
+            return JsonUtil.jsonToObj(pgObject.getValue(), Map.class);
+        }
+
+        throw new ElasticsearchException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val));
+    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/ESUtil.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/util/ESUtil.java

@@ -1,62 +1,72 @@
-package org.dbsyncer.connector.util;
-
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.TrustAllStrategy;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
-import org.apache.http.ssl.SSLContextBuilder;
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorException;
-import org.dbsyncer.connector.config.ESConfig;
-import org.dbsyncer.connector.es.EasyRestHighLevelClient;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.client.RestHighLevelClient;
-
-import javax.net.ssl.SSLContext;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Objects;
-
-public abstract class ESUtil {
-
-    public static final String PROPERTIES = "properties";
-
-    private ESUtil() {
-    }
-
-    public static EasyRestHighLevelClient getConnection(ESConfig config) {
-        String[] ipAddress = StringUtil.split(config.getUrl(), ",");
-        HttpHost[] hosts = Arrays.stream(ipAddress).map(node -> HttpHost.create(node)).filter(Objects::nonNull).toArray(
-                HttpHost[]::new);
-        RestClientBuilder builder = RestClient.builder(hosts);
-        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.getUsername(), config.getPassword()));
-        try {
-            SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(new TrustAllStrategy()).build();
-            SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslContext, NoopHostnameVerifier.INSTANCE);
-            builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setSSLStrategy(sessionStrategy));
-            final EasyRestHighLevelClient client = new EasyRestHighLevelClient(builder);
-            client.ping(RequestOptions.DEFAULT);
-            return client;
-        } catch (Exception e) {
-            throw new ConnectorException(String.format("Failed to connect to ElasticSearch on %s, %s", config.getUrl(), e.getMessage()));
-        }
-    }
-
-    public static void close(RestHighLevelClient client) {
-        if (null != client) {
-            try {
-                client.close();
-            } catch (IOException e) {
-                throw new ConnectorException(e.getMessage());
-            }
-        }
-    }
-
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.util;
+
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustAllStrategy;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.elasticsearch.api.EasyRestHighLevelClient;
+import org.dbsyncer.connector.elasticsearch.config.ESConfig;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * ES连接器工具类
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
+ */
+public abstract class ESUtil {
+
+    public static final String PROPERTIES = "properties";
+
+    private ESUtil() {
+    }
+
+    public static EasyRestHighLevelClient getConnection(ESConfig config) {
+        String[] ipAddress = StringUtil.split(config.getUrl(), ",");
+        HttpHost[] hosts = Arrays.stream(ipAddress).map(node -> HttpHost.create(node)).filter(Objects::nonNull).toArray(
+                HttpHost[]::new);
+        RestClientBuilder builder = RestClient.builder(hosts);
+        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.getUsername(), config.getPassword()));
+        try {
+            SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(new TrustAllStrategy()).build();
+            SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslContext, NoopHostnameVerifier.INSTANCE);
+            builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setSSLStrategy(sessionStrategy));
+            final EasyRestHighLevelClient client = new EasyRestHighLevelClient(builder);
+            client.ping(RequestOptions.DEFAULT);
+            return client;
+        } catch (Exception e) {
+            throw new ElasticsearchException(String.format("Failed to connect to ElasticSearch on %s, %s", config.getUrl(), e.getMessage()));
+        }
+    }
+
+    public static void close(RestHighLevelClient client) {
+        if (null != client) {
+            try {
+                client.close();
+            } catch (IOException e) {
+                throw new ElasticsearchException(e.getMessage());
+            }
+        }
+    }
+
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ElasticsearchConfigChecker.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/validator/ESConfigValidator.java

@@ -1,19 +1,22 @@
-package org.dbsyncer.biz.checker.impl.connector;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.elasticsearch.validator;
 
-import org.dbsyncer.biz.checker.ConnectorConfigChecker;
-import org.dbsyncer.connector.config.ESConfig;
-import org.springframework.stereotype.Component;
+import org.dbsyncer.connector.elasticsearch.config.ESConfig;
+import org.dbsyncer.sdk.connector.ConfigValidator;
 import org.springframework.util.Assert;
 
 import java.util.Map;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2021/8/25 23:30
+ * ES连接配置校验器实现
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2021-08-25 23:30
  */
-@Component
-public class ElasticsearchConfigChecker implements ConnectorConfigChecker<ESConfig> {
+public final class ESConfigValidator implements ConfigValidator<ESConfig> {
 
     @Override
     public void modify(ESConfig connectorConfig, Map<String, String> params) {

dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService

@@ -0,0 +1 @@
+org.dbsyncer.connector.elasticsearch.ElasticsearchConnector

dbsyncer-listener/src/main/test/ESClientTest.java → dbsyncer-connector/dbsyncer-connector-elasticsearch/src/test/java/ESClientTest.java

@@ -1,5 +1,9 @@
-import org.dbsyncer.connector.config.ESConfig;
-import org.dbsyncer.connector.util.ESUtil;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+
+import org.dbsyncer.connector.elasticsearch.config.ESConfig;
+import org.dbsyncer.connector.elasticsearch.util.ESUtil;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -39,6 +43,13 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
+/**
+ * ES测试用例
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-11-25 23:10
+ */
 public class ESClientTest {
     private final Logger logger = LoggerFactory.getLogger(getClass());
     private RestHighLevelClient client;

dbsyncer-listener/pom.xml → dbsyncer-connector/dbsyncer-connector-file/pom.xml

@@ -3,29 +3,27 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>dbsyncer</artifactId>
+        <artifactId>dbsyncer-connector</artifactId>
         <groupId>org.ghi</groupId>
-        <version>1.2.7_1201</version>
+        <version>2.0.0_1226-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-    <artifactId>dbsyncer-listener</artifactId>
+
+    <artifactId>dbsyncer-connector-file</artifactId>
 
     <dependencies>
-        <!-- Connector 连接器 -->
+        <!-- sdk -->
         <dependency>
             <groupId>org.ghi</groupId>
-            <artifactId>dbsyncer-connector</artifactId>
+            <artifactId>dbsyncer-sdk</artifactId>
             <version>${project.parent.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>com.zendesk</groupId>
-            <artifactId>mysql-binlog-connector-java</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-log4j2</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>

dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileBitValueMapper.java

@@ -0,0 +1,26 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file;
+
+import org.dbsyncer.sdk.connector.AbstractValueMapper;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+
+/**
+ * @author AE86
+ * @version 1.0.0
+ * @date 2023/11/15 0:07
+ */
+public class FileBitValueMapper extends AbstractValueMapper<Integer> {
+
+    @Override
+    protected Integer convert(ConnectorInstance connectorInstance, Object val) {
+        if (val instanceof Boolean) {
+            Boolean b = (Boolean) val;
+            return b ? 1 : 0;
+        }
+
+        throw new FileException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val));
+    }
+
+}

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnector.java

@@ -1,22 +1,30 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.connector.file;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.LineIterator;
 import org.dbsyncer.common.model.Result;
-import org.dbsyncer.common.spi.ConnectorMapper;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.AbstractConnector;
-import org.dbsyncer.connector.Connector;
-import org.dbsyncer.connector.ConnectorException;
-import org.dbsyncer.connector.config.CommandConfig;
-import org.dbsyncer.connector.config.FileConfig;
-import org.dbsyncer.connector.config.ReaderConfig;
-import org.dbsyncer.connector.config.WriterBatchConfig;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.connector.model.FileSchema;
-import org.dbsyncer.connector.model.MetaInfo;
-import org.dbsyncer.connector.model.Table;
+import org.dbsyncer.connector.file.cdc.FileListener;
+import org.dbsyncer.connector.file.model.FileResolver;
+import org.dbsyncer.connector.file.config.FileConfig;
+import org.dbsyncer.connector.file.model.FileSchema;
+import org.dbsyncer.connector.file.validator.FileConfigValidator;
+import org.dbsyncer.sdk.config.CommandConfig;
+import org.dbsyncer.sdk.config.ReaderConfig;
+import org.dbsyncer.sdk.config.WriterBatchConfig;
+import org.dbsyncer.sdk.connector.AbstractConnector;
+import org.dbsyncer.sdk.connector.ConfigValidator;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.enums.ListenerTypeEnum;
+import org.dbsyncer.sdk.listener.Listener;
+import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.model.MetaInfo;
+import org.dbsyncer.sdk.model.Table;
+import org.dbsyncer.sdk.spi.ConnectorService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
@@ -37,42 +45,71 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/5 23:19
+ * 文件连接器实现
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
  */
-public final class FileConnector extends AbstractConnector implements Connector<FileConnectorMapper, FileConfig> {
+public final class FileConnector extends AbstractConnector implements ConnectorService<FileConnectorInstance, FileConfig> {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    private static final String FILE_NAME = "fileName";
-    private static final String FILE_PATH = "filePath";
+    private final String TYPE = "File";
+    private final String FILE_NAME = "fileName";
+    private final String FILE_PATH = "filePath";
     private final FileResolver fileResolver = new FileResolver();
+    private final FileConfigValidator configValidator = new FileConfigValidator();
 
     public FileConnector() {
         VALUE_MAPPERS.put(Types.BIT, new FileBitValueMapper());
     }
 
     @Override
-    public ConnectorMapper connect(FileConfig config) {
-        return new FileConnectorMapper(config);
+    public String getConnectorType() {
+        return TYPE;
     }
 
     @Override
-    public void disconnect(FileConnectorMapper connectorMapper) {
+    public boolean isSupportedTiming() {
+        return false;
+    }
 
+    @Override
+    public boolean isSupportedLog() {
+        return true;
     }
 
     @Override
-    public boolean isAlive(FileConnectorMapper connectorMapper) {
-        String fileDir = connectorMapper.getConnection();
+    public Class<FileConfig> getConfigClass() {
+        return FileConfig.class;
+    }
+
+    @Override
+    public ConnectorInstance connect(FileConfig config) {
+        return new FileConnectorInstance(config);
+    }
+
+    @Override
+    public ConfigValidator getConfigValidator() {
+        return configValidator;
+    }
+
+    @Override
+    public void disconnect(FileConnectorInstance connectorInstance) {
+
+    }
+
+    @Override
+    public boolean isAlive(FileConnectorInstance connectorInstance) {
+        String fileDir = connectorInstance.getConnection();
         boolean alive = new File(fileDir).exists();
         if (!alive) {
             logger.warn("can not find fileDir:{}", fileDir);
             return false;
         }
-        for (FileSchema fileSchema : connectorMapper.getFileSchemaList()) {
-            String filePath = connectorMapper.getFilePath(fileSchema.getFileName());
+        for (FileSchema fileSchema : connectorInstance.getFileSchemaList()) {
+            String filePath = connectorInstance.getFilePath(fileSchema.getFileName());
             if (!new File(filePath).exists()) {
                 logger.warn("can not find file:{}", filePath);
                 alive = false;
@@ -82,7 +119,7 @@ public final class FileConnector extends AbstractConnector implements Connector<
     }
 
     @Override
-    public String getConnectorMapperCacheKey(FileConfig config) {
+    public String getConnectorInstanceCacheKey(FileConfig config) {
         String localIP;
         try {
             localIP = InetAddress.getLocalHost().getHostAddress();
@@ -94,18 +131,18 @@ public final class FileConnector extends AbstractConnector implements Connector<
     }
 
     @Override
-    public List<Table> getTable(FileConnectorMapper connectorMapper) {
-        return connectorMapper.getFileSchemaList().stream().map(fileSchema -> new Table(fileSchema.getFileName())).collect(Collectors.toList());
+    public List<Table> getTable(FileConnectorInstance connectorInstance) {
+        return connectorInstance.getFileSchemaList().stream().map(fileSchema -> new Table(fileSchema.getFileName())).collect(Collectors.toList());
     }
 
     @Override
-    public MetaInfo getMetaInfo(FileConnectorMapper connectorMapper, String tableName) {
-        FileSchema fileSchema = connectorMapper.getFileSchema(tableName);
+    public MetaInfo getMetaInfo(FileConnectorInstance connectorInstance, String tableName) {
+        FileSchema fileSchema = connectorInstance.getFileSchema(tableName);
         return new MetaInfo().setColumn(fileSchema.getFields());
     }
 
     @Override
-    public long getCount(FileConnectorMapper connectorMapper, Map<String, String> command) {
+    public long getCount(FileConnectorInstance connectorInstance, Map<String, String> command) {
         AtomicLong count = new AtomicLong();
         FileReader reader = null;
         try {
@@ -116,7 +153,7 @@ public final class FileConnector extends AbstractConnector implements Connector<
                 count.addAndGet(1);
             }
         } catch (IOException e) {
-            throw new ConnectorException(e.getCause());
+            throw new FileException(e.getCause());
         } finally {
             IOUtils.closeQuietly(reader);
         }
@@ -124,12 +161,12 @@ public final class FileConnector extends AbstractConnector implements Connector<
     }
 
     @Override
-    public Result reader(FileConnectorMapper connectorMapper, ReaderConfig config) {
+    public Result reader(FileConnectorInstance connectorInstance, ReaderConfig config) {
         List<Map<String, Object>> list = new ArrayList<>();
         FileReader reader = null;
         try {
-            FileConfig fileConfig = connectorMapper.getConfig();
-            FileSchema fileSchema = connectorMapper.getFileSchema(config.getCommand().get(FILE_NAME));
+            FileConfig fileConfig = connectorInstance.getConfig();
+            FileSchema fileSchema = connectorInstance.getFileSchema(config.getCommand().get(FILE_NAME));
             final List<Field> fields = fileSchema.getFields();
             Assert.notEmpty(fields, "The fields of file schema is empty.");
             final char separator = fileConfig.getSeparator();
@@ -151,7 +188,7 @@ public final class FileConnector extends AbstractConnector implements Connector<
                 }
             }
         } catch (IOException e) {
-            throw new ConnectorException(e.getCause());
+            throw new FileException(e.getCause());
         } finally {
             IOUtils.closeQuietly(reader);
         }
@@ -159,20 +196,20 @@ public final class FileConnector extends AbstractConnector implements Connector<
     }
 
     @Override
-    public Result writer(FileConnectorMapper connectorMapper, WriterBatchConfig config) {
+    public Result writer(FileConnectorInstance connectorInstance, WriterBatchConfig config) {
         List<Map> data = config.getData();
         if (CollectionUtils.isEmpty(data)) {
             logger.error("writer data can not be empty.");
-            throw new ConnectorException("writer data can not be empty.");
+            throw new FileException("writer data can not be empty.");
         }
 
         final List<Field> fields = config.getFields();
-        final String separator = new String(new char[] {connectorMapper.getConfig().getSeparator()});
+        final String separator = new String(new char[]{connectorInstance.getConfig().getSeparator()});
 
         Result result = new Result();
         OutputStream output = null;
         try {
-            final String filePath = connectorMapper.getFilePath(config.getCommand().get(FILE_NAME));
+            final String filePath = connectorInstance.getFilePath(config.getCommand().get(FILE_NAME));
             output = new FileOutputStream(filePath, true);
             List<String> lines = data.stream().map(row -> {
                 List<String> array = new ArrayList<>();
@@ -215,4 +252,12 @@ public final class FileConnector extends AbstractConnector implements Connector<
         return command;
     }
 
+    @Override
+    public Listener getListener(String listenerType) {
+        if (ListenerTypeEnum.isLog(listenerType)) {
+            return new FileListener();
+        }
+        return null;
+    }
+
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorMapper.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java

@@ -1,10 +1,13 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.connector.file;
 
-import org.dbsyncer.common.spi.ConnectorMapper;
 import org.dbsyncer.common.util.JsonUtil;
-import org.dbsyncer.connector.config.FileConfig;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.connector.model.FileSchema;
+import org.dbsyncer.connector.file.config.FileConfig;
+import org.dbsyncer.connector.file.model.FileSchema;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.model.Field;
 import org.springframework.util.Assert;
 
 import java.io.File;
@@ -17,13 +20,13 @@ import java.util.concurrent.ConcurrentHashMap;
  * @version 1.0.0
  * @date 2022/5/5 23:19
  */
-public final class FileConnectorMapper implements ConnectorMapper<FileConfig, String> {
+public final class FileConnectorInstance implements ConnectorInstance<FileConfig, String> {
 
     private FileConfig config;
     private List<FileSchema> fileSchemaList;
     private Map<String, FileResolver> fileSchemaMap = new ConcurrentHashMap<>();
 
-    public FileConnectorMapper(FileConfig config) {
+    public FileConnectorInstance(FileConfig config) {
         this.config = config;
         fileSchemaList = JsonUtil.jsonToArray(config.getSchema(), FileSchema.class);
         Assert.notEmpty(fileSchemaList, "The schema is empty.");
@@ -37,6 +40,11 @@ public final class FileConnectorMapper implements ConnectorMapper<FileConfig, St
         }
     }
 
+    @Override
+    public String getServiceUrl() {
+        return config.getFileDir();
+    }
+
     public FileConfig getConfig() {
         return config;
     }
@@ -88,4 +96,4 @@ public final class FileConnectorMapper implements ConnectorMapper<FileConfig, St
             Assert.isTrue(file.exists(), String.format("found not file '%s'", filePath));
         }
     }
-}
+}

dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileException.java

@@ -0,0 +1,30 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
+ */
+public class FileException extends RuntimeException {
+
+	private static final long serialVersionUID = 1L;
+
+	public FileException(String message) {
+        super(message);
+    }
+
+    public FileException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public FileException(Throwable cause) {
+        super(cause);
+    }
+
+    protected FileException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

dbsyncer-common/src/main/java/org/dbsyncer/common/file/BufferedRandomAccessFile.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/BufferedRandomAccessFile.java

@@ -1,4 +1,7 @@
-package org.dbsyncer.common.file;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.cdc;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -7,9 +10,9 @@ import java.io.RandomAccessFile;
 import java.util.Arrays;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/7 22:27
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-07 22:27
  */
 public class BufferedRandomAccessFile extends RandomAccessFile {
     static final int LogBuffSz_ = 16; // 64K buffer

dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java

@@ -1,20 +1,22 @@
-package org.dbsyncer.listener.file;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.cdc;
 
 import org.apache.commons.io.IOUtils;
-import org.dbsyncer.common.event.ChangedOffset;
-import org.dbsyncer.common.event.RowChangedEvent;
-import org.dbsyncer.common.file.BufferedRandomAccessFile;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.config.FileConfig;
-import org.dbsyncer.connector.constant.ConnectorConstant;
-import org.dbsyncer.connector.file.FileConnectorMapper;
-import org.dbsyncer.connector.file.FileResolver;
-import org.dbsyncer.connector.model.Field;
-import org.dbsyncer.connector.model.FileSchema;
-import org.dbsyncer.listener.AbstractExtractor;
-import org.dbsyncer.listener.ListenerException;
+import org.dbsyncer.connector.file.FileConnectorInstance;
+import org.dbsyncer.connector.file.FileException;
+import org.dbsyncer.connector.file.model.FileSchema;
+import org.dbsyncer.connector.file.config.FileConfig;
+import org.dbsyncer.connector.file.model.FileResolver;
+import org.dbsyncer.sdk.constant.ConnectorConstant;
+import org.dbsyncer.sdk.listener.AbstractListener;
+import org.dbsyncer.sdk.listener.event.RowChangedEvent;
+import org.dbsyncer.sdk.model.ChangedOffset;
+import org.dbsyncer.sdk.model.Field;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
@@ -40,11 +42,11 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/6 21:42
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
  */
-public class FileExtractor extends AbstractExtractor {
+public class FileListener extends AbstractListener {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -52,7 +54,7 @@ public class FileExtractor extends AbstractExtractor {
     private static final String CHARSET_NAME = "UTF-8";
     private final Lock connectLock = new ReentrantLock();
     private volatile boolean connected;
-    private FileConnectorMapper connectorMapper;
+    private FileConnectorInstance instance;
     private WatchService watchService;
     private Worker worker;
     private Map<String, PipelineResolver> pipeline = new ConcurrentHashMap<>();
@@ -68,9 +70,9 @@ public class FileExtractor extends AbstractExtractor {
                 return;
             }
 
-            connectorMapper = (FileConnectorMapper) connectorFactory.connect(connectorConfig);
-            final FileConfig config = connectorMapper.getConfig();
-            final String mapperCacheKey = connectorFactory.getConnector(connectorMapper).getConnectorMapperCacheKey(connectorConfig);
+            instance = (FileConnectorInstance) connectorInstance;
+            final FileConfig config = instance.getConfig();
+            final String cacheKey = connectorService.getConnectorInstanceCacheKey(config);
             connected = true;
 
             separator = config.getSeparator();
@@ -84,20 +86,20 @@ public class FileExtractor extends AbstractExtractor {
             }
 
             worker = new Worker();
-            worker.setName(new StringBuilder("file-parser-").append(mapperCacheKey).append("_").append(worker.hashCode()).toString());
+            worker.setName(new StringBuilder("file-parser-").append(cacheKey).append("_").append(worker.hashCode()).toString());
             worker.setDaemon(false);
             worker.start();
         } catch (Exception e) {
             logger.error("启动失败:{}", e.getMessage());
             closePipelineAndWatch();
-            throw new ListenerException(e);
+            throw new FileException(e);
         } finally {
             connectLock.unlock();
         }
     }
 
     private void initPipeline(String fileDir) throws IOException {
-        for (FileSchema fileSchema : connectorMapper.getFileSchemaList()) {
+        for (FileSchema fileSchema : instance.getFileSchemaList()) {
             String fileName = fileSchema.getFileName();
             String file = fileDir.concat(fileName);
             Assert.isTrue(new File(file).exists(), String.format("found not file '%s'", file));
@@ -105,7 +107,7 @@ public class FileExtractor extends AbstractExtractor {
             final RandomAccessFile raf = new BufferedRandomAccessFile(file, "r");
             final String filePosKey = getFilePosKey(fileName);
             if (snapshot.containsKey(filePosKey)) {
-                raf.seek(NumberUtil.toLong(snapshot.get(filePosKey), 0L));
+                raf.seek(NumberUtil.toLong((String) snapshot.get(filePosKey), 0L));
             } else {
                 raf.seek(raf.length());
                 snapshot.put(filePosKey, String.valueOf(raf.getFilePointer()));

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java

@@ -1,12 +1,15 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.connector.file.column;
 
 import java.sql.Date;
 import java.sql.Timestamp;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/5 22:39
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
  */
 public interface ColumnValue {
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/impl/FileColumnValue.java

@@ -1,17 +1,20 @@
-package org.dbsyncer.connector.file.column;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.column.impl;
 
 import org.dbsyncer.common.util.DateFormatUtil;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.file.column.ColumnValue;
 
 import java.sql.Date;
 import java.sql.Timestamp;
-import java.time.LocalDateTime;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/6 15:48
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
  */
 public class FileColumnValue implements ColumnValue {
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/config/FileConfig.java

@@ -1,13 +1,16 @@
-package org.dbsyncer.connector.config;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.config;
 
-import org.dbsyncer.common.model.AbstractConnectorConfig;
+import org.dbsyncer.sdk.model.ConnectorConfig;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/5 23:19
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
  */
-public class FileConfig extends AbstractConnectorConfig {
+public class FileConfig extends ConnectorConfig {
 
     /**
      * 文件目录