!96 优化自定义主键

Merge pull request !96 from AE86/lumingbao

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java

@@ -1,5 +1,6 @@
 package org.dbsyncer.biz.checker.impl.connector;
 
+import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.config.DatabaseConfig;
 import org.springframework.stereotype.Component;
 
@@ -17,5 +18,8 @@ public class DqlOracleConfigChecker extends AbstractDataBaseConfigChecker {
     public void modify(DatabaseConfig connectorConfig, Map<String, String> params) {
         super.modify(connectorConfig, params);
         super.modifyDql(connectorConfig, params);
+
+        String schema = params.get("schema");
+        connectorConfig.setSchema(StringUtil.isBlank(schema) ? connectorConfig.getUsername().toUpperCase() : schema.toUpperCase());
     }
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java

@@ -3,6 +3,7 @@ package org.dbsyncer.biz.checker.impl.connector;
 import org.dbsyncer.biz.enums.OracleIncrementEnum;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.model.Field;
 import org.dbsyncer.connector.model.MetaInfo;
 import org.dbsyncer.connector.model.Table;
@@ -34,6 +35,14 @@ public class OracleConfigChecker extends AbstractDataBaseConfigChecker {
     @Autowired
     private Manager manager;
 
+    @Override
+    public void modify(DatabaseConfig connectorConfig, Map<String, String> params) {
+        super.modify(connectorConfig, params);
+
+        String schema = params.get("schema");
+        connectorConfig.setSchema(StringUtil.isBlank(schema) ? connectorConfig.getUsername().toUpperCase() : schema.toUpperCase());
+    }
+
     @Override
     public void dealIncrementStrategy(Mapping mapping, TableGroup tableGroup) {
         String rowIdLabelName = OracleIncrementEnum.ROW_ID_LABEL_NAME.getName();

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java

@@ -49,6 +49,8 @@ public class TableGroupChecker extends AbstractChecker {
         String mappingId = params.get("mappingId");
         String sourceTable = params.get("sourceTable");
         String targetTable = params.get("targetTable");
+        String sourceTablePK = params.get("sourceTablePK");
+        String targetTablePK = params.get("targetTablePK");
         Assert.hasText(mappingId, "tableGroup mappingId is empty.");
         Assert.hasText(sourceTable, "tableGroup sourceTable is empty.");
         Assert.hasText(targetTable, "tableGroup targetTable is empty.");
@@ -64,8 +66,8 @@ public class TableGroupChecker extends AbstractChecker {
         tableGroup.setName(ConfigConstant.TABLE_GROUP);
         tableGroup.setType(ConfigConstant.TABLE_GROUP);
         tableGroup.setMappingId(mappingId);
-        tableGroup.setSourceTable(getTable(mapping.getSourceConnectorId(), sourceTable));
-        tableGroup.setTargetTable(getTable(mapping.getTargetConnectorId(), targetTable));
+        tableGroup.setSourceTable(getTable(mapping.getSourceConnectorId(), sourceTable, sourceTablePK));
+        tableGroup.setTargetTable(getTable(mapping.getTargetConnectorId(), targetTable, targetTablePK));
         tableGroup.setParams(new HashMap<>());
 
         // 修改基本配置
@@ -130,10 +132,10 @@ public class TableGroupChecker extends AbstractChecker {
         checker.dealIncrementStrategy(mapping, tableGroup);
     }
 
-    private Table getTable(String connectorId, String tableName) {
+    private Table getTable(String connectorId, String tableName, String primaryKey) {
         MetaInfo metaInfo = manager.getMetaInfo(connectorId, tableName);
         Assert.notNull(metaInfo, "无法获取连接器表信息.");
-        return new Table(tableName, metaInfo.getTableType(), metaInfo.getColumn());
+        return new Table(tableName, metaInfo.getTableType(), primaryKey, metaInfo.getColumn());
     }
 
     private void checkRepeatedTable(String mappingId, String sourceTable, String targetTable) {

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java

@@ -211,9 +211,10 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem
         map.put(delete, buildSql(delete, commandConfig, schema, null));
 
         // 获取查询数据行是否存在
-        String pk = findOriginalTablePrimaryKey(commandConfig, quotation);
-        StringBuilder queryCount = new StringBuilder().append("SELECT COUNT(1) FROM ").append(schema).append(quotation).append(
-                commandConfig.getTable().getName()).append(quotation).append(" WHERE ").append(pk).append(" = ?");
+        String tableName = commandConfig.getTable().getName();
+        String pk = findOriginalTablePrimaryKey(commandConfig);
+        StringBuilder queryCount = new StringBuilder("SELECT COUNT(1) FROM ").append(schema).append(quotation).append(tableName).append(quotation)
+                .append(" WHERE ").append(quotation).append(pk).append(quotation).append(" = ?");
         String queryCountExist = ConnectorConstant.OPERTION_QUERY_COUNT_EXIST;
         map.put(queryCountExist, queryCount.toString());
         return map;
@@ -270,28 +271,29 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem
      * 返回主键名称
      *
      * @param commandConfig
-     * @param quotation
      * @return
      */
-    protected String findOriginalTablePrimaryKey(CommandConfig commandConfig, String quotation) {
+    protected String findOriginalTablePrimaryKey(CommandConfig commandConfig) {
+        // 获取自定义主键
+        String pk = commandConfig.getTable().getPrimaryKey();
+        if (StringUtil.isNotBlank(pk)) {
+            return pk;
+        }
+
+        // 获取表原始主键
         Table table = commandConfig.getOriginalTable();
         if (null != table) {
             List<Field> column = table.getColumn();
             if (!CollectionUtils.isEmpty(column)) {
                 for (Field c : column) {
                     if (c.isPk()) {
-                        return new StringBuilder(quotation).append(c.getName()).append(quotation).toString();
+                        return c.getName();
                     }
                 }
             }
         }
 
-        DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
-        if (StringUtil.isBlank(cfg.getPrimaryKey())) {
-            throw new ConnectorException("Table primary key can not be empty.");
-        }
-
-        return new StringBuilder(quotation).append(cfg.getPrimaryKey()).append(quotation).toString();
+        throw new ConnectorException(String.format("The primary key of table '%s' is null.", commandConfig.getTable().getName()));
     }
 
     /**
@@ -382,13 +384,13 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem
      */
     protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
         String table = commandConfig.getTable().getName();
-        String pk = findOriginalTablePrimaryKey(commandConfig, quotation);
+        String pk = findOriginalTablePrimaryKey(commandConfig);
         StringBuilder queryCount = new StringBuilder();
         queryCount.append("SELECT COUNT(1) FROM (SELECT 1 FROM ").append(schema).append(quotation).append(table).append(quotation);
         if (StringUtil.isNotBlank(queryFilterSql)) {
             queryCount.append(queryFilterSql);
         }
-        queryCount.append(" GROUP BY ").append(pk).append(") DBSYNCER_T");
+        queryCount.append(" GROUP BY ").append(quotation).append(pk).append(quotation).append(") DBSYNCER_T");
         return queryCount.toString();
     }
 
@@ -508,7 +510,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem
             throw new ConnectorException("Table name can not be empty.");
         }
         if (StringUtil.isBlank(pk)) {
-            pk = findOriginalTablePrimaryKey(commandConfig, "");
+            pk = findOriginalTablePrimaryKey(commandConfig);
         }
 
         SqlBuilderConfig config = new SqlBuilderConfig(this, schema, tableName, pk, fields, queryFilterSQL, buildSqlWithQuotation());

dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Table.java

@@ -21,6 +21,11 @@ public class Table {
      */
     private String type;
 
+    /**
+     * 主键
+     */
+    private String primaryKey;
+
     /**
      * 属性字段
      * 格式：[{"name":"ID","typeName":"INT","type":"4"},{"name":"NAME","typeName":"VARCHAR","type":"12"}]
@@ -34,18 +39,17 @@ public class Table {
     }
 
     public Table(String name) {
-        this.name = name;
-        this.type = TableTypeEnum.TABLE.getCode();
+        this(name, TableTypeEnum.TABLE.getCode());
     }
 
     public Table(String name, String type) {
-        this.name = name;
-        this.type = type;
+        this(name, type, null, null);
     }
 
-    public Table(String name, String type, List<Field> column) {
+    public Table(String name, String type, String primaryKey, List<Field> column) {
         this.name = name;
         this.type = type;
+        this.primaryKey = primaryKey;
         this.column = column;
     }
 
@@ -67,6 +71,14 @@ public class Table {
         return this;
     }
 
+    public String getPrimaryKey() {
+        return primaryKey;
+    }
+
+    public void setPrimaryKey(String primaryKey) {
+        this.primaryKey = primaryKey;
+    }
+
     public List<Field> getColumn() {
         return column;
     }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java

@@ -58,8 +58,4 @@ public final class OracleConnector extends AbstractDatabaseConnector {
         return String.format("SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = '%s' AND TABLE_NAME = '%s'", cfg.getUsername().toUpperCase(), table.getName());
     }
 
-    @Override
-    protected String getSchema(DatabaseConfig config) {
-        return config.getUsername().toUpperCase();
-    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java

@@ -1,6 +1,5 @@
 package org.dbsyncer.connector.sql;
 
-import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.config.ReaderConfig;
 import org.dbsyncer.connector.constant.DatabaseConstant;
 import org.dbsyncer.connector.model.PageSql;
@@ -29,8 +28,4 @@ public final class DQLOracleConnector extends AbstractDQLConnector {
         return "select 1 from dual";
     }
 
-    @Override
-    protected String getSchema(DatabaseConfig config) {
-        return config.getUsername().toUpperCase();
-    }
 }

dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java

@@ -143,7 +143,7 @@ public class IncrementPuller extends AbstractPuller implements ScheduledTaskJob
             AbstractQuartzExtractor extractor = listener.getExtractor(ListenerTypeEnum.TIMING, connectorConfig.getConnectorType(), AbstractQuartzExtractor.class);
             extractor.setCommands(list.stream().map(t -> {
                 Picker picker = new Picker(t.getFieldMapping());
-                return new TableGroupCommand(picker.getSourcePrimaryKeyName(connectorConfig), t.getCommand());
+                return new TableGroupCommand(picker.getSourcePrimaryKeyName(t.getSourceTable()), t.getCommand());
             }).collect(Collectors.toList()));
             setExtractorConfig(extractor, connectorConfig, listenerConfig, meta.getSnapshot(), new QuartzListener(mapping, list));
             return extractor;

dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserFactory.java

@@ -27,13 +27,7 @@ import org.dbsyncer.parser.enums.ConvertEnum;
 import org.dbsyncer.parser.event.FullRefreshEvent;
 import org.dbsyncer.parser.logger.LogService;
 import org.dbsyncer.parser.logger.LogType;
-import org.dbsyncer.parser.model.BatchWriter;
-import org.dbsyncer.parser.model.Connector;
-import org.dbsyncer.parser.model.FieldMapping;
-import org.dbsyncer.parser.model.Mapping;
-import org.dbsyncer.parser.model.Picker;
-import org.dbsyncer.parser.model.TableGroup;
-import org.dbsyncer.parser.model.Task;
+import org.dbsyncer.parser.model.*;
 import org.dbsyncer.parser.strategy.FlushStrategy;
 import org.dbsyncer.parser.strategy.ParserStrategy;
 import org.dbsyncer.parser.util.ConvertUtil;
@@ -153,8 +147,8 @@ public class ParserFactory implements Parser {
         AbstractConnectorConfig tConnConfig = getConnectorConfig(mapping.getTargetConnectorId());
         Table sourceTable = tableGroup.getSourceTable();
         Table targetTable = tableGroup.getTargetTable();
-        Table sTable = new Table(sourceTable.getName(), sourceTable.getType(), new ArrayList<>());
-        Table tTable = new Table(targetTable.getName(), targetTable.getType(), new ArrayList<>());
+        Table sTable = new Table(sourceTable.getName(), sourceTable.getType(), sourceTable.getPrimaryKey(), new ArrayList<>());
+        Table tTable = new Table(targetTable.getName(), targetTable.getType(), targetTable.getPrimaryKey(), new ArrayList<>());
         List<FieldMapping> fieldMapping = tableGroup.getFieldMapping();
         if (!CollectionUtils.isEmpty(fieldMapping)) {
             fieldMapping.forEach(m -> {
@@ -253,7 +247,7 @@ public class ParserFactory implements Parser {
         Assert.notEmpty(fieldMapping, String.format("数据源表[%s]同步到目标源表[%s], 映射关系不能为空.", sTableName, tTableName));
         // 获取同步字段
         Picker picker = new Picker(fieldMapping);
-        String pk = picker.getSourcePrimaryKeyName(sConfig);
+        String pk = picker.getSourcePrimaryKeyName(tableGroup.getSourceTable());
 
         int pageSize = mapping.getReadNum();
         int batchSize = mapping.getBatchNum();

dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Picker.java

@@ -1,8 +1,8 @@
 package org.dbsyncer.parser.model;
 
-import org.dbsyncer.common.model.AbstractConnectorConfig;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.connector.model.Field;
+import org.dbsyncer.connector.model.Table;
 import org.springframework.util.Assert;
 
 import java.util.ArrayList;
@@ -57,14 +57,14 @@ public class Picker {
         }
     }
 
-    public String getSourcePrimaryKeyName(AbstractConnectorConfig config) {
+    public String getSourcePrimaryKeyName(Table sourceTable) {
         for (Field f : sourceFields) {
             if (null != f && f.isPk()) {
                 return f.getName();
             }
         }
 
-        String primaryKey = config.getPrimaryKey();
+        String primaryKey = sourceTable.getPrimaryKey();
         Assert.hasText(primaryKey, "主键为空");
         return primaryKey;
     }

dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/TableGroupController.java

@@ -44,7 +44,7 @@ public class TableGroupController extends BaseController {
             Map<String, String> params = getParams(request);
             return RestResult.restSuccess(tableGroupService.add(params));
         } catch (Exception e) {
-            logger.error(e.getLocalizedMessage(), e.getClass());
+            logger.error(e.getLocalizedMessage(), e);
             return RestResult.restFail(e.getMessage());
         }
     }

dbsyncer-web/src/main/resources/public/connector/addDqlOracle.html

@@ -47,6 +47,13 @@
             <textarea name="url" class="form-control dbsyncer_textarea_resize_none" maxlength="512" dbsyncer-valid="require" rows="5" th:text="${connector?.config?.url}?:'jdbc:oracle:thin:@127.0.0.1:1521:ORCL'"></textarea>
         </div>
     </div>
+    <div class="form-group">
+        <label class="col-sm-2 control-label">架构名 </label>
+        <div class="col-sm-4">
+            <input class="form-control" name="schema" type="text" maxlength="32" th:value="${connector?.config?.schema}"/>
+        </div>
+        <div class="col-sm-6"></div>
+    </div>
     <div class="form-group">
         <label class="col-sm-2 control-label">驱动 </label>
         <div class="col-sm-10">

dbsyncer-web/src/main/resources/public/connector/addMysql.html

@@ -22,14 +22,6 @@
                       th:text="${connector?.config?.url} ?: 'jdbc:mysql://127.0.0.1:3306/test?rewriteBatchedStatements=true&useUnicode=true&amp;characterEncoding=UTF8&amp;serverTimezone=Asia/Shanghai&amp;useSSL=false&amp;verifyServerCertificate=false&amp;autoReconnect=true&amp;failOverReadOnly=false'"></textarea>
         </div>
     </div>
-    <div class="form-group">
-        <label class="col-sm-2 control-label">主键 <i class="fa fa-question-circle fa_gray" aria-hidden="true" title="如果使用的表没有主键，可以自定义主键（大小写必须一致，添加表映射关系时，主键不能为空）。"></i></label>
-        <div class="col-sm-4">
-            <input class="form-control" name="primaryKey" type="text" maxlength="32" placeholder="ID"
-                   th:value="${connector?.config?.primaryKey}"/>
-        </div>
-        <div class="col-sm-6"></div>
-    </div>
     <div class="form-group">
         <label class="col-sm-2 control-label">驱动 </label>
         <div class="col-sm-10">

dbsyncer-web/src/main/resources/public/connector/addOracle.html

@@ -23,10 +23,9 @@
         </div>
     </div>
     <div class="form-group">
-        <label class="col-sm-2 control-label">主键 <i class="fa fa-question-circle fa_gray" aria-hidden="true" title="如果使用的表没有主键，可以自定义主键（大小写必须一致，添加表映射关系时，主键不能为空）。"></i></label>
+        <label class="col-sm-2 control-label">架构名 </label>
         <div class="col-sm-4">
-            <input class="form-control" name="primaryKey" type="text" maxlength="32" placeholder="ID"
-                   th:value="${connector?.config?.primaryKey}"/>
+            <input class="form-control" name="schema" type="text" maxlength="32" th:value="${connector?.config?.schema}"/>
         </div>
         <div class="col-sm-6"></div>
     </div>

dbsyncer-web/src/main/resources/public/connector/addPostgreSQL.html

@@ -29,11 +29,7 @@
             <input class="form-control" dbsyncer-valid="require" maxlength="32" name="schema" placeholder="public"
                    th:value="${connector?.config?.schema} ?: 'public'" type="text"/>
         </div>
-        <label class="col-sm-2 control-label">主键 <i class="fa fa-question-circle fa_gray" aria-hidden="true" title="如果使用的表没有主键，可以自定义主键（大小写必须一致，添加表映射关系时，主键不能为空）。"></i></label>
-        <div class="col-sm-4">
-            <input class="form-control" name="primaryKey" type="text" maxlength="32" placeholder="ID"
-                   th:value="${connector?.config?.primaryKey}"/>
-        </div>
+        <div class="col-sm-6"></div>
     </div>
     <div class="form-group">
         <label class="col-sm-2 control-label">删除Slot <i aria-hidden="true" class="fa fa-question-circle fa_gray" title="增量同步，停止驱动自动删除Slot"></i></label>

dbsyncer-web/src/main/resources/public/connector/addSqlServer.html

@@ -25,11 +25,7 @@
             <input class="form-control" name="schema" type="text" maxlength="32" dbsyncer-valid="require"
                    placeholder="dbo" th:value="${connector?.config?.schema} ?: 'dbo'"/>
         </div>
-        <label class="col-sm-2 control-label">主键 <i class="fa fa-question-circle fa_gray" aria-hidden="true" title="如果使用的表没有主键，可以自定义主键（大小写必须一致，添加表映射关系时，主键不能为空）。"></i></label>
-        <div class="col-sm-4">
-            <input class="form-control" name="primaryKey" type="text" maxlength="32" placeholder="ID"
-                   th:value="${connector?.config?.primaryKey}"/>
-        </div>
+        <div class="col-sm-6"></div>
     </div>
     <div class="form-group">
         <label class="col-sm-2 control-label">驱动 </label>

dbsyncer-web/src/main/resources/public/mapping/editTable.html

@@ -31,6 +31,28 @@
             </div>
         </div>
     </div>
+    <div class="row">
+        <!-- 数据源配置 -->
+        <div class="col-md-5">
+            <label class="col-sm-3 control-label text-right">标记主键 <i class="fa fa-question-circle fa_gray" aria-hidden="true" title="如果使用的数据源表或视图没有主键，可以自定义主键（大小写必须一致）"></i></label>
+            <div class="col-sm-9">
+                <input id="sourceTablePK" class="form-control" type="text" />
+            </div>
+        </div>
+        <!-- 中间图标 -->
+        <div class="col-md-2 text-center">
+            <span class="fa fa-angle-double-right fa-2x"></span>
+        </div>
+        <!-- 目标源配置 -->
+        <div class="col-md-5">
+            <div class="form-group">
+                <label class="col-sm-3 control-label text-right">标记主键 <i class="fa fa-question-circle fa_gray" aria-hidden="true" title="如果使用的目标源表或视图没有主键，可以自定义主键（大小写必须一致）"></i></label>
+                <div class="col-sm-9">
+                    <input id="targetTablePK" class="form-control" type="text" />
+                </div>
+            </div>
+        </div>
+    </div>
 
     <div class="form-group">
         <div class="row">

dbsyncer-web/src/main/resources/static/js/mapping/edit.js

@@ -154,6 +154,8 @@ function bindMappingTableGroupAddClick($sourceSelect, $targetSelect) {
         }
         m.sourceTable = m.sourceTable.join('|');
         m.targetTable = m.targetTable.join('|');
+        m.sourceTablePK = $("#sourceTablePK").val();
+        m.targetTablePK = $("#targetTablePK").val();
 
         doPoster("/tableGroup/add", m, function (data) {
             if (data.success == true) {