优化慢查询SQL

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java

@@ -94,10 +94,10 @@ public abstract class AbstractDatabaseConnector implements Database {
         map.put(query, buildSql(query, table, null, queryFilterSql));
 
         // 获取查询总数SQL
-        StringBuilder queryCount = new StringBuilder();
         String quotation = buildSqlWithQuotation();
         String pk = DatabaseUtil.findTablePrimaryKey(commandConfig.getOriginalTable(), quotation);
-        queryCount.append("SELECT COUNT(").append(pk).append(") FROM ").append(quotation).append(table.getName()).append(quotation);
+        StringBuilder queryCount = new StringBuilder();
+        queryCount.append("SELECT COUNT(1) FROM (SELECT 1 FROM ").append(quotation).append(table.getName()).append(quotation).append(" GROUP BY ").append(quotation).append(pk).append(quotation).append(") _T");
         if (StringUtils.isNotBlank(queryFilterSql)) {
             queryCount.append(queryFilterSql);
         }

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/MysqlStorageServiceImpl.java

@@ -233,9 +233,8 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
     }
 
     private String buildQueryCountSql(Query query, Executor executor, List<Object> args) {
-        StringBuilder sql = new StringBuilder("SELECT COUNT(ID) FROM (").append(executor.getQuery());
+        StringBuilder sql = new StringBuilder("SELECT COUNT(1) FROM ").append(executor.getTable());
         buildQuerySqlWithParams(query, args, sql);
-        sql.append(") _T");
         return sql.toString();
     }
 
@@ -249,7 +248,7 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
                     sql.append(" AND ");
                 }
                 // name=?
-                sql.append(p.getKey()).append(p.isHighlighter() ? " like ?" : "=?");
+                sql.append(p.getKey()).append(p.isHighlighter() ? " LIKE ?" : "=?");
                 args.add(p.isHighlighter() ? new StringBuilder("%").append(p.getValue()).append("%") : p.getValue());
                 flag.compareAndSet(false, true);
             });
@@ -342,7 +341,7 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
         String insert = SqlBuilderEnum.INSERT.getSqlBuilder().buildSql(config);
         String update = SqlBuilderEnum.UPDATE.getSqlBuilder().buildSql(config);
         String delete = SqlBuilderEnum.DELETE.getSqlBuilder().buildSql(config);
-        executor.setQuery(query).setInsert(insert).setUpdate(update).setDelete(delete);
+        executor.setTable(table).setQuery(query).setInsert(insert).setUpdate(update).setDelete(delete);
     }
 
     private String readSql(String filePath) {
@@ -412,6 +411,7 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
     }
 
     class Executor {
+        private String table;
         private String query;
         private String insert;
         private String update;
@@ -432,6 +432,15 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
             this.orderByUpdateTime = orderByUpdateTime;
         }
 
+        public Executor setTable(String table) {
+            this.table = table;
+            return this;
+        }
+
+        public String getTable() {
+            return table;
+        }
+
         public String getQuery() {
             return query;
         }

dbsyncer-storage/src/main/resources/dbsyncer_config.sql

@@ -5,5 +5,6 @@ CREATE TABLE `dbsyncer_config`  (
   `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
   `UPDATE_TIME` bigint(0) NOT NULL COMMENT '修改时间',
   `JSON` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '配置信息',
-  PRIMARY KEY (`ID`) USING BTREE
+  PRIMARY KEY (`ID`) USING BTREE,
+  INDEX `IDX_TYPE_UPDATE_CREATE_TIME`(`TYPE`, `UPDATE_TIME`, `CREATE_TIME`) USING BTREE
 ) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '配置信息表' ROW_FORMAT = Dynamic;

dbsyncer-storage/src/main/resources/dbsyncer_data.sql

@@ -5,5 +5,8 @@ CREATE TABLE `dbsyncer_data` (
   `ERROR` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT '异常信息',
   `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
   `JSON` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '同步数据',
-  PRIMARY KEY (`ID`) USING BTREE
+  PRIMARY KEY (`ID`) USING BTREE,
+  INDEX `IDX_SUCCESS`(`SUCCESS`) USING BTREE,
+  INDEX `IDX_SUCCESS_CREATE_TIME`(`SUCCESS`, `CREATE_TIME`) USING BTREE,
+  FULLTEXT INDEX `FULL_TEXT_ERROR`(`ERROR`)
 ) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '同步数据表' ROW_FORMAT = Dynamic;

dbsyncer-storage/src/main/resources/dbsyncer_log.sql

@@ -3,5 +3,6 @@ CREATE TABLE `dbsyncer_log`  (
   `TYPE` varchar(24) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '连接器、映射配置、表映射、元信息、系统日志',
   `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
   `JSON` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '日志信息',
-  PRIMARY KEY (`ID`) USING BTREE
+  PRIMARY KEY (`ID`) USING BTREE,
+  FULLTEXT INDEX `FULL_TEXT_JSON`(`JSON`)
 ) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '操作日志表' ROW_FORMAT = Dynamic;

dbsyncer-web/src/main/resources/static/js/monitor/index.js

@@ -130,7 +130,7 @@ function showData($dataList, arr, append){
             html += '<td>' + (start + i + 1) + '</td>';
             html += '<td>' + arr[i].event + '</td>';
             html += '<td>' + (arr[i].success ? '<span class="label label-success">成功</span>' : '<span class="label label-warning">失败</span>') + '</td>';
-            html += '<td style="max-width:100px;" class="dbsyncer_over_hidden">' + arr[i].error + '</td>';
+            html += '<td title="' + arr[i].error + ''" style="max-width:100px;" class="dbsyncer_over_hidden">' + arr[i].error + '</td>';
             html += '<td>' + formatDate(arr[i].createTime) + '</td>';
             html += '<td><a json=' + arr[i].json + ' href="javascript:;" class="label label-info queryData">查看数据</a></td>';
             html += '</tr>';