优化count

dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java

@@ -167,27 +167,15 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
     @Override
     public Map<String, String> getSourceCommand(CommandConfig commandConfig) {
         // 获取过滤SQL
-        List<Filter> filter = commandConfig.getFilter();
-        String queryFilterSql = getQueryFilterSql(filter);
+        final String queryFilterSql = getQueryFilterSql(commandConfig.getFilter());
+        final String quotation = buildSqlWithQuotation();
 
         // 获取查询SQL
-        Table table = commandConfig.getTable();
         Map<String, String> map = new HashMap<>();
-
-        String query = ConnectorConstant.OPERTION_QUERY;
-        String quotation = buildSqlWithQuotation();
         String schema = getSchema((DatabaseConfig) commandConfig.getConnectorConfig(), quotation);
-        map.put(query, buildSql(query, commandConfig, schema, queryFilterSql));
-
+        map.put(ConnectorConstant.OPERTION_QUERY, buildSql(ConnectorConstant.OPERTION_QUERY, commandConfig, schema, queryFilterSql));
         // 获取查询总数SQL
-        String pk = findOriginalTablePrimaryKey(commandConfig, quotation);
-        StringBuilder queryCount = new StringBuilder();
-        queryCount.append("SELECT COUNT(1) FROM (SELECT 1 FROM ").append(schema).append(quotation).append(table.getName()).append(quotation);
-        if (StringUtil.isNotBlank(queryFilterSql)) {
-            queryCount.append(queryFilterSql);
-        }
-        queryCount.append(" GROUP BY ").append(pk).append(") DBSYNCER_T");
-        map.put(ConnectorConstant.OPERTION_QUERY_COUNT, queryCount.toString());
+        map.put(ConnectorConstant.OPERTION_QUERY_COUNT, getQueryCountSql(commandConfig, schema, quotation, queryFilterSql));
         return map;
     }
 
@@ -264,6 +252,27 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector
         return Collections.EMPTY_LIST;
     }
 
+    /**
+     * 获取查询总数SQL
+     *
+     * @param commandConfig
+     * @param schema
+     * @param quotation
+     * @param queryFilterSql
+     * @return
+     */
+    protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
+        String table = commandConfig.getTable().getName();
+        String pk = findOriginalTablePrimaryKey(commandConfig, quotation);
+        StringBuilder queryCount = new StringBuilder();
+        queryCount.append("SELECT COUNT(1) FROM (SELECT 1 FROM ").append(schema).append(quotation).append(table).append(quotation);
+        if (StringUtil.isNotBlank(queryFilterSql)) {
+            queryCount.append(queryFilterSql);
+        }
+        queryCount.append(" GROUP BY ").append(pk).append(") DBSYNCER_T");
+        return queryCount.toString();
+    }
+
     /**
      * 获取查询条件SQL
      *

dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java

@@ -1,6 +1,9 @@
 package org.dbsyncer.connector.oracle;
 
 import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.config.CommandConfig;
+import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.model.PageSql;
 import org.dbsyncer.connector.model.Table;
 import org.dbsyncer.connector.constant.DatabaseConstant;
@@ -43,4 +46,17 @@ public final class OracleConnector extends AbstractDatabaseConnector {
     protected String getValidationQuery() {
         return "select 1 from dual";
     }
+
+    @Override
+    protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
+        // 有过滤条件，走默认方式
+        if (StringUtil.isNotBlank(queryFilterSql)) {
+            return super.getQueryCountSql(commandConfig, schema, quotation, queryFilterSql);
+        }
+
+        // 从系统表查询
+        final String table = commandConfig.getTable().getName();
+        DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
+        return String.format("SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = '%s' AND TABLE_NAME = '%s'", cfg.getUsername().toUpperCase(), table);
+    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java

@@ -1,6 +1,8 @@
 package org.dbsyncer.connector.postgresql;
 
 import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.config.CommandConfig;
 import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.model.PageSql;
 import org.dbsyncer.connector.model.Table;
@@ -46,4 +48,17 @@ public final class PostgreSQLConnector extends AbstractDatabaseConnector {
     protected String buildSqlWithQuotation() {
         return "\"";
     }
-}
+
+    @Override
+    protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
+        // 有过滤条件，走默认方式
+        if (StringUtil.isNotBlank(queryFilterSql)) {
+            return super.getQueryCountSql(commandConfig, schema, quotation, queryFilterSql);
+        }
+
+        // 从系统表查询
+        final String table = commandConfig.getTable().getName();
+        DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
+        return String.format("SELECT N_LIVE_TUP FROM PG_STAT_USER_TABLES WHERE SCHEMANAME='%s' AND RELNAME='%s'", cfg.getSchema(), table);
+    }
+}

dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java

@@ -3,16 +3,13 @@ package org.dbsyncer.connector.sqlserver;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.config.CommandConfig;
 import org.dbsyncer.connector.config.DatabaseConfig;
-import org.dbsyncer.connector.constant.ConnectorConstant;
 import org.dbsyncer.connector.constant.DatabaseConstant;
 import org.dbsyncer.connector.database.AbstractDatabaseConnector;
 import org.dbsyncer.connector.database.DatabaseConnectorMapper;
 import org.dbsyncer.connector.model.PageSql;
 import org.dbsyncer.connector.model.Table;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 public final class SqlServerConnector extends AbstractDatabaseConnector {
 
@@ -29,34 +26,19 @@ public final class SqlServerConnector extends AbstractDatabaseConnector {
 
     @Override
     public Object[] getPageArgs(int pageIndex, int pageSize) {
-        return new Object[]{(pageIndex - 1) * pageSize + 1, pageIndex * pageSize};
+        return new Object[] {(pageIndex - 1) * pageSize + 1, pageIndex * pageSize};
     }
 
     @Override
-    public Map<String, String> getSourceCommand(CommandConfig commandConfig) {
-        // 获取过滤SQL
-        String queryFilterSql = this.getQueryFilterSql(commandConfig.getFilter());
-
-        // 获取查询SQL
-        Table table = commandConfig.getTable();
-        String schema = getSchema((DatabaseConfig) commandConfig.getConnectorConfig(), buildSqlWithQuotation());
-        Map<String, String> map = new HashMap<>();
-
-        String query = ConnectorConstant.OPERTION_QUERY;
-        map.put(query, this.buildSql(query, commandConfig, schema, queryFilterSql));
-
-        // 获取查询总数SQL
-        StringBuilder queryCount = new StringBuilder();
+    protected String getQueryCountSql(CommandConfig commandConfig, String schema, String quotation, String queryFilterSql) {
+        // 有过滤条件，走默认方式
         if (StringUtil.isNotBlank(queryFilterSql)) {
-            queryCount.append("SELECT COUNT(*) FROM ").append(schema).append(table.getName()).append(queryFilterSql);
-        } else {
-            DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
-            // 从存储过程查询（定时更新总数，可能存在误差）
-            queryCount.append("SELECT ROWS FROM SYSINDEXES WHERE ID = OBJECT_ID('").append(cfg.getSchema()).append(".").append(table.getName()).append(
-                    "') AND INDID IN (0, 1)");
+            return super.getQueryCountSql(commandConfig, schema, quotation, queryFilterSql);
         }
-        map.put(ConnectorConstant.OPERTION_QUERY_COUNT, queryCount.toString());
-        return map;
-    }
 
+        String table = commandConfig.getTable().getName();
+        DatabaseConfig cfg = (DatabaseConfig) commandConfig.getConnectorConfig();
+        // 从存储过程查询（定时更新总数，可能存在误差）
+        return String.format("SELECT ROWS FROM SYSINDEXES WHERE ID = OBJECT_ID('%s.%s') AND INDID IN (0, 1)", cfg.getSchema(), table);
+    }
 }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/DatabaseUtil.java

@@ -6,6 +6,9 @@ import org.dbsyncer.connector.ConnectorException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.regex.Matcher;
+
+import static java.util.regex.Pattern.compile;
 
 public abstract class DatabaseUtil {
 
@@ -33,4 +36,19 @@ public abstract class DatabaseUtil {
         }
     }
 
+    public static String getDatabaseName(String url) {
+        Matcher matcher = compile("(//)(?!(\\?)).+?(\\?)").matcher(url);
+        while (matcher.find()) {
+            url = matcher.group(0);
+            break;
+        }
+        int s = url.lastIndexOf("/");
+        int e = url.lastIndexOf("?");
+        if (s > 0 && e > 0) {
+            return StringUtil.substring(url, s + 1, e);
+        }
+
+        throw new ConnectorException("database is invalid");
+    }
+
 }

dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MysqlExtractor.java

@@ -6,6 +6,7 @@ import org.dbsyncer.common.event.RowChangedEvent;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.config.DatabaseConfig;
 import org.dbsyncer.connector.constant.ConnectorConstant;
+import org.dbsyncer.connector.util.DatabaseUtil;
 import org.dbsyncer.listener.AbstractDatabaseExtractor;
 import org.dbsyncer.listener.ListenerException;
 import org.dbsyncer.listener.config.Host;
@@ -86,7 +87,7 @@ public class MysqlExtractor extends AbstractDatabaseExtractor {
         if (StringUtil.isBlank(config.getUrl())) {
             throw new ListenerException("url is invalid");
         }
-        database = readDatabaseName(config.getUrl());
+        database = DatabaseUtil.getDatabaseName(config.getUrl());
         cluster = readNodes(config.getUrl());
         Assert.notEmpty(cluster, "Mysql连接地址有误.");
 
@@ -104,21 +105,6 @@ public class MysqlExtractor extends AbstractDatabaseExtractor {
         client.connect();
     }
 
-    private String readDatabaseName(String url) {
-        Matcher matcher = compile("(//)(?!(\\?)).+?(\\?)").matcher(url);
-        while (matcher.find()) {
-            url = matcher.group(0);
-            break;
-        }
-        int s = url.lastIndexOf("/");
-        int e = url.lastIndexOf("?");
-        if (s > 0 && e > 0) {
-            return StringUtil.substring(url, s + 1, e);
-        }
-
-        throw new ListenerException("database is invalid");
-    }
-
     private List<Host> readNodes(String url) {
         Matcher matcher = compile("(//)(?!(/)).+?(/)").matcher(url);
         while (matcher.find()) {

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/MysqlStorageServiceImpl.java

@@ -13,10 +13,10 @@ import org.dbsyncer.connector.constant.ConnectorConstant;
 import org.dbsyncer.connector.constant.DatabaseConstant;
 import org.dbsyncer.connector.database.Database;
 import org.dbsyncer.connector.database.DatabaseConnectorMapper;
-import org.dbsyncer.connector.database.ds.SimpleConnection;
 import org.dbsyncer.connector.enums.ConnectorEnum;
 import org.dbsyncer.connector.enums.SetterEnum;
 import org.dbsyncer.connector.enums.SqlBuilderEnum;
+import org.dbsyncer.connector.util.DatabaseUtil;
 import org.dbsyncer.storage.AbstractStorageService;
 import org.dbsyncer.storage.StorageException;
 import org.dbsyncer.storage.constant.ConfigConstant;
@@ -34,8 +34,6 @@ import org.springframework.util.Assert;
 
 import javax.annotation.PostConstruct;
 import java.io.*;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -87,27 +85,7 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
         config.setConnectorType(ConnectorEnum.MYSQL.getType());
         connectorMapper = (DatabaseConnectorMapper) connectorFactory.connect(config);
         connector = (Database) connectorFactory.getConnector(connectorMapper);
-
-        // 获取数据库名称
-        database = connectorMapper.execute(databaseTemplate -> {
-            Connection conn = databaseTemplate.getConnection();
-            DatabaseMetaData metaData = conn.getMetaData();
-            String driverVersion = metaData.getDriverVersion();
-            String databaseProductVersion = metaData.getDatabaseProductVersion();
-            boolean driverThanMysql8 = StringUtil.startsWith(driverVersion, "mysql-connector-java-8");
-            boolean dbThanMysql8 = StringUtil.startsWith(databaseProductVersion, "8");
-            Assert.isTrue(driverThanMysql8 == dbThanMysql8, String.format("当前驱动%s和数据库%s版本不一致.", driverVersion, databaseProductVersion));
-
-            if(conn instanceof SimpleConnection){
-                SimpleConnection simpleConnection = (SimpleConnection) conn;
-                conn = simpleConnection.getConnection();
-            }
-            Class clazz = dbThanMysql8 ? conn.getClass() : conn.getClass().getSuperclass();
-            java.lang.reflect.Field field = clazz.getDeclaredField("database");
-            field.setAccessible(true);
-            Object value = field.get(conn);
-            return String.valueOf(value);
-        });
+        database = DatabaseUtil.getDatabaseName(config.getUrl());
 
         // 初始化表
         initTable();

dbsyncer-web/src/main/resources/public/connector/addDqlPostgreSQL.html

@@ -46,7 +46,7 @@
         <label class="col-sm-2 control-label">URL <strong class="driverVerifcateRequired">*</strong></label>
         <div class="col-sm-10">
             <textarea class="form-control dbsyncer_textarea_resize_none" dbsyncer-valid="require" maxlength="1024" name="url" rows="5"
-                      th:text="${connector?.config?.url} ?: 'jdbc:postgresql://127.0.0.1:5432/test'"></textarea>
+                      th:text="${connector?.config?.url} ?: 'jdbc:postgresql://127.0.0.1:5432/postgres'"></textarea>
         </div>
     </div>
     <div class="form-group">

dbsyncer-web/src/main/resources/public/connector/addPostgreSQL.html

@@ -20,7 +20,7 @@
         <div class="col-sm-10">
             <textarea class="form-control dbsyncer_textarea_resize_none" dbsyncer-valid="require" maxlength="1024"
                       name="url" rows="5"
-                      th:text="${connector?.config?.url} ?: 'jdbc:postgresql://127.0.0.1:5432/test'"></textarea>
+                      th:text="${connector?.config?.url} ?: 'jdbc:postgresql://127.0.0.1:5432/postgres'"></textarea>
         </div>
     </div>
     <div class="form-group">