废弃存储binlog

dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageEnum.java

@@ -22,6 +22,7 @@ public enum StorageEnum {
     /**
      * Binlog：缓存队列数据
      */
+    @Deprecated
     BINLOG("binlog");
 
     private String type;

dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/BinlogStrategy.java

@@ -1,21 +0,0 @@
-package org.dbsyncer.storage.strategy.impl;
-
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.strategy.Strategy;
-import org.springframework.stereotype.Component;
-
-/**
- * 缓存队列数据
- *
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/15 22:39
- */
-@Component
-public class BinlogStrategy implements Strategy {
-
-    @Override
-    public String createSharding(String separator, String collectionId) {
-        return StorageEnum.BINLOG.getType();
-    }
-}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/DiskStorageServiceImpl.java

@@ -1,8 +1,13 @@
 package org.dbsyncer.storage.support;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.highlight.Highlighter;
 import org.apache.lucene.search.highlight.QueryScorer;
 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
@@ -24,7 +29,11 @@ import org.dbsyncer.storage.util.DocumentUtil;
 import javax.annotation.PostConstruct;
 import java.io.File;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -46,10 +55,11 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
 
     @PostConstruct
     private void init() {
+        // 废弃binlog
+        FileUtils.deleteQuietly(new File(PATH + "binlog"));
         // 创建配置和日志索引shard
         getShard(getSharding(StorageEnum.CONFIG, null));
         getShard(getSharding(StorageEnum.LOG, null));
-        getShard(getSharding(StorageEnum.BINLOG, null));
     }
 
     @Override
@@ -211,9 +221,6 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
                 case CONFIG:
                     docs.add(DocumentUtil.convertConfig2Doc(r));
                     break;
-                case BINLOG:
-                    docs.add(DocumentUtil.convertBinlog2Doc(r));
-                    break;
                 default:
                     break;
             }

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/MysqlStorageServiceImpl.java

@@ -65,7 +65,6 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
     private static final String SHOW_DATA_TABLE = "show tables where Tables_in_%s like \"%s\"";
     private static final String DROP_TABLE = "DROP TABLE %s";
     private static final String TRUNCATE_TABLE = "TRUNCATE TABLE %s";
-    private static final String UPGRADE_SQL = "upgrade";
 
     @Autowired
     private ConnectorFactory connectorFactory;
@@ -333,38 +332,18 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
     }
 
     private void initUpgradeSql() {
-        // show tables where Tables_in_dbsyncer like "dbsyncer_data%"
-        String sql = String.format(SHOW_DATA_TABLE, database, PREFIX_TABLE.concat(StorageEnum.DATA.getType()).concat("%"));
-        List<String> tables = null;
         try {
-            tables = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(sql, String.class));
-        } catch (EmptyResultDataAccessException e) {
-            // 没有可更新的表
-        }
-        if (CollectionUtils.isEmpty(tables)) {
-            return;
-        }
-        final String queryColumnCount = "SELECT count(*) FROM information_schema.columns WHERE table_name = '%s' and column_name = 'DATA'";
-        tables.forEach(table -> {
-            try {
-                String query = String.format(queryColumnCount, table);
-                // 是否已升级
-                int count = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(query, Integer.class));
-                if (count == 0) {
-                    String ddlSql = readSql(UPGRADE_SQL, false, table);
-                    Stream.of(StringUtil.split(ddlSql, ";")).forEach(ddl -> executeSql(ddl));
+            executeSql("drop table if exists `dbsyncer_binlog`;");
+        } catch (Exception e) {
+            if (e.getCause() instanceof SQLSyntaxErrorException) {
+                SQLSyntaxErrorException ex = (SQLSyntaxErrorException) e.getCause();
+                if (ex.getSQLState().equals("42S21")) {
+                    // ignore
+                    return;
                 }
-            } catch (Exception e) {
-                if (e.getCause() instanceof SQLSyntaxErrorException) {
-                    SQLSyntaxErrorException ex = (SQLSyntaxErrorException) e.getCause();
-                    if (ex.getSQLState().equals("42S21")) {
-                        // ignore
-                        return;
-                    }
-                }
-                logger.error(e.getMessage());
             }
-        });
+            logger.error(e.getMessage());
+        }
     }
 
     private void initTable() throws InterruptedException {
@@ -377,17 +356,12 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
         builder.build(ConfigConstant.CONFIG_MODEL_ID, ConfigConstant.CONFIG_MODEL_TYPE, ConfigConstant.CONFIG_MODEL_CREATE_TIME, ConfigConstant.CONFIG_MODEL_JSON);
         List<Field> logFields = builder.getFields();
 
-        // 缓存任务
-        builder.build(ConfigConstant.CONFIG_MODEL_ID, ConfigConstant.BINLOG_STATUS, ConfigConstant.CONFIG_MODEL_CREATE_TIME, ConfigConstant.BINLOG_DATA);
-        List<Field> binlogFields = builder.getFields();
-
         // 数据
         builder.build(ConfigConstant.CONFIG_MODEL_ID, ConfigConstant.DATA_SUCCESS, ConfigConstant.DATA_TABLE_GROUP_ID, ConfigConstant.DATA_TARGET_TABLE_NAME, ConfigConstant.DATA_EVENT, ConfigConstant.DATA_ERROR, ConfigConstant.CONFIG_MODEL_CREATE_TIME, ConfigConstant.BINLOG_DATA);
         List<Field> dataFields = builder.getFields();
 
         tables.computeIfAbsent(StorageEnum.CONFIG.getType(), k -> new Executor(k, configFields, true, true));
         tables.computeIfAbsent(StorageEnum.LOG.getType(), k -> new Executor(k, logFields, true, false));
-        tables.computeIfAbsent(StorageEnum.BINLOG.getType(), k -> new Executor(k, binlogFields, true, false));
         tables.computeIfAbsent(StorageEnum.DATA.getType(), k -> new Executor(k, dataFields, false, false));
         // 创建表
         tables.forEach((tableName, e) -> {

dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/DocumentUtil.java

@@ -119,24 +119,4 @@ public abstract class DocumentUtil {
         return doc;
     }
 
-    public static Document convertBinlog2Doc(Map params) {
-        Document doc = new Document();
-        String id = (String) params.get(ConfigConstant.CONFIG_MODEL_ID);
-        doc.add(new StringField(ConfigConstant.CONFIG_MODEL_ID, id, Field.Store.YES));
-
-        Integer status = (Integer) params.get(ConfigConstant.BINLOG_STATUS);
-        doc.add(new IntPoint(ConfigConstant.BINLOG_STATUS, status));
-        doc.add(new StoredField(ConfigConstant.BINLOG_STATUS, status));
-
-        byte[] bytes = (byte[]) params.get(ConfigConstant.BINLOG_DATA);
-        doc.add(new BinaryDocValuesField(ConfigConstant.BINLOG_DATA, new BytesRef(bytes)));
-        doc.add(new StoredField(ConfigConstant.BINLOG_DATA, bytes));
-
-        Long createTime = (Long) params.get(ConfigConstant.CONFIG_MODEL_CREATE_TIME);
-        doc.add(new LongPoint(ConfigConstant.CONFIG_MODEL_CREATE_TIME, createTime));
-        doc.add(new StoredField(ConfigConstant.CONFIG_MODEL_CREATE_TIME, createTime));
-        doc.add(new NumericDocValuesField(ConfigConstant.CONFIG_MODEL_CREATE_TIME, createTime));
-        return doc;
-    }
-
 }