支持删除过期同步数据和日志https://gitee.com/ghi/dbsyncer/issues/I7S852

Signed-off-by: AE86 <836391306@qq.com>

dbsyncer-biz/src/main/java/org/dbsyncer/biz/MonitorService.java

@@ -1,7 +1,6 @@
 package org.dbsyncer.biz;
 
 import org.dbsyncer.biz.vo.AppReportMetricVo;
-import org.dbsyncer.biz.vo.MessageVo;
 import org.dbsyncer.biz.vo.MetaVo;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.monitor.enums.MetricEnum;
@@ -71,6 +70,11 @@ public interface MonitorService {
      */
     String clearLog();
 
+    /**
+     * 删除过期的数据和日志
+     */
+    void deleteExpiredDataAndLog();
+
     /**
      * 获取所有同步数据状态类型
      *

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java

@@ -53,12 +53,13 @@ public class SystemConfigChecker extends AbstractChecker {
         SystemConfig systemConfig = manager.getSystemConfig();
         Assert.notNull(systemConfig, "配置文件为空.");
 
+        // 同步数据过期时间（天）
+        systemConfig.setExpireDataDays(NumberUtil.toInt(params.get("expireDataDays"), systemConfig.getExpireDataDays()));
+        // 系统日志过期时间（天）
+        systemConfig.setExpireLogDays(NumberUtil.toInt(params.get("expireLogDays"), systemConfig.getExpireLogDays()));
         // 刷新监控间隔（秒）
-        String refreshInterval = params.get("refreshInterval");
-        if (StringUtil.isNotBlank(refreshInterval)) {
-            int time = NumberUtil.toInt(refreshInterval, 10);
-            systemConfig.setRefreshInterval(time);
-        }
+        systemConfig.setRefreshIntervalSeconds(NumberUtil.toInt(params.get("refreshIntervalSeconds"), systemConfig.getRefreshIntervalSeconds()));
+
         logService.log(LogType.SystemLog.INFO, "修改系统配置");
 
         // 修改基本配置

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java

@@ -2,6 +2,7 @@ package org.dbsyncer.biz.impl;
 
 import org.dbsyncer.biz.DataSyncService;
 import org.dbsyncer.biz.MonitorService;
+import org.dbsyncer.biz.SystemConfigService;
 import org.dbsyncer.biz.metric.MetricDetailFormatter;
 import org.dbsyncer.biz.metric.impl.CpuMetricDetailFormatter;
 import org.dbsyncer.biz.metric.impl.DiskMetricDetailFormatter;
@@ -21,6 +22,7 @@ import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.enums.FilterEnum;
 import org.dbsyncer.monitor.Monitor;
 import org.dbsyncer.monitor.enums.DiskMetricEnum;
 import org.dbsyncer.monitor.enums.MetricEnum;
@@ -31,8 +33,13 @@ import org.dbsyncer.monitor.model.MetricResponse;
 import org.dbsyncer.parser.enums.ModelEnum;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
+import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
+import org.dbsyncer.storage.enums.StorageEnum;
+import org.dbsyncer.storage.query.BooleanFilter;
+import org.dbsyncer.storage.query.Query;
+import org.dbsyncer.storage.query.filter.LongFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
@@ -41,6 +48,8 @@ import org.springframework.util.Assert;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
@@ -67,6 +76,12 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
     @Resource
     private ScheduledTaskService scheduledTaskService;
 
+    @Resource
+    private StorageService storageService;
+
+    @Resource
+    private SystemConfigService systemConfigService;
+
     private Map<String, MetricDetailFormatter> metricDetailFormatterMap = new LinkedHashMap<>();
 
     @PostConstruct
@@ -169,6 +184,12 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
         return "清空日志成功";
     }
 
+    @Override
+    public void deleteExpiredDataAndLog() {
+        deleteExpiredData();
+        deleteExpiredLog();
+    }
+
     @Override
     public List<StorageDataStatusEnum> getStorageDataStatusEnumAll() {
         return monitor.getStorageDataStatusEnumAll();
@@ -219,6 +240,32 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
         }
     }
 
+    private void deleteExpiredData() {
+        List<MetaVo> metaAll = getMetaAll();
+        if (!CollectionUtils.isEmpty(metaAll)) {
+            Query query = new Query();
+            query.setType(StorageEnum.DATA);
+            int expireDataDays = systemConfigService.getSystemConfigVo().getExpireDataDays();
+            long expiredTime = Timestamp.valueOf(LocalDateTime.now().minusDays(expireDataDays)).getTime();
+            LongFilter expiredFilter = new LongFilter(ConfigConstant.CONFIG_MODEL_CREATE_TIME, FilterEnum.LT, expiredTime);
+            query.setBooleanFilter(new BooleanFilter().add(expiredFilter));
+            metaAll.forEach(metaVo -> {
+                query.setMetaId(metaVo.getId());
+                storageService.delete(query);
+            });
+        }
+    }
+
+    private void deleteExpiredLog() {
+        Query query = new Query();
+        query.setType(StorageEnum.LOG);
+        int expireLogDays = systemConfigService.getSystemConfigVo().getExpireLogDays();
+        long expiredTime = Timestamp.valueOf(LocalDateTime.now().minusDays(expireLogDays)).getTime();
+        LongFilter expiredFilter = new LongFilter(ConfigConstant.CONFIG_MODEL_CREATE_TIME, FilterEnum.LT, expiredTime);
+        query.setBooleanFilter(new BooleanFilter().add(expiredFilter));
+        storageService.delete(query);
+    }
+
     private MetaVo convertMeta2Vo(Meta meta) {
         Mapping mapping = monitor.getMapping(meta.getMappingId());
         Assert.notNull(mapping, "驱动不存在.");

dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java

@@ -15,14 +15,42 @@ public class SystemConfig extends ConfigModel {
         super.setType(ConfigConstant.SYSTEM);
     }
 
-    private int refreshInterval = 5;
+    /**
+     * 同步数据过期时间（天）
+     */
+    private int expireDataDays = 7;
 
-    public int getRefreshInterval() {
-        return refreshInterval;
+    /**
+     * 系统日志过期时间（天）
+     */
+    private int expireLogDays = 30;
+
+    /**
+     * 刷新页面间隔（秒）
+     */
+    private int refreshIntervalSeconds = 5;
+
+    public int getExpireDataDays() {
+        return expireDataDays;
     }
 
-    public void setRefreshInterval(int refreshInterval) {
-        this.refreshInterval = refreshInterval;
+    public void setExpireDataDays(int expireDataDays) {
+        this.expireDataDays = expireDataDays;
     }
 
+    public int getExpireLogDays() {
+        return expireLogDays;
+    }
+
+    public void setExpireLogDays(int expireLogDays) {
+        this.expireLogDays = expireLogDays;
+    }
+
+    public int getRefreshIntervalSeconds() {
+        return refreshIntervalSeconds;
+    }
+
+    public void setRefreshIntervalSeconds(int refreshIntervalSeconds) {
+        this.refreshIntervalSeconds = refreshIntervalSeconds;
+    }
 }

dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java

@@ -37,6 +37,8 @@ public abstract class AbstractStorageService implements StorageService, Disposab
 
     protected abstract Paging select(String sharding, Query query);
 
+    protected abstract void delete(String sharding, Query query);
+
     protected abstract void deleteAll(String sharding);
 
     protected abstract void batchInsert(StorageEnum type, String sharding, List<Map> list);
@@ -79,6 +81,24 @@ public abstract class AbstractStorageService implements StorageService, Disposab
         return new Paging(query.getPageNum(), query.getPageSize());
     }
 
+    @Override
+    public void delete(Query query) {
+        boolean locked = false;
+        try {
+            locked = lock.tryLock(3, TimeUnit.SECONDS);
+            if (locked) {
+                String sharding = getSharding(query.getType(), query.getMetaId());
+                delete(sharding, query);
+            }
+        } catch (InterruptedException e) {
+            logger.warn("tryLock error", e.getLocalizedMessage());
+        } finally {
+            if (locked) {
+                lock.unlock();
+            }
+        }
+    }
+
     @Override
     public void clear(StorageEnum type, String metaId) {
         boolean locked = false;

dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java

@@ -22,6 +22,13 @@ public interface StorageService {
      */
     Paging query(Query query);
 
+    /**
+     * 根据条件删除
+     *
+     * @param query
+     */
+    void delete(Query query);
+
     /**
      * 清空数据/日志
      *

dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Shard.java

@@ -91,6 +91,18 @@ public class Shard {
         }
     }
 
+    public void delete(Query query) {
+        try {
+            indexWriter.deleteDocuments(query);
+            indexWriter.flush();
+            indexWriter.commit();
+            indexWriter.forceMergeDeletes();
+            indexWriter.deleteUnusedFiles();
+        } catch (IOException e) {
+            logger.error(e.getLocalizedMessage(), e);
+        }
+    }
+
     public void deleteAll() {
         // Fix Bug: this IndexReader is closed. 直接删除文件
         try {
@@ -247,6 +259,10 @@ public class Shard {
         indexReader = DirectoryReader.open(indexWriter);
     }
 
+    public Analyzer getAnalyzer() {
+        return analyzer;
+    }
+
     interface Callback {
 
         /**

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/DiskStorageServiceImpl.java

@@ -75,13 +75,7 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
             }
 
             Set<String> highLightKeys = new HashSet<>();
-            BooleanQuery build = null;
-            if (!CollectionUtils.isEmpty(filters)) {
-                build = buildQueryWithFilters(filters, highLightKeys);
-            } else {
-                build = buildQueryWithBooleanFilters(clauses, highLightKeys);
-            }
-
+            BooleanQuery build = buildQuery(filters, clauses, highLightKeys);
             option.setQuery(build);
 
             // 高亮查询
@@ -98,6 +92,18 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
         }
     }
 
+    @Override
+    protected void delete(String sharding, Query query) {
+        Shard shard = getShard(sharding);
+        BooleanFilter baseQuery = query.getBooleanFilter();
+        List<AbstractFilter> filters = baseQuery.getFilters();
+        List<BooleanFilter> clauses = baseQuery.getClauses();
+        if (CollectionUtils.isEmpty(clauses) && CollectionUtils.isEmpty(filters)) {
+            throw new StorageException("必须包含删除条件");
+        }
+        shard.delete(buildQuery(filters, clauses, new HashSet<>()));
+    }
+
     @Override
     public void deleteAll(String sharding) {
         shards.computeIfPresent(sharding, (k, v) -> {
@@ -140,6 +146,13 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
         shards.clear();
     }
 
+    private BooleanQuery buildQuery(List<AbstractFilter> filters, List<BooleanFilter> clauses, Set<String> highLightKeys) {
+        if (!CollectionUtils.isEmpty(filters)) {
+            return buildQueryWithFilters(filters, highLightKeys);
+        }
+        return buildQueryWithBooleanFilters(clauses, highLightKeys);
+    }
+
     private BooleanQuery buildQueryWithFilters(List<AbstractFilter> filters, Set<String> highLightKeys) {
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
         filters.forEach(p -> {

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/MysqlStorageServiceImpl.java

@@ -121,6 +121,11 @@ public class MysqlStorageServiceImpl extends AbstractStorageService {
         return paging;
     }
 
+    @Override
+    protected void delete(String sharding, Query query) {
+
+    }
+
     @Override
     protected void deleteAll(String sharding) {
         tables.computeIfPresent(sharding, (k, executor) -> {

dbsyncer-storage/src/main/test/LuceneFactoryTest.java

@@ -2,26 +2,47 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.*;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.search.highlight.Highlighter;
 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
 import org.apache.lucene.search.highlight.QueryScorer;
 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
 import org.apache.lucene.util.BytesRef;
 import org.dbsyncer.common.model.Paging;
+import org.dbsyncer.common.snowflake.SnowflakeIdWorker;
 import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.RandomUtil;
 import org.dbsyncer.connector.constant.ConnectorConstant;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
+import org.dbsyncer.storage.lucene.Option;
 import org.dbsyncer.storage.lucene.Shard;
-import org.dbsyncer.storage.query.Option;
+import org.dbsyncer.storage.util.BinlogMessageUtil;
 import org.dbsyncer.storage.util.DocumentUtil;
 import org.junit.After;
 import org.junit.Before;
@@ -32,15 +53,24 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.io.StringReader;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.*;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 public class LuceneFactoryTest {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
-    private       Shard  shard;
+
+    private Shard shard;
+
+    private SnowflakeIdWorker snowflakeIdWorker = new SnowflakeIdWorker();
 
     @Before
     public void setUp() throws IOException {
@@ -60,7 +90,7 @@ public class LuceneFactoryTest {
         final CyclicBarrier barrier = new CyclicBarrier(threadSize);
         final CountDownLatch latch = new CountDownLatch(threadSize);
         for (int i = 0; i < threadSize; i++) {
-            final int k = i + 3;
+            final int k = i + 1;
             pool.submit(() -> {
                 try {
                     barrier.await();
@@ -68,10 +98,9 @@ public class LuceneFactoryTest {
                     // 模拟操作
                     System.out.println(String.format("%s:%s", Thread.currentThread().getName(), k));
 
-                    Document data = DocumentUtil.convertData2Doc(createMap(k));
-                    //IndexableField field = data.getField(ConfigConstant.CONFIG_MODEL_ID);
-                    //shard.update(new Term(ConfigConstant.CONFIG_MODEL_ID, field.stringValue()), data);
-                    shard.insert(data);
+                    List<Document> docs = new ArrayList<>();
+                    docs.add(DocumentUtil.convertData2Doc(createMap(k)));
+                    shard.insertBatch(docs);
 
                 } catch (InterruptedException e) {
                     logger.error(e.getMessage());
@@ -92,7 +121,9 @@ public class LuceneFactoryTest {
             check();
             Sort sort = new Sort(new SortField(ConfigConstant.CONFIG_MODEL_UPDATE_TIME, SortField.Type.LONG, true),
                     new SortField(ConfigConstant.CONFIG_MODEL_CREATE_TIME, SortField.Type.LONG, true));
-            Paging paging = shard.query(new Option(new MatchAllDocsQuery()), 1, 20, sort);
+            Option option = new Option();
+            option.setQuery(new MatchAllDocsQuery());
+            Paging paging = shard.query(option, 1, 20, sort);
             if (!CollectionUtils.isEmpty(paging.getData())) {
                 List<Map> data = (List<Map>) paging.getData();
                 data.stream().forEach(r -> System.out.println(r));
@@ -109,17 +140,19 @@ public class LuceneFactoryTest {
 
     private Map<String, Object> createMap(int i) {
         Map<String, Object> params = new HashMap();
-        params.put(ConfigConstant.CONFIG_MODEL_ID, "953402886828589057");
+        params.put(ConfigConstant.CONFIG_MODEL_ID, String.valueOf(snowflakeIdWorker.nextId()));
         params.put(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.SUCCESS.getValue());
         params.put(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_UPDATE);
+        params.put(ConfigConstant.DATA_TABLE_GROUP_ID, "" + i);
+        params.put(ConfigConstant.DATA_TARGET_TABLE_NAME, "MY_USER");
         params.put(ConfigConstant.DATA_ERROR, "");
         Map<String, Object> row = new HashMap<>();
         row.put("id", i);
         row.put("name", "中文");
         row.put("tel", "15800001234");
-        row.put("update_time", System.currentTimeMillis());
+        row.put("update_time", Instant.now().toEpochMilli());
         row.put("remark", "test" + i);
-        params.put(ConfigConstant.CONFIG_MODEL_JSON, JsonUtil.objToJson(row));
+        params.put(ConfigConstant.BINLOG_DATA, BinlogMessageUtil.toBinlogMap(row).toByteArray());
         params.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, Instant.now().toEpochMilli());
         return params;
     }
@@ -127,6 +160,7 @@ public class LuceneFactoryTest {
     @Test
     public void testQuery() throws IOException {
         int size = 3;
+        List<Document> docs = new ArrayList<>();
         for (int i = size; i > 0; i--) {
             Document doc = new Document();
             doc.add(new StringField("id", String.valueOf(i), Field.Store.YES));
@@ -147,16 +181,18 @@ public class LuceneFactoryTest {
             doc.add(new LongPoint("createTime", createTime));
             doc.add(new StoredField("createTime", createTime));
             doc.add(new NumericDocValuesField("createTime", createTime));
-
-            shard.insert(doc);
+            docs.add(doc);
         }
+        shard.insertBatch(docs);
         // 范围查询 IntPoint.newRangeQuery("id", 1, 100)
         // 集合查询 IntPoint.newSetQuery("id", 2, 3)
         // 单个查询 IntPoint.newExactQuery("id", 3)
         BooleanQuery query = new BooleanQuery.Builder()
                 .add(IntPoint.newRangeQuery("age", 1, 100), BooleanClause.Occur.MUST)
                 .build();
-        Paging paging = shard.query(new Option(query), 1, 20, new Sort(new SortField("createTime", SortField.Type.LONG, true)));
+        Option option = new Option();
+        option.setQuery(query);
+        Paging paging = shard.query(option, 1, 20, new Sort(new SortField("createTime", SortField.Type.LONG, true)));
         paging.getData().forEach(m -> System.out.println(m));
 
         // 清空
@@ -175,7 +211,9 @@ public class LuceneFactoryTest {
         String id = "100";
         doc.add(new StringField("id", id, Field.Store.YES));
         doc.add(new TextField("content", "这是一款大规模数据处理软件，名字叫做Apache Spark", Field.Store.YES));
-        shard.insert(doc);
+        List<Document> docs = new ArrayList<>();
+        docs.add(doc);
+        shard.insertBatch(docs);
         System.out.println("新增后：");
         maps = query(new MatchAllDocsQuery());
         maps.forEach(m -> System.out.println(m));
@@ -190,7 +228,7 @@ public class LuceneFactoryTest {
         check();
 
         // 删除
-        shard.delete(new Term("id", id));
+        shard.deleteBatch(new Term("id", id));
         System.out.println("删除后：");
         maps = query(new MatchAllDocsQuery());
         maps.forEach(m -> System.out.println(m));
@@ -213,7 +251,9 @@ public class LuceneFactoryTest {
         doc.add(new StringField("id", id, Field.Store.YES));
         BytesRef bytesRef = new BytesRef("中文".getBytes());
         doc.add(new StoredField("content", bytesRef));
-        shard.insert(doc);
+        List<Document> docs = new ArrayList<>();
+        docs.add(doc);
+        shard.insertBatch(docs);
         System.out.println("新增后：");
         maps = query(new MatchAllDocsQuery());
         maps.forEach(m -> {
@@ -444,7 +484,9 @@ public class LuceneFactoryTest {
     }
 
     private List<Map> query(Query query) throws IOException {
-        Paging paging = shard.query(new Option(query), 1, 20, null);
+        Option option = new Option();
+        option.setQuery(query);
+        Paging paging = shard.query(option, 1, 20, null);
         return (List<Map>) paging.getData();
     }
 

dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java

@@ -101,6 +101,11 @@ public class MonitorController extends BaseController {
         connectorService.refreshHealth();
     }
 
+    @Scheduled(fixedRate = 15000)
+    public void deleteExpiredDataAndLog() {
+        monitorService.deleteExpiredDataAndLog();
+    }
+
     @GetMapping("/queryData")
     @ResponseBody
     public RestResult queryData(HttpServletRequest request) {
@@ -180,11 +185,11 @@ public class MonitorController extends BaseController {
     }
 
     @ResponseBody
-    @GetMapping("/getRefreshInterval")
+    @GetMapping("/getRefreshIntervalSeconds")
     public RestResult getRefreshInterval() {
         try {
             SystemConfigVo config = systemConfigService.getSystemConfigVo();
-            return RestResult.restSuccess(config.getRefreshInterval());
+            return RestResult.restSuccess(config.getRefreshIntervalSeconds());
         } catch (Exception e) {
             logger.error(e.getLocalizedMessage(), e.getClass());
             return RestResult.restFail(e.getMessage());

dbsyncer-web/src/main/resources/public/index/index.html

@@ -192,11 +192,11 @@
                                         <div class="dropdown">
                                             <a data-toggle="dropdown" href="javascript:;"><span class="well-sign-operation"><i class="fa fa-gears fa-2x"></i></span></a>
                                             <ul class="dropdown-menu" role="menu" aria-labelledby="dLabel">
-                                                <li th:url="'/mapping/copy?id='+${m?.id}"><a href="javascript:;"><i class="fa fa-copy"></i>&nbsp;复制</a></li>
                                                 <!-- 未运行 -->
                                                 <li th:if="${m?.meta?.state ne 1}" th:url="'/mapping/start?id='+${m?.id}"><a href="javascript:;"><i class="fa fa-check-circle-o well-sign-green"></i>&nbsp;&nbsp;启动</a></li>
                                                 <!-- 运行中 -->
                                                 <li th:if="${m?.meta?.state eq 1}" th:url="'/mapping/stop?id='+${m?.id}"><a href="javascript:;"><i class="fa fa-times-circle-o well-sign-red"></i>&nbsp;&nbsp;停止</a></li>
+                                                <li th:url="'/mapping/copy?id='+${m?.id}"><a href="javascript:;"><i class="fa fa-copy"></i>&nbsp;复制</a></li>
                                                 <!-- 未运行 -->
                                                 <li th:if="${m?.meta?.state ne 1}" th:url="'/mapping/remove?id='+${m?.id}" confirm="true" confirmMessage="确认删除驱动?"><a href="javascript:;"><i class="fa fa-trash well-sign-red"></i>&nbsp;&nbsp;删除</a></li>
                                             </ul>

dbsyncer-web/src/main/resources/public/mapping/editFull.html

@@ -10,19 +10,19 @@
             <div class="col-md-4">
                 <label class="col-sm-3 control-label text-right">批量读取<strong class="driverVerifcateRequired">*</strong></label>
                 <div class="col-sm-9">
-                    <input type="number" name="readNum" class="form-control" min="1" dbsyncer-valid="require" th:value="${mapping?.readNum}">
+                    <input type="number" name="readNum" class="form-control" min="1" max="200000" dbsyncer-valid="require" th:value="${mapping?.readNum}">
                 </div>
             </div>
             <div class="col-md-4">
                 <label class="col-sm-3 control-label text-right">单次写入<strong class="driverVerifcateRequired">*</strong></label>
                 <div class="col-sm-9">
-                    <input type="number" name="batchNum" class="form-control" min="1" dbsyncer-valid="require" th:value="${mapping?.batchNum}">
+                    <input type="number" name="batchNum" class="form-control" min="1" max="20000" dbsyncer-valid="require" th:value="${mapping?.batchNum}">
                 </div>
             </div>
             <div class="col-md-4">
                 <label class="col-sm-3 control-label text-right">线程数<strong class="driverVerifcateRequired">*</strong></label>
                 <div class="col-sm-9">
-                    <input type="number" name="threadNum" class="form-control" min="1" dbsyncer-valid="require" th:value="${mapping?.threadNum}">
+                    <input type="number" name="threadNum" class="form-control" min="1" max="64" dbsyncer-valid="require" th:value="${mapping?.threadNum}">
                 </div>
             </div>
         </div>

dbsyncer-web/src/main/resources/public/system/system.html

@@ -15,10 +15,22 @@
             <!-- 系统参数配置 -->
             <div class="col-md-3"></div>
             <div class="col-md-6">
+                <div class="form-group">
+                    <label class="col-sm-4 control-label">同步数据过期时间(天) <strong class="driverVerifcateRequired">*</strong></label>
+                    <div class="col-sm-8">
+                        <input type="number" class="form-control" min="1" max="180" dbsyncer-valid="require" name="expireDataDays" th:value="${config?.expireDataDays}"/>
+                    </div>
+                </div>
+                <div class="form-group">
+                    <label class="col-sm-4 control-label">系统日志过期时间(天) <strong class="driverVerifcateRequired">*</strong></label>
+                    <div class="col-sm-8">
+                        <input type="number" class="form-control" min="1" max="180" dbsyncer-valid="require" name="expireLogDays" th:value="${config?.expireLogDays}"/>
+                    </div>
+                </div>
                 <div class="form-group">
                     <label class="col-sm-4 control-label">刷新监控频率(秒) <strong class="driverVerifcateRequired">*</strong></label>
                     <div class="col-sm-8">
-                        <input type="number" class="form-control" max="100" min="1" dbsyncer-valid="require" name="refreshInterval" th:value="${config?.refreshInterval}"/>
+                        <input type="number" class="form-control" min="1" max="60" dbsyncer-valid="require" name="refreshIntervalSeconds" th:value="${config?.refreshIntervalSeconds}"/>
                     </div>
                 </div>
                 <div class="form-group">

dbsyncer-web/src/main/resources/static/js/index/index.js

@@ -188,7 +188,7 @@ function doPost(url) {
 
 // 创建定时器
 function createTimer($projectGroupSelect){
-    doGetWithoutLoading("/monitor/getRefreshInterval",{}, function (data) {
+    doGetWithoutLoading("/monitor/getRefreshIntervalSeconds",{}, function (data) {
         if (data.success == true) {
             timer = setInterval(function(){
                 backIndexPage($projectGroupSelect.selectpicker('val'));
@@ -218,4 +218,4 @@ $(function () {
 
     bindConnectorDropdownMenu();
     bindMappingDropdownMenu();
-});
+});

dbsyncer-web/src/main/resources/static/js/monitor/index.js

@@ -477,7 +477,7 @@ function showChartTable() {
 // 创建定时器
 function createTimer() {
     showChartTable();
-    doGetWithoutLoading("/monitor/getRefreshInterval", {}, function (data) {
+    doGetWithoutLoading("/monitor/getRefreshIntervalSeconds", {}, function (data) {
         if (data.success == true) {
             timer = setInterval(function () {
                 showChartTable();

dbsyncer-web/src/main/resources/static/plugins/js/formValidate/formValidate.js

@@ -23,10 +23,20 @@ $.fn.formValidate = function(opt) {
 }
 
 var formValidateMethod = function($this){
+	let errorClassName = "dbsyncerVerifcateError";
 	if ($this.val() == "") {
-		$this.addClass("dbsyncerVerifcateError").attr("data-original-title", "必填").tooltip({trigger : 'manual'}).tooltip('show');
+		$this.addClass(errorClassName).attr("data-original-title", "必填").tooltip({trigger : 'manual'}).tooltip('show');
 		return false;
 	}
-	$this.tooltip('hide').removeClass("dbsyncerVerifcateError");
+	// 数字类型校验
+	if ($this.attr("type") == "number") {
+		let max = parseInt($this.attr("max"));
+		let min = parseInt($this.attr("min"));
+		if($this.val() > max || $this.val() < min){
+			$this.addClass(errorClassName).attr("data-original-title", "有效范围应在" + min + "-" + max).tooltip({trigger: 'manual'}).tooltip('show');
+			return false;
+		}
+	}
+	$this.tooltip('hide').removeClass(errorClassName);
 	return true;
-}
+}