add log

dbsyncer-biz/src/main/java/org/dbsyncer/biz/ConnectorService.java

@@ -31,7 +31,7 @@ public interface ConnectorService {
      *
      * @param id
      */
-    boolean remove(String id);
+    String remove(String id);
 
     /**
      * 获取连接器

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java

@@ -2,6 +2,10 @@ package org.dbsyncer.biz.impl;
 
 import org.dbsyncer.manager.Manager;
 import org.dbsyncer.parser.enums.MetaEnum;
+import org.dbsyncer.parser.enums.ModelEnum;
+import org.dbsyncer.parser.logger.LogService;
+import org.dbsyncer.parser.logger.LogType;
+import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
@@ -13,6 +17,9 @@ public class BaseServiceImpl {
     @Autowired
     protected Manager manager;
 
+    @Autowired
+    private LogService logService;
+
     /**
      * 驱动启停锁
      */
@@ -38,4 +45,33 @@ public class BaseServiceImpl {
         }
     }
 
+    protected void log(LogType log, ConfigModel model) {
+        if (null != model) {
+            // 新增连接器:知识库
+            logService.log(log, "%s%s:%s", log.getMessage(), log.getName(), model.getName());
+        }
+    }
+
+    protected void log(LogType log, Mapping mapping) {
+        if (null != mapping) {
+            // 新增驱动:知识库(全量)
+            String model = ModelEnum.getModelEnum(mapping.getModel()).getName();
+            logService.log(log, "%s%s:%s(%s)", log.getMessage(), log.getName(), mapping.getName(), model);
+        }
+    }
+
+    protected void log(LogType log, TableGroup tableGroup) {
+        if (null != tableGroup) {
+            Mapping mapping = manager.getMapping(tableGroup.getMappingId());
+            if (null != mapping) {
+                // 新增驱动知识库(全量)映射关系:[My_User] >> [My_User_Target]
+                String name = mapping.getName();
+                String model = ModelEnum.getModelEnum(mapping.getModel()).getName();
+                String s = tableGroup.getSourceTable().getName();
+                String t = tableGroup.getTargetTable().getName();
+                logService.log(log, "%s驱动%s(%s)%s:[%s] >> [%s]", log.getMessage(), name, model, log.getName(), s, t);
+            }
+        }
+    }
+
 }

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java

@@ -4,10 +4,9 @@ import org.apache.commons.lang.StringUtils;
 import org.dbsyncer.biz.ConnectorService;
 import org.dbsyncer.biz.checker.Checker;
 import org.dbsyncer.manager.Manager;
+import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Connector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -23,9 +22,7 @@ import java.util.stream.Collectors;
  * @date 2019/10/17 23:20
  */
 @Service
-public class ConnectorServiceImpl implements ConnectorService {
-
-    private final Logger logger = LoggerFactory.getLogger(getClass());
+public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorService {
 
     @Autowired
     private Manager manager;
@@ -36,20 +33,26 @@ public class ConnectorServiceImpl implements ConnectorService {
     @Override
     public String add(Map<String, String> params) {
         ConfigModel model = connectorChecker.checkAddConfigModel(params);
+        log(LogType.ConnectorLog.INSERT, model);
+
         return manager.addConnector(model);
     }
 
     @Override
     public String edit(Map<String, String> params) {
         ConfigModel model = connectorChecker.checkEditConfigModel(params);
+        log(LogType.ConnectorLog.UPDATE, model);
+
         return manager.editConnector(model);
     }
 
     @Override
-    public boolean remove(String id) {
-        logger.info("remove:{}", id);
+    public String remove(String id) {
+        Connector connector = manager.getConnector(id);
+        log(LogType.ConnectorLog.DELETE, connector);
+
         manager.removeConnector(id);
-        return true;
+        return "删除连接器成功!";
     }
 
     @Override

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java

@@ -7,7 +7,6 @@ import org.dbsyncer.biz.vo.ConnectorVo;
 import org.dbsyncer.biz.vo.MappingVo;
 import org.dbsyncer.biz.vo.MetaVo;
 import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.parser.logger.LogService;
 import org.dbsyncer.monitor.Monitor;
 import org.dbsyncer.parser.enums.ModelEnum;
 import org.dbsyncer.parser.logger.LogType;
@@ -41,13 +40,11 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
     @Autowired
     private Checker mappingChecker;
 
-    @Autowired
-    private LogService logService;
-
     @Override
     public String add(Map<String, String> params) {
         ConfigModel model = mappingChecker.checkAddConfigModel(params);
-        log((Mapping) model, LogType.MappingLog.INSERT);
+        log(LogType.MappingLog.INSERT, (Mapping) model);
+
         return manager.addMapping(model);
     }
 
@@ -58,7 +55,8 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
         synchronized (LOCK) {
             assertRunning(mapping.getMetaId());
             ConfigModel model = mappingChecker.checkEditConfigModel(params);
-            log(mapping, LogType.MappingLog.UPDATE);
+            log(LogType.MappingLog.UPDATE, (Mapping) model);
+
             return manager.editMapping(model);
         }
     }
@@ -67,14 +65,17 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
     public String remove(String id) {
         Mapping mapping = assertMappingExist(id);
         String metaId = mapping.getMetaId();
+        Meta meta = manager.getMeta(metaId);
         synchronized (LOCK) {
             assertRunning(metaId);
 
             // 删除数据
             manager.clearData(metaId);
-            
+            log(LogType.MetaLog.CLEAR, meta);
+
             // 删除meta
             manager.removeMeta(metaId);
+            log(LogType.MetaLog.DELETE, meta);
 
             // 删除tableGroup
             List<TableGroup> groupList = manager.getTableGroupAll(id);
@@ -84,7 +85,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
 
             // 删除驱动
             manager.removeMapping(id);
-            log(mapping, LogType.MappingLog.DELETE);
+            log(LogType.MappingLog.DELETE, mapping);
         }
         return "驱动删除成功";
     }
@@ -118,9 +119,10 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
             meta.getSuccess().set(0);
             manager.editMeta(meta);
 
+            // 启动
             manager.start(mapping);
 
-            log(mapping, LogType.MappingLog.RUNNING);
+            log(LogType.MappingLog.RUNNING, mapping);
         }
         return "驱动启动成功";
     }
@@ -134,17 +136,11 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic
             }
             manager.close(mapping);
 
-            log(mapping, LogType.MappingLog.STOP);
+            log(LogType.MappingLog.STOP, mapping);
         }
         return "驱动停止成功";
     }
 
-    private void log(Mapping mapping, LogType logType) {
-        String model = ModelEnum.getModelEnum(mapping.getModel()).getName();
-        String msg = String.format("%s:%s(%s)", logType.getMessage(), mapping.getName(), model);
-        logService.log(logType, msg);
-    }
-
     private MappingVo convertMapping2Vo(Mapping mapping) {
         String model = mapping.getModel();
         Assert.notNull(mapping, "Mapping can not be null.");

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java

@@ -4,7 +4,7 @@ import org.dbsyncer.biz.TableGroupService;
 import org.dbsyncer.biz.checker.Checker;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.connector.config.Field;
-import org.dbsyncer.manager.Manager;
+import org.dbsyncer.parser.logger.LogType;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
 import org.slf4j.Logger;
@@ -25,9 +25,6 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    @Autowired
-    private Manager manager;
-
     @Autowired
     private Checker tableGroupChecker;
 
@@ -35,10 +32,13 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
     public String add(Map<String, String> params) {
         TableGroup model = (TableGroup) tableGroupChecker.checkAddConfigModel(params);
         assertRunning(model);
+        log(LogType.TableGroupLog.INSERT, model);
+
         String id = manager.addTableGroup(model);
 
         // 合并驱动公共字段
         mergeMappingColumn(model.getMappingId());
+
         return id;
     }
 
@@ -46,6 +46,8 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
     public String edit(Map<String, String> params) {
         TableGroup model = (TableGroup) tableGroupChecker.checkEditConfigModel(params);
         assertRunning(model);
+        log(LogType.TableGroupLog.UPDATE, model);
+
         return manager.editTableGroup(model);
     }
 
@@ -54,9 +56,12 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
         TableGroup tableGroup = manager.getTableGroup(id);
         Assert.notNull(tableGroup, "tableGroup can not be null.");
         assertRunning(tableGroup);
+        log(LogType.TableGroupLog.DELETE, tableGroup);
 
         manager.removeTableGroup(id);
+        // 合并驱动公共字段
         mergeMappingColumn(tableGroup.getMappingId());
+
         return true;
     }
 

dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/impl/IncrementPuller.java

@@ -132,7 +132,7 @@ public class IncrementPuller extends AbstractPuller {
             // TODO 更新待优化，存在性能问题
             DefaultExtractor extractor = map.get(metaId);
             if (null != extractor) {
-                logger.info("flushEvent map:{}", extractor.getMap());
+                logger.debug("flushEvent map:{}", extractor.getMap());
                 Meta meta = manager.getMeta(metaId);
                 if (null != meta) {
                     meta.setMap(extractor.getMap());

dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserFactory.java

@@ -273,7 +273,7 @@ public class ParserFactory implements Parser {
      * @param data
      */
     private void flush(Task task, Result writer, List<Map<String, Object>> data) {
-        flush(task.getId(), writer, ConnectorConstant.OPERTION_DELETE, data);
+        flush(task.getId(), writer, ConnectorConstant.OPERTION_INSERT, data);
 
         // 发布刷新事件给FullExtractor
         task.setEndTime(System.currentTimeMillis());

dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogService.java

@@ -11,5 +11,5 @@ public interface LogService {
 
     void log(LogType logType);
 
-    void log(LogType logType, String msg);
+    void log(LogType logType, String format, Object... args);
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogServiceImpl.java

@@ -26,7 +26,7 @@ public class LogServiceImpl implements LogService {
     }
 
     @Override
-    public void log(LogType logType, String msg) {
-        flushService.asyncWrite(logType.getType(), msg);
+    public void log(LogType logType, String format, Object... args) {
+        flushService.asyncWrite(logType.getType(), String.format(format, args));
     }
 }

dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogType.java

@@ -14,7 +14,7 @@ public interface LogType {
      *
      * @return
      */
-    String getTypeName();
+    String getName();
 
     /**
      * 类型
@@ -56,7 +56,7 @@ public interface LogType {
         }
 
         @Override
-        public String getTypeName() {
+        public String getName() {
             return "系统日志";
         }
 
@@ -89,7 +89,7 @@ public interface LogType {
         }
 
         @Override
-        public String getTypeName() {
+        public String getName() {
             return "连接器";
         }
 
@@ -123,7 +123,7 @@ public interface LogType {
         }
 
         @Override
-        public String getTypeName() {
+        public String getName() {
             return "驱动";
         }
 
@@ -155,7 +155,7 @@ public interface LogType {
         }
 
         @Override
-        public String getTypeName() {
+        public String getName() {
             return "映射关系";
         }
 
@@ -174,9 +174,9 @@ public interface LogType {
      * 元信息5
      */
     enum MetaLog implements LogType {
-        UPDATE("41", "修改"),
-        DELETE("42", "删除"),
-        CLEAR("43", "删除数据");
+        DELETE("50", "删除"),
+        CLEAR("51", "删除数据"),
+        TASK("52", "任务");
 
         private String type;
         private String message;
@@ -187,7 +187,7 @@ public interface LogType {
         }
 
         @Override
-        public String getTypeName() {
+        public String getName() {
             return "元信息";
         }
 

dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/LuceneFactoryTest.java → dbsyncer-storage/src/main/test/LuceneFactoryTest.java

@@ -1,5 +1,3 @@
-package org.dbsyncer.storage.lucene;
-
 import org.apache.commons.lang.math.RandomUtils;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
@@ -15,6 +13,7 @@ import org.apache.lucene.search.highlight.Highlighter;
 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
 import org.apache.lucene.search.highlight.QueryScorer;
 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
+import org.dbsyncer.storage.lucene.Shard;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;

dbsyncer-web/src/main/resources/templates/index/index.html

@@ -127,11 +127,17 @@
                                                     <tr>
                                                         <td class="text-left">
                                                             同步结果>
-                                                            总数:[[${m?.meta?.total}]]
-                                                            <span th:if="${m?.meta?.model eq '全量' and (m?.meta?.success + m?.meta?.fail) gt 0}">
-                                                            ,进度:[[${#numbers.formatDecimal(((m?.meta?.success + m?.meta?.fail) / m?.meta?.total * 100.00),0 ,2)}]]%
-                                                            ,耗时:[[${(m?.meta?.endTime - m?.meta?.beginTime) / 1000}]]秒
+                                                            <span th:if="${m?.meta?.model eq '全量'}">
+                                                                总数:[[${m?.meta?.total}]]
+                                                                <span th:if="${m?.meta?.total gt 0 and (m?.meta?.success + m?.meta?.fail) gt 0}">
+                                                                ,进度:[[${#numbers.formatDecimal(((m?.meta?.success + m?.meta?.fail) / m?.meta?.total * 100.00),0 ,2)}]]%
+                                                                </span>
+                                                                ,耗时:[[${(m?.meta?.endTime - m?.meta?.beginTime) / 1000}]]秒
                                                             </span>
+                                                            <span th:if="${m?.meta?.model eq '增量'}">
+                                                                总数:[[${#numbers.formatDecimal((m?.meta?.success + m?.meta?.fail),0 ,0)}]]
+                                                            </span>
+
                                                             <span th:if="${m?.meta?.success gt 0}">,成功:[[${m?.meta?.success}]]</span>
                                                             <span th:if="${m?.meta?.fail gt 0}">,失败:[[${m?.meta?.fail}]] <a th:id="${m?.meta?.id}" href="javascript:;" class="label label-danger queryData">日志</a></span>
                                                         </td>