impl search

dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java

@@ -14,6 +14,7 @@ import org.dbsyncer.parser.enums.ModelEnum;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.storage.query.Query;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
@@ -79,8 +80,14 @@ public class MonitorServiceImpl implements MonitorService {
 
         int pageNum = NumberUtils.toInt(params.get("pageNum"), 1);
         int pageSize = NumberUtils.toInt(params.get("pageSize"), 20);
-        List<DataVo> list = manager.queryData(id, pageNum, pageSize)
-                .stream()
+        Query query = new Query(pageNum, pageSize);
+        // 查询异常信息
+        String error = params.get(ConfigConstant.DATA_ERROR);
+        if(StringUtils.isNotBlank(error)){
+            query.put(ConfigConstant.DATA_ERROR, error, true);
+        }
+
+        List<DataVo> list = manager.queryData(query, id).stream()
                 .map(m -> convert2Vo(m, DataVo.class))
                 .collect(Collectors.toList());
         return list;
@@ -95,11 +102,15 @@ public class MonitorServiceImpl implements MonitorService {
 
     @Override
     public List<LogVo> queryLog(Map<String, String> params) {
-        String type = params.get(ConfigConstant.CONFIG_MODEL_TYPE);
         int pageNum = NumberUtils.toInt(params.get("pageNum"), 1);
         int pageSize = NumberUtils.toInt(params.get("pageSize"), 20);
-        List<LogVo> list = manager.queryLog(type, pageNum, pageSize)
-                .stream()
+        Query query = new Query(pageNum, pageSize);
+        // 查询日志内容
+        String json = params.get(ConfigConstant.CONFIG_MODEL_JSON);
+        if(StringUtils.isNotBlank(json)){
+            query.put(ConfigConstant.CONFIG_MODEL_JSON, json, true);
+        }
+        List<LogVo> list = manager.queryLog(query).stream()
                 .map(m -> convert2Vo(m, LogVo.class))
                 .collect(Collectors.toList());
         return list;

dbsyncer-manager/src/main/java/org/dbsyncer/manager/Manager.java

@@ -9,6 +9,7 @@ import org.dbsyncer.listener.enums.QuartzFilterEnum;
 import org.dbsyncer.parser.enums.ConvertEnum;
 import org.dbsyncer.parser.model.*;
 import org.dbsyncer.plugin.config.Plugin;
+import org.dbsyncer.storage.query.Query;
 
 import java.util.List;
 import java.util.Map;
@@ -88,12 +89,12 @@ public interface Manager extends Executor {
     List<Config> getConfigAll();
 
     // Data
-    List<Map> queryData(String id, int pageNum, int pageSize);
+    List<Map> queryData(Query query, String collectionId);
 
-    void clearData(String id);
+    void clearData(String collectionId);
 
     // Log
-    List<Map> queryLog(String type, int pageNum, int pageSize);
+    List<Map> queryLog(Query query);
 
     void clearLog();
 

dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java

@@ -12,8 +12,8 @@ import org.dbsyncer.manager.config.QueryConfig;
 import org.dbsyncer.manager.enums.GroupStrategyEnum;
 import org.dbsyncer.manager.enums.HandlerEnum;
 import org.dbsyncer.manager.puller.Puller;
-import org.dbsyncer.manager.template.impl.OperationTemplate;
 import org.dbsyncer.manager.template.impl.DataTemplate;
+import org.dbsyncer.manager.template.impl.OperationTemplate;
 import org.dbsyncer.parser.Parser;
 import org.dbsyncer.parser.enums.ConvertEnum;
 import org.dbsyncer.parser.enums.MetaEnum;
@@ -22,6 +22,7 @@ import org.dbsyncer.plugin.PluginFactory;
 import org.dbsyncer.plugin.config.Plugin;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageEnum;
+import org.dbsyncer.storage.query.Query;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
@@ -235,18 +236,18 @@ public class ManagerFactory implements Manager, ApplicationContextAware, Applica
     }
 
     @Override
-    public List<Map> queryData(String id, int pageNum, int pageSize) {
-        return dataTemplate.query(StorageEnum.DATA, id, pageNum, pageSize);
+    public List<Map> queryData(Query query, String collectionId) {
+        return dataTemplate.query(StorageEnum.DATA, query, collectionId);
     }
 
     @Override
-    public void clearData(String id) {
-        dataTemplate.clear(StorageEnum.DATA, id);
+    public void clearData(String collectionId) {
+        dataTemplate.clear(StorageEnum.DATA, collectionId);
     }
 
     @Override
-    public List<Map> queryLog(String type, int pageNum, int pageSize) {
-        return dataTemplate.query(StorageEnum.LOG, type, pageNum, pageSize);
+    public List<Map> queryLog(Query query) {
+        return dataTemplate.query(StorageEnum.LOG, query, null);
     }
 
     @Override

dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/impl/DataTemplate.java

@@ -26,8 +26,7 @@ public final class DataTemplate {
     @Autowired
     private StorageService storageService;
 
-    public List<Map> query(StorageEnum type, String collectionId, int pageNum, int pageSize) {
-        Query query = new Query(pageNum, pageSize);
+    public List<Map> query(StorageEnum type, Query query, String collectionId) {
         return storageService.query(type, query, collectionId);
     }
 

dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Shard.java

@@ -2,12 +2,15 @@ package org.dbsyncer.storage.lucene;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.*;
 import org.apache.lucene.search.*;
+import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.dbsyncer.storage.query.Option;
 
 import java.io.File;
 import java.io.IOException;
@@ -116,23 +119,19 @@ public class Shard {
     }
 
     public List<Map> query(Query query) throws IOException {
-        return query(query, 1, 20);
+        final IndexSearcher searcher = getSearcher();
+        final TopDocs topDocs = searcher.search(query, MAX_SIZE);
+        return search(searcher, topDocs, new Option(), 1, 20);
     }
 
     public List<Map> query(Query query, Sort sort) throws IOException {
-        return query(query, 1, 20, sort);
+        return query(new Option(query), 1, 20, sort);
     }
 
-    public List<Map> query(Query query, int pageNum, int pageSize) throws IOException {
+    public List<Map> query(Option option, int pageNum, int pageSize, Sort sort) throws IOException {
         final IndexSearcher searcher = getSearcher();
-        final TopDocs topDocs = searcher.search(query, MAX_SIZE);
-        return search(searcher, topDocs, pageNum, pageSize);
-    }
-
-    public List<Map> query(Query query, int pageNum, int pageSize, Sort sort) throws IOException {
-        final IndexSearcher searcher = getSearcher();
-        final TopDocs topDocs = searcher.search(query, MAX_SIZE, sort);
-        return search(searcher, topDocs, pageNum, pageSize);
+        final TopDocs topDocs = searcher.search(option.getQuery(), MAX_SIZE, sort);
+        return search(searcher, topDocs, option, pageNum, pageSize);
     }
 
     /**
@@ -140,11 +139,12 @@ public class Shard {
      *
      * @param searcher
      * @param topDocs
+     * @param option
      * @param pageNum
      * @param pageSize
      * @throws IOException
      */
-    private List<Map> search(final IndexSearcher searcher, final TopDocs topDocs, int pageNum, int pageSize) throws IOException {
+    private List<Map> search(IndexSearcher searcher, TopDocs topDocs, Option option, int pageNum, int pageSize) throws IOException {
         ScoreDoc[] docs = topDocs.scoreDocs;
         int total = docs.length;
         int begin = (pageNum - 1) * pageSize;
@@ -166,6 +166,23 @@ public class Shard {
             r = new LinkedHashMap<>();
             while (iterator.hasNext()) {
                 f = iterator.next();
+
+                // 开启高亮
+                if (option.isEnableHighLightSearch()) {
+                    try {
+                        final String key = f.name();
+                        if (option.getHighLightKeys().contains(key)) {
+                            String content = doc.get(key);
+                            TokenStream tokenStream = analyzer.tokenStream("", content);
+                            content = option.getHighlighter().getBestFragment(tokenStream, content);
+                            r.put(key, content);
+                            continue;
+                        }
+                    } catch (InvalidTokenOffsetsException e) {
+                        e.printStackTrace();
+                    }
+                }
+
                 r.put(f.name(), f.stringValue());
             }
             list.add(r);

dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/Option.java

@@ -1,8 +1,12 @@
 package org.dbsyncer.storage.query;
 
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.highlight.Highlighter;
+import org.apache.lucene.search.highlight.QueryScorer;
 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
 import org.dbsyncer.common.util.CollectionUtils;
 
+import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -13,24 +17,37 @@ import java.util.stream.Collectors;
  */
 public class Option {
 
+    private Query       query;
     private Set<String> highLightKeys;
-    private boolean enableHighLightSearch;
-    private SimpleHTMLFormatter formatter = null;
+    private boolean     enableHighLightSearch;
+    private Highlighter highlighter = null;
 
     public Option() {
     }
 
     public Option(Query query) {
-        this.highLightKeys = query.getParams().stream()
-                .filter(p -> p.isHighlighter())
-                .map(p -> p.getKey())
-                .collect(Collectors.toSet());
+        this.query = query;
+    }
+
+    public Option(Query query, List<Param> params) {
+        this.query = query;
+        if (!CollectionUtils.isEmpty(params)) {
+            this.highLightKeys = params.stream()
+                    .filter(p -> p.isHighlighter())
+                    .map(p -> p.getKey())
+                    .collect(Collectors.toSet());
+        }
         if (!CollectionUtils.isEmpty(highLightKeys)) {
             enableHighLightSearch = true;
-            formatter = new SimpleHTMLFormatter(query.getPreTag(), query.getPostTag());
+            SimpleHTMLFormatter formatter = new SimpleHTMLFormatter("<span style='color:red'>", "</span>");
+            highlighter = new Highlighter(formatter, new QueryScorer(query));
         }
     }
 
+    public Query getQuery() {
+        return query;
+    }
+
     public Set<String> getHighLightKeys() {
         return highLightKeys;
     }
@@ -39,7 +56,7 @@ public class Option {
         return enableHighLightSearch;
     }
 
-    public SimpleHTMLFormatter getFormatter() {
-        return formatter;
+    public Highlighter getHighlighter() {
+        return highlighter;
     }
-}
+}

dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/Query.java

@@ -12,10 +12,6 @@ public class Query {
 
     private List<Param> params;
 
-    private String preTag = "<span style='color:red'>";
-
-    private String postTag = "</span>";
-
     private int pageNum = 1;
 
     private int pageSize = 20;
@@ -46,28 +42,12 @@ public class Query {
         this.params = params;
     }
 
-    public String getPreTag() {
-        return preTag;
-    }
-
-    public String getPostTag() {
-        return postTag;
-    }
-
     public int getPageNum() {
         return pageNum;
     }
 
-    public void setPageNum(int pageNum) {
-        this.pageNum = pageNum;
-    }
-
     public int getPageSize() {
         return pageSize;
     }
 
-    public void setPageSize(int pageSize) {
-        this.pageSize = pageSize;
-    }
-
 }

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/DiskStorageServiceImpl.java

@@ -10,6 +10,7 @@ import org.dbsyncer.storage.StorageException;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageEnum;
 import org.dbsyncer.storage.lucene.Shard;
+import org.dbsyncer.storage.query.Option;
 import org.dbsyncer.storage.query.Param;
 import org.dbsyncer.storage.query.Query;
 import org.dbsyncer.storage.util.ParamsUtil;
@@ -78,10 +79,10 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
                 BooleanQuery.Builder builder = new BooleanQuery.Builder();
                 params.forEach(p -> builder.add(new TermQuery(new Term(p.getKey(), p.getValue())), BooleanClause.Occur.MUST));
                 BooleanQuery q = builder.build();
-                return shard.query(q, pageNum, pageSize, sort);
+                return shard.query(new Option(q, query.getParams()), pageNum, pageSize, sort);
             }
 
-            return shard.query(new MatchAllDocsQuery(), pageNum, pageSize, sort);
+            return shard.query(new Option(new MatchAllDocsQuery()), pageNum, pageSize, sort);
         }
         return Collections.emptyList();
     }

dbsyncer-web/src/main/resources/static/js/monitor/index.js

@@ -1,5 +1,29 @@
+function formatDate(time) {
+   var date = new Date(time);
+  var YY = date.getFullYear() + '-';
+  var MM = (date.getMonth() + 1 < 10 ? '0' + (date.getMonth() + 1) : date.getMonth() + 1) + '-';
+  var DD = (date.getDate() < 10 ? '0' + (date.getDate()) : date.getDate());
+  var hh = (date.getHours() < 10 ? '0' + date.getHours() : date.getHours()) + ':';
+  var mm = (date.getMinutes() < 10 ? '0' + date.getMinutes() : date.getMinutes()) + ':';
+  var ss = (date.getSeconds() < 10 ? '0' + date.getSeconds() : date.getSeconds());
+
+  return YY + MM + DD +" "+hh + mm + ss;
+}
+
+// 全局Ajax get
+function doGetter(url, params, action) {
+    $.loadingT(true);
+    $.get(url, params, function (data) {
+        $.loadingT(false);
+        action(data);
+    }).error(function (xhr, status, info) {
+        $.loadingT(false);
+        bootGrowl("访问异常，请刷新或重试.", "danger");
+    });
+}
+
 // 查看详细数据
-function bindQueryDataEvent() {
+function bindQueryDataDetailEvent() {
     $(".metaDataList .queryData").click(function () {
         var json = $(this).attr("json");
         var html = '<div class="row driver_break_word">' + json + '</div>';
@@ -52,6 +76,65 @@ function bindClearEvent($btn, $title, $msg, $url){
     });
 }
 
+// 查看数据
+function bindQueryDataEvent() {
+    $("#queryDataBtn").click(function () {
+        var keyword = $("#searchDataKeyword").val();
+        var id = $("select[name='metaData']").select2("val");
+        doGetter('/monitor/queryData', {"error": keyword, "id" : id}, function (data) {
+            if (data.success == true) {
+                showDataList(data.resultValue);
+            } else {
+                bootGrowl(data.resultValue, "danger");
+            }
+        });
+    });
+}
+
+// 查看日志
+function bindQueryLogEvent() {
+    $("#queryLogBtn").click(function () {
+        var keyword = $("#searchLogKeyword").val();
+        doGetter('/monitor/queryLog', {"json": keyword}, function (data) {
+            if (data.success == true) {
+                showLogList(data.resultValue);
+            } else {
+                bootGrowl(data.resultValue, "danger");
+            }
+        });
+    });
+}
+
+function showDataList(arr){
+    var size = arr.length;
+    var html = '';
+    for(i = 0; i < size; i++) {
+        html += '<tr>';
+        html += '<td>' + (i + 1) + '</td>';
+        html += '<td>' + arr[i].event + '</td>';
+        html += '<td>' + (arr[i].success ? '<span class="label label-success">成功</span>' : '<span class="label label-warning">失败</span>') + '</td>';
+        html += '<td>' + arr[i].error + '</td>';
+        html += '<td>' + formatDate(arr[i].createTime) + '</td>';
+        html += '<td><a json=' + arr[i].json + ' href="javascript:;" class="label label-info queryData">查看数据</a></td>';
+        html += '</tr>';
+    }
+    $("#dataList").html(html);
+    bindQueryDataDetailEvent();
+}
+
+function showLogList(arr){
+    var size = arr.length;
+    var html = '';
+    for(i = 0; i < size; i++) {
+        html += '<tr>';
+        html += '<td>' + (i + 1) + '</td>';
+        html += '<td>' + arr[i].json + '</td>';
+        html += '<td>' + formatDate(arr[i].createTime) + '</td>';
+        html += '</tr>';
+    }
+    $("#logList").html(html);
+}
+
 $(function () {
     // 初始化select2插件
     $(".select-control").select2({
@@ -65,7 +148,9 @@ $(function () {
         $initContainer.load('/monitor?id=' + $id);
     });
 
+    bindQueryLogEvent();
     bindQueryDataEvent();
+    bindQueryDataDetailEvent();
     bindClearEvent($(".clearDataBtn"), "确认清空数据？", "清空数据成功!", "/monitor/clearData");
     bindClearEvent($(".clearLogBtn"), "确认清空日志？", "清空日志成功!", "/monitor/clearLog");
 

dbsyncer-web/src/main/resources/templates/monitor/monitor.html

@@ -17,10 +17,10 @@
                         </select>
                     </div>
                     <div class="col-sm-4">
-                        <input class="form-control" type="text" maxlength="32" placeholder="请输入内容关键字(最多32个字)." />
+                        <input id="searchDataKeyword" class="form-control" type="text" maxlength="32" placeholder="请输入内容关键字(最多32个字)." />
                     </div>
                     <div class="col-md-1">
-                        <button type="button" class="btn btn-primary">查询数据</button>
+                        <button id="queryDataBtn" type="button" class="btn btn-primary">查询数据</button>
                     </div>
                     <div class="col-md-4 text-right">
                         <button th:id="${metaId}" type="button" class="btn btn-default clearDataBtn">清空数据</button>
@@ -38,17 +38,19 @@
                         <th style="width:10%;">详情</th>
                     </tr>
                     </thead>
-                    <tr th:each="d,s : ${data}">
-                        <td th:text="${s.index}+1"></td>
-                        <td th:text="${d?.event}"></td>
-                        <td>
-                            <span th:if="${d?.success}" class="label label-success">成功</span>
-                            <span th:if="${not d?.success}" class="label label-warning">失败</span>
-                        </td>
-                        <td th:text="${d?.error}"></td>
-                        <td th:text="${#dates.format(d?.createTime, 'yyyy-MM-dd HH:mm:ss')}"></td>
-                        <td><a th:json="${d?.json}" href="javascript:;" class="label label-info queryData">查看数据</a></td>
-                    </tr>
+                    <tbody id="dataList">
+                        <tr th:each="d,s : ${data}">
+                            <td th:text="${s.index}+1"></td>
+                            <td th:text="${d?.event}"></td>
+                            <td>
+                                <span th:if="${d?.success}" class="label label-success">成功</span>
+                                <span th:if="${not d?.success}" class="label label-warning">失败</span>
+                            </td>
+                            <td th:text="${d?.error}"></td>
+                            <td th:text="${#dates.format(d?.createTime, 'yyyy-MM-dd HH:mm:ss')}"></td>
+                            <td><a th:json="${d?.json}" href="javascript:;" class="label label-info queryData">查看数据</a></td>
+                        </tr>
+                    </tbody>
                 </table>
             </div>
 
@@ -80,10 +82,10 @@
             <div class="col-md-12">
                 <div class="form-group">
                     <div class="col-sm-4">
-                        <input class="form-control" type="text" maxlength="32" placeholder="请输入内容关键字(最多32个字)." />
+                        <input id="searchLogKeyword" class="form-control" type="text" maxlength="32" placeholder="请输入内容关键字(最多32个字)." />
                     </div>
                     <div class="col-md-1">
-                        <button type="button" class="btn btn-primary">查询日志</button>
+                        <button id="queryLogBtn" type="button" class="btn btn-primary">查询日志</button>
                     </div>
                     <div class="col-md-4"></div>
                     <div class="col-md-3 text-right">
@@ -99,11 +101,13 @@
                         <th style="width:27%;">时间</th>
                     </tr>
                     </thead>
-                    <tr th:each="l,s : ${log}">
-                        <td th:text="${s.index}+1"></td>
-                        <td th:text="${l?.json}"></td>
-                        <td th:text="${#dates.format(l?.createTime, 'yyyy-MM-dd HH:mm:ss')}"></td>
-                    </tr>
+                    <tbody id="logList">
+                        <tr th:each="l,s : ${log}">
+                            <td th:text="${s.index}+1"></td>
+                            <td th:text="${l?.json}"></td>
+                            <td th:text="${#dates.format(l?.createTime, 'yyyy-MM-dd HH:mm:ss')}"></td>
+                        </tr>
+                    </tbody>
                 </table>
             </div>