!118 merge

Merge pull request !118 from AE86/V_1.0.0_RC

dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Shard.java

@@ -5,11 +5,22 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.util.IOUtils;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.storage.StorageException;
 import org.slf4j.Logger;
@@ -19,7 +30,11 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * @author AE86
@@ -54,40 +69,23 @@ public class Shard {
             directory = FSDirectory.open(dir);
             // 分词器
             analyzer = new SmartChineseAnalyzer();
-            // 创建索引写入配置
-            config = new IndexWriterConfig(analyzer);
-            // 默认32M, 减少合并次数
-            config.setRAMBufferSizeMB(32);
-            // 创建索引写入对象
-            indexWriter = new IndexWriter(directory, config);
-            // 创建索引的读取器
-            indexReader = DirectoryReader.open(indexWriter);
+            reopen();
         } catch (IOException e) {
             throw new StorageException(e);
         }
     }
 
-    public void insert(Document doc) throws IOException {
-        execute(doc, () -> indexWriter.addDocument(doc));
-    }
-
-    public void insertBatch(List<Document> docs) throws IOException {
+    public void insertBatch(List<Document> docs) {
         execute(docs, () -> indexWriter.addDocuments(docs));
     }
 
-    public void update(Term term, Document doc) throws IOException {
+    public void update(Term term, Document doc) {
         if (null != term) {
             execute(doc, () -> indexWriter.updateDocument(term, doc));
         }
     }
 
-    public void delete(Term term) throws IOException {
-        if (null != term) {
-            execute(term, () -> indexWriter.deleteDocuments(term));
-        }
-    }
-
-    public void deleteBatch(Term... terms) throws IOException {
+    public void deleteBatch(Term... terms) {
         if (null != terms) {
             execute(terms, () -> indexWriter.deleteDocuments(terms));
         }
@@ -203,14 +201,50 @@ public class Shard {
         return list;
     }
 
-    private void execute(Object value, Callback callback) throws IOException {
-        if (null != value && indexWriter.isOpen()) {
-            callback.execute();
-            indexWriter.flush();
-            indexWriter.commit();
+    private void execute(Object value, Callback callback) {
+        if (value == null) {
+            return;
+        }
+
+        if (indexWriter.isOpen()) {
+            try {
+                doExecute(callback);
+            } catch (IOException e) {
+                // 程序异常导致文件锁未关闭 java.nio.channels.ClosedChannelException
+                logger.error(String.format("索引异常：%s", indexPath.getAbsolutePath()), e);
+            }
             return;
         }
-        logger.error(value.toString());
+
+        // 索引异常关闭
+        try {
+            reopen();
+            doExecute(callback);
+        } catch (IOException e) {
+            // 重试失败打印异常数据
+            logger.error(value.toString());
+        }
+    }
+
+    private void doExecute(Callback callback) throws IOException {
+        callback.execute();
+        indexWriter.flush();
+        indexWriter.commit();
+    }
+
+    private void reopen() throws IOException {
+        Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME);
+        if (writeLock != null) {
+            IOUtils.close(writeLock); // release write lock
+        }
+        // 创建索引写入配置
+        config = new IndexWriterConfig(analyzer);
+        // 默认32M, 减少合并次数
+        config.setRAMBufferSizeMB(32);
+        // 创建索引写入对象
+        indexWriter = new IndexWriter(directory, config);
+        // 创建索引的读取器
+        indexReader = DirectoryReader.open(indexWriter);
     }
 
     interface Callback {

dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/DiskStorageServiceImpl.java

@@ -128,11 +128,7 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
         for (int i = 0; i < size; i++) {
             terms[i] = getPrimaryKeyTerm(ids.get(i));
         }
-        try {
-            shard.deleteBatch(terms);
-        } catch (IOException e) {
-            throw new StorageException(e);
-        }
+        shard.deleteBatch(terms);
     }
 
     @Override

dbsyncer-web/src/main/resources/application.properties

@@ -31,7 +31,7 @@ dbsyncer.storage.binlog.recorder.queue-capacity=10000
 dbsyncer.storage.binlog.recorder.writer-period-millisecond=500
 dbsyncer.storage.binlog.recorder.reader-period-millisecond=2000
 dbsyncer.storage.support.mysql.enabled=false
-dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false
+dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
 dbsyncer.storage.support.mysql.config.username=root
 dbsyncer.storage.support.mysql.config.password=123
 
@@ -51,6 +51,4 @@ spring.thymeleaf.prefix=classpath:/public/
 spring.thymeleaf.suffix=.html
 spring.thymeleaf.mode=HTML
 spring.thymeleaf.encoding=UTF-8
-spring.thymeleaf.cache=false
-spring.resources.chain.strategy.content.enabled=true
-spring.resources.chain.strategy.content.paths=/**
+spring.thymeleaf.cache=false