add module dbsyncer-connector-file

dbsyncer-connector/dbsyncer-connector-file/pom.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>dbsyncer-connector</artifactId>
+        <groupId>org.ghi</groupId>
+        <version>2.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>dbsyncer-connector-file</artifactId>
+
+    <dependencies>
+        <!-- sdk -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-sdk</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-log4j2</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+    </dependencies>
+</project>

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnector.java

@@ -8,8 +8,11 @@ import org.apache.commons.io.LineIterator;
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorException;
-import org.dbsyncer.connector.config.FileConfig;
+import org.dbsyncer.connector.file.cdc.FileListener;
+import org.dbsyncer.connector.file.model.FileResolver;
+import org.dbsyncer.connector.file.config.FileConfig;
+import org.dbsyncer.connector.file.model.FileSchema;
+import org.dbsyncer.connector.file.validator.FileConfigValidator;
 import org.dbsyncer.sdk.config.CommandConfig;
 import org.dbsyncer.sdk.config.ReaderConfig;
 import org.dbsyncer.sdk.config.WriterBatchConfig;
@@ -24,7 +27,6 @@ import org.dbsyncer.sdk.model.Table;
 import org.dbsyncer.sdk.spi.ConnectorService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
 import java.io.File;
@@ -48,7 +50,6 @@ import java.util.stream.Collectors;
  * @Version 1.0.0
  * @Date 2022-05-05 23:19
  */
-@Component
 public final class FileConnector extends AbstractConnector implements ConnectorService<FileConnectorInstance, FileConfig> {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
@@ -57,6 +58,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS
     private final String FILE_NAME = "fileName";
     private final String FILE_PATH = "filePath";
     private final FileResolver fileResolver = new FileResolver();
+    private final FileConfigValidator configValidator = new FileConfigValidator();
 
     @Override
     public String getConnectorType() {
@@ -85,7 +87,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS
 
     @Override
     public ConfigValidator getConfigValidator() {
-        return null;
+        return configValidator;
     }
 
     @Override
@@ -146,7 +148,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS
                 count.addAndGet(1);
             }
         } catch (IOException e) {
-            throw new ConnectorException(e.getCause());
+            throw new FileException(e.getCause());
         } finally {
             IOUtils.closeQuietly(reader);
         }
@@ -181,7 +183,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS
                 }
             }
         } catch (IOException e) {
-            throw new ConnectorException(e.getCause());
+            throw new FileException(e.getCause());
         } finally {
             IOUtils.closeQuietly(reader);
         }
@@ -193,7 +195,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS
         List<Map> data = config.getData();
         if (CollectionUtils.isEmpty(data)) {
             logger.error("writer data can not be empty.");
-            throw new ConnectorException("writer data can not be empty.");
+            throw new FileException("writer data can not be empty.");
         }
 
         final List<Field> fields = config.getFields();

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java

@@ -1,7 +1,11 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.connector.file;
 
 import org.dbsyncer.common.util.JsonUtil;
-import org.dbsyncer.connector.config.FileConfig;
+import org.dbsyncer.connector.file.config.FileConfig;
+import org.dbsyncer.connector.file.model.FileSchema;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
 import org.dbsyncer.sdk.model.Field;
 import org.springframework.util.Assert;

dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileException.java

@@ -0,0 +1,30 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
+ */
+public class FileException extends RuntimeException {
+
+	private static final long serialVersionUID = 1L;
+
+	public FileException(String message) {
+        super(message);
+    }
+
+    public FileException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public FileException(Throwable cause) {
+        super(cause);
+    }
+
+    protected FileException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/BufferedRandomAccessFile.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/BufferedRandomAccessFile.java

@@ -1,4 +1,7 @@
-package org.dbsyncer.connector.file;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.cdc;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -7,9 +10,9 @@ import java.io.RandomAccessFile;
 import java.util.Arrays;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/7 22:27
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-07 22:27
  */
 public class BufferedRandomAccessFile extends RandomAccessFile {
     static final int LogBuffSz_ = 16; // 64K buffer

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java

@@ -1,11 +1,17 @@
-package org.dbsyncer.connector.file;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.cdc;
 
 import org.apache.commons.io.IOUtils;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorException;
-import org.dbsyncer.connector.config.FileConfig;
+import org.dbsyncer.connector.file.FileConnectorInstance;
+import org.dbsyncer.connector.file.FileException;
+import org.dbsyncer.connector.file.model.FileSchema;
+import org.dbsyncer.connector.file.config.FileConfig;
+import org.dbsyncer.connector.file.model.FileResolver;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.listener.AbstractListener;
 import org.dbsyncer.sdk.listener.event.RowChangedEvent;
@@ -36,9 +42,9 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/6 21:42
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
  */
 public class FileListener extends AbstractListener {
 
@@ -86,7 +92,7 @@ public class FileListener extends AbstractListener {
         } catch (Exception e) {
             logger.error("启动失败:{}", e.getMessage());
             closePipelineAndWatch();
-            throw new ConnectorException(e);
+            throw new FileException(e);
         } finally {
             connectLock.unlock();
         }

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java

@@ -1,12 +1,15 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.connector.file.column;
 
 import java.sql.Date;
 import java.sql.Timestamp;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/5 22:39
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
  */
 public interface ColumnValue {
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/impl/FileColumnValue.java

@@ -1,16 +1,20 @@
-package org.dbsyncer.connector.file.column;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.column.impl;
 
 import org.dbsyncer.common.util.DateFormatUtil;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.file.column.ColumnValue;
 
 import java.sql.Date;
 import java.sql.Timestamp;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/6 15:48
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
  */
 public class FileColumnValue implements ColumnValue {
 

dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/config/FileConfig.java

@@ -0,0 +1,53 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.config;
+
+import org.dbsyncer.sdk.model.ConnectorConfig;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-05 23:19
+ */
+public class FileConfig extends ConnectorConfig {
+
+    /**
+     * 文件目录
+     */
+    private String fileDir;
+
+    /**
+     * 分隔符
+     */
+    private char separator;
+
+    /**
+     * 文件描述信息
+     */
+    private String schema;
+
+    public String getFileDir() {
+        return fileDir;
+    }
+
+    public void setFileDir(String fileDir) {
+        this.fileDir = fileDir;
+    }
+
+    public char getSeparator() {
+        return separator;
+    }
+
+    public void setSeparator(char separator) {
+        this.separator = separator;
+    }
+
+    public String getSchema() {
+        return schema;
+    }
+
+    public void setSchema(String schema) {
+        this.schema = schema;
+    }
+}

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileResolver.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model/FileResolver.java

@@ -1,8 +1,11 @@
-package org.dbsyncer.connector.file;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.model;
 
 import org.dbsyncer.common.column.Lexer;
 import org.dbsyncer.connector.file.column.ColumnValue;
-import org.dbsyncer.connector.file.column.FileColumnValue;
+import org.dbsyncer.connector.file.column.impl.FileColumnValue;
 import org.dbsyncer.sdk.model.Field;
 
 import java.util.ArrayList;
@@ -11,9 +14,9 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/6 15:46
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-06 00:04
  */
 public class FileResolver {
 

dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileSchema.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model/FileSchema.java

@@ -1,13 +1,16 @@
-package org.dbsyncer.connector.file;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.file.model;
 
 import org.dbsyncer.sdk.model.Field;
 
 import java.util.List;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/5/6 0:04
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-05-06 00:04
  */
 public class FileSchema {
 

dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigValidator.java → dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/validator/FileConfigValidator.java

@@ -1,14 +1,13 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.biz.checker.impl.connector;
+package org.dbsyncer.connector.file.validator;
 
-import org.dbsyncer.sdk.connector.ConfigValidator;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.config.FileConfig;
-import org.dbsyncer.connector.file.FileSchema;
-import org.springframework.stereotype.Component;
+import org.dbsyncer.connector.file.model.FileSchema;
+import org.dbsyncer.connector.file.config.FileConfig;
+import org.dbsyncer.sdk.connector.ConfigValidator;
 import org.springframework.util.Assert;
 
 import java.io.File;
@@ -22,7 +21,6 @@ import java.util.Map;
  * @Version 1.0.0
  * @Date 2022-05-06 00:04
  */
-@Component
 public class FileConfigValidator implements ConfigValidator<FileConfig> {
 
     @Override

dbsyncer-connector/dbsyncer-connector-file/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService

@@ -0,0 +1 @@
+org.dbsyncer.connector.file.FileConnector

dbsyncer-connector/pom.xml

@@ -12,6 +12,7 @@
     <packaging>pom</packaging>
     <modules>
         <module>dbsyncer-connector-elasticsearch</module>
+        <module>dbsyncer-connector-file</module>
     </modules>
 
     <dependencies>
@@ -22,11 +23,18 @@
             <version>${project.parent.version}</version>
         </dependency>
 
-        <!-- dbsyncer-connector-elasticsearch -->
+        <!-- dbsyncer-connector-elasticsearch
         <dependency>
             <groupId>org.ghi</groupId>
             <artifactId>dbsyncer-connector-elasticsearch</artifactId>
             <version>${project.parent.version}</version>
+        </dependency> -->
+
+        <!-- dbsyncer-connector-file -->
+        <dependency>
+            <groupId>org.ghi</groupId>
+            <artifactId>dbsyncer-connector-file</artifactId>
+            <version>${project.parent.version}</version>
         </dependency>
 
         <!-- Spring-JDBC -->