Przeglądaj źródła

优化GC & 批量删表关系,重置总数

穿云 3 miesięcy temu
rodzic
commit
208219c34c

+ 4 - 6
dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java

@@ -82,9 +82,7 @@ public class MetricReporter implements ScheduledTaskJob {
 
     private final MappingReportMetric mappingReportMetric = new MappingReportMetric();
 
-    private AppReportMetric report = new AppReportMetric();
-
-    private final int SHOW_BUFFER_ACTUATOR_SIZE = 7;
+    private final AppReportMetric report = new AppReportMetric();
 
     @PostConstruct
     private void init() {
@@ -107,10 +105,10 @@ public class MetricReporter implements ScheduledTaskJob {
                     tableList.add(collect(bufferActuator, tableGroupCode, mapping.getName(), bufferActuator.getTableName()))
                 );
             });
-            List<MetricResponseInfo> sortList = tableList.stream()
+            list.addAll(tableList.stream()
                     .sorted(Comparator.comparing(MetricResponseInfo::getQueueUp).reversed())
-                    .collect(Collectors.toList());
-            list.addAll(sortList.size() <= SHOW_BUFFER_ACTUATOR_SIZE ? sortList : sortList.subList(0, SHOW_BUFFER_ACTUATOR_SIZE));
+                    .limit(7)
+                    .collect(Collectors.toList()));
         }
         return list.stream().map(MetricResponseInfo::getResponse).collect(Collectors.toList());
     }

+ 2 - 0
dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java

@@ -113,6 +113,8 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup
 
         // 合并驱动公共字段
         mergeMappingColumn(mapping);
+        // 更新meta
+        updateMeta(mapping, null);
 
         // 重置排序
         resetTableGroupAllIndex(mappingId);

+ 5 - 13
dbsyncer-connector/dbsyncer-connector-base/src/test/java/ConnectionTest.java

@@ -35,6 +35,7 @@ import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 /**
  * @Author AE86
@@ -333,22 +334,13 @@ public class ConnectionTest {
         int total = dataList.size();
         int taskSize = total % batchSize == 0 ? total / batchSize : total / batchSize + 1;
         final CountDownLatch latch = new CountDownLatch(taskSize);
-        int fromIndex = 0;
-        int toIndex = batchSize;
+        int offset = 0;
         for (int i = 0; i < taskSize; i++) {
-            final List<Object[]> data;
-            if (toIndex > total) {
-                toIndex = fromIndex + (total % batchSize);
-                data = dataList.subList(fromIndex, toIndex);
-            } else {
-                data = dataList.subList(fromIndex, toIndex);
-                fromIndex += batchSize;
-                toIndex += batchSize;
-            }
-
+            List<Object[]> slice = dataList.stream().skip(offset).limit(batchSize).collect(Collectors.toList());
+            offset += batchSize;
             pool.submit(() -> {
                 try {
-                    connectorInstance.execute(databaseTemplate -> databaseTemplate.batchUpdate(sql, data));
+                    connectorInstance.execute(databaseTemplate -> databaseTemplate.batchUpdate(sql, slice));
                 } catch (Exception e) {
                     logger.error(e.getMessage());
                 } finally {

+ 7 - 22
dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java

@@ -43,6 +43,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
+import java.util.stream.Collectors;
 
 /**
  * @author AE86
@@ -214,35 +215,19 @@ public class ParserComponentImpl implements ParserComponent {
             return result;
         }
 
-        List<Map> dataList = context.getTargetList();
         int batchSize = context.getBatchSize();
         // 总数
-        int total = dataList.size();
-        // 单次任务
-        if (total <= batchSize) {
-            return connectorFactory.writer(context);
-        }
-
+        int total = context.getTargetList().size();
         // 批量任务, 拆分
         int taskSize = total % batchSize == 0 ? total / batchSize : total / batchSize + 1;
-
-        final CountDownLatch latch = new CountDownLatch(taskSize);
-        int fromIndex = 0;
-        int toIndex = batchSize;
+        CountDownLatch latch = new CountDownLatch(taskSize);
+        int offset = 0;
         for (int i = 0; i < taskSize; i++) {
-            final List<Map> data;
-            if (toIndex > total) {
-                toIndex = fromIndex + (total % batchSize);
-                data = dataList.subList(fromIndex, toIndex);
-            } else {
-                data = dataList.subList(fromIndex, toIndex);
-                fromIndex += batchSize;
-                toIndex += batchSize;
-            }
-
             try {
                 PluginContext tmpContext = (PluginContext) context.clone();
-                tmpContext.setTargetList(data);
+                List<Map> slice = context.getTargetList().stream().skip(offset).limit(batchSize).collect(Collectors.toList());
+                offset += batchSize;
+                tmpContext.setTargetList(slice);
                 executor.execute(() -> {
                     try {
                         Result w = connectorFactory.writer(tmpContext);