Merge pull request #20 from F-ca7/feature/v1_3_5

BatchTool V1.3.5
polardb · Dec 13, 2023 · bfb6f29 · bfb6f29
2 parents 3986c0e + 5575026
commit bfb6f29
Show file tree

Hide file tree

Showing 10 changed files with 64 additions and 38 deletions.
diff --git a/batch-tool/pom.xml b/batch-tool/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.alibaba.polardbx</groupId>
     <artifactId>batch-tool</artifactId>
-    <version>1.3.4</version>
+    <version>1.3.5</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

diff --git a/batch-tool/src/main/java/cmd/BaseOperateCommand.java b/batch-tool/src/main/java/cmd/BaseOperateCommand.java
@@ -16,6 +16,8 @@
 
 package cmd;
 
+import org.apache.commons.collections4.CollectionUtils;
+
 import javax.validation.constraints.NotNull;
 import java.util.List;
 
@@ -26,6 +28,8 @@ public class BaseOperateCommand {
 
     private final String dbName;
 
+    private boolean isDbOperation;
+
     /**
      * 支持同一个库下的多张表
      */
@@ -44,18 +48,25 @@ public class BaseOperateCommand {
     public BaseOperateCommand(@NotNull String dbName, boolean shardingEnabled) {
         this.dbName = dbName;
         this.shardingEnabled = shardingEnabled;
+        this.isDbOperation = true;
     }
 
-    public BaseOperateCommand(@NotNull String dbName, List<String> tableName, boolean shardingEnabled) {
+    public BaseOperateCommand(@NotNull String dbName, List<String> tableNames, boolean shardingEnabled) {
         this.dbName = dbName;
-        this.tableNames = tableName;
+        this.tableNames = tableNames;
         this.shardingEnabled = shardingEnabled;
+        this.isDbOperation = CollectionUtils.isEmpty(tableNames);
     }
 
     public List<String> getTableNames() {
         return tableNames;
     }
 
+    public void setTableNamesInCmd(List<String> tableNames) {
+        this.tableNames = tableNames;
+        this.isDbOperation = CollectionUtils.isEmpty(tableNames);
+    }
+
     public void setTableNames(List<String> tableNames) {
         this.tableNames = tableNames;
     }
@@ -77,7 +88,7 @@ public String getDbName() {
     }
 
     public boolean isDbOperation() {
-        return this.tableNames == null;
+        return isDbOperation;
     }
 
     public List<String> getColumnNames() {

diff --git a/batch-tool/src/main/java/cmd/CommandUtil.java b/batch-tool/src/main/java/cmd/CommandUtil.java
@@ -292,7 +292,7 @@ private static BaseOperateCommand initCommand(ConfigResult result) {
         default:
             throw new IllegalArgumentException("Unsupported command: " + commandTypeStr);
         }
-        command.setTableNames(getTableNames(result));
+        command.setTableNamesInCmd(getTableNames(result));
         command.setColumnNames(getColumnNames(result));
         return command;
     }

diff --git a/batch-tool/src/main/java/cmd/ConfigArgOption.java b/batch-tool/src/main/java/cmd/ConfigArgOption.java
@@ -105,7 +105,7 @@ private static ConfigArgOption of(String argShort, String argLong, String desc,
         of("ringsize", "ringSize", "Ring buffer size.", "size (power of 2)");
    public static final ConfigArgOption ARG_SHORT_QUOTE_ENCLOSE_MODE =
         of("quote", "quoteMode",
-            "The mode of how field values are enclosed by double-quotes when exporting table (default AUTO).",
+            "The mode of how field values are enclosed by double-quotes when exporting table (default FORCE).",
             "AUTO | FORCE | NONE");
     public static final ConfigArgOption ARG_SHORT_TPS_LIMIT =
         of("tps", "tpsLimit", "Configure of tps limit (default -1: no limit).", "tps limit");

diff --git a/batch-tool/src/main/java/exec/export/SingleThreadExportExecutor.java b/batch-tool/src/main/java/exec/export/SingleThreadExportExecutor.java
@@ -66,7 +66,7 @@ private void doDefaultExport(String tableName, ExecutorService executor) {
             TableFieldMetaInfo tableFieldMetaInfo = DbUtil.getTableFieldMetaInfo(connection,
                 getSchemaName(), tableName, command.getColumnNames());
             DirectExportWorker directExportWorker = ExportWorkerFactory.buildDefaultDirectExportWorker(dataSource,
-                new TableTopology("", tableName), tableFieldMetaInfo,
+                new TableTopology(tableName), tableFieldMetaInfo,
                 fileName, config);
             executor.submit(directExportWorker);
             logger.info("开始导出表 {} 到文件 {}", tableName, fileName);

diff --git a/batch-tool/src/main/java/model/config/ConfigConstant.java b/batch-tool/src/main/java/model/config/ConfigConstant.java
@@ -49,7 +49,7 @@ public class ConfigConstant {
     /**
      * 默认引号转义模式
      */
-    public static final QuoteEncloseMode DEFAULT_QUOTE_ENCLOSE_MODE = QuoteEncloseMode.AUTO;
+    public static final QuoteEncloseMode DEFAULT_QUOTE_ENCLOSE_MODE = QuoteEncloseMode.FORCE;
 
     public static final int DEFAULT_READ_BLOCK_SIZE_IN_MB = 2;
 

diff --git a/batch-tool/src/main/java/model/config/QuoteEncloseMode.java b/batch-tool/src/main/java/model/config/QuoteEncloseMode.java
@@ -18,16 +18,18 @@
 
 public enum QuoteEncloseMode {
     /**
-     * 默认模式
      * 根据字段（字符型）内是否含有特殊字符 如分隔符、换行符
      * 来添加双引号
      */
     AUTO,
     /**
+     * 性能最差
+     * 默认模式
      * 不管字段类型与内容，都添加双引号
      */
     FORCE,
     /**
+     * 性能最好
      * 不管字段类型与内容，都不添加双引号
      * 如 已知表的字段都是数值型、或已知字段内容无特殊字符
      */

diff --git a/batch-tool/src/main/java/model/db/TableTopology.java b/batch-tool/src/main/java/model/db/TableTopology.java
@@ -16,6 +16,8 @@
 
 package model.db;
 
+import org.apache.commons.lang3.StringUtils;
+
 public class TableTopology {
     /**
      * 分库名
@@ -44,6 +46,10 @@ public String getTableName() {
         return tableName;
     }
 
+    public boolean hasGroup() {
+        return StringUtils.isNotEmpty(groupName);
+    }
+
     @Override
     public String toString() {
         if (groupName == null || groupName.length() == 0) {

diff --git a/batch-tool/src/main/java/worker/common/BaseShardedConsumer.java b/batch-tool/src/main/java/worker/common/BaseShardedConsumer.java
@@ -90,8 +90,7 @@ public void onProxyEvent(BatchLineEvent event) {
                 }
             }
         } catch (Exception e) {
-            e.printStackTrace();
-            logger.error(e.getMessage());
+            logger.error(e.getMessage(), e);
             // 认为无法恢复
             System.exit(1);
         } finally {
@@ -123,8 +122,7 @@ protected void execSqlWithShardingHint(TableTopology topology, StringBuilder dat
             String sql = getSqlWithHint(topology, data);
             stmt.execute(sql);
         } catch (SQLException e) {
-            e.printStackTrace();
-            logger.error(e.getMessage());
+            logger.error(e.getMessage(), e);
             System.exit(1);
         } finally {
             JdbcUtils.close(stmt);

diff --git a/batch-tool/src/main/java/worker/util/ExportUtil.java b/batch-tool/src/main/java/worker/util/ExportUtil.java
@@ -40,19 +40,24 @@ public static String getDirectSql(TableTopology topology,
             return getDirectSql(topology, fieldMetaInfoList);
         }
 
-        return String.format(DIRECT_NODE_HINT + "select %s from %s where %s;",
-            topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+        if (topology.hasGroup()) {
+            return String.format(DIRECT_NODE_HINT + "select %s from %s where %s;",
+                topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+                topology.getTableName(), whereCondition);
+        }
+        return String.format("select %s from %s where %s;",
+            formatFieldWithDateType(fieldMetaInfoList),
             topology.getTableName(), whereCondition);
     }
 
     private static String getDirectSql(TableTopology topology,
                                        List<FieldMetaInfo> fieldMetaInfoList) {
-        if (topology.getGroupName().isEmpty()) {
-            return String.format("select %s from %s;", formatFieldWithDateType(fieldMetaInfoList),
+        if (topology.hasGroup()) {
+            return String.format(DIRECT_NODE_HINT + "select %s from %s;",
+                topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
                 topology.getTableName());
         }
-        return String.format(DIRECT_NODE_HINT + "select %s from %s;",
-            topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+        return String.format("select %s from %s;", formatFieldWithDateType(fieldMetaInfoList),
             topology.getTableName());
     }
 
@@ -89,34 +94,32 @@ public static String getOrderBySql(TableTopology topology,
                                        List<FieldMetaInfo> fieldMetaInfoList,
                                        String columnName, boolean isAscending) {
         String orderType = isAscending ? "asc" : "desc";
-        return String.format(DIRECT_NODE_HINT + "select %s from %s order by %s " + orderType,
-            topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+        if (topology.hasGroup()) {
+            return String.format(DIRECT_NODE_HINT + "select %s from %s order by %s " + orderType,
+                topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+                topology.getTableName(), columnName);
+        }
+        return String.format("select %s from %s order by %s " + orderType,
+            formatFieldWithDateType(fieldMetaInfoList),
             topology.getTableName(), columnName);
     }
 
     public static String getOrderBySql(TableTopology topology,
                                        List<FieldMetaInfo> fieldMetaInfoList,
                                        List<String> columnNameList, boolean isAscending) {
         String orderType = isAscending ? "asc" : "desc";
-        return String.format(DIRECT_NODE_HINT + "select %s from %s order by %s " + orderType,
-            topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+        if (topology.hasGroup()) {
+            return String.format(DIRECT_NODE_HINT + "select %s from %s order by %s " + orderType,
+                topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+                topology.getTableName(),
+                StringUtils.join(columnNameList, ","));
+        }
+        return String.format("select %s from %s order by %s " + orderType,
+            formatFieldWithDateType(fieldMetaInfoList),
             topology.getTableName(),
             StringUtils.join(columnNameList, ","));
     }
 
-    public static String getOrderBySql(TableTopology topology,
-                                       List<FieldMetaInfo> fieldMetaInfoList,
-                                       String columnName,
-                                       String whereCondition, boolean isAscending) {
-        if (StringUtils.isEmpty(whereCondition)) {
-            return getOrderBySql(topology, fieldMetaInfoList, columnName, isAscending);
-        }
-        String orderType = isAscending ? "asc" : "desc";
-        return String.format(DIRECT_NODE_HINT + "select %s from %s where %s order by %s " + orderType,
-            topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
-            topology.getTableName(), whereCondition, columnName);
-    }
-
     public static String getOrderBySql(TableTopology topology,
                                        List<FieldMetaInfo> fieldMetaInfoList,
                                        List<String> columnNameList,
@@ -125,8 +128,14 @@ public static String getOrderBySql(TableTopology topology,
             return getOrderBySql(topology, fieldMetaInfoList, columnNameList, isAscending);
         }
         String orderType = isAscending ? "asc" : "desc";
-        return String.format(DIRECT_NODE_HINT + "select %s from %s where %s order by %s " + orderType,
-            topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+        if (topology.hasGroup()) {
+            return String.format(DIRECT_NODE_HINT + "select %s from %s where %s order by %s " + orderType,
+                topology.getGroupName(), formatFieldWithDateType(fieldMetaInfoList),
+                topology.getTableName(), whereCondition,
+                StringUtils.join(columnNameList, ","));
+        }
+        return String.format("select %s from %s where %s order by %s " + orderType,
+            formatFieldWithDateType(fieldMetaInfoList),
             topology.getTableName(), whereCondition,
             StringUtils.join(columnNameList, ","));
     }