|
|
@@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory;
|
|
8
|
8
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
9
|
9
|
import org.springframework.stereotype.Service;
|
|
10
|
10
|
|
|
|
11
|
+import java.nio.charset.StandardCharsets;
|
|
11
|
12
|
import java.sql.*;
|
|
12
|
13
|
import java.time.ZoneOffset;
|
|
13
|
14
|
import java.util.*;
|
|
|
@@ -47,9 +48,6 @@ public class TdEngineService {
|
|
47
|
48
|
// 每批次最大列数(防止 TDengine 行超限)
|
|
48
|
49
|
private static final int MAX_COLUMNS_PER_INSERT = 100;
|
|
49
|
50
|
|
|
50
|
|
- // 拆分超级表的列数阈值
|
|
51
|
|
- private static final int SPLIT_STABLE_COLUMN_THRESHOLD = 100;
|
|
52
|
|
-
|
|
53
|
51
|
// 默认 VARCHAR 字段长度限制
|
|
54
|
52
|
private static final int DEFAULT_VARCHAR_LENGTH = 128;
|
|
55
|
53
|
|
|
|
@@ -128,7 +126,7 @@ public class TdEngineService {
|
|
128
|
126
|
if (name == null || name.isEmpty()) {
|
|
129
|
127
|
return "`unknown`";
|
|
130
|
128
|
}
|
|
131
|
|
- return "`" + name.replaceAll("`", "") + "`";
|
|
|
129
|
+ return "`" + name.replace("`", "") + "`";
|
|
132
|
130
|
}
|
|
133
|
131
|
|
|
134
|
132
|
private boolean isValidFieldName(String name) {
|
|
|
@@ -291,17 +289,18 @@ public class TdEngineService {
|
|
291
|
289
|
private void splitAndInsertToMultipleStables(String dbName, String stableName, String tableName,
|
|
292
|
290
|
List<Map<String, Object>> dataList,
|
|
293
|
291
|
Map<String, String> columnTypes) throws SQLException {
|
|
|
292
|
+
|
|
294
|
293
|
// 按首字符 UTF-8 值分组
|
|
295
|
|
- Map<Integer, Map<String, String>> groupColumnTypes = new LinkedHashMap<>();
|
|
|
294
|
+ Map<Integer, Map<String, String>> groupColumnTypeMap = new LinkedHashMap<>();
|
|
296
|
295
|
for (Map.Entry<String, String> entry : columnTypes.entrySet()) {
|
|
297
|
296
|
int groupId = getFirstCharGroupId(entry.getKey());
|
|
298
|
|
- groupColumnTypes.computeIfAbsent(groupId, k -> new LinkedHashMap<>()).put(entry.getKey(), entry.getValue());
|
|
|
297
|
+ groupColumnTypeMap.computeIfAbsent(groupId, k -> new LinkedHashMap<>()).put(entry.getKey(), entry.getValue());
|
|
299
|
298
|
}
|
|
300
|
299
|
|
|
301
|
|
- log.info("拆分后超级表数量: {} | 分组: {}", groupColumnTypes.size(), groupColumnTypes.keySet());
|
|
|
300
|
+ log.info("拆分后超级表数量: {} | 分组: {}", groupColumnTypeMap.size(), groupColumnTypeMap.keySet());
|
|
302
|
301
|
|
|
303
|
302
|
// 每个分组作为一个超级表,处理其对应的列和数据
|
|
304
|
|
- for (Map.Entry<Integer, Map<String, String>> group : groupColumnTypes.entrySet()) {
|
|
|
303
|
+ for (Map.Entry<Integer, Map<String, String>> group : groupColumnTypeMap.entrySet()) {
|
|
305
|
304
|
int groupId = group.getKey();
|
|
306
|
305
|
// 超级表名 = superTableName + groupId (例如 charge + 3 = charge3)
|
|
307
|
306
|
String newStableName = stableName+"_"+groupId;
|
|
|
@@ -329,7 +328,7 @@ public class TdEngineService {
|
|
329
|
328
|
return 0;
|
|
330
|
329
|
}
|
|
331
|
330
|
String firstChar = columnName.substring(0, 1);
|
|
332
|
|
- byte[] bytes = firstChar.getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
|
|
331
|
+ byte[] bytes = firstChar.getBytes(StandardCharsets.UTF_8);
|
|
333
|
332
|
return (bytes[0] & 0xFF) % 10;
|
|
334
|
333
|
}
|
|
335
|
334
|
|
|
|
@@ -339,7 +338,9 @@ public class TdEngineService {
|
|
339
|
338
|
private List<Map<String, Object>> filterDataByColumnGroup(List<Map<String, Object>> dataList, Set<String> columns) {
|
|
340
|
339
|
List<Map<String, Object>> filtered = new ArrayList<>();
|
|
341
|
340
|
for (Map<String, Object> data : dataList) {
|
|
342
|
|
- if (data == null) continue;
|
|
|
341
|
+ if (data == null) {
|
|
|
342
|
+ continue;
|
|
|
343
|
+ }
|
|
343
|
344
|
Map<String, Object> filteredRow = new LinkedHashMap<>();
|
|
344
|
345
|
for (String col : columns) {
|
|
345
|
346
|
if (data.containsKey(col)) {
|
|
|
@@ -351,11 +352,6 @@ public class TdEngineService {
|
|
351
|
352
|
return filtered;
|
|
352
|
353
|
}
|
|
353
|
354
|
|
|
354
|
|
- private String extractSuperTableName(String table) {
|
|
355
|
|
- int idx = table.lastIndexOf('_');
|
|
356
|
|
- return idx > 0 ? table.substring(0, idx) : table;
|
|
357
|
|
- }
|
|
358
|
|
-
|
|
359
|
355
|
/**
|
|
360
|
356
|
* 收集数据中所有动态列及其类型
|
|
361
|
357
|
*/
|
|
|
@@ -431,7 +427,7 @@ public class TdEngineService {
|
|
431
|
427
|
if (!hasData) {
|
|
432
|
428
|
return null;
|
|
433
|
429
|
}
|
|
434
|
|
- log.info("生成的 INSERT SQL | 列类型: {} | SQL 前100字符: {}", columnTypes, sql.toString().substring(0, Math.min(100, sql.toString().length())));
|
|
|
430
|
+ log.info("生成的 INSERT SQL | 列类型: {} | SQL 前100字符: {}", columnTypes, sql.substring(0, Math.min(100, sql.toString().length())));
|
|
435
|
431
|
sql.setLength(sql.length() - 1);
|
|
436
|
432
|
return sql.toString();
|
|
437
|
433
|
}
|
|
|
@@ -563,7 +559,7 @@ public class TdEngineService {
|
|
563
|
559
|
/**
|
|
564
|
560
|
* 根据 TdEngine 列类型获取创建列的 SQL 类型
|
|
565
|
561
|
*/
|
|
566
|
|
- private String getColumnTypeForDDL(String tdType, String columnName) {
|
|
|
562
|
+ private String getColumnTypeForDDL(String tdType) {
|
|
567
|
563
|
switch (tdType) {
|
|
568
|
564
|
case "BOOL":
|
|
569
|
565
|
case "BIGINT":
|
|
|
@@ -654,7 +650,7 @@ public class TdEngineService {
|
|
654
|
650
|
}
|
|
655
|
651
|
|
|
656
|
652
|
if (!existingColumns.contains(col)) {
|
|
657
|
|
- String ddlType = getColumnTypeForDDL(colType, col);
|
|
|
653
|
+ String ddlType = getColumnTypeForDDL(colType);
|
|
658
|
654
|
String alterSql = String.format(
|
|
659
|
655
|
"ALTER STABLE %s.%s ADD COLUMN %s %s",
|
|
660
|
656
|
wrapName(dbName),
|