|
|
@@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory;
|
|
8
|
8
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
9
|
9
|
import org.springframework.stereotype.Service;
|
|
10
|
10
|
|
|
|
11
|
+import java.nio.charset.StandardCharsets;
|
|
11
|
12
|
import java.math.BigDecimal;
|
|
12
|
13
|
import java.sql.*;
|
|
13
|
14
|
import java.time.ZoneOffset;
|
|
|
@@ -48,11 +49,11 @@ public class TdEngineService {
|
|
48
|
49
|
// 每批次最大列数(防止 TDengine 行超限)
|
|
49
|
50
|
private static final int MAX_COLUMNS_PER_INSERT = 100;
|
|
50
|
51
|
|
|
51
|
|
- // 拆分超级表的列数阈值
|
|
52
|
|
- private static final int SPLIT_STABLE_COLUMN_THRESHOLD = 100;
|
|
|
52
|
+ // 默认 VARCHAR 字段长度限制(用于未提供实际长度时的兜底)
|
|
|
53
|
+ private static final int DEFAULT_VARCHAR_LENGTH = 16;
|
|
53
|
54
|
|
|
54
|
|
- // 默认 VARCHAR 字段长度限制
|
|
55
|
|
- private static final int DEFAULT_VARCHAR_LENGTH = 128;
|
|
|
55
|
+ // TDengine 数据保留天数(超过此天数的数据自动删除)
|
|
|
56
|
+ private static final int DATA_RETENTION_DAYS = 180;
|
|
56
|
57
|
|
|
57
|
58
|
// 东八区时区偏移(避免重复创建)
|
|
58
|
59
|
private static final ZoneOffset ZONE_OFFSET_8 = ZoneOffset.of("+8");
|
|
|
@@ -237,7 +238,7 @@ public class TdEngineService {
|
|
237
|
238
|
try (Connection conn = getConnection(); Statement stmt = conn.createStatement()) {
|
|
238
|
239
|
stmt.setQueryTimeout(10);
|
|
239
|
240
|
|
|
240
|
|
- stmt.executeUpdate("CREATE DATABASE IF NOT EXISTS " + wrapName(dbName));
|
|
|
241
|
+ stmt.executeUpdate("CREATE DATABASE IF NOT EXISTS " + wrapName(dbName) + " KEEP " + DATA_RETENTION_DAYS);
|
|
241
|
242
|
|
|
242
|
243
|
// 创建超级表:固定 ts + controller_id 列(TDengine 要求至少 2 列)
|
|
243
|
244
|
String stableSql = String.format(
|
|
|
@@ -292,17 +293,18 @@ public class TdEngineService {
|
|
292
|
293
|
private void splitAndInsertToMultipleStables(String dbName, String stableName, String controllerId,
|
|
293
|
294
|
List<Map<String, Object>> dataList,
|
|
294
|
295
|
Map<String, String> columnTypes) throws SQLException {
|
|
|
296
|
+
|
|
295
|
297
|
// 按首字符 UTF-8 值分组
|
|
296
|
|
- Map<Integer, Map<String, String>> groupColumnTypes = new LinkedHashMap<>();
|
|
|
298
|
+ Map<Integer, Map<String, String>> groupColumnTypeMap = new LinkedHashMap<>();
|
|
297
|
299
|
for (Map.Entry<String, String> entry : columnTypes.entrySet()) {
|
|
298
|
300
|
int groupId = getFirstCharGroupId(entry.getKey());
|
|
299
|
|
- groupColumnTypes.computeIfAbsent(groupId, k -> new LinkedHashMap<>()).put(entry.getKey(), entry.getValue());
|
|
|
301
|
+ groupColumnTypeMap.computeIfAbsent(groupId, k -> new LinkedHashMap<>()).put(entry.getKey(), entry.getValue());
|
|
300
|
302
|
}
|
|
301
|
303
|
|
|
302
|
|
- log.info("拆分后超级表数量: {} | 分组: {}", groupColumnTypes.size(), groupColumnTypes.keySet());
|
|
|
304
|
+ log.info("拆分后超级表数量: {} | 分组: {}", groupColumnTypeMap.size(), groupColumnTypeMap.keySet());
|
|
303
|
305
|
|
|
304
|
306
|
// 每个分组作为一个超级表,处理其对应的列和数据
|
|
305
|
|
- for (Map.Entry<Integer, Map<String, String>> group : groupColumnTypes.entrySet()) {
|
|
|
307
|
+ for (Map.Entry<Integer, Map<String, String>> group : groupColumnTypeMap.entrySet()) {
|
|
306
|
308
|
int groupId = group.getKey();
|
|
307
|
309
|
// 超级表名 = superTableName + groupId (例如 charge + 3 = charge3)
|
|
308
|
310
|
String newStableName = stableName+"_"+groupId;
|
|
|
@@ -330,7 +332,7 @@ public class TdEngineService {
|
|
330
|
332
|
return 0;
|
|
331
|
333
|
}
|
|
332
|
334
|
String firstChar = columnName.substring(0, 1);
|
|
333
|
|
- byte[] bytes = firstChar.getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
|
|
335
|
+ byte[] bytes = firstChar.getBytes(StandardCharsets.UTF_8);
|
|
334
|
336
|
return (bytes[0] & 0xFF) % 10;
|
|
335
|
337
|
}
|
|
336
|
338
|
|
|
|
@@ -340,7 +342,9 @@ public class TdEngineService {
|
|
340
|
342
|
private List<Map<String, Object>> filterDataByColumnGroup(List<Map<String, Object>> dataList, Set<String> columns) {
|
|
341
|
343
|
List<Map<String, Object>> filtered = new ArrayList<>();
|
|
342
|
344
|
for (Map<String, Object> data : dataList) {
|
|
343
|
|
- if (data == null) continue;
|
|
|
345
|
+ if (data == null) {
|
|
|
346
|
+ continue;
|
|
|
347
|
+ }
|
|
344
|
348
|
Map<String, Object> filteredRow = new LinkedHashMap<>();
|
|
345
|
349
|
for (String col : columns) {
|
|
346
|
350
|
if (data.containsKey(col)) {
|
|
|
@@ -352,11 +356,6 @@ public class TdEngineService {
|
|
352
|
356
|
return filtered;
|
|
353
|
357
|
}
|
|
354
|
358
|
|
|
355
|
|
- private String extractSuperTableName(String table) {
|
|
356
|
|
- int idx = table.lastIndexOf('_');
|
|
357
|
|
- return idx > 0 ? table.substring(0, idx) : table;
|
|
358
|
|
- }
|
|
359
|
|
-
|
|
360
|
359
|
/**
|
|
361
|
360
|
* 收集数据中所有动态列及其类型
|
|
362
|
361
|
*/
|
|
|
@@ -379,6 +378,31 @@ public class TdEngineService {
|
|
379
|
378
|
}
|
|
380
|
379
|
|
|
381
|
380
|
/**
|
|
|
381
|
+ * 收集数据中所有动态列的最大字符串长度(用于动态 VARCHAR 长度)
|
|
|
382
|
+ */
|
|
|
383
|
+ private Map<String, Integer> collectColumnMaxLengths(List<Map<String, Object>> dataList) {
|
|
|
384
|
+ Map<String, Integer> maxLengths = new HashMap<>();
|
|
|
385
|
+ for (Map<String, Object> data : dataList) {
|
|
|
386
|
+ if (data == null) {
|
|
|
387
|
+ continue;
|
|
|
388
|
+ }
|
|
|
389
|
+ for (Map.Entry<String, Object> entry : data.entrySet()) {
|
|
|
390
|
+ String key = entry.getKey();
|
|
|
391
|
+ if (!isValidFieldName(key) || isReservedColumn(key)) {
|
|
|
392
|
+ continue;
|
|
|
393
|
+ }
|
|
|
394
|
+ Object value = entry.getValue();
|
|
|
395
|
+ if (value != null) {
|
|
|
396
|
+ int len = value.toString().length();
|
|
|
397
|
+ maxLengths.merge(key, len, Math::max);
|
|
|
398
|
+ }
|
|
|
399
|
+ }
|
|
|
400
|
+ }
|
|
|
401
|
+ log.debug("收集到的列最大长度: {}", maxLengths);
|
|
|
402
|
+ return maxLengths;
|
|
|
403
|
+ }
|
|
|
404
|
+
|
|
|
405
|
+ /**
|
|
382
|
406
|
* 构建批量插入 SQL
|
|
383
|
407
|
*/
|
|
384
|
408
|
private String buildInsertSql(String dbName, String table, String superTableName,
|
|
|
@@ -432,7 +456,7 @@ public class TdEngineService {
|
|
432
|
456
|
if (!hasData) {
|
|
433
|
457
|
return null;
|
|
434
|
458
|
}
|
|
435
|
|
- log.info("生成的 INSERT SQL | 列类型: {} | SQL 前100字符: {}", columnTypes, sql.toString().substring(0, Math.min(100, sql.toString().length())));
|
|
|
459
|
+ log.info("生成的 INSERT SQL | 列类型: {} | SQL 前100字符: {}", columnTypes, sql.substring(0, Math.min(100, sql.toString().length())));
|
|
436
|
460
|
sql.setLength(sql.length() - 1);
|
|
437
|
461
|
return sql.toString();
|
|
438
|
462
|
}
|
|
|
@@ -456,6 +480,8 @@ public class TdEngineService {
|
|
456
|
480
|
return;
|
|
457
|
481
|
}
|
|
458
|
482
|
|
|
|
483
|
+ Map<String, Integer> columnMaxLengths = collectColumnMaxLengths(dataList);
|
|
|
484
|
+
|
|
459
|
485
|
log.info("收集到的列类型: {}", columnTypes);
|
|
460
|
486
|
Set<String> existingColumns = getStableColumns(dbName, superTableName);
|
|
461
|
487
|
log.info("超级表已有列: {}", existingColumns);
|
|
|
@@ -473,7 +499,7 @@ public class TdEngineService {
|
|
473
|
499
|
return;
|
|
474
|
500
|
}
|
|
475
|
501
|
|
|
476
|
|
- ensureColumnsExist(dbName, superTableName, columnTypes);
|
|
|
502
|
+ ensureColumnsExist(dbName, superTableName, columnTypes, columnMaxLengths);
|
|
477
|
503
|
|
|
478
|
504
|
existingColumns = getStableColumns(dbName, superTableName);
|
|
479
|
505
|
if (existingColumns.size() >= MAX_COLUMNS_PER_STABLE) {
|
|
|
@@ -564,7 +590,7 @@ public class TdEngineService {
|
|
564
|
590
|
/**
|
|
565
|
591
|
* 根据 TdEngine 列类型获取创建列的 SQL 类型
|
|
566
|
592
|
*/
|
|
567
|
|
- private String getColumnTypeForDDL(String tdType, String columnName) {
|
|
|
593
|
+ private String getColumnTypeForDDL(String tdType, Integer maxLen) {
|
|
568
|
594
|
switch (tdType) {
|
|
569
|
595
|
case "BOOL":
|
|
570
|
596
|
case "BIGINT":
|
|
|
@@ -573,7 +599,8 @@ public class TdEngineService {
|
|
573
|
599
|
return tdType;
|
|
574
|
600
|
case "VARCHAR":
|
|
575
|
601
|
default:
|
|
576
|
|
- return "VARCHAR(" + DEFAULT_VARCHAR_LENGTH + ")";
|
|
|
602
|
+ int len = (maxLen != null && maxLen > 0) ? maxLen + 5 : DEFAULT_VARCHAR_LENGTH;
|
|
|
603
|
+ return "VARCHAR(" + len + ")";
|
|
577
|
604
|
}
|
|
578
|
605
|
}
|
|
579
|
606
|
|
|
|
@@ -617,20 +644,16 @@ public class TdEngineService {
|
|
617
|
644
|
}
|
|
618
|
645
|
}
|
|
619
|
646
|
|
|
620
|
|
- // 字符串类型校验长度
|
|
621
|
|
- if (strValue.length() > DEFAULT_VARCHAR_LENGTH) {
|
|
622
|
|
- log.debug("字段值超长,截断存储 | 列: {} | 值长度: {} | 最大: {} | 截断后: {}...",
|
|
623
|
|
- columnName, strValue.length(), DEFAULT_VARCHAR_LENGTH, strValue.substring(0, DEFAULT_VARCHAR_LENGTH));
|
|
624
|
|
- strValue = strValue.substring(0, DEFAULT_VARCHAR_LENGTH);
|
|
625
|
|
- }
|
|
|
647
|
+ // 字符串类型:使用实际值长度 + 5 冗余,不截断
|
|
|
648
|
+ // (TDengine VARCHAR 实际限制在创建列时已保证充足)
|
|
626
|
649
|
return "'" + escapeValue(strValue) + "'";
|
|
627
|
650
|
}
|
|
628
|
651
|
|
|
629
|
652
|
/**
|
|
630
|
653
|
* 确保列存在,如有新列则 ALTER 添加(根据类型添加对应列)
|
|
631
|
654
|
*/
|
|
632
|
|
- private void ensureColumnsExist(String dbName, String superTableName, Map<String, String> columnTypes)
|
|
633
|
|
- throws SQLException {
|
|
|
655
|
+ private void ensureColumnsExist(String dbName, String superTableName, Map<String, String> columnTypes,
|
|
|
656
|
+ Map<String, Integer> columnMaxLengths) throws SQLException {
|
|
634
|
657
|
|
|
635
|
658
|
Set<String> existingColumns = getStableColumns(dbName, superTableName);
|
|
636
|
659
|
|
|
|
@@ -646,6 +669,7 @@ public class TdEngineService {
|
|
646
|
669
|
for (Map.Entry<String, String> entry : columnTypes.entrySet()) {
|
|
647
|
670
|
String col = entry.getKey();
|
|
648
|
671
|
String colType = entry.getValue();
|
|
|
672
|
+ Integer maxLen = columnMaxLengths != null ? columnMaxLengths.get(col) : null;
|
|
649
|
673
|
|
|
650
|
674
|
// 再次检查列数上限
|
|
651
|
675
|
if (existingColumns.size() >= MAX_COLUMNS_PER_STABLE) {
|
|
|
@@ -655,7 +679,7 @@ public class TdEngineService {
|
|
655
|
679
|
}
|
|
656
|
680
|
|
|
657
|
681
|
if (!existingColumns.contains(col)) {
|
|
658
|
|
- String ddlType = getColumnTypeForDDL(colType, col);
|
|
|
682
|
+ String ddlType = getColumnTypeForDDL(colType, maxLen);
|
|
659
|
683
|
String alterSql = String.format(
|
|
660
|
684
|
"ALTER STABLE %s.%s ADD COLUMN %s %s",
|
|
661
|
685
|
wrapName(dbName),
|
|
|
@@ -752,6 +776,96 @@ public class TdEngineService {
|
|
752
|
776
|
log.info("清除了 TdEngine 超级表结构缓存");
|
|
753
|
777
|
}
|
|
754
|
778
|
|
|
|
779
|
+ /**
|
|
|
780
|
+ * 根据 controllerId、deviceId 查询 TDengine 最新数据
|
|
|
781
|
+ * @param controllerId 控制器ID
|
|
|
782
|
+ * @param deviceId 设备ID
|
|
|
783
|
+ * @param key 列名,不传则返回所有列
|
|
|
784
|
+ */
|
|
|
785
|
+ public Object queryLatestData(String controllerId, String deviceId, String key) throws SQLException {
|
|
|
786
|
+ // 构建 dbName 前缀
|
|
|
787
|
+ String dbNamePrefix = "pe_iot_" + controllerId.substring(0, 2);
|
|
|
788
|
+
|
|
|
789
|
+ // 遍历 10 个分组查找数据
|
|
|
790
|
+ for (int groupId = 0; groupId < 10; groupId++) {
|
|
|
791
|
+ // 表名格式: {deviceId}_{groupId}_{controllerId} (与 insertBatch 保持一致)
|
|
|
792
|
+ String tableName = deviceId + "_" + groupId + "_" + controllerId;
|
|
|
793
|
+
|
|
|
794
|
+ // 检查表是否存在
|
|
|
795
|
+ if (!isTableExists(dbNamePrefix, tableName)) {
|
|
|
796
|
+ continue;
|
|
|
797
|
+ }
|
|
|
798
|
+
|
|
|
799
|
+ // 构建查询 SQL
|
|
|
800
|
+ String colPart = (key != null && !key.trim().isEmpty()) ? wrapName(key) : "*";
|
|
|
801
|
+ String sql = String.format("SELECT %s FROM %s.%s ORDER BY ts DESC LIMIT 1",
|
|
|
802
|
+ colPart, wrapName(dbNamePrefix), wrapName(tableName));
|
|
|
803
|
+
|
|
|
804
|
+ try (Connection conn = getConnection();
|
|
|
805
|
+ Statement stmt = conn.createStatement();
|
|
|
806
|
+ ResultSet rs = stmt.executeQuery(sql)) {
|
|
|
807
|
+
|
|
|
808
|
+ if (rs.next()) {
|
|
|
809
|
+ if (key != null && !key.trim().isEmpty()) {
|
|
|
810
|
+ // 返回指定列的值
|
|
|
811
|
+ return rs.getObject(1);
|
|
|
812
|
+ } else {
|
|
|
813
|
+ // 返回整行数据
|
|
|
814
|
+ java.util.Map<String, Object> row = new java.util.LinkedHashMap<>();
|
|
|
815
|
+ ResultSetMetaData metaData = rs.getMetaData();
|
|
|
816
|
+ int columnCount = metaData.getColumnCount();
|
|
|
817
|
+ for (int i = 1; i <= columnCount; i++) {
|
|
|
818
|
+ row.put(metaData.getColumnLabel(i), rs.getObject(i));
|
|
|
819
|
+ }
|
|
|
820
|
+ return row;
|
|
|
821
|
+ }
|
|
|
822
|
+ }
|
|
|
823
|
+ } catch (SQLException e) {
|
|
|
824
|
+ // 列不存在,继续查找下一个分组
|
|
|
825
|
+ if (e.getMessage().contains("Invalid column")) {
|
|
|
826
|
+ log.debug("表 {} 中无列 {},继续查找其他分组", tableName, key);
|
|
|
827
|
+ continue;
|
|
|
828
|
+ }
|
|
|
829
|
+ throw e;
|
|
|
830
|
+ }
|
|
|
831
|
+ }
|
|
|
832
|
+ return null;
|
|
|
833
|
+ }
|
|
|
834
|
+
|
|
|
835
|
+ /**
|
|
|
836
|
+ * 检查表是否存在
|
|
|
837
|
+ */
|
|
|
838
|
+ private boolean isTableExists(String dbName, String tableName) throws SQLException {
|
|
|
839
|
+ // 先检查数据库是否存在
|
|
|
840
|
+ String checkDbSql = "SHOW DATABASES";
|
|
|
841
|
+ try (Connection conn = getConnection();
|
|
|
842
|
+ Statement stmt = conn.createStatement()) {
|
|
|
843
|
+ stmt.setQueryTimeout(5);
|
|
|
844
|
+ boolean dbExists = false;
|
|
|
845
|
+ try (ResultSet rs = stmt.executeQuery(checkDbSql)) {
|
|
|
846
|
+ while (rs.next()) {
|
|
|
847
|
+ if (dbName.equals(rs.getString(1))) {
|
|
|
848
|
+ dbExists = true;
|
|
|
849
|
+ break;
|
|
|
850
|
+ }
|
|
|
851
|
+ }
|
|
|
852
|
+ }
|
|
|
853
|
+ if (!dbExists) {
|
|
|
854
|
+ return false;
|
|
|
855
|
+ }
|
|
|
856
|
+ // 检查表是否存在
|
|
|
857
|
+ String sql = String.format("SHOW %s.TABLES", dbName);
|
|
|
858
|
+ try (ResultSet rs = stmt.executeQuery(sql)) {
|
|
|
859
|
+ while (rs.next()) {
|
|
|
860
|
+ if (tableName.equals(rs.getString(1))) {
|
|
|
861
|
+ return true;
|
|
|
862
|
+ }
|
|
|
863
|
+ }
|
|
|
864
|
+ }
|
|
|
865
|
+ return false;
|
|
|
866
|
+ }
|
|
|
867
|
+ }
|
|
|
868
|
+
|
|
755
|
869
|
public void close() {
|
|
756
|
870
|
log.info("关闭 TdEngine 服务...");
|
|
757
|
871
|
if (dataSource != null) {
|