feat(record): 增加物联网数据导出功能并优化性能

- 新增专用导出接口支持多设备批量导出
- 实现动态列导出根据设备类型选择对应数据列
- 添加数据过滤功能过滤无效和零值数据
- 增加最大导出记录数限制防止内存溢出
- 优化SQL查询移除ORDER BY提升大数据量导出性能
- 添加表名格式校验防止SQL注入安全风险
- 支持按日期分表查询导出历史数据
- 实现设备类型参数解析和数据有效性验证
boardTest
zangch@mesnac.com 5 days ago
parent bd86f818a5
commit 7f04ba13c4

@ -1,14 +1,16 @@
package com.os.ems.record.controller;
import java.math.BigDecimal;
import java.text.ParseException;
import java.util.Date;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletResponse;
import com.os.common.exception.ServiceException;
import com.os.common.core.page.PageDomain;
import com.os.common.core.page.TableSupport;
import org.checkerframework.checker.units.qual.A;
import org.springframework.format.annotation.DateTimeFormat;
import com.os.common.utils.StringUtils;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
@ -23,7 +25,7 @@ import com.os.common.core.page.TableDataInfo;
/**
* Controller
*
*
* @author zch
* @date 2025-04-28
*/
@ -34,6 +36,9 @@ public class RecordIotenvInstantController extends BaseController
@Autowired
private IRecordIotenvInstantService recordIotenvInstantService;
/** 最大导出记录数限制10万条 */
private static final int MAX_EXPORT_RECORDS = 100000;
/**
*
*/
@ -48,15 +53,196 @@ public class RecordIotenvInstantController extends BaseController
/**
*
*
*/
@Log(title = "物联网数据", businessType = BusinessType.EXPORT)
@PostMapping("/export")
public void export(HttpServletResponse response, RecordIotenvInstant recordIotenvInstant)throws ParseException
public void export(HttpServletResponse response, RecordIotenvInstant recordIotenvInstant) throws ParseException
{
List<RecordIotenvInstant> list = recordIotenvInstantService.selectRecordIotenvInstantList(recordIotenvInstant);
// 校验时间范围参数
Map<String, Object> params = recordIotenvInstant.getParams();
if (params == null || params.get("beginRecordTime") == null || params.get("endRecordTime") == null) {
throw new ServiceException("导出失败:请选择记录时间范围");
}
// 解析设备类型参数(用于动态导出列和数据过滤)
Set<Integer> monitorTypes = parseMonitorTypes(params);
// 使用专用导出方法性能优化移除ORDER BY支持多设备批量导出
List<RecordIotenvInstant> list = recordIotenvInstantService.selectRecordIotenvInstantListForExport(recordIotenvInstant);
// 根据设备类型过滤无效数据(零值和超范围值)
list = filterInvalidData(list, monitorTypes);
// 添加数据量检查,防止导出数据量过大导致内存溢出
if (list.size() > MAX_EXPORT_RECORDS) {
throw new ServiceException("导出数据量过大(" + list.size() + "条),请缩小时间范围或减少设备数量");
}
// 根据设备类型动态选择导出列
String[] includeColumns = getExportColumnsByType(monitorTypes);
ExcelUtil<RecordIotenvInstant> util = new ExcelUtil<RecordIotenvInstant>(RecordIotenvInstant.class);
util.exportExcel(response, list, "物联网数据数据");
// 使用showColumn方法指定需要导出的列
util.showColumn(includeColumns);
util.exportExcel(response, list, "物联网数据");
}
/**
*
*/
private Set<Integer> parseMonitorTypes(Map<String, Object> params) {
Set<Integer> types = new HashSet<>();
Object monitorTypesObj = params.get("monitorTypes");
if (monitorTypesObj != null && StringUtils.isNotEmpty(monitorTypesObj.toString())) {
String[] typeStrs = monitorTypesObj.toString().split(",");
for (String typeStr : typeStrs) {
try {
types.add(Integer.parseInt(typeStr.trim()));
} catch (NumberFormatException ignored) {
}
}
}
return types;
}
/**
*
* 使OR
* 1. (type=5)>0<=79
* 2. 湿(type=6)湿>0<=79
* 3. (type=7)>0<=79
* 4. (type=10)>0
*/
private List<RecordIotenvInstant> filterInvalidData(List<RecordIotenvInstant> list, Set<Integer> monitorTypes) {
if (monitorTypes.isEmpty()) {
// 未指定类型时,保留有任何有效数据的记录
return list.stream().filter(this::hasValidData).collect(Collectors.toList());
}
// 单一类型:严格按该类型过滤
if (monitorTypes.size() == 1) {
Integer type = monitorTypes.iterator().next();
return list.stream().filter(record -> isValidForType(record, type)).collect(Collectors.toList());
}
// 多类型使用OR关系满足任一选中类型的有效性即保留
return list.stream().filter(record -> {
for (Integer type : monitorTypes) {
if (isValidForType(record, type)) {
return true;
}
}
return false;
}).collect(Collectors.toList());
}
/** 判断记录对于指定类型是否有效 */
private boolean isValidForType(RecordIotenvInstant record, Integer type) {
switch (type) {
case 5: // 温度设备
return isValidTemperature(record.getTemperature());
case 6: // 温湿度设备
return isValidTemperature(record.getTemperature()) || isValidHumidity(record.getHumidity());
case 7: // 噪声设备
return isValidNoise(record.getNoise());
case 10: // 振动设备
return hasValidVibrationData(record);
default:
return hasValidData(record);
}
}
/** 判断温度是否有效:>0且<=79 */
private boolean isValidTemperature(BigDecimal temperature) {
return temperature != null && temperature.compareTo(BigDecimal.ZERO) > 0 && temperature.compareTo(new BigDecimal("79")) <= 0;
}
/** 判断湿度是否有效:>0且<=79 */
private boolean isValidHumidity(BigDecimal humidity) {
return humidity != null && humidity.compareTo(BigDecimal.ZERO) > 0 && humidity.compareTo(new BigDecimal("79")) <= 0;
}
/** 判断噪声是否有效:>0且<=79 */
private boolean isValidNoise(BigDecimal noise) {
return noise != null && noise.compareTo(BigDecimal.ZERO) > 0 && noise.compareTo(new BigDecimal("79")) <= 0;
}
/** 判断是否有有效的振动数据 */
private boolean hasValidVibrationData(RecordIotenvInstant record) {
return (record.getVibrationSpeed() != null && record.getVibrationSpeed().compareTo(BigDecimal.ZERO) > 0) ||
(record.getVibrationDisplacement() != null && record.getVibrationDisplacement().compareTo(BigDecimal.ZERO) > 0) ||
(record.getVibrationAcceleration() != null && record.getVibrationAcceleration().compareTo(BigDecimal.ZERO) > 0) ||
(record.getVibrationTemp() != null && record.getVibrationTemp().compareTo(BigDecimal.ZERO) > 0);
}
/** 判断记录是否有任何有效数据 */
private boolean hasValidData(RecordIotenvInstant record) {
return isValidTemperature(record.getTemperature()) ||
isValidHumidity(record.getHumidity()) ||
isValidNoise(record.getNoise()) ||
hasValidVibrationData(record);
}
/**
*
*
* type=5: ->
* type=6: 湿 -> 湿
* type=7: ->
* type=10: ->
* :
*/
private String[] getExportColumnsByType(Set<Integer> monitorTypes) {
List<String> columns = new ArrayList<>();
// 固定列:设备编号、设备名称
columns.add("monitorCode");
columns.add("monitorName");
if (monitorTypes.isEmpty() || monitorTypes.size() > 1) {
// 混合类型或未指定:导出所有数据列
columns.add("temperature");
columns.add("humidity");
columns.add("noise");
columns.add("vibrationSpeed");
columns.add("vibrationDisplacement");
columns.add("vibrationAcceleration");
columns.add("vibrationTemp");
} else {
// 单一类型:只导出对应的列
Integer type = monitorTypes.iterator().next();
switch (type) {
case 5: // 温度设备
columns.add("temperature");
break;
case 6: // 温湿度设备
columns.add("temperature");
columns.add("humidity");
break;
case 7: // 噪声设备
columns.add("noise");
break;
case 10: // 振动设备
columns.add("vibrationSpeed");
columns.add("vibrationDisplacement");
columns.add("vibrationAcceleration");
columns.add("vibrationTemp");
break;
default:
// 其他类型:导出所有列
columns.add("temperature");
columns.add("humidity");
columns.add("noise");
columns.add("vibrationSpeed");
columns.add("vibrationDisplacement");
columns.add("vibrationAcceleration");
columns.add("vibrationTemp");
break;
}
}
// 固定列:记录时间
columns.add("recodeTime");
return columns.toArray(new String[0]);
}
/**

@ -27,9 +27,19 @@ public class RecordIotenvInstant extends BaseEntity
private Long objid;
/** 计量设备编号 */
@Excel(name = "计量设备编号")
// @Excel(name = "计量设备编号")
private String monitorId;
/** 计量设备编号 */
@Excel(name = "计量设备编号")
private String monitorCode;
/** 设备名称 */
@Excel(name = "设备名称")
private String monitorName;
/** 温度 */
@Excel(name = "温度")
private BigDecimal temperature;
@ -68,7 +78,7 @@ public class RecordIotenvInstant extends BaseEntity
/** 采集时间 */
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
@Excel(name = "采集时间", width = 30, dateFormat = "yyyy-MM-dd HH:mm:ss")
// @Excel(name = "采集时间", width = 30, dateFormat = "yyyy-MM-dd HH:mm:ss")
private Date collectTime;
/** 记录时间 */
@ -79,12 +89,7 @@ public class RecordIotenvInstant extends BaseEntity
private String[] monitorIds;
//关联查询设备名称
private String monitorName;
/** 计量设备编号 */
@Excel(name = "计量设备编号")
private String monitorCode;
/** 能源类型 */
private Long monitorType;

@ -158,11 +158,25 @@ public interface RecordIotenvInstantMapper
/**
*
*
*
*
* @param tableNames
* @param recordIotenvInstant samplingInterval
* @return
*/
List<RecordIotenvInstant> selectRecordIotenvInstantListFromTablesWithSampling(@Param("tableNames") List<String> tableNames,
@Param("recordIotenvInstant") RecordIotenvInstant recordIotenvInstant);
/**
*
*
* 1. ORDER BY
* 2.
* 3.
*
* @param tableNames
* @param recordIotenvInstant
* @return
*/
List<RecordIotenvInstant> selectRecordIotenvInstantListForExport(@Param("tableNames") List<String> tableNames,
@Param("recordIotenvInstant") RecordIotenvInstant recordIotenvInstant);
}

@ -78,10 +78,22 @@ public interface IRecordIotenvInstantService
/**
* ID
*
*
* @param parentId ID
* @return
*/
public List<RecordIotenvInstant> selectRecordListByParentId(Long parentId);
/**
*
*
* 1. 使MapperORDER BY
* 2.
* 3.
*
* @param recordIotenvInstant
* @return
*/
public List<RecordIotenvInstant> selectRecordIotenvInstantListForExport(RecordIotenvInstant recordIotenvInstant) throws ParseException;
}

@ -8,6 +8,7 @@ import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.regex.Pattern;
import com.github.pagehelper.PageInfo;
import com.os.common.constant.HttpStatus;
@ -29,7 +30,7 @@ import com.os.ems.record.service.IRecordIotenvInstantService;
* @date 2025-04-28
*/
@Service
public class RecordIotenvInstantServiceImpl implements IRecordIotenvInstantService
public class RecordIotenvInstantServiceImpl implements IRecordIotenvInstantService
{
@Autowired
private RecordIotenvInstantMapper recordIotenvInstantMapper;
@ -37,6 +38,9 @@ public class RecordIotenvInstantServiceImpl implements IRecordIotenvInstantServi
@Autowired
private EmsBaseMonitorInfoMapper emsBaseMonitorInfoMapper;
/** 表名格式校验正则表达式record_iotenv_instant_yyyyMMdd */
private static final Pattern TABLE_NAME_PATTERN = Pattern.compile("^record_iotenv_instant_\\d{8}$");
/**
*
*
@ -159,6 +163,12 @@ public class RecordIotenvInstantServiceImpl implements IRecordIotenvInstantServi
while (!currentDate.after(endDateOnly)) {
String tableSuffix = tableFormat.format(currentDate);
String tableName = "record_iotenv_instant_" + tableSuffix;
// 添加表名格式校验防止SQL注入
if (!TABLE_NAME_PATTERN.matcher(tableName).matches()) {
throw new ServiceException("非法的表名格式: " + tableName);
}
if (isTableExists(tableName)){
tableNames.add(tableName);
}
@ -452,7 +462,7 @@ public class RecordIotenvInstantServiceImpl implements IRecordIotenvInstantServi
/**
*
*
*
* @return
*/
private String getTodayTableName() {
@ -461,5 +471,49 @@ public class RecordIotenvInstantServiceImpl implements IRecordIotenvInstantServi
String dateSuffix = today.format(formatter);
return "record_iotenv_instant_" + dateSuffix;
}
/**
*
*
* 1. 使MapperORDER BY
* 2.
* 3.
*
* @param recordIotenvInstant
* @return
*/
@Override
public List<RecordIotenvInstant> selectRecordIotenvInstantListForExport(RecordIotenvInstant recordIotenvInstant) throws ParseException {
Map<String, Object> params = recordIotenvInstant.getParams();
// 添加null检查防止空指针异常
if (params == null) {
throw new ServiceException("导出参数不能为空");
}
Object beginTimeObj = params.get("beginRecordTime");
Object endTimeObj = params.get("endRecordTime");
if (beginTimeObj == null || endTimeObj == null) {
throw new ServiceException("导出时间范围不能为空");
}
String beginTimeStr = beginTimeObj.toString();
String endTimeStr = endTimeObj.toString();
// 解析日期
SimpleDateFormat fullFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date beginDate = fullFormat.parse(beginTimeStr);
Date endDate = fullFormat.parse(endTimeStr);
// 获取需要查询的表名列表
List<String> tableNames = getTableNamesByDateRange(beginDate, endDate);
if (tableNames.isEmpty()) {
return new ArrayList<>();
}
// 使用专用导出查询无ORDER BY性能优化
return recordIotenvInstantMapper.selectRecordIotenvInstantListForExport(tableNames, recordIotenvInstant);
}
}

@ -19,6 +19,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<result property="collectTime" column="collectTime" />
<result property="recodeTime" column="recodeTime" />
<result property="monitorName" column="monitor_name" />
<result property="monitorCode" column="monitor_code" />
</resultMap>
<sql id="selectRecordIotenvInstantVo">
@ -165,7 +166,10 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
AND (t.temperature IS NULL OR t.temperature BETWEEN 0 AND 79)
AND (t.humidity IS NULL OR t.humidity BETWEEN 0 AND 79)
AND (t.noise IS NULL OR t.noise BETWEEN 0 AND 79)
<!-- 过滤虚拟设备忽略is_ammeter为0的虚拟设备 -->
-- AND (ebmi.is_ammeter IS NULL OR ebmi.is_ammeter != '0')
<!-- 根据设备类型过滤掉负责字段为0的数据 -->
<!-- 对于没有设备信息的记录保留所有非0数据 -->
AND (
@ -266,13 +270,16 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
#{monitorId}
</foreach>
</if>
<!-- 过滤异常数据:温度、湿度、噪声范围 [1, 79] -->
<!-- 优化使用BETWEEN和简化NULL检查提高TiDB性能 -->
AND (t.temperature IS NULL OR t.temperature BETWEEN 0 AND 79)
AND (t.humidity IS NULL OR t.humidity BETWEEN 0 AND 79)
AND (t.noise IS NULL OR t.noise BETWEEN 0 AND 79)
<!-- 过滤虚拟设备忽略is_ammeter为0的虚拟设备 -->
-- AND (ebmi.is_ammeter IS NULL OR ebmi.is_ammeter != '0')
<!-- 根据设备类型过滤掉负责字段为0的数据 -->
<!-- 对于没有设备信息的记录保留所有非0数据 -->
AND (
@ -334,13 +341,16 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
#{monitorId}
</foreach>
</if>
<!-- 过滤异常数据:温度、湿度、噪声范围 [1, 79] -->
<!-- 优化使用BETWEEN和简化NULL检查提高TiDB性能 -->
AND (t.temperature IS NULL OR t.temperature BETWEEN 0 AND 79)
AND (t.humidity IS NULL OR t.humidity BETWEEN 0 AND 79)
AND (t.noise IS NULL OR t.noise BETWEEN 0 AND 79)
<!-- 过滤虚拟设备忽略is_ammeter为0的虚拟设备 -->
-- AND (ebmi.is_ammeter IS NULL OR ebmi.is_ammeter != '0')
<!-- 根据设备类型过滤掉负责字段为0的数据 -->
<!-- 对于没有设备信息的记录保留所有非0数据 -->
AND (
@ -447,7 +457,10 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
AND (t.temperature IS NULL OR t.temperature BETWEEN 0 AND 79)
AND (t.humidity IS NULL OR t.humidity BETWEEN 0 AND 79)
AND (t.noise IS NULL OR t.noise BETWEEN 0 AND 79)
<!-- 过滤虚拟设备忽略is_ammeter为0的虚拟设备 -->
-- AND (ebmi2.is_ammeter IS NULL OR ebmi2.is_ammeter != '0')
<!-- 根据设备类型过滤掉负责字段为0的数据 -->
<!-- 对于没有设备信息的记录保留所有非0数据 -->
AND (
@ -469,4 +482,43 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
ORDER BY monitorId ASC, recodeTime ASC
</select>
<!-- 专用导出查询多设备、时间范围、无ORDER BY优化性能 -->
<!-- 性能优化移除复杂的monitor_type过滤条件前端已过滤虚拟设备避免OR条件导致的全表扫描 -->
<select id="selectRecordIotenvInstantListForExport" resultMap="RecordIotenvInstantResult">
<foreach collection="tableNames" item="tableName" separator=" UNION ALL ">
SELECT
t.objid, t.monitorId, t.temperature, t.humidity, t.illuminance, t.noise, t.concentration,
t.vibration_speed, t.vibration_displacement, t.vibration_acceleration, t.vibration_temp,
t.collectTime, t.recodeTime,
COALESCE(ebmi.monitor_name, t.monitorId) as monitor_name,
t.monitorId as monitor_code
FROM ${tableName} t
LEFT JOIN ems_base_monitor_info ebmi ON t.monitorId = ebmi.monitor_code
<where>
<!-- 单个设备查询 -->
<if test="recordIotenvInstant.monitorId != null and recordIotenvInstant.monitorId != ''">
AND t.monitorId = #{recordIotenvInstant.monitorId}
</if>
<!-- 多设备批量查询(导出核心功能) -->
<if test="recordIotenvInstant.monitorIds != null and recordIotenvInstant.monitorIds.length > 0">
AND t.monitorId IN
<foreach collection="recordIotenvInstant.monitorIds" item="monitorId" open="(" separator="," close=")">
#{monitorId}
</foreach>
</if>
<!-- 时间范围过滤 -->
<if test="recordIotenvInstant.params.beginRecordTime != null and recordIotenvInstant.params.endRecordTime != null">
AND t.recodeTime BETWEEN #{recordIotenvInstant.params.beginRecordTime} AND #{recordIotenvInstant.params.endRecordTime}
</if>
<!-- 过滤异常数据:温度、湿度、噪声范围 [0, 79] -->
AND (t.temperature IS NULL OR t.temperature BETWEEN 0 AND 79)
AND (t.humidity IS NULL OR t.humidity BETWEEN 0 AND 79)
AND (t.noise IS NULL OR t.noise BETWEEN 0 AND 79)
</where>
</foreach>
</select>
</mapper>
Loading…
Cancel
Save