liwenlong/fys-Multi-tenant
1
0
Fork 0
forked from dyf/fys-Multi-tenant
Code Pull Requests Activity

导出设备数据

Browse Source
This commit is contained in:
daiyongfei
2025-11-21 13:36:13 +08:00
parent 00a4394b43
commit b18ab98feb
5 changed files with 257 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
fys-modules/fys-equipment/src/main/java/com/fuyuanshen/equipment/converter/IgnoreFailedImageConverter.java
View File

@ -4,6 +4,8 @@ import com.alibaba.excel.converters.Converter;
import com.alibaba.excel.metadata.GlobalConfiguration;
import com.alibaba.excel.metadata.data.WriteCellData;
import com.alibaba.excel.metadata.property.ExcelContentProperty;
import com.fuyuanshen.common.core.utils.file.ImageCompressUtil;
import com.fuyuanshen.common.redis.utils.RedisUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -12,44 +14,73 @@ import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.Base64;
import java.util.HashSet;
import java.util.Set;
/**
* @author: 默苍璃
* @date: 2025-06-0618:56
*/
public class IgnoreFailedImageConverter implements Converter<URL> {
private static final Logger logger = LoggerFactory.getLogger(IgnoreFailedImageConverter.class);
// 重试次数
private static final int MAX_RETRIES = 3;
// 指数退避初始延迟（毫秒）
private static final int INITIAL_DELAY = 1000;
// 图片压缩阈值(1MB)
private static final int COMPRESSION_THRESHOLD = 1024 * 1024;
// 压缩目标大小(100KB)
private static final int COMPRESSION_TARGET = 100 * 1024;
// 用于跟踪本次任务中使用到的URL缓存键
private static final ThreadLocal<Set<String>> USED_CACHE_KEYS = new ThreadLocal<Set<String>>() {
@Override
protected Set<String> initialValue() {
return new HashSet<>();
}
};
@Override
public Class<?> supportJavaTypeKey() {
return URL.class;
}
@Override
public WriteCellData<?> convertToExcelData(URL value, ExcelContentProperty contentProperty, GlobalConfiguration globalConfiguration) {
if (value == null) {
logger.debug("图片URL为空");
logger.info("图片URL为空");
return new WriteCellData<>(new byte[0]);
}
String cacheKey = "excel:image:" + value.toString();
// 将当前使用的缓存键添加到集合中
USED_CACHE_KEYS.get().add(cacheKey);
// 尝试从缓存获取
String cachedData = RedisUtils.getCacheObject(cacheKey);
if (cachedData != null) {
// 从缓存中读取Base64编码的数据并解码
byte[] cachedBytes = Base64.getDecoder().decode(cachedData);
logger.info("从缓存获取图片数据: {}, 大小: {} 字节", value, cachedBytes.length);
return new WriteCellData<>(cachedBytes);
}
// 缓存未命中，从URL加载
// 尝试多次加载图片
for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
logger.debug("开始加载图片: {}, 尝试次数: {}", value, attempt);
logger.info("开始加载图片: {}, 尝试次数: {}", value, attempt);
URLConnection conn = value.openConnection();
// 增加连接和读取超时时间
conn.setConnectTimeout(10000); // 10秒连接超时
conn.setReadTimeout(30000); // 30秒读取超时
// 添加User-Agent避免被服务器拦截
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 ExcelExporter/1.0");
// 如果是HTTP连接，设置一些额外的属性
if (conn instanceof HttpURLConnection) {
HttpURLConnection httpConn = (HttpURLConnection) conn;
@ -58,38 +89,44 @@ public class IgnoreFailedImageConverter implements Converter<URL> {
httpConn.setUseCaches(false);
// 跟随重定向
httpConn.setInstanceFollowRedirects(true);
// 检查响应码
int responseCode = httpConn.getResponseCode();
if (responseCode != HttpURLConnection.HTTP_OK) {
logger.warn("HTTP响应码异常: {}, URL: {}", responseCode, value);
logger.info("HTTP响应码异常: {}, URL: {}", responseCode, value);
if (attempt < MAX_RETRIES) {
// 等待后重试
waitForRetry(attempt);
continue;
} else {
// 将空数据写入缓存
RedisUtils.setCacheObject(cacheKey, "");
return new WriteCellData<>(new byte[0]);
}
}
}
long contentLength = conn.getContentLengthLong();
logger.debug("连接建立成功，图片大小: {} 字节", contentLength);
logger.info("连接建立成功，图片大小: {} 字节", contentLength);
// 检查内容长度是否有效
if (contentLength == 0) {
logger.warn("图片文件为空: {}", value);
logger.info("图片文件为空: {}", value);
if (attempt < MAX_RETRIES) {
waitForRetry(attempt);
continue;
} else {
// 将空数据写入缓存
RedisUtils.setCacheObject(cacheKey, "");
return new WriteCellData<>(new byte[0]);
}
}
// 限制图片大小（防止过大文件导致内存问题）
if (contentLength > 10 * 1024 * 1024) { // 10MB限制
logger.warn("图片文件过大 ({} bytes)，跳过加载: {}", contentLength, value);
logger.info("图片文件过大 ({} bytes)，跳过加载: {}", contentLength, value);
// 将空数据写入缓存
RedisUtils.setCacheObject(cacheKey, "");
return new WriteCellData<>(new byte[0]);
}
@ -97,52 +134,98 @@ public class IgnoreFailedImageConverter implements Converter<URL> {
// byte[] bytes = FileUtils.readInputStream(inputStream, value.toString());
// 替代 FileUtils.readInputStream 的自定义方法
byte[] bytes = readInputStream(inputStream);
// 检查读取到的数据是否为空
if (bytes == null || bytes.length == 0) {
logger.warn("读取到空的图片数据: {}", value);
logger.info("读取到空的图片数据: {}", value);
if (attempt < MAX_RETRIES) {
waitForRetry(attempt);
continue;
} else {
// 将空数据写入缓存
RedisUtils.setCacheObject(cacheKey, "");
return new WriteCellData<>(new byte[0]);
}
}
logger.debug("成功读取图片数据，大小: {} 字节", bytes.length);
// 如果图片大于1MB，则进行压缩
if (bytes.length > COMPRESSION_THRESHOLD) {
logger.info("图片大小超过1MB ({} bytes)，开始压缩", bytes.length);
long beforeCompressSize = bytes.length;
bytes = ImageCompressUtil.compressImage(bytes, COMPRESSION_TARGET);
long afterCompressSize = bytes.length;
logger.info("图片压缩完成，压缩前大小: {} bytes, 压缩后大小: {} bytes, 压缩率: {}%",
beforeCompressSize, afterCompressSize,
String.format("%.2f", (1.0 - (double) afterCompressSize / beforeCompressSize) * 100));
}
logger.info("成功读取图片数据，大小: {} 字节", bytes.length);
// 将数据写入缓存，不设置过期时间，使用Base64编码存储
String encodedData = Base64.getEncoder().encodeToString(bytes);
RedisUtils.setCacheObject(cacheKey, encodedData);
return new WriteCellData<>(bytes);
}
} catch (Exception e) {
logger.warn("图片加载失败: {}, 尝试次数: {}, 原因: {}", value, attempt, e.getMessage(), e);
logger.info("图片加载失败: {}, 尝试次数: {}, 原因: {}", value, attempt, e.getMessage(), e);
if (attempt < MAX_RETRIES) {
// 等待后重试
waitForRetry(attempt);
} else {
// 最后一次尝试也失败了
logger.error("图片加载最终失败，已重试 {} 次: {}", MAX_RETRIES, value, e);
logger.info("图片加载最终失败，已重试 {} 次: {}", MAX_RETRIES, value, e);
// 将空数据写入缓存
RedisUtils.setCacheObject(cacheKey, "");
return new WriteCellData<>(new byte[0]); // 返回空数组而不是 null
}
}
}
// 所有尝试都失败了
// 将空数据写入缓存
RedisUtils.setCacheObject(cacheKey, "");
return new WriteCellData<>(new byte[0]);
}
/**
* 清理未使用的缓存
* 任务结束后调用此方法，删除本次任务中未使用的URL缓存
*/
public static void cleanUnusedCache() {
Set<String> usedKeys = USED_CACHE_KEYS.get();
if (usedKeys != null && !usedKeys.isEmpty()) {
// 获取所有图片缓存键
Iterable<String> allKeys = RedisUtils.keys("excel:image:*");
if (allKeys != null) {
// 删除未使用的缓存
for (String key : allKeys) {
if (!usedKeys.contains(key)) {
RedisUtils.deleteObject(key);
logger.info("删除未使用的缓存: {}", key);
}
}
}
// 清理ThreadLocal
USED_CACHE_KEYS.remove();
}
}
/**
* 等待重试，使用指数退避策略
*
* @param attempt 当前尝试次数
*/
private void waitForRetry(int attempt) {
try {
long delay = (long) INITIAL_DELAY * (1L << (attempt - 1)); // 指数退避
logger.debug("等待 {} 毫秒后重试...", delay);
logger.info("等待 {} 毫秒后重试...", delay);
Thread.sleep(delay);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
}
/**
* 替代 FileUtils.readInputStream 的自定义方法
*
@ -159,14 +242,15 @@ public class IgnoreFailedImageConverter implements Converter<URL> {
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
totalBytes += bytesRead;
// 如果读取的数据过大，提前终止
if (totalBytes > 10 * 1024 * 1024) { // 10MB限制
logger.warn("读取的图片数据超过10MB限制，提前终止");
logger.info("读取的图片数据超过10MB限制，提前终止");
break;
}
}
return outputStream.toByteArray();
}
}

2
fys-modules/fys-equipment/src/main/java/com/fuyuanshen/equipment/service/impl/DeviceExportService.java
View File

@ -83,6 +83,7 @@ public class DeviceExportService {
}
/**
* 导出设备数据（包含完整设备类型信息）
*
@ -183,6 +184,7 @@ public class DeviceExportService {
}
}
/**
* 转换定位方式代码为中文描述
*