diff --git a/das/pom.xml b/das/pom.xml index 61fde9d6..d4d40234 100644 --- a/das/pom.xml +++ b/das/pom.xml @@ -31,6 +31,7 @@ 3.2.10 3.4.4 5.4.3 + 8.4.3 @@ -201,6 +202,13 @@ + + + io.minio + minio + ${minio.version} + + diff --git a/das/src/main/java/com/das/common/constant/FileConstants.java b/das/src/main/java/com/das/common/constant/FileConstants.java new file mode 100644 index 00000000..656d425c --- /dev/null +++ b/das/src/main/java/com/das/common/constant/FileConstants.java @@ -0,0 +1,27 @@ +package com.das.common.constant; + +/** + * @Author liuyuxia + * @ClassName FilesvrConstants + * @Date 2018/12/10 0010 11:02 + * @Version 2.5 + * @Description 文件服务基础常量 + **/ +public class FileConstants { + + public static final String FILE_SEPARATOR = "/"; + + public static final String FILE_CHARSET = "UTF-8"; + + /** + * 递归 + */ + public static final Integer YES_RECURSIVE=1; + public static final Integer NO_RECURSIVE=0; + + public static final int META_R = 1001; + public static final int META_W = 1002; + + public static final String META_R_NAME = "读"; + public static final String META_W_NAME = "写"; +} diff --git a/das/src/main/java/com/das/modules/fdr/config/MinioConfig.java b/das/src/main/java/com/das/modules/fdr/config/MinioConfig.java new file mode 100644 index 00000000..92941ef5 --- /dev/null +++ b/das/src/main/java/com/das/modules/fdr/config/MinioConfig.java @@ -0,0 +1,71 @@ +package com.das.modules.fdr.config; + +import io.minio.*; +import io.minio.errors.*; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Slf4j +@Configuration +@ConditionalOnClass(MinioClient.class) +public class MinioConfig { + + @Resource + private MinioProperties minioAutoProperties; + @Bean + public MinioClient minioClient() { + log.info("开始初始化MinioClient, url为{}, accessKey为:{}", minioAutoProperties.getUrl(), minioAutoProperties.getAccessKey()); + MinioClient minioClient = MinioClient + .builder() + .endpoint(minioAutoProperties.getUrl()) + .credentials(minioAutoProperties.getAccessKey(), minioAutoProperties.getSecretKey()) + .build(); + + minioClient.setTimeout( + minioAutoProperties.getConnectTimeout(), + minioAutoProperties.getWriteTimeout(), + minioAutoProperties.getReadTimeout() + ); + // Start detection + if (minioAutoProperties.isCheckBucket()) { + log.info("checkBucket为{}, 开始检测桶是否存在", minioAutoProperties.isCheckBucket()); + String bucketName = minioAutoProperties.getBucket(); + if (!checkBucket(bucketName, minioClient)) { + log.info("文件桶[{}]不存在, 开始检查是否可以新建桶", bucketName); + if (minioAutoProperties.isCreateBucket()) { + log.info("createBucket为{},开始新建文件桶", minioAutoProperties.isCreateBucket()); + createBucket(bucketName, minioClient); + } + } + log.info("文件桶[{}]已存在, minio客户端连接成功!", bucketName); + } else { + throw new RuntimeException("桶不存在, 请检查桶名称是否正确或者将checkBucket属性改为false"); + } + return minioClient; + } + + private boolean checkBucket(String bucketName, MinioClient minioClient) { + boolean isExists = false; + try { + isExists = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build()); + } catch (Exception e) { + throw new RuntimeException("failed to check if the bucket exists", e); + } + return isExists; + } + + private void createBucket(String bucketName, MinioClient minioClient) { + try { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); + log.info("文件桶[{}]新建成功, minio客户端已连接", bucketName); + } catch (Exception e) { + throw new RuntimeException("failed to create default bucket", e); + } + } + +} diff --git a/das/src/main/java/com/das/modules/fdr/config/MinioProperties.java b/das/src/main/java/com/das/modules/fdr/config/MinioProperties.java new file mode 100644 index 00000000..3ce7cfed --- /dev/null +++ b/das/src/main/java/com/das/modules/fdr/config/MinioProperties.java @@ -0,0 +1,70 @@ +package com.das.modules.fdr.config; + +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; +import org.hibernate.validator.constraints.URL; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; + +@Data +@Validated +@Component +public class MinioProperties { + /** + * 服务地址 + */ + @NotEmpty(message = "minio服务地址不可为空") + @URL(message = "minio服务地址格式错误") + @Value("${minio.url}") + private String url; + + /** + * 认证账户 + */ + @NotEmpty(message = "minio认证账户不可为空") + @Value("${minio.accessKey}") + private String accessKey; + + /** + * 认证密码 + */ + @NotEmpty(message = "minio认证密码不可为空") + @Value("${minio.secretKey}") + private String secretKey; + + /** + * 桶名称, 优先级最低 + */ + @Value("${minio.bucket}") + private String bucket; + + /** + * 桶不在的时候是否新建桶 + */ + private boolean createBucket = true; + + /** + * 启动的时候检查桶是否存在 + */ + private boolean checkBucket = true; + + /** + * 设置HTTP连接、写入和读取超时。值为0意味着没有超时 + * HTTP连接超时，以毫秒为单位。 + */ + private long connectTimeout; + + /** + * 设置HTTP连接、写入和读取超时。值为0意味着没有超时 + * HTTP写超时，以毫秒为单位。 + */ + private long writeTimeout; + + /** + * 设置HTTP连接、写入和读取超时。值为0意味着没有超时 + * HTTP读取超时，以毫秒为单位。 + */ + private long readTimeout; +} diff --git a/das/src/main/java/com/das/modules/fdr/controller/FaultRecorderController.java b/das/src/main/java/com/das/modules/fdr/controller/FaultRecorderController.java new file mode 100644 index 00000000..0aff2c33 --- /dev/null +++ b/das/src/main/java/com/das/modules/fdr/controller/FaultRecorderController.java @@ -0,0 +1,31 @@ +package com.das.modules.fdr.controller; + +import com.das.common.result.R; +import com.das.modules.fdr.domain.FileNode; +import com.das.modules.fdr.service.FaultRecorderService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * 故障录波controller + */ +@Slf4j +@RequestMapping("/api/fdr") +@RestController +public class FaultRecorderController { + + @Autowired + private FaultRecorderService faultRecorderService; + + @RequestMapping(value = "/files", method = RequestMethod.POST) + public R> findList(@RequestParam(required = false) String name, String startTime, String endTime) { + List result = faultRecorderService.getDirOrFileList(name,startTime,endTime); + return R.success(result); + } +} diff --git a/das/src/main/java/com/das/modules/fdr/domain/FileNode.java b/das/src/main/java/com/das/modules/fdr/domain/FileNode.java new file mode 100644 index 00000000..8a2894b5 --- /dev/null +++ b/das/src/main/java/com/das/modules/fdr/domain/FileNode.java @@ -0,0 +1,22 @@ +package com.das.modules.fdr.domain; + +import lombok.*; + +@Getter +@Setter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FileNode { + //节点名称 + private String name; + + // 0代表文件夹，1代表文件 + private int type; + + private String size; + + private String lastModified; + + private String path; +} diff --git a/das/src/main/java/com/das/modules/fdr/service/FaultRecorderService.java b/das/src/main/java/com/das/modules/fdr/service/FaultRecorderService.java new file mode 100644 index 00000000..b282c184 --- /dev/null +++ b/das/src/main/java/com/das/modules/fdr/service/FaultRecorderService.java @@ -0,0 +1,19 @@ +package com.das.modules.fdr.service; + +import com.das.modules.fdr.domain.FileNode; +import org.springframework.web.multipart.MultipartFile; + +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.List; + +public interface FaultRecorderService { + List getDirOrFileList(String name,String startTime, String endTime); + + String upload(String parent, String folderName, MultipartFile file); + + void readFileToSteam(String path, OutputStream stream); + + void download(String path, Path tempDir); + +} diff --git a/das/src/main/java/com/das/modules/fdr/service/MinioViewsServcie.java b/das/src/main/java/com/das/modules/fdr/service/MinioViewsServcie.java new file mode 100644 index 00000000..0c4f33f5 --- /dev/null +++ b/das/src/main/java/com/das/modules/fdr/service/MinioViewsServcie.java @@ -0,0 +1,218 @@ +package com.das.modules.fdr.service; + +import cn.hutool.core.io.FileUtil; +import com.das.common.constant.FileConstants; +import com.das.modules.fdr.config.MinioProperties; +import com.das.modules.fdr.domain.FileNode; +import io.micrometer.common.util.StringUtils; +import io.minio.*; +import io.minio.messages.Item; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.*; +import java.nio.file.Path; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +@Service +@Slf4j +public class MinioViewsServcie { + @Autowired + private MinioClient minioClient; + + @Autowired + private MinioProperties minioProperties; + + /** + * 删除文件 + * + * @param bucketName 存储桶 + * @param objectName 文件名称 + */ + public void removeFile(String bucketName, String objectName, Boolean recursive) throws Exception { + Iterable> results = minioClient.listObjects( + ListObjectsArgs.builder().bucket(bucketName).prefix(objectName).recursive(true).build()); + List> list = StreamSupport.stream(results.spliterator(), false) + .collect(Collectors.toList()); + if (list.size() >= 2 && !recursive) { + throw new IOException("请清空文件后再删除目录"); + } + for (Result result : results) { + Item item = result.get(); + minioClient.removeObject( + RemoveObjectArgs.builder() + .bucket(bucketName) + .object(item.objectName()) + .build()); + } + } + + + public Boolean deleteFileViews(String path, Boolean recursive) throws IOException { + if (StringUtils.isBlank(path)) { + throw new IOException("请确认删除文件路径"); + } + Boolean success = null; + try { + path = path.substring(path.indexOf("/") + 1); + removeFile(minioProperties.getBucket(), path, recursive); + } catch (Exception e) { + throw new RuntimeException(e); + } + return success; + } + + public boolean deleteFile(File file) { + return file.delete(); + } + + + public String upload(String path, String folderName,MultipartFile file) { + String targetFile = null; + try { + // 上传一个空对象来模拟文件夹 + if (!StringUtils.isBlank(folderName)){ + targetFile = path + folderName + FileConstants.FILE_SEPARATOR; + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); + minioClient.putObject( + PutObjectArgs.builder() + .bucket(minioProperties.getBucket()) + .object(targetFile) + .stream(bais, 0, -1) + .build()); + } + else { + targetFile= path +"/" + file.getOriginalFilename(); + uploadFile(minioProperties.getBucket(), file, targetFile, "application/octet-stream"); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return targetFile; + } + + + /** + * 使用MultipartFile进行文件上传 + * + * @param bucketName 存储桶 + * @param file 文件名 + * @param objectName 对象名 + * @param contentType 类型 + * @throws Exception + */ + public void uploadFile(String bucketName, MultipartFile file, String objectName, String contentType) throws Exception { + InputStream inputStream = file.getInputStream(); + try{ + minioClient.putObject( + PutObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .contentType(contentType) + .stream(inputStream, inputStream.available(), -1) + .build()); + }catch (Exception e){ + log.error("minio文件上传失败{}", e); + } + + } + + //获取路径下的文件夹文件列表 + public List getFileTree(String directoryName) { + List fileNodes = new ArrayList<>(); + ListObjectsArgs build; + + try { + if (StringUtils.isBlank(directoryName)) { + build = ListObjectsArgs.builder().bucket(minioProperties.getBucket()).recursive(false).build(); + } else { + build = ListObjectsArgs.builder().bucket(minioProperties.getBucket()).prefix(directoryName+"/").recursive(false).build(); + } + Iterable> results = minioClient.listObjects(build); + for (Result result : results) { + Item item = result.get(); + String itemName = item.objectName(); + boolean isDir = item.isDir(); + String size = FileUtil.readableFileSize(item.size()); + String relativePath = null; + String[] parts = null; + if (!StringUtils.isBlank(directoryName)){ + relativePath = itemName.substring(directoryName.length()); + parts = relativePath.split("/"); + } + else { + parts = itemName.split("/"); + } + String lastModifyTime = null; + DateTimeFormatter dateFormat =DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm:ss"); + if (!isDir){ + ZonedDateTime zonedDateTime = item.lastModified(); + lastModifyTime = zonedDateTime.format(dateFormat); + } + if (parts.length > 0) { + String nodeName = parts[0]; + int type = isDir ? 0 : 1; + itemName= isDir ? itemName.substring(0,itemName.lastIndexOf("/")) : itemName; + FileNode node = new FileNode(nodeName, type,size,lastModifyTime,"/"+itemName); + if (!fileNodes.contains(node)) { + fileNodes.add(node); + } + } + } + } catch (Exception e) { + log.error("minio获取树列表失败", e); + } + return fileNodes; + } + + public void readFileToStream(String path, OutputStream stream) { + + try ( GetObjectResponse res = minioClient.getObject( + GetObjectArgs.builder().bucket(minioProperties.getBucket()).object(path).build())){ + res.transferTo(stream); + } catch (Exception e) { + log.error("minio读取文件失败", e); + } + } + + public void download(String path, Path tempDir) { + + try (InputStream inputStream = minioClient.getObject(GetObjectArgs.builder() + .bucket(minioProperties.getBucket()) + .object(path) + .build())) { + + // 保存到临时文件夹 + File tempFile = tempDir.resolve(tempDir+path).toFile(); + FileUtil.writeFromStream(inputStream,tempFile); + } + catch (Exception ignored){ + + } + } + + + // 递归方式 计算文件的大小 + public long getTotalSizeOfFilesInDir(File file) { + if (file.isFile()) { + return file.length(); + } + + File[] children = file.listFiles(); + long total = 0; + if (children != null) { + + for (final File child : children) { + total += getTotalSizeOfFilesInDir(child); + } + } + return total; + } +} diff --git a/das/src/main/java/com/das/modules/fdr/service/impl/FaultRecorderServiceImpl.java b/das/src/main/java/com/das/modules/fdr/service/impl/FaultRecorderServiceImpl.java new file mode 100644 index 00000000..53d35767 --- /dev/null +++ b/das/src/main/java/com/das/modules/fdr/service/impl/FaultRecorderServiceImpl.java @@ -0,0 +1,74 @@ +package com.das.modules.fdr.service.impl; + +import com.das.common.constant.FileConstants; +import com.das.modules.fdr.domain.FileNode; +import com.das.modules.fdr.service.FaultRecorderService; +import com.das.modules.fdr.service.MinioViewsServcie; +import io.micrometer.common.util.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.OutputStream; +import java.nio.file.Path; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class FaultRecorderServiceImpl implements FaultRecorderService { + + @Autowired + private MinioViewsServcie minioViewsServcie; + + + @Override + public List getDirOrFileList(String name, String startTime, String endTime) { + List fileResult = new ArrayList<>(); + List monthsBetween = getMonthsBetween(startTime, endTime); + for (String item : monthsBetween){ + String directoryName = name + FileConstants.FILE_SEPARATOR + item.substring(0,item.indexOf("-")) + FileConstants.FILE_SEPARATOR + item.substring(item.indexOf("-")+1); + List fileTree = minioViewsServcie.getFileTree(directoryName); + fileResult.addAll(fileTree); + } + Comparator fileNodeComparator = Comparator.comparing(FileNode::getLastModified) + .thenComparing(FileNode::getName); + fileResult.sort(fileNodeComparator); + return fileResult; + } + + @Override + public String upload(String parent, String folderName, MultipartFile file) { + return minioViewsServcie.upload(parent, folderName, file); + } + + @Override + public void readFileToSteam(String path, OutputStream stream) { + minioViewsServcie.readFileToStream(path, stream); + } + + @Override + public void download(String path, Path tempDir) { + minioViewsServcie.download(path, tempDir); + } + + + private List getMonthsBetween(String startTime, String endTime) { + List months = new ArrayList<>(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + LocalDate start = LocalDate.parse(startTime + "-01", formatter); + LocalDate end = LocalDate.parse(endTime + "-01", formatter); + + DateTimeFormatter monthFormatter = DateTimeFormatter.ofPattern("yyyy-MM"); + + while (!start.isAfter(end)) { + months.add(start.format(monthFormatter)); + start = start.plusMonths(1); + } + return months; + } +} diff --git a/das/src/main/resources/application.yml b/das/src/main/resources/application.yml index 5dba9c52..3988874e 100644 --- a/das/src/main/resources/application.yml +++ b/das/src/main/resources/application.yml @@ -98,4 +98,10 @@ logging: tdengine: password: taosdata url: jdbc:TAOS-RS://192.168.109.160:6041/das - username: root \ No newline at end of file + username: root + +minio: + url: http://192.168.109.187:9000 + bucket: das + accessKey: das + secretKey: zaq12WSX \ No newline at end of file