#include "ftp2minio.h" #include #include #include FtpManage::FtpManage(const std::string user, const std::string password, const std::string id) :Ftp_ip(id), User(user), Password(password) { CURLcode res = curl_global_init(CURL_GLOBAL_ALL); // 初始化全局环境只需要初始化一次 if (res != CURLE_OK) { vLog(LOG_ERROR, "curl_global_init() falied %s\n", curl_easy_strerror(res)); return; } } FtpManage::~FtpManage() { if (curl) { curl_easy_cleanup(curl); } curl_global_cleanup(); } //接收消息回调函数 size_t WriteCallback(void* contents, size_t size, size_t nmemb, std::string* output) { size_t totalSize = size * nmemb; output->append(static_cast(contents), totalSize); return totalSize; } // 初始化curl void FtpManage::SetURL() { curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L); // 设置长连接 curl_easy_setopt(curl, CURLOPT_USERPWD, (User + ':' + Password).c_str()); //设置用户名和密码 } // 把带有路径的文件转换为文件名 D:/ggbond/123.jpg->123.jpg std::string FtpManage::GetFileNameFromPath(const std::string& filePath) { if (filePath == "") return filePath; //去除路径最后的空格 std::string retPath; size_t lastBlank = filePath.find_last_not_of(' '); if (lastBlank != std::string::npos) { retPath = filePath.substr(0, lastBlank+1); } else { return ""; } size_t lastSlashPos = retPath.find_last_of("/\\"); if (lastSlashPos != std::string::npos) { return retPath.substr(lastSlashPos + 1); } else { // 如果没有找到斜杠或反斜杠,整个路径就是文件名 return retPath; } } // 下载单一文件,把remoteFilePath的文件下载到localDirectory里面 bool FtpManage::DownloadFile(const char* remoteFilePath, const char* localDirectory) { SetURL(); _URL = Ftp_ip + remoteFilePath; curl_easy_setopt(curl, CURLOPT_URL, _URL.c_str()); // 设置请求的URL if (curl) { FILE* fp = fopen((localDirectory + GetFileNameFromPath(remoteFilePath)).c_str(), "wb"); if (fp) { // 设置文件写入地址 curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); // 执行任务 CURLcode res = curl_easy_perform(curl); if (res != CURLE_OK) { vLog(LOG_ERROR, "DownLoad file: %s failed<%s>.", remoteFilePath, curl_easy_strerror(res)); fclose(fp); curl_easy_cleanup(curl); return false; } else { vLog(LOG_DEBUG, "DownLoad file: %s successfully!\n", remoteFilePath); fclose(fp); } } else { vLog(LOG_ERROR, "file open failed!\n"); curl_easy_cleanup(curl); // 清除curl return false; } } curl_easy_cleanup(curl); return true; } // 下载全部文件 bool FtpManage::DownloadAllFiles(const char* remoteFilePath, const char* localDirectory) { if (GetfilenameFromftp(remoteFilePath)) { for (const auto& fns : fNs) { std::string filename = remoteFilePath + fns; bool res = DownloadFile(filename.c_str(), localDirectory); if (res) continue; else { return false; } } } return true; } // 获取ftp某个文件夹内文件名 bool FtpManage::GetfilenameFromftp(const std::string filePath) { SetURL(); std::string path = Ftp_ip + filePath; std::string fileName; // 文件名列表保存位置 if (curl) { curl_easy_setopt(curl, CURLOPT_URL, path.c_str()); // 设置访问URL curl_easy_setopt(curl, CURLOPT_DIRLISTONLY, 1L); // 设置只返回文件 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &fileName); // 设置只获取文件名列表 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); // 设置get的回调函数 if (curl_easy_perform(curl) == CURLE_OK) //执行任务 { fNs.clear(); size_t startPos = 0, endPos = 0; while ((endPos = fileName.find('\n', startPos)) != std::string::npos ) { std::string name = fileName.substr(startPos, endPos - startPos); fNs.emplace_back(name); startPos = endPos + 1; } } } else { return false; } curl_easy_cleanup(curl); return true; } // 获取FTP文件夹内所有文件名的接口函数 const std::vector& FtpManage::GetFilesName(const std::string filepath) { if (GetfilenameFromftp(filepath)) { return fNs; } return std::vector(); } CFtp2MinioProcess::CFtp2MinioProcess() { m_pAftp = NULL; } CFtp2MinioProcess::~CFtp2MinioProcess() { if (m_pAftp) delete m_pAftp; m_pAftp = NULL; saveMapToFile(); } void CFtp2MinioProcess::saveMapToFile(void) { char fileName[260]; snprintf(fileName, sizeof(fileName), "minioftp_%d.mem", GetCurID()); std::ofstream outFile(fileName); if (!outFile) { vLog(LOG_ERROR, "Unable to open file for writing!\n"); return; } for (const auto& pair : fileName2Id_map) { outFile << pair.first << " " << pair.second << std::endl; } outFile.close(); } void CFtp2MinioProcess::loadMapFromFile(void) { char fileName[260]; snprintf(fileName, sizeof(fileName), "minioftp_%d.mem", GetCurID()); std::ifstream inFile(fileName); if (!inFile) { vLog(LOG_ERROR, "Unable to open file for reading!\n"); return; } std::string key; int value; fileName2Id_map.clear(); while (inFile >> key >> value) { fileName2Id_map[key] = value; //如果键已经存在,则会更新其值 } inFile.close(); } BOOLEAN CFtp2MinioProcess::OnPreCreate(int id) { if (!CProcess::OnPreCreate(id)) return FALSE; DWORD target_addr; if (!GetOption(&m_nOptions, sizeof(m_nOptions))) { //获取新配置 vLog(LOG_DEBUG, "润阳ftp转modbus读取配置错误。"); return FALSE; } if (m_pAftp) m_pAftp = NULL; target_addr = m_nOptions.net.target_addr; char user[128] = "administrator"; char password[128] = "123456"; char ipaddress[128] = "127.0.0.1"; char remotePath[128] = "\0"; char localPath[128]; char url[256]; //char listpath[256]; snprintf(user, sizeof(user), "%s", m_nOptions.ftp.user); snprintf(password, sizeof(password), "%s", m_nOptions.ftp.password); snprintf(localPath, sizeof(localPath), "%s", m_nOptions.ftp.localPath); m_localPath = std::string(localPath); if (m_localPath.back() != '/') m_localPath += std::string("/"); char *escaped_string = escape_char_in_string(m_nOptions.ftp.remotePath, ' '); if (!escaped_string) return FALSE; snprintf(remotePath, sizeof(remotePath), "%s", escaped_string); free(escaped_string); memset(ipaddress, '\0', sizeof(ipaddress)); inet_ntop(AF_INET, &target_addr, ipaddress, 16); snprintf(url, sizeof(url), "ftp://%s", ipaddress); std::string listpaths(remotePath); std::vector splits = split(listpaths, ';'); for (std::vector::iterator it = splits.begin(); it != splits.end(); it++) { if ((*it).back() != '/') { m_listPaths.push_back((*it) + "/"); } else { m_listPaths.push_back((*it)); } } m_pAftp = new FtpManage(user, password, url); fileName2Id_map.clear(); //读取列表 loadMapFromFile(); last_count = fileName2Id_map.size(); return TRUE; } BOOLEAN CFtp2MinioProcess::Run(void) { if (!CProcess::Run()) return FALSE; return TRUE; } std::optional CFtp2MinioProcess::extractDate(const std::string& filename, const std::string& regex) { std::regex datePattern(regex);//(R"(\d{4}-\d{2}-\d{2})"); std::smatch match; if (std::regex_search(filename, match, datePattern)) { return match.str(); // 返回匹配到的日期字符串 } return std::nullopt; // 没有找到日期则返回 nullopt } BOOLEAN CFtp2MinioProcess::OnTimer(void) { if (!CProcess::OnTimer()) return FALSE; BOOLEAN min_changed = FALSE; if (last_min != system32.now.minute) { last_min = system32.now.minute; min_changed = TRUE; } if (min_changed) { for (std::vector::iterator it = m_listPaths.begin(); it != m_listPaths.end(); it++) { m_listPath = (*it); if (m_listPath.empty()) continue; // 检查字符串的最后一个字符是否为 '/' vLog(LOG_DEBUG, "准备读取文件夹%s的内容\n", m_listPath.c_str()); for (const auto& n : m_pAftp->GetFilesName(m_listPath)) { std::string remotefile = m_listPath + n; std::string localpath = m_localPath; if (fileName2Id_map.find(n) == fileName2Id_map.end()) { fileName2Id_map.insert(fileName2Idmap::value_type(n, 0)); if (m_pAftp->DownloadFile(remotefile.c_str(), localpath.c_str())) { vLog(LOG_DEBUG, "下载成功!\n"); std::vector tokens = split(m_listPath, '/'); push2minio(tokens[tokens.size() - 1], n); } } else { vLog(LOG_WARN, "该文件:%s已经被下载。\n", n.c_str()); } } if (last_count != fileName2Id_map.size()) { last_count = fileName2Id_map.size(); saveMapToFile(); } } } return TRUE; } BOOLEAN CFtp2MinioProcess::push2minio(std::string parentDir, std::string pathName) { //创建URL //minio::s3::BaseUrl base_url("http://192.168.109.187:9000"); minio::s3::BaseUrl base_url(m_nOptions.minio.url); base_url.https = false; //创建鉴权对象 //minio::creds::StaticProvider provider("das", "zaq12WSX"); minio::creds::StaticProvider provider(m_nOptions.minio.user, m_nOptions.minio.password); // 创建客户端 minio::s3::Client client(base_url, &provider); //std::string bucket_name = "test"; std::string bucket_name = std::string(m_nOptions.minio.bucket); // 检查test桶是否存在 bool exist; { minio::s3::BucketExistsArgs args; args.bucket = bucket_name; minio::s3::BucketExistsResponse resp = client.BucketExists(args); if (!resp) { vLog(LOG_ERROR, "unable to do bucket existence check; %s\n", resp.Error()); return FALSE; } exist = resp.exist; } // 如果test桶不存在,则创建test桶 if (!exist) { minio::s3::MakeBucketArgs args; args.bucket = bucket_name; minio::s3::MakeBucketResponse resp = client.MakeBucket(args); if (!resp) { vLog(LOG_ERROR, "unable to create bucket; %s\n", resp.Error()); return FALSE; } } // 上传文件 minio::s3::UploadObjectArgs args; args.bucket = bucket_name; //上传到桶中的绝对路径 //char *pYMDhms = strchr((char *)pathName.c_str(), '.'); //args.object = "/A-001/Alarm/" + pathName; //args.object = std::string(m_nOptions.minio.object) + pathName; auto date = extractDate(pathName, R"(\d{4}-\d{2}-\d{2})"); std::string date_string; if (date.has_value()) { //vLog(LOG_DEBUG, "fileName is: %s, Date is: %s\n", pathName.c_str(), date.value().c_str()); date_string = replaceChar(date.value(), '-', '/'); //vLog(LOG_DEBUG, "date_string is: %s\n", date_string.c_str()); } else { date = extractDate(pathName, R"(\d{4}\d{2}\d{2})"); if (date.has_value()) { //vLog(LOG_DEBUG, "fileName is: %s, Date is: %s\n", pathName.c_str(), date.value().c_str()); date_string = date.value().substr(0, 4) + std::string("/") + date.value().substr(4, 2) + std::string("/") + date.value().substr(6, 2) + std::string("/"); //vLog(LOG_DEBUG, "date_string is: %s\n", date_string.c_str()); } else { vLog(LOG_DEBUG, "文件不存在时间格式\n"); return TRUE; } } //return TRUE; args.object = std::string(m_nOptions.minio.object) + std::string("/") + parentDir + std::string("/") + date_string + std::string("/") + pathName; //本地文件系统中的绝对路径 args.filename = m_localPath + pathName; minio::s3::UploadObjectResponse resp = client.UploadObject(args); if (!resp) { vLog(LOG_ERROR, "unable to upload object; %s\n", resp.Error()); return FALSE; } vLog(LOG_DEBUG, "'%s' is successfully uploaded as object '%s' to bucket 'test'.", args.filename.c_str(), args.object.c_str()); return TRUE; }