#include "ftp2minio.h" #include #include FtpManage::FtpManage(const std::string user, const std::string password, const std::string id) :Ftp_ip(id), User(user), Password(password) { CURLcode res = curl_global_init(CURL_GLOBAL_ALL); // 初始化全局环境只需要初始化一次 if (res != CURLE_OK) { vLog(LOG_ERROR, "curl_global_init() falied %s\n", curl_easy_strerror(res)); return; } } FtpManage::~FtpManage() { if (curl) { curl_easy_cleanup(curl); } curl_global_cleanup(); } //接收消息回调函数 size_t WriteCallback(void* contents, size_t size, size_t nmemb, std::string* output) { size_t totalSize = size * nmemb; output->append(static_cast(contents), totalSize); return totalSize; } // 初始化curl void FtpManage::SetURL() { curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L); // 设置长连接 curl_easy_setopt(curl, CURLOPT_USERPWD, (User + ':' + Password).c_str()); //设置用户名和密码 } // 把带有路径的文件转换为文件名 D:/ggbond/123.jpg->123.jpg std::string FtpManage::GetFileNameFromPath(const std::string& filePath) { if (filePath == "") return filePath; //去除路径最后的空格 std::string retPath; size_t lastBlank = filePath.find_last_not_of(' '); if (lastBlank != std::string::npos) { retPath = filePath.substr(0, lastBlank+1); } else { return ""; } size_t lastSlashPos = retPath.find_last_of("/\\"); if (lastSlashPos != std::string::npos) { return retPath.substr(lastSlashPos + 1); } else { // 如果没有找到斜杠或反斜杠,整个路径就是文件名 return retPath; } } // 下载单一文件,把remoteFilePath的文件下载到localDirectory里面 bool FtpManage::DownloadFile(const char* remoteFilePath, const char* localDirectory) { SetURL(); _URL = Ftp_ip + remoteFilePath; curl_easy_setopt(curl, CURLOPT_URL, _URL.c_str()); // 设置请求的URL if (curl) { FILE* fp = fopen((localDirectory + GetFileNameFromPath(remoteFilePath)).c_str(), "wb"); if (fp) { // 设置文件写入地址 curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); // 执行任务 CURLcode res = curl_easy_perform(curl); if (res != CURLE_OK) { vLog(LOG_ERROR, "DownLoad file: %s failed<%s>.", remoteFilePath, curl_easy_strerror(res)); fclose(fp); curl_easy_cleanup(curl); return false; } else { vLog(LOG_DEBUG, "DownLoad file: %s successfully!\n", remoteFilePath); fclose(fp); } } else { vLog(LOG_ERROR, "file open failed!\n"); curl_easy_cleanup(curl); // 清除curl return false; } } curl_easy_cleanup(curl); return true; } // 下载全部文件 bool FtpManage::DownloadAllFiles(const char* remoteFilePath, const char* localDirectory) { if (GetfilenameFromftp(remoteFilePath)) { for (const auto& fns : fNs) { std::string filename = remoteFilePath + fns; bool res = DownloadFile(filename.c_str(), localDirectory); if (res) continue; else { return false; } } } return true; } // 获取ftp某个文件夹内文件名 bool FtpManage::GetfilenameFromftp(const std::string filePath) { SetURL(); std::string path = Ftp_ip + filePath; std::string fileName; // 文件名列表保存位置 vLog(LOG_DEBUG, "path is: %s", path.c_str()); if (curl) { curl_easy_setopt(curl, CURLOPT_URL, path.c_str()); // 设置访问URL curl_easy_setopt(curl, CURLOPT_DIRLISTONLY, 1L); // 设置只返回文件 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &fileName); // 设置只获取文件名列表 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); // 设置get的回调函数 if (curl_easy_perform(curl) == CURLE_OK) //执行任务 { fNs.clear(); size_t startPos = 0, endPos = 0; while ((endPos = fileName.find('\n', startPos)) != std::string::npos ) { std::string name = fileName.substr(startPos, endPos - startPos); fNs.emplace_back(name); startPos = endPos + 1; } } } else { return false; } curl_easy_cleanup(curl); return true; } // 获取FTP文件夹内所有文件名的接口函数 const std::vector& FtpManage::GetFilesName(const std::string filepath) { if (GetfilenameFromftp(filepath)) { return fNs; } return std::vector(); } CFtp2MinioProcess::CFtp2MinioProcess() { m_pAftp = NULL; } CFtp2MinioProcess::~CFtp2MinioProcess() { if (m_pAftp) delete m_pAftp; m_pAftp = NULL; saveMapToFile(); } void CFtp2MinioProcess::saveMapToFile(void) { char fileName[260]; snprintf(fileName, sizeof(fileName), "minioftp_%d.mem", GetCurID()); std::ofstream outFile(fileName); if (!outFile) { vLog(LOG_ERROR, "Unable to open file for writing!\n"); return; } for (const auto& pair : fileName2Id_map) { outFile << pair.first << " " << pair.second << std::endl; } outFile.close(); } void CFtp2MinioProcess::loadMapFromFile(void) { char fileName[260]; snprintf(fileName, sizeof(fileName), "minioftp_%d.mem", GetCurID()); std::ifstream inFile(fileName); if (!inFile) { vLog(LOG_ERROR, "Unable to open file for reading!\n"); return; } std::string key; int value; fileName2Id_map.clear(); while (inFile >> key >> value) { fileName2Id_map[key] = value; //如果键已经存在,则会更新其值 } inFile.close(); } BOOLEAN CFtp2MinioProcess::OnPreCreate(int id) { if (!CProcess::OnPreCreate(id)) return FALSE; DWORD target_addr; if (!GetOption(&m_nOptions, sizeof(m_nOptions))) { //获取新配置 vLog(LOG_DEBUG, "润阳ftp转modbus读取配置错误。"); return FALSE; } if (m_pAftp) m_pAftp = NULL; target_addr = m_nOptions.net.target_addr; char user[128] = "administrator"; char password[128] = "123456"; char ipaddress[128] = "127.0.0.1"; char remotePath[128] = "\0"; char localPath[128]; char url[256]; char listpath[256]; snprintf(user, sizeof(user), "%s", m_nOptions.ftp.user); snprintf(password, sizeof(password), "%s", m_nOptions.ftp.password); snprintf(localPath, sizeof(localPath), "%s", m_nOptions.ftp.localPath); m_localPath = std::string(localPath); char *escaped_string = escape_char_in_string(m_nOptions.ftp.remotePath, ' '); if (!escaped_string) return FALSE; snprintf(remotePath, sizeof(remotePath), "%s", escaped_string); free(escaped_string); memset(ipaddress, '\0', sizeof(ipaddress)); inet_ntop(AF_INET, &target_addr, ipaddress, 16); snprintf(url, sizeof(url), "ftp://%s", ipaddress); snprintf(listpath, sizeof(listpath), "/%s", remotePath); m_pAftp = new FtpManage(user, password, url); m_listPath = std::string(listpath); fileName2Id_map.clear(); //读取列表 loadMapFromFile(); last_count = fileName2Id_map.size(); return TRUE; } BOOLEAN CFtp2MinioProcess::Run(void) { if (!CProcess::Run()) return FALSE; return TRUE; } BOOLEAN CFtp2MinioProcess::OnTimer(void) { if (!CProcess::OnTimer()) return FALSE; BOOLEAN min_changed = FALSE; if (last_min != system32.now.minute) { last_min = system32.now.minute; min_changed = TRUE; } if (min_changed) { vLog(LOG_DEBUG, "准备读取文件夹%s的内容\n", m_listPath.c_str()); for (const auto& n : m_pAftp->GetFilesName(m_listPath)) { std::string remotefile = m_listPath + "/" + n; std::string localpath = m_localPath; if (fileName2Id_map.find(n) == fileName2Id_map.end()) { fileName2Id_map.insert(fileName2Idmap::value_type(n, 0)); if (m_pAftp->DownloadFile(remotefile.c_str(), localpath.c_str())) { vLog(LOG_DEBUG, "下载成功!\n"); push2minio(n); } } else { vLog(LOG_WARN, "该文件:%s已经被下载。\n", n.c_str()); return TRUE; } } if (last_count != fileName2Id_map.size()) { last_count = fileName2Id_map.size(); saveMapToFile(); } } return TRUE; } BOOLEAN CFtp2MinioProcess::push2minio(std::string pathName) { //创建URL //minio::s3::BaseUrl base_url("http://192.168.109.187:9000"); minio::s3::BaseUrl base_url(m_nOptions.minio.url); base_url.https = false; //创建鉴权对象 //minio::creds::StaticProvider provider("das", "zaq12WSX"); minio::creds::StaticProvider provider(m_nOptions.minio.user, m_nOptions.minio.password); // 创建客户端 minio::s3::Client client(base_url, &provider); //std::string bucket_name = "test"; std::string bucket_name = std::string(m_nOptions.minio.bucket); // 检查test桶是否存在 bool exist; { minio::s3::BucketExistsArgs args; args.bucket = bucket_name; minio::s3::BucketExistsResponse resp = client.BucketExists(args); if (!resp) { vLog(LOG_ERROR, "unable to do bucket existence check; %s\n", resp.Error()); return FALSE; } exist = resp.exist; } // 如果test桶不存在,则创建test桶 if (!exist) { minio::s3::MakeBucketArgs args; args.bucket = bucket_name; minio::s3::MakeBucketResponse resp = client.MakeBucket(args); if (!resp) { vLog(LOG_ERROR, "unable to create bucket; %s\n", resp.Error()); return FALSE; } } // 上传文件 minio::s3::UploadObjectArgs args; args.bucket = bucket_name; //上传到桶中的绝对路径 //char *pYMDhms = strchr((char *)pathName.c_str(), '.'); //args.object = "/A-001/Alarm/" + pathName; args.object = std::string(m_nOptions.minio.object) + pathName; //本地文件系统中的绝对路径 args.filename = pathName;//"/das/minio-example/test.txt"; minio::s3::UploadObjectResponse resp = client.UploadObject(args); if (!resp) { vLog(LOG_ERROR, "unable to upload object; %s\n", resp.Error()); return FALSE; } vLog(LOG_DEBUG, "'%s' is successfully uploaded as object '%s' to bucket 'test'.", args.filename.c_str(), args.object.c_str()); return TRUE; }