443 lines
13 KiB
C++
443 lines
13 KiB
C++
#include "ftp2minio.h"
|
||
#include <iostream>
|
||
#include <fstream>
|
||
#include <regex>
|
||
|
||
FtpManage::FtpManage(const std::string user, const std::string password, const std::string id)
|
||
:Ftp_ip(id), User(user), Password(password)
|
||
{
|
||
CURLcode res = curl_global_init(CURL_GLOBAL_ALL); // 初始化全局环境只需要初始化一次
|
||
if (res != CURLE_OK)
|
||
{
|
||
vLog(LOG_ERROR, "curl_global_init() falied %s\n", curl_easy_strerror(res));
|
||
return;
|
||
}
|
||
}
|
||
|
||
FtpManage::~FtpManage()
|
||
{
|
||
if (curl)
|
||
{
|
||
curl_easy_cleanup(curl);
|
||
}
|
||
curl_global_cleanup();
|
||
}
|
||
|
||
//接收消息回调函数
|
||
size_t WriteCallback(void* contents, size_t size, size_t nmemb, std::string* output)
|
||
{
|
||
size_t totalSize = size * nmemb;
|
||
output->append(static_cast<char*>(contents), totalSize);
|
||
return totalSize;
|
||
}
|
||
|
||
// 初始化curl
|
||
void FtpManage::SetURL()
|
||
{
|
||
curl = curl_easy_init();
|
||
curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L); // 设置长连接
|
||
curl_easy_setopt(curl, CURLOPT_USERPWD, (User + ':' + Password).c_str()); //设置用户名和密码
|
||
}
|
||
|
||
// 把带有路径的文件转换为文件名 D:/ggbond/123.jpg->123.jpg
|
||
std::string FtpManage::GetFileNameFromPath(const std::string& filePath)
|
||
{
|
||
if (filePath == "") return filePath;
|
||
|
||
//去除路径最后的空格
|
||
std::string retPath;
|
||
size_t lastBlank = filePath.find_last_not_of(' ');
|
||
if (lastBlank != std::string::npos)
|
||
{
|
||
retPath = filePath.substr(0, lastBlank+1);
|
||
}
|
||
else
|
||
{
|
||
return "";
|
||
}
|
||
size_t lastSlashPos = retPath.find_last_of("/\\");
|
||
if (lastSlashPos != std::string::npos)
|
||
{
|
||
return retPath.substr(lastSlashPos + 1);
|
||
}
|
||
else
|
||
{ // 如果没有找到斜杠或反斜杠，整个路径就是文件名
|
||
return retPath;
|
||
}
|
||
}
|
||
// 下载单一文件,把remoteFilePath的文件下载到localDirectory里面
|
||
bool FtpManage::DownloadFile(const char* remoteFilePath, const char* localDirectory)
|
||
{
|
||
SetURL();
|
||
_URL = Ftp_ip + remoteFilePath;
|
||
curl_easy_setopt(curl, CURLOPT_URL, _URL.c_str()); // 设置请求的URL
|
||
if (curl)
|
||
{
|
||
FILE* fp = fopen((localDirectory + GetFileNameFromPath(remoteFilePath)).c_str(), "wb");
|
||
if (fp)
|
||
{ // 设置文件写入地址
|
||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
|
||
// 执行任务
|
||
CURLcode res = curl_easy_perform(curl);
|
||
if (res != CURLE_OK)
|
||
{
|
||
vLog(LOG_ERROR, "DownLoad file: %s failed<%s>.", remoteFilePath, curl_easy_strerror(res));
|
||
fclose(fp);
|
||
curl_easy_cleanup(curl);
|
||
return false;
|
||
}
|
||
else
|
||
{
|
||
vLog(LOG_DEBUG, "DownLoad file: %s successfully!\n", remoteFilePath);
|
||
fclose(fp);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
vLog(LOG_ERROR, "file open failed!\n");
|
||
curl_easy_cleanup(curl); // 清除curl
|
||
return false;
|
||
}
|
||
}
|
||
curl_easy_cleanup(curl);
|
||
return true;
|
||
}
|
||
// 下载全部文件
|
||
bool FtpManage::DownloadAllFiles(const char* remoteFilePath, const char* localDirectory)
|
||
{
|
||
if (GetfilenameFromftp(remoteFilePath))
|
||
{
|
||
for (const auto& fns : fNs)
|
||
{
|
||
std::string filename = remoteFilePath + fns;
|
||
bool res = DownloadFile(filename.c_str(), localDirectory);
|
||
if (res) continue;
|
||
else
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
}
|
||
return true;
|
||
}
|
||
|
||
// 获取ftp某个文件夹内文件名
|
||
bool FtpManage::GetfilenameFromftp(const std::string filePath)
|
||
{
|
||
SetURL();
|
||
std::string path = Ftp_ip + filePath;
|
||
std::string fileName; // 文件名列表保存位置
|
||
|
||
if (curl)
|
||
{
|
||
curl_easy_setopt(curl, CURLOPT_URL, path.c_str()); // 设置访问URL
|
||
curl_easy_setopt(curl, CURLOPT_DIRLISTONLY, 1L); // 设置只返回文件
|
||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &fileName); // 设置只获取文件名列表
|
||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); // 设置get的回调函数
|
||
if (curl_easy_perform(curl) == CURLE_OK) //执行任务
|
||
{
|
||
fNs.clear();
|
||
size_t startPos = 0, endPos = 0;
|
||
while ((endPos = fileName.find('\n', startPos)) != std::string::npos )
|
||
{
|
||
std::string name = fileName.substr(startPos, endPos - startPos);
|
||
fNs.emplace_back(name);
|
||
startPos = endPos + 1;
|
||
}
|
||
}
|
||
}
|
||
else
|
||
{
|
||
return false;
|
||
}
|
||
curl_easy_cleanup(curl);
|
||
return true;
|
||
}
|
||
|
||
// 获取FTP文件夹内所有文件名的接口函数
|
||
const std::vector<std::string>& FtpManage::GetFilesName(const std::string filepath)
|
||
{
|
||
if (GetfilenameFromftp(filepath))
|
||
{
|
||
return fNs;
|
||
}
|
||
return std::vector<std::string>();
|
||
}
|
||
|
||
CFtp2MinioProcess::CFtp2MinioProcess()
|
||
{
|
||
m_pAftp = NULL;
|
||
}
|
||
|
||
CFtp2MinioProcess::~CFtp2MinioProcess()
|
||
{
|
||
if (m_pAftp) delete m_pAftp;
|
||
m_pAftp = NULL;
|
||
saveMapToFile();
|
||
}
|
||
|
||
void CFtp2MinioProcess::saveMapToFile(void)
|
||
{
|
||
char fileName[260];
|
||
snprintf(fileName, sizeof(fileName), "minioftp_%d.mem", GetCurID());
|
||
std::ofstream outFile(fileName);
|
||
if (!outFile)
|
||
{
|
||
vLog(LOG_ERROR, "Unable to open file for writing!\n");
|
||
return;
|
||
}
|
||
for (const auto& pair : fileName2Id_map)
|
||
{
|
||
outFile << pair.first << " " << pair.second << std::endl;
|
||
}
|
||
outFile.close();
|
||
}
|
||
|
||
void CFtp2MinioProcess::loadMapFromFile(void)
|
||
{
|
||
char fileName[260];
|
||
snprintf(fileName, sizeof(fileName), "minioftp_%d.mem", GetCurID());
|
||
std::ifstream inFile(fileName);
|
||
if (!inFile)
|
||
{
|
||
vLog(LOG_ERROR, "Unable to open file for reading!\n");
|
||
return;
|
||
}
|
||
std::string key;
|
||
int value;
|
||
fileName2Id_map.clear();
|
||
while (inFile >> key >> value) {
|
||
fileName2Id_map[key] = value; //如果键已经存在，则会更新其值
|
||
}
|
||
inFile.close();
|
||
}
|
||
|
||
BOOLEAN CFtp2MinioProcess::OnPreCreate(int id)
|
||
{
|
||
if (!CProcess::OnPreCreate(id)) return FALSE;
|
||
|
||
DWORD target_addr;
|
||
if (!GetOption(&m_nOptions, sizeof(m_nOptions)))
|
||
{ //获取新配置
|
||
vLog(LOG_DEBUG, "润阳ftp转modbus读取配置错误。");
|
||
return FALSE;
|
||
}
|
||
|
||
if (m_pAftp) m_pAftp = NULL;
|
||
|
||
target_addr = m_nOptions.net.target_addr;
|
||
|
||
char user[128] = "administrator";
|
||
char password[128] = "123456";
|
||
char ipaddress[128] = "127.0.0.1";
|
||
char remotePath[128] = "\0";
|
||
char localPath[128];
|
||
char url[256];
|
||
//char listpath[256];
|
||
snprintf(user, sizeof(user), "%s", m_nOptions.ftp.user);
|
||
snprintf(password, sizeof(password), "%s", m_nOptions.ftp.password);
|
||
snprintf(localPath, sizeof(localPath), "%s", m_nOptions.ftp.localPath);
|
||
m_localPath = std::string(localPath);
|
||
if (m_localPath.back() != '/') m_localPath += std::string("/");
|
||
|
||
char *escaped_string = escape_char_in_string(m_nOptions.ftp.remotePath, ' ');
|
||
if (!escaped_string) return FALSE;
|
||
snprintf(remotePath, sizeof(remotePath), "%s", escaped_string);
|
||
free(escaped_string);
|
||
|
||
memset(ipaddress, '\0', sizeof(ipaddress));
|
||
inet_ntop(AF_INET, &target_addr, ipaddress, 16);
|
||
|
||
snprintf(url, sizeof(url), "ftp://%s", ipaddress);
|
||
|
||
std::string listpaths(remotePath);
|
||
std::vector<std::string> splits = split(listpaths, ';');
|
||
for (std::vector<std::string>::iterator it = splits.begin(); it != splits.end(); it++)
|
||
{
|
||
if ((*it).back() != '/')
|
||
{
|
||
m_listPaths.push_back((*it) + "/");
|
||
}
|
||
else
|
||
{
|
||
m_listPaths.push_back((*it));
|
||
}
|
||
}
|
||
|
||
m_pAftp = new FtpManage(user, password, url);
|
||
|
||
fileName2Id_map.clear();
|
||
//读取列表
|
||
loadMapFromFile();
|
||
|
||
last_count = fileName2Id_map.size();
|
||
return TRUE;
|
||
}
|
||
|
||
BOOLEAN CFtp2MinioProcess::Run(void)
|
||
{
|
||
if (!CProcess::Run()) return FALSE;
|
||
|
||
return TRUE;
|
||
}
|
||
|
||
std::optional<std::string> CFtp2MinioProcess::extractDate(const std::string& filename, const std::string& regex)
|
||
{
|
||
std::regex datePattern(regex);//(R"(\d{4}-\d{2}-\d{2})");
|
||
std::smatch match;
|
||
if (std::regex_search(filename, match, datePattern))
|
||
{
|
||
return match.str(); // 返回匹配到的日期字符串
|
||
}
|
||
return std::nullopt; // 没有找到日期则返回 nullopt
|
||
}
|
||
|
||
BOOLEAN CFtp2MinioProcess::OnTimer(void)
|
||
{
|
||
if (!CProcess::OnTimer()) return FALSE;
|
||
|
||
BOOLEAN sec_changed = FALSE;
|
||
if (last_sec != system32.timers)
|
||
{
|
||
last_sec = system32.timers;
|
||
sec_changed = TRUE;
|
||
}
|
||
if (sec_changed)
|
||
{
|
||
FeedDog();
|
||
}
|
||
BOOLEAN min_changed = FALSE;
|
||
if (last_min != system32.now.minute)
|
||
{
|
||
last_min = system32.now.minute;
|
||
min_changed = TRUE;
|
||
}
|
||
if (min_changed)
|
||
{
|
||
for (std::vector<std::string>::iterator it = m_listPaths.begin(); it != m_listPaths.end(); it++)
|
||
{
|
||
m_listPath = (*it);
|
||
if (m_listPath.empty()) continue;
|
||
// 检查字符串的最后一个字符是否为 '/'
|
||
vLog(LOG_DEBUG, "准备读取文件夹%s的内容\n", m_listPath.c_str());
|
||
for (const auto& n : m_pAftp->GetFilesName(m_listPath))
|
||
{
|
||
std::string remotefile = m_listPath + n;
|
||
std::string localpath = m_localPath;
|
||
if (fileName2Id_map.find(n) == fileName2Id_map.end())
|
||
{
|
||
fileName2Id_map.insert(fileName2Idmap::value_type(n, 0));
|
||
if (m_pAftp->DownloadFile(remotefile.c_str(), localpath.c_str()))
|
||
{
|
||
vLog(LOG_DEBUG, "下载成功！\n");
|
||
std::vector<std::string> tokens = split(m_listPath, '/');
|
||
push2minio(tokens[tokens.size() - 1], n);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
vLog(LOG_WARN, "该文件：%s已经被下载。\n", n.c_str());
|
||
}
|
||
}
|
||
if (last_count != fileName2Id_map.size())
|
||
{
|
||
last_count = fileName2Id_map.size();
|
||
saveMapToFile();
|
||
}
|
||
}
|
||
}
|
||
return TRUE;
|
||
}
|
||
|
||
BOOLEAN CFtp2MinioProcess::push2minio(std::string parentDir, std::string pathName)
|
||
{
|
||
//创建URL
|
||
//minio::s3::BaseUrl base_url("http://192.168.109.187:9000");
|
||
minio::s3::BaseUrl base_url(m_nOptions.minio.url);
|
||
base_url.https = false;
|
||
|
||
//创建鉴权对象
|
||
//minio::creds::StaticProvider provider("das", "zaq12WSX");
|
||
minio::creds::StaticProvider provider(m_nOptions.minio.user, m_nOptions.minio.password);
|
||
|
||
// 创建客户端
|
||
minio::s3::Client client(base_url, &provider);
|
||
//std::string bucket_name = "test";
|
||
std::string bucket_name = std::string(m_nOptions.minio.bucket);
|
||
|
||
// 检查test桶是否存在
|
||
bool exist;
|
||
{
|
||
minio::s3::BucketExistsArgs args;
|
||
args.bucket = bucket_name;
|
||
|
||
minio::s3::BucketExistsResponse resp = client.BucketExists(args);
|
||
if (!resp) {
|
||
vLog(LOG_ERROR, "unable to do bucket existence check; %s\n", resp.Error());
|
||
return FALSE;
|
||
}
|
||
|
||
exist = resp.exist;
|
||
}
|
||
|
||
// 如果test桶不存在，则创建test桶
|
||
if (!exist) {
|
||
minio::s3::MakeBucketArgs args;
|
||
args.bucket = bucket_name;
|
||
|
||
minio::s3::MakeBucketResponse resp = client.MakeBucket(args);
|
||
if (!resp) {
|
||
vLog(LOG_ERROR, "unable to create bucket; %s\n", resp.Error());
|
||
return FALSE;
|
||
}
|
||
}
|
||
|
||
// 上传文件
|
||
minio::s3::UploadObjectArgs args;
|
||
args.bucket = bucket_name;
|
||
//上传到桶中的绝对路径
|
||
//char *pYMDhms = strchr((char *)pathName.c_str(), '.');
|
||
//args.object = "/A-001/Alarm/" + pathName;
|
||
//args.object = std::string(m_nOptions.minio.object) + pathName;
|
||
|
||
auto date = extractDate(pathName, R"(\d{4}-\d{2}-\d{2})");
|
||
std::string date_string;
|
||
if (date.has_value())
|
||
{
|
||
//vLog(LOG_DEBUG, "fileName is: %s, Date is: %s\n", pathName.c_str(), date.value().c_str());
|
||
date_string = replaceChar(date.value(), '-', '/');
|
||
//vLog(LOG_DEBUG, "date_string is: %s\n", date_string.c_str());
|
||
}
|
||
else
|
||
{
|
||
date = extractDate(pathName, R"(\d{4}\d{2}\d{2})");
|
||
if (date.has_value())
|
||
{
|
||
//vLog(LOG_DEBUG, "fileName is: %s, Date is: %s\n", pathName.c_str(), date.value().c_str());
|
||
date_string = date.value().substr(0, 4) + std::string("/") + date.value().substr(4, 2) + std::string("/") + date.value().substr(6, 2) + std::string("/");
|
||
//vLog(LOG_DEBUG, "date_string is: %s\n", date_string.c_str());
|
||
}
|
||
else
|
||
{
|
||
vLog(LOG_DEBUG, "文件不存在时间格式\n");
|
||
return TRUE;
|
||
}
|
||
}
|
||
|
||
//return TRUE;
|
||
args.object = std::string(m_nOptions.minio.object) + std::string("/") + parentDir + std::string("/") + date_string + std::string("/") + pathName;
|
||
|
||
//本地文件系统中的绝对路径
|
||
args.filename = m_localPath + pathName;
|
||
|
||
minio::s3::UploadObjectResponse resp = client.UploadObject(args);
|
||
if (!resp) {
|
||
vLog(LOG_ERROR, "unable to upload object; %s\n", resp.Error());
|
||
return FALSE;
|
||
}
|
||
|
||
vLog(LOG_DEBUG, "'%s' is successfully uploaded as object '%s' to bucket 'test'.", args.filename.c_str(), args.object.c_str());
|
||
|
||
return TRUE;
|
||
}
|