map/das-dn/minio/ftp2minio.cpp

#include "ftp2minio.h"
#include <iostream>
#include <fstream>
#include <regex>

FtpManage::FtpManage(const std::string user, const std::string password, const std::string id)
	:Ftp_ip(id), User(user), Password(password)
{
    CURLcode res = curl_global_init(CURL_GLOBAL_ALL);	// 初始化全局环境只需要初始化一次
    if (res != CURLE_OK)
    {
        vLog(LOG_ERROR, "curl_global_init() falied %s\n", curl_easy_strerror(res));
        return;
    }
}

FtpManage::~FtpManage()
{
    if (curl)
    {
        curl_easy_cleanup(curl);
    }
    curl_global_cleanup();
}

//接收消息回调函数
size_t WriteCallback(void* contents, size_t size, size_t nmemb, std::string* output)
{
    size_t totalSize = size * nmemb;
    output->append(static_cast<char*>(contents), totalSize);
    return totalSize;
}

// 初始化curl
void FtpManage::SetURL()
{
    curl = curl_easy_init();
    curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);		// 设置长连接
    curl_easy_setopt(curl, CURLOPT_USERPWD, (User + ':' + Password).c_str());	//设置用户名和密码
}

// 把带有路径的文件转换为文件名 D:/ggbond/123.jpg->123.jpg
std::string FtpManage::GetFileNameFromPath(const std::string& filePath)
{
    if (filePath == "") return filePath;
    
    //去除路径最后的空格
    std::string retPath;
    size_t lastBlank = filePath.find_last_not_of(' ');
    if (lastBlank != std::string::npos)
    {
        retPath = filePath.substr(0, lastBlank+1);
    }
    else
    {
        return "";
    }
    size_t lastSlashPos = retPath.find_last_of("/\\");
    if (lastSlashPos != std::string::npos)
    {
        return retPath.substr(lastSlashPos + 1);
    }
    else
    {   // 如果没有找到斜杠或反斜杠，整个路径就是文件名
        return retPath;
    }
}
// 下载单一文件,把remoteFilePath的文件下载到localDirectory里面
bool FtpManage::DownloadFile(const char* remoteFilePath, const char* localDirectory)
{
    SetURL();
    _URL = Ftp_ip + remoteFilePath;
    curl_easy_setopt(curl, CURLOPT_URL, _URL.c_str());		// 设置请求的URL
    if (curl)
    {
        FILE* fp = fopen((localDirectory + GetFileNameFromPath(remoteFilePath)).c_str(), "wb");
        if (fp)
        {   // 设置文件写入地址
            curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
            // 执行任务
            CURLcode res = curl_easy_perform(curl);
            if (res != CURLE_OK)
            {
                vLog(LOG_ERROR, "DownLoad file: %s failed<%s>.", remoteFilePath, curl_easy_strerror(res));
                fclose(fp);
                curl_easy_cleanup(curl);
                return false;
            }
            else
            {
                vLog(LOG_DEBUG, "DownLoad file: %s successfully!\n", remoteFilePath);
                fclose(fp);
            }
        }
        else
        {
            vLog(LOG_ERROR, "file open failed!\n");
            curl_easy_cleanup(curl);		// 清除curl
            return false;
        }
    }
    curl_easy_cleanup(curl);
    return true;
}
// 下载全部文件
bool FtpManage::DownloadAllFiles(const char* remoteFilePath, const char* localDirectory)
{
    if (GetfilenameFromftp(remoteFilePath))
    {
        for (const auto& fns : fNs)
        {
            std::string filename = remoteFilePath + fns;
            bool res = DownloadFile(filename.c_str(), localDirectory);
            if (res) continue;
            else
            {
                return false;
            }
        }
    }
    return true;
}

// 获取ftp某个文件夹内文件名
bool FtpManage::GetfilenameFromftp(const std::string filePath)
{
    SetURL();
    std::string path = Ftp_ip + filePath;
    std::string fileName;	// 文件名列表保存位置
    
    if (curl)
    {
        curl_easy_setopt(curl, CURLOPT_URL, path.c_str());	// 设置访问URL
        curl_easy_setopt(curl, CURLOPT_DIRLISTONLY, 1L);	// 设置只返回文件
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &fileName);	// 设置只获取文件名列表
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);	// 设置get的回调函数
        if (curl_easy_perform(curl) == CURLE_OK)				//执行任务
        {
            fNs.clear();
            size_t startPos = 0, endPos = 0;
            while ((endPos = fileName.find('\n', startPos)) != std::string::npos )
            {
                std::string name = fileName.substr(startPos, endPos - startPos);
                fNs.emplace_back(name);
                startPos = endPos + 1;
            }
        }
    }
    else
    {
        return false;
    }
    curl_easy_cleanup(curl);
    return true;
}

// 获取FTP文件夹内所有文件名的接口函数
const std::vector<std::string>& FtpManage::GetFilesName(const std::string filepath)
{
    if (GetfilenameFromftp(filepath))
    {
        return fNs;
    }
    return std::vector<std::string>();
}

CFtp2MinioProcess::CFtp2MinioProcess()
{
    m_pAftp = NULL;
}

CFtp2MinioProcess::~CFtp2MinioProcess()
{
    if (m_pAftp) delete m_pAftp;
    m_pAftp = NULL;
    saveMapToFile();
}

void CFtp2MinioProcess::saveMapToFile(void)
{
    char fileName[260];
    snprintf(fileName, sizeof(fileName), "minioftp_%d.mem", GetCurID());
    std::ofstream outFile(fileName);
    if (!outFile)
    {
        vLog(LOG_ERROR, "Unable to open file for writing!\n");
        return;
    }
    for (const auto& pair : fileName2Id_map)
    {
        outFile << pair.first << " " << pair.second << std::endl;
    }
    outFile.close();
}

void CFtp2MinioProcess::loadMapFromFile(void)
{
    char fileName[260];
    snprintf(fileName, sizeof(fileName), "minioftp_%d.mem", GetCurID());
    std::ifstream inFile(fileName);
    if (!inFile)
    {
        vLog(LOG_ERROR, "Unable to open file for reading!\n");
        return;
    }
    std::string key;
    int value;
    fileName2Id_map.clear();
    while (inFile >> key >> value) {
        fileName2Id_map[key] = value; //如果键已经存在，则会更新其值
    }
    inFile.close();
}

BOOLEAN CFtp2MinioProcess::OnPreCreate(int id)
{
    if (!CProcess::OnPreCreate(id)) return FALSE;

    DWORD   target_addr;
    if (!GetOption(&m_nOptions, sizeof(m_nOptions)))
    { //获取新配置
        vLog(LOG_DEBUG, "润阳ftp转modbus读取配置错误。");
        return FALSE;
    }

    if (m_pAftp) m_pAftp = NULL;

    target_addr = m_nOptions.net.target_addr;

    char user[128] = "administrator";
    char password[128] = "123456";
    char ipaddress[128] = "127.0.0.1";
    char remotePath[128] = "\0";
    char localPath[128];
    char url[256];
    //char listpath[256];
    snprintf(user, sizeof(user), "%s", m_nOptions.ftp.user);
    snprintf(password, sizeof(password), "%s", m_nOptions.ftp.password);
    snprintf(localPath, sizeof(localPath), "%s", m_nOptions.ftp.localPath);
    m_localPath = std::string(localPath);
    if (m_localPath.back() != '/') m_localPath += std::string("/");
    
    char *escaped_string = escape_char_in_string(m_nOptions.ftp.remotePath, ' ');
    if (!escaped_string) return FALSE;
    snprintf(remotePath, sizeof(remotePath), "%s", escaped_string);
    free(escaped_string);
    
    memset(ipaddress, '\0', sizeof(ipaddress));
    inet_ntop(AF_INET, &target_addr, ipaddress, 16);

    snprintf(url, sizeof(url), "ftp://%s", ipaddress);

    std::string listpaths(remotePath);
    std::vector<std::string> splits = split(listpaths, ';');
    for (std::vector<std::string>::iterator it = splits.begin(); it != splits.end(); it++)
    {
        if ((*it).back() != '/')
        {
            m_listPaths.push_back((*it) + "/");
        }
        else
        {
            m_listPaths.push_back((*it));
        }
    }
    
    m_pAftp = new FtpManage(user, password, url);

    fileName2Id_map.clear();
    //读取列表
    loadMapFromFile();

    last_count = fileName2Id_map.size();
    return TRUE;
}

BOOLEAN CFtp2MinioProcess::Run(void)
{
    if (!CProcess::Run()) return FALSE;

    return TRUE;
}

std::optional<std::string> CFtp2MinioProcess::extractDate(const std::string& filename, const std::string& regex)
{
    std::regex datePattern(regex);//(R"(\d{4}-\d{2}-\d{2})");
    std::smatch match;
    if (std::regex_search(filename, match, datePattern))
    {
        return match.str(); // 返回匹配到的日期字符串
    }
    return std::nullopt; // 没有找到日期则返回 nullopt
}

BOOLEAN CFtp2MinioProcess::OnTimer(void)
{
    if (!CProcess::OnTimer()) return FALSE;

    BOOLEAN sec_changed = FALSE;
    if (last_sec != system32.timers)
    {
        last_sec = system32.timers;
        sec_changed = TRUE;
    }
    if (sec_changed)
    {
        FeedDog();
    }
    BOOLEAN min_changed = FALSE;
    if (last_min != system32.now.minute)
    {
        last_min = system32.now.minute;
        min_changed = TRUE;
    }
    if (min_changed)
    {
        for (std::vector<std::string>::iterator it = m_listPaths.begin(); it != m_listPaths.end(); it++)
        {
            m_listPath = (*it);
            if (m_listPath.empty()) continue;
            // 检查字符串的最后一个字符是否为 '/'
            vLog(LOG_DEBUG, "准备读取文件夹%s的内容\n", m_listPath.c_str());
            for (const auto& n : m_pAftp->GetFilesName(m_listPath))
            {
                std::string remotefile = m_listPath + n;
                std::string localpath = m_localPath;
                if (fileName2Id_map.find(n) == fileName2Id_map.end())
                {
                    fileName2Id_map.insert(fileName2Idmap::value_type(n, 0));
                    if (m_pAftp->DownloadFile(remotefile.c_str(), localpath.c_str()))
                    {
                        vLog(LOG_DEBUG, "下载成功！\n");
                        std::vector<std::string> tokens = split(m_listPath, '/');
                        push2minio(tokens[tokens.size() - 1], n);
                    }
                }
                else
                {
                    vLog(LOG_WARN, "该文件：%s已经被下载。\n", n.c_str());
                }
            }
            if (last_count != fileName2Id_map.size())
            {
                last_count = fileName2Id_map.size();
                saveMapToFile();
            }
        }
    }
    return TRUE;
}

BOOLEAN CFtp2MinioProcess::push2minio(std::string parentDir, std::string pathName)
{
    //创建URL
    //minio::s3::BaseUrl base_url("http://192.168.109.187:9000");
    minio::s3::BaseUrl base_url(m_nOptions.minio.url);
    base_url.https = false;
    
    //创建鉴权对象
    //minio::creds::StaticProvider provider("das", "zaq12WSX");
    minio::creds::StaticProvider provider(m_nOptions.minio.user, m_nOptions.minio.password);
    
    // 创建客户端
    minio::s3::Client client(base_url, &provider);
    //std::string bucket_name = "test";
    std::string bucket_name = std::string(m_nOptions.minio.bucket);

    // 检查test桶是否存在
    bool exist;
    {
        minio::s3::BucketExistsArgs args;
        args.bucket = bucket_name;

        minio::s3::BucketExistsResponse resp = client.BucketExists(args);
        if (!resp) {
            vLog(LOG_ERROR, "unable to do bucket existence check; %s\n", resp.Error());
            return FALSE;
        }

        exist = resp.exist;
    }

    // 如果test桶不存在，则创建test桶
    if (!exist) {
        minio::s3::MakeBucketArgs args;
        args.bucket = bucket_name;

        minio::s3::MakeBucketResponse resp = client.MakeBucket(args);
        if (!resp) {
            vLog(LOG_ERROR, "unable to create bucket; %s\n", resp.Error());
            return FALSE;
        }
    }

    // 上传文件
    minio::s3::UploadObjectArgs args;
    args.bucket = bucket_name;
    //上传到桶中的绝对路径
    //char *pYMDhms = strchr((char *)pathName.c_str(), '.');
    //args.object = "/A-001/Alarm/" + pathName;
    //args.object = std::string(m_nOptions.minio.object) + pathName;

    auto date = extractDate(pathName, R"(\d{4}-\d{2}-\d{2})");
    std::string date_string;
    if (date.has_value())
    {
        //vLog(LOG_DEBUG, "fileName is: %s, Date is: %s\n", pathName.c_str(), date.value().c_str());
        date_string = replaceChar(date.value(), '-', '/');
        //vLog(LOG_DEBUG, "date_string is: %s\n", date_string.c_str());
    }
    else
    {
        date = extractDate(pathName, R"(\d{4}\d{2}\d{2})");
        if (date.has_value())
        {
            //vLog(LOG_DEBUG, "fileName is: %s, Date is: %s\n", pathName.c_str(), date.value().c_str());
            date_string = date.value().substr(0, 4) + std::string("/") + date.value().substr(4, 2) + std::string("/") + date.value().substr(6, 2) + std::string("/");
            //vLog(LOG_DEBUG, "date_string is: %s\n", date_string.c_str());
        }
        else
        {
            vLog(LOG_DEBUG, "文件不存在时间格式\n");
            return TRUE;
        }
    }

    //return TRUE;
    args.object = std::string(m_nOptions.minio.object) + std::string("/") + parentDir + std::string("/") + date_string + std::string("/") + pathName;

    //本地文件系统中的绝对路径
    args.filename = m_localPath + pathName;

    minio::s3::UploadObjectResponse resp = client.UploadObject(args);
    if (!resp) {
        vLog(LOG_ERROR, "unable to upload object; %s\n", resp.Error());
        return FALSE;
    }

    vLog(LOG_DEBUG, "'%s' is successfully uploaded as object '%s' to bucket 'test'.", args.filename.c_str(), args.object.c_str());

    return TRUE;
}