c++ - 使用 libcurl 的线程安全代理

Question

    #pragma once

    #ifndef __CURL_CURL_H
    #include "curl.h"
    #endif

    #ifndef __CURL_EASY_H
    #include "easy.h"
    #endif

    #include <stdint.h>
    #include <memory>
    #include <string>

    namespace CommUnit
    {
        enum ERR_PROXY
        {
            ERR_CURL_INIT_FAILED = 0xA0,
            ERR_SET_PROXY_FAILED = 0xA1,
        };


        class MyProxy
        {
        public:
            static MyProxy & GetInstance()  //Meyers' Singlton
            {
                static MyProxy ProxySigleton;
                return ProxySigleton;
            }
        public:
            /*
            * @bref:Get request
            * @param[in] sUrl:Access URL
            * @param[in] sProxyIp:Proxy IP
            * @param[in] uProxyPort:Proxy Port
            * @param[in] uTimeOut:Time out
            * @param[in] isSSL:HTTPS true,else false
            * @param[out] sRetContent:Return the URL content
            */
            uint32_t Get(const std::string &sUrl,
                const std::string& sProxyIp,
                uint32_t uProxyPort,
                uint32_t uTimeOut,
                bool isSSL,
                std::string &sRetContent);

        private:
            MyProxy();                              //Constructor hidden
            MyProxy(MyProxy const &);               //Copy-Constructor hidden
            MyProxy & operator= (MyProxy const &);  //Assign operator hidden
            ~MyProxy();                             //Destructor hidden

            inline void _setCurlopt(CURL *pCurl,
                const std::string &sUrl,
                std::string &sWriterData,
                const uint32_t uTimeOut,
                bool isSSL);

            //Callback function, write data to writerData
            static int Writer(char *data,
                uint32_t size,
                uint32_t nmemb,
                std::string *writerData);


        private:
            std::string m_sErrMsg;
            static char s_ErrBuffer[CURL_ERROR_SIZE];
            static const uint32_t m_MAXBUF = 2 * 1024 * 1024 - 128; 
        };
    }


//////////////////////////////////////////////////////////////////////////
#include <iostream>
#include <string>
#include "MyProxy.h"
#include "Log.h"
#include <curl.h>

using namespace CommUnit;

char MyProxy::s_ErrBuffer[CURL_ERROR_SIZE] = { 0 };


MyProxy::MyProxy(void)
{
    CURLcode oCURLcode = curl_global_init(CURL_GLOBAL_ALL);
    if (oCURLcode != CURLE_OK)
    {
        Log("ERR: %s curl_init failed!", __func__);
    }
}

MyProxy::~MyProxy(void)
{
    curl_global_cleanup();
}


uint32_t MyProxy::Get(const std::string &sUrl,
    const std::string& sProxyIp,
    uint32_t uProxyPort,
    uint32_t uTimeOut,
    bool isSSL,
    std::string &sRetContent)
{
    sRetContent.clear();
    CURL *pCurl = curl_easy_init();
    CURLcode oCURLcode;
    if (nullptr == pCurl)
    {
        Log("ERR: %s curl_easy_init failed!", __func__);
        return ERR_CURL_INIT_FAILED;
    }
    _setCurlopt(pCurl, sUrl, sRetContent, uTimeOut, isSSL);
    if (0 == sProxyIp.length()|| 0 == uProxyPort)
    {
        Log("ERR: %s SetProxy: ProxyIp [%s], ProxyPort[%u] failed",__func__, sProxyIp.c_str(), uProxyPort);
        return ERR_SET_PROXY_FAILED;
    }
    Log("INFO: %s SetProxy: ProxyIp [%s], ProxyPort[%u] failed", __func__, sProxyIp.c_str(), uProxyPort);
    curl_easy_setopt(pCurl, CURLOPT_PROXY, sProxyIp.c_str());
    curl_easy_setopt(pCurl, CURLOPT_PROXYPORT, uProxyPort);
    int iTimes = 0;
    while (true)
    {
        oCURLcode = curl_easy_perform(pCurl);
        if (oCURLcode != CURLE_OK && ++iTimes < 3)
            usleep(5);
        else
            break;
    }
    if (oCURLcode != CURLE_OK)
    {
        Log("ERR: %s curl_easy_perform failed!", __func__);
    }
    curl_easy_cleanup(pCurl);
    return oCURLcode;
}

void MyProxy::_setCurlopt(CURL *pCurl,
    const std::string &sUrl,
    std::string &sWriterData,
    const uint32_t uTimeOut,
    bool isSSL)
{
    curl_easy_setopt(pCurl, CURLOPT_ERRORBUFFER, s_ErrBuffer);
    curl_easy_setopt(pCurl, CURLOPT_FOLLOWLOCATION, 1);
    curl_easy_setopt(pCurl, CURLOPT_URL, sUrl.c_str());
    curl_easy_setopt(pCurl, CURLOPT_TIMEOUT, uTimeOut);
    Log("INFO: %s Set Url:[%s],TimeOut:[%d]", __func__, sUrl.c_str(), uTimeOut);
    curl_easy_setopt(pCurl, CURLOPT_WRITEFUNCTION, MyProxy::Writer);
    curl_easy_setopt(pCurl, CURLOPT_WRITEDATA, &sWriterData);

    //Skip peer and hostname verification
    if (isSSL)
    {
        curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYPEER, 0L);
        curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYHOST, 0L);
    }
}


int MyProxy::Writer(char *data,
    uint32_t size,
    uint32_t nmemb,
    std::string *writerData)
{
    if (writerData == nullptr)
    {
        Log("ERR: %s writerData is null!", __func__);
        return 0;
    }
    int len = size * nmemb;
    if ((writerData->size() + len) > m_MAXBUF)
    {
        Log("ERR: %s writerData size over MAXBUF!", __func__);
        return 0;
    }
    writerData->append(data, len);
    return len;
}

我想用libcurl实现一个代理，可以获取给定url(https)的内容。此外，它需要是线程安全的。但是当我用 pthreads 创建 200 个线程来测试我的代码时，有时会发生段错误。我怎么解决这个问题？与 sRetContent(std::string) 有关系吗？谢谢!

错误信息:双自由或腐败 (!prev): 0x0ac72840 ***段错误

Answer 1

我的理解是，如果您使用 https(看起来您是)，libcurl 不是线程安全的，因为它使用的是底层 ssl 库。 See libcurl documentation和 OpenSSL documentation了解更多信息。

如果您的 libcurl 是使用 OpenSSL 编译的，那么您必须初始化一些回调函数，否则您可能会遇到问题。这是您需要做的事情(在 Windows 上编译):

#include <curl/curl.h>
#include <openssl/crypto.h>

void win32_locking_callback(int mode, int type, const char *file, int line)
{
    if (mode & CRYPTO_LOCK)
    {
        WaitForSingleObject(lock_cs[type],INFINITE);
    }
    else
    {
        ReleaseMutex(lock_cs[type]);
    }
}

void thread_setup(void)
{
    int i;

    lock_cs=(HANDLE*)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(HANDLE));
    for (i=0; i<CRYPTO_num_locks(); i++)
    {
        lock_cs[i]=CreateMutex(NULL,FALSE,NULL);
    }

    CRYPTO_set_locking_callback((void (*)(int,int,const char *,int))win32_locking_callback);
}

void thread_cleanup(void)
{
    int i;

    CRYPTO_set_locking_callback(NULL);
    for (i=0; i<CRYPTO_num_locks(); i++)
        CloseHandle(lock_cs[i]);
    OPENSSL_free(lock_cs);
}

我总是在调用 curl_global_init(CURL_GLOBAL_ALL) 之后调用 thread_setup()然后是 thread_cleanup()，就在我调用 curl_global_cleanup() 之前。

我经常在负载测试场景中将这种代码与 libcurl 一起使用，并且从未遇到过任何问题。如果您继续遇到问题，那不是 libcurl，而是您的代码中没有正确完成某些事情。