c++ - 对等体断开连接后 SSL 内存未释放

Question

我正在开发一个简单的多线程 HTTP/s 服务器，使用非阻塞套接字来处理基于 Linux Epoll 的 HTTP 请求。它创建了 4/8 个线程(struct SSLWorker)，每个线程都可以接受和处理连接，除了 SSL_CTX(struct SSLContext)之外，连接之间没有共享数据。

随着每个新的连接，内存都会增长，并且在断开连接后，内存永远不会被释放。我想不出原因。似乎它与 SSL 代码有关，因为它在使用 HTTP 请求时不会发生。

使用 valgrind、heaptrack 和泄漏清理程序并没有帮助，因为似乎内存仍然可以访问并且没有检测到泄漏。
清理 Connection struct SSL 和 BIO 数据的正确方法和顺序是什么？ (freeSsl())。
对我所缺少的有什么帮助吗？

笔记:
- 尝试了来自 Debian 和自建的 Openssl 1.1 和 1.1.1，它们的行为相同。
- 当客户端使用 TLS 1.2 时，内存增长更加明显
- 尝试禁用内部 SSL SESSION 缓存并设置 SSL_MODE_RELEASE_BUFFERS。
- 断开连接由客户端开始。

struct SSLContext{   //Shared between worker threads
    SSL_CTX *ssl_ctx{nullptr};
    SSLContext(){
        ERR_load_crypto_strings();
        ERR_load_SSL_strings();
        SSL_load_error_strings();
        OpenSSL_add_all_algorithms();
        int r = SSL_library_init();
        ssl_ctx = SSL_CTX_new(SSLv23_method());
        int r = SSL_CTX_use_certificate_file(ssl_ctx, global::cert_file.c_str(),
            SSL_FILETYPE_PEM); 
        r = SSL_CTX_use_PrivateKey_file(ssl_ctx, global::key_file.c_str(), SSL_FILETYPE_PEM); 
        r = SSL_CTX_check_private_key(ssl_ctx); 
    }

    virtual ~SSLContext(){
        SSL_CTX_free(ssl_ctx);
        ERR_free_strings();
    }
};

struct Connection //Data associated to every connection
{
    int sock_fd;
    SSL *ssl{nullptr}; 
    BIO *sbio{nullptr};// socket bio  
    BIO *io{nullptr};// buffer bio  
    BIO *ssl_bio{nullptr};// ssl bio
    ~Connection(){ //free ssl objects
        if (ssl != nullptr) {
            SSL_set_shutdown(ssl, SSL_RECEIVED_SHUTDOWN);
            freeSsl();
            if(sock_fd > 0) ::close(sock_fd);
        }
    }
    void freeSsl(){ //free connection       
        //SSL_shutdown(ssl); it seems to be done by bellow free call.
        //SSL_free(ssl); it seems to be done by bellow free call.
        if(sbio != nullptr) {
            BIO_free(sbio);
            sbio = nullptr;
        }
        if(io != nullptr) {
            BIO_flush(io);
            BIO_free(io);
            io = nullptr;
        }
        if(ssl_bio != nullptr) {
            BIO_flush(ssl_bio);
            BIO_free(ssl_bio);     
            ssl_bio = nullptr; 
        }
        ssl = nullptr;
    }
    bool enableReadEvent(); //enable socket read events, set EPOLLIN | EPOLLET
    bool enableWriteEvent();//enable socket write event, set EPOLLOUT | EPOLLET 
};

struct SSLConnectionManager //ssl operations hanldler.
{
    static SSLContext ssl_context; 

    bool handleHandshake(Connection &ssl_connection) //Initialize connection and do hanshake
    {
        if(ssl_connection.ssl != nullptr) ssl_connection.freeSsl();     
        ssl_connection.ssl = SSL_new(ssl_context->ssl_ctx);
        ssl_connection.sbio = BIO_new_socket(ssl_connection.sock_fd, BIO_CLOSE);
        SSL_set_bio(ssl_connection.ssl, ssl_connection.sbio, ssl_connection.sbio);
        ssl_connection.io = BIO_new(BIO_f_buffer());
        ssl_connection.ssl_bio = BIO_new(BIO_f_ssl());
        BIO_set_ssl(ssl_connection.ssl_bio, ssl_connection.ssl, BIO_CLOSE);
        BIO_push(ssl_connection.io, ssl_connection.ssl_bio);
        SSL_set_accept_state(ssl_connection.ssl);

        int r = SSL_do_handshake(ssl_connection.ssl); 
        if (r == 1) {
            ssl_connection.ssl_connected = true;   
            ssl_connection.enableReadEvent();
            return true;
        }

        int err = SSL_get_error(ssl_connection.ssl, r);
        if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) {  
            ssl_connection.enableReadEvent();
            return true;
        } 
        //"SSL_do_handshake error, abort connection
        return false;    

    }

    IO::IO_RESULT SSLConnectionManager::handleRead(Connection &ssl_connection) {
        if (!ssl_connection.ssl_connected) {
            return IO::IO_RESULT::SSL_NEED_HANDSHAKE;
        }
        int rc = -1;
        int bytes_read = 0;
        for (;;) {
            rc = BIO_read(ssl_connection.io,
                ssl_connection.buffer + ssl_connection.buffer_size,
                static_cast<int>(MAX_DATA_SIZE - ssl_connection.buffer_size));        
            if (rc == 0) {
                return bytes_read > 0 ? IO::IO_RESULT::SUCCESS : IO::IO_RESULT::ZERO_DATA_READ;            
            }else if (rc < 0) {
                if (BIO_should_retry(ssl_connection.io)) {
                    return IO::IO_RESULT::DONE_TRY_AGAIN;
                }
                return IO::IO_RESULT::ERROR;
            }
            bytes_read += rc;
            ssl_connection.buffer_size += static_cast<size_t>(rc);
        }
        return IO::IO_RESULT::SUCCESS;
    }

    IO::IO_RESULT SSLConnectionManager::handleWrite(Connection &ssl_connection,
        const char *data, size_t data_size, size_t &written) {
        if (!ssl_connection.ssl_connected) {
            return IO::IO_RESULT::SSL_NEED_HANDSHAKE; // after we call handleHanshake
        }

        IO::IO_RESULT result;
        int rc = -1;
        written = 0;
        for (;;) {
            rc = BIO_write(ssl_connection.io, data + written, static_cast<int>(data_size - written));        
            if (rc == 0) {
                result = IO::IO_RESULT::DONE_TRY_AGAIN;
                break;
            } else if (rc < 0) {
                if (BIO_should_retry(ssl_connection.io)) {
                    result = IO::IO_RESULT::DONE_TRY_AGAIN;             
                    break;
                } else {
                    return IO::IO_RESULT::ERROR;                    
                }
            } else {
                written += rc;
                if ((data_size - written) == 0) {
                    result = IO::IO_RESULT::SUCCESS;
                    break;
                };
            }
        }
        BIO_flush(ssl_connection.io);  
        return result;
    }   
};

struct SSLWorker : EpollManager{ //Thread worker task.

    SSLConnectionManager ssl_connection_manager;

    bool onConnectEvent(Connection &ssl_connection){
        auto sock_fd  = accept(...);    
        Connection * new_connection = new  Connection(sock_fd);
        addToEventManager(*new_connection, EV_READ);
    } 

    void doWork() {
        is_running = true;
        int res = 0;
        epoll_event events[1024];
        while (is_running) {
            res = ::epoll_wait(epoll_fd, events, 1024, -1);
            if (res < 0)
                return;
            for (int i = 0; i < res; i++) {
                auto conn = static_cast<Connection *>(events[i].data.ptr);
                if (events[i].events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) {
                    delete conn; //remote closed connection, free all
                } else {
                    if (events[i].events & EPOLLIN) {
                        if (conn->fd == global::listen_fd) {            
                            onConnectEvent();
                            continue;
                        } else {            
                            ssl_connection_manager.handleRead(*conn);
                            processRequest();
                            conn.enableWriteEvent();
                        }
                    }
                    if (events[i].events & EPOLLOUT) {   
                        ssl_connection_manager.handleWrite(*conn);
                    }
                }
            }
        }
    }


};

在这里，您可以在短暂执行后看到什么 heaptrack 报告泄漏。

leak stack

Answer 1

如果您是多线程并且静态链接到 openssl，您可能需要调用 OPENSSL_thread_stop释放内存。有关详细信息，请参阅观察 SSL_accept 中分配的数据泄漏的人提出的以下问题:
https://github.com/openssl/openssl/issues/9605