python - 导入 Pybind11/C++ 编译模块不起作用

Question

我对 python setuptools 和 dist 还很陌生。我似乎无法获取要导入的 C++ 包装器模块，以便我可以使用这些函数。

已编译的 .so 文件在 pip 安装后显示在 installed-files.txt 中，但导入包装器却没有。

setup.py

import subprocess
import os
from pathlib import Path

from setuptools import setup, Extension, find_packages
from setuptools.command.build_ext import build_ext


class CMakeExtension(Extension):
    def __init__(self, name):
        Extension.__init__(self, name, sources=[])


class CMakeBuild(build_ext):
    def run(self):
        for ext in self.extensions:
            self.build_cmake(ext)
        super().run()

    def build_cmake(self, ext):
        try:
            subprocess.check_output(["cmake", "--version"])
        except OSError:
            raise RuntimeError(
                "CMake must be installed to build the following extensions: "
                + ", ".join(e.name for e in self.extensions)
            )

        cwd = Path().absolute()

        # these dirs will be created in build_py, so if you don't have
        # any python sources to bundle, the dirs will be missing
        build_temp = Path(self.build_temp)
        build_temp.mkdir(parents=True, exist_ok=True)
        extdir = Path(self.get_ext_fullpath(ext.name))
        extdir.mkdir(parents=True, exist_ok=True)

        pyenv_root = os.environ.get("PYENV_ROOT")

        cmake_args = [
            f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}",
            "-DCMAKE_BUILD_TYPE=Release",
            "-DTRANSIT_INCLUDE_TESTS:BOOL=OFF",
        ]

        if pyenv_root is not None:
            cmake_args += [f"-DPYTHON_EXECUTABLE={pyenv_root}/shims/python"]

        build_args = ["--config", "Release", "--", "-j2"]

        env = os.environ.copy()

        self.announce("Running CMake prepare", level=3)
        subprocess.check_call(["cmake", cwd] + cmake_args, cwd=build_temp, env=env)

        self.announce("Building extensions")
        cmake_cmd = ["cmake", "--build", "."] + build_args
        subprocess.check_call(cmake_cmd, cwd=build_temp)


setup(
    name="bgtfs_py_lib",
    version="3.2.2",
    long_description="",
    zip_safe=False,
    install_requires=[
        "redis==2.10.6",
        "cffi==1.11.5",
        "numpy==1.15.3",
        "patricia-trie==10",
        "PuLP==1.6.8",
        "py-lz4framed==0.13.0",
        "pycparser==2.19",
        "pyparsing==2.2.2",
        "pypng==0.0.18",
        "pyproj==1.9.5.1",
        "python-graph-core==1.8.2",
        "pytz==2018.6",
        "redis==2.10.6",
        "requests==2.21.0",
        "six==1.11.0",
        "tabulate==0.8.2",
        "unicodecsv==0.14.1",
        "Unidecode==1.0.22",
    ],
    ext_modules=[CMakeExtension("bgtfs_py_lib.bgtfs_py_lib_wrapper")],
    cmdclass=dict(build_ext=CMakeBuild),
    packages=find_packages(exclude=["tests"]),
    package_data={"": "*.so"},
)

CMakeLists.txt

cmake_minimum_required(VERSION 3.8)

project(bgtfs_py_lib_wrapper)
include(submodules/transitLib/bgtfs/bgtfsLib/TransitUtils/transit_shared.cmake)

# bgtfsPyLib
set(PYBIND11_CPP_STANDARD -std=c++14)
set(PYBIND11_PYTHON_VERSION 3.6)

add_subdirectory(submodules/transitLib transitLib)
add_subdirectory(pybind11)

include_directories(
    cpp
    submodules/transitLib/bgtfs/bgtfsLib/
    submodules/transitLib/bgtfs/bgtfsLib/bgtfsLib
)

pybind11_add_module(bgtfs_py_lib_wrapper MODULE NO_EXTRAS
    cpp/pybindCasts.cpp
    cpp/bgtfsPyLibWrapper.cpp
    cpp/BgtfsFeedHandler.cpp
)

target_link_libraries(bgtfs_py_lib_wrapper PRIVATE transitLib)
set_target_properties(bgtfs_py_lib_wrapper PROPERTIES
    LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/bgtfs_py_lib
    LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_LIST_DIR}/bgtfs_py_lib
    LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_LIST_DIR}/bgtfs_py_lib
)
target_compile_definitions(bgtfs_py_lib_wrapper PRIVATE TRANSIT_SERVER)
target_compile_definitions(transitLib PRIVATE TRANSIT_SERVER)
target_compile_definitions(bgtfsLib PRIVATE ENABLE_BACKWARD_FILE_COMPATIBILITY YES)
set_default_target_properties(bgtfs_py_lib_wrapper)

我正在尝试使用 virtualenv 来隔离在我的项目中运行所需的模块。

这是文件结构:

.
|-- CMakeLists.txt
|-- README.md
|-- bgtfs_py_lib
|   |-- __init__.py
|   |-- bgtfs_handler
|-- cpp
|   |-- BgtfsFeedHandler.cpp
|   |-- BgtfsFeedHandler.h
|   |-- bgtfsPyLibWrapper.cpp
|   `-- pybindCasts.cpp
|-- deploy.sh
|-- make.sh
|-- pybind11
|-- setup.py
|-- submodules
|-- test.sh
`-- tests
    |-- __init__.py
    |-- __pycache__
    |-- fixtures
    |-- test.py
    `-- test_functions.py

bgtfs_py_lib 中的 init.py 文件如下所示。包装器的功能正在公开。

import bgtfs_py_lib_wrapper as _bgtfs_py_lib
from bgtfs_py_lib.bgtfs_handler.bgtfs_handler import BgtfsHandler

在另一个项目中，它使用 git+ssh 和 egg 进行 pip 安装。

git+ssh://git@github.com/path/to/project.git@build/production/setup#egg=bgtfs_py_lib

当我在 pyCharm 中 ctrl+space 时，会找到包装器模块并且存在类。
该模块位于 Binary Skeletons 目录中，但尚未

import bgtfs_py_lib_wrapper as _bgtfs_py_lib 根本不起作用并抛出以下异常:ModuleNotFoundError: No module named 'bgtfs_py_lib_wrapper'

有人可以帮我弄清楚如何正确构建 C++/Pybind11 模块并在带有 virtualenv 的 pip 安装包中使用它们吗？

Answer 1

终于解决了

原来需要更改 CMakeLists.txt，因为它在 cmake_args 和 build_args 之间存在不一致。所以 CMakeList.txt 文件现在看起来像这样:

CMakeLists.txt

cmake_minimum_required(VERSION 3.8)

project(bgtfs_py_lib_wrapper)
include(submodules/transitLib/bgtfs/bgtfsLib/TransitUtils/transit_shared.cmake)

# bgtfsPyLib
set(PYBIND11_CPP_STANDARD -std=c++14)
set(PYBIND11_PYTHON_VERSION 3.6)

add_subdirectory(submodules/transitLib transitLib)
add_subdirectory(pybind11)

include_directories(
    cpp
    submodules/transitLib/bgtfs/bgtfsLib/
    submodules/transitLib/bgtfs/bgtfsLib/bgtfsLib
)

pybind11_add_module(bgtfs_py_lib_wrapper MODULE NO_EXTRAS
    cpp/pybindCasts.cpp
    cpp/bgtfsPyLibWrapper.cpp
    cpp/BgtfsFeedHandler.cpp
)

target_link_libraries(bgtfs_py_lib_wrapper PRIVATE transitLib)
target_compile_definitions(bgtfs_py_lib_wrapper PRIVATE TRANSIT_SERVER)
target_compile_definitions(transitLib PRIVATE TRANSIT_SERVER)
target_compile_definitions(bgtfsLib PRIVATE ENABLE_BACKWARD_FILE_COMPATIBILITY YES)
set_default_target_properties(bgtfs_py_lib_wrapper)

setup.py 文件现在是:

setup.py

import subprocess
import os
from pathlib import Path

from setuptools import setup, Extension, find_packages
from setuptools.command.build_ext import build_ext


class CMakeExtension(Extension):
    def __init__(self, name):
        Extension.__init__(self, name, sources=[])


class CMakeBuild(build_ext):
    def run(self):
        for ext in self.extensions:
            self.build_cmake(ext)
        super().run()

    def build_cmake(self, ext):
        try:
            subprocess.check_output(["cmake", "--version"])
        except OSError:
            raise RuntimeError(
                "CMake must be installed to build the following extensions: "
                + ", ".join(e.name for e in self.extensions)
            )

        cwd = Path().absolute()

        # these dirs will be created in build_py, so if you don't have
        # any python sources to bundle, the dirs will be missing
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp)
        extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))

        pyenv_root = os.environ.get("PYENV_ROOT")

        cmake_args = [
            f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}",
            "-DCMAKE_BUILD_TYPE=Release",
            "-DTRANSIT_INCLUDE_TESTS:BOOL=OFF",
        ]

        if pyenv_root is not None:
            cmake_args += [f"-DPYTHON_EXECUTABLE={pyenv_root}/shims/python"]

        build_args = ["--config", "Release", "--", "-j2"]

        env = os.environ.copy()

        self.announce("Running CMake prepare", level=3)
        subprocess.check_call(["cmake", cwd] + cmake_args, cwd=self.build_temp, env=env)

        self.announce("Building extensions")
        cmake_cmd = ["cmake", "--build", "."] + build_args
        subprocess.check_call(cmake_cmd, cwd=self.build_temp)


setup(
    name="bgtfs_py_lib",
    version="3.2.2",
    author="Transit App",
    author_email="juan@transitapp.com",
    description="A python wrapper for the transitLib",
    long_description="",
    zip_safe=False,
    license="Transit",
    install_requires=[
        "bgtfs_py_lib",
        "redis==2.10.6",
        "cffi==1.11.5",
        "numpy==1.15.3",
        "patricia-trie==10",
        "PuLP==1.6.8",
        "py-lz4framed==0.13.0",
        "pycparser==2.19",
        "pyparsing==2.2.2",
        "pypng==0.0.18",
        "pyproj==1.9.5.1",
        "python-graph-core==1.8.2",
        "pytz==2018.6",
        "redis==2.10.6",
        "requests==2.21.0",
        "six==1.11.0",
        "tabulate==0.8.2",
        "unicodecsv==0.14.1",
        "Unidecode==1.0.22",
    ],
    ext_modules=[CMakeExtension("bgtfs_py_lib_wrapper")],
    cmdclass=dict(build_ext=CMakeBuild),
    packages=find_packages(exclude=["tests", "*.plist"]),
    package_data={"": "*.so"},
)

扩展文件不会输出到 bgtfs_py_lib 目录中，而是输出到虚拟环境中，并且需要项目本身

大声喊Sergei帮助解决问题