00070-pybind11 杂项

杂项

发布日期: 2023-06-04

更新日期: 2024-12-28

前言

pybind11 — Seamless operability between C++11 and Python

pybind11 官方文档: https://pybind11.readthedocs.io/en/latest/ .

操作系统：Ubuntu 20.04.4 LTS

参考文档

pybind11 官方文档

安装

pip install pybind11

传递 numpy 数组

arr.cpp 文件:

#include <iostream>
#include <thread>
#include <vector>
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
namespace py = pybind11;

// 单个线程执行的运算
void update(unsigned long id, py::array_t<int64_t> x){
    auto r = x.mutable_unchecked<1>();
    for (int i = 1; i < r.shape(0); i++)
    {
        if (i % id == 0)
        {
            r(i) = id;
        }
        // if (PyErr_CheckSignals() != 0)
        //     throw py::error_already_set();
    }
}

// 利用多线程分配 numpy
void launch(py::array_t<int64_t> x) {
    // 计算机 cpu 核心数
    unsigned long const hardware_threads = std::thread::hardware_concurrency();
    std::cout << "cpus: " << hardware_threads << std::endl;
    std::vector<std::thread> threads;
    for (unsigned long i = 0; i < hardware_threads; i++)
    {
        threads.emplace_back(update, i+1, x);
    }
    for(auto& entry: threads)
        entry.join();
}

// 测试设置 int 变量
int bernFlag = 0;
void setBern(int con) {
	bernFlag = con;
    std::cout << "Bern: " << bernFlag << std::endl;
}

// 测试设置字符串变量
std::string inPath = "../data/FB15K/";
void setInPath(std::string path) {
	inPath = std::move(path);
    std::cout << "Input Files Path : "
              << inPath << std::endl;
}

// Python 接口
PYBIND11_MODULE(arr, m) {
    // 分配类型为 int64 numpy 数组
    m.def("assign", [](py::array_t<int64_t> x) {
        auto r = x.mutable_unchecked<1>();
        for (py::ssize_t i = 0; i < r.shape(0); i++)
            r(i) = i;
    }, py::arg().noconvert());

    // 分配类型为 float32 numpy 数组
    m.def("assign", [](py::array_t<float> x) {
        auto r = x.mutable_unchecked<1>();
        for (py::ssize_t i = 0; i < r.shape(0); i++)
            r(i) = i;
    }, py::arg().noconvert());
    
    // 利用多线程分配 numpy
    m.def("launch", &launch, py::arg().noconvert(),
        py::call_guard<py::gil_scoped_release>());
    
    // 测试设置 int 变量
    m.def("setBern", &setBern);

    // 测试设置字符串变量
    m.def("setInPath", &setInPath);
}

test_arr.py 文件:

import numpy as np
import arr as m

def test_from_python():
    int64_arr = np.zeros(3, dtype=np.int64)
    float32_arr = np.zeros(3, dtype=np.float32)
    print(int64_arr)
    print(float32_arr)
    m.assign(int64_arr)
    m.assign(float32_arr)
    print(int64_arr)
    print(float32_arr)

    large = np.zeros(10**8, dtype=np.int64)
    print(large.size)
    print(large)
    m.launch(large)
    print(large)

    path = "/home/luyanfeng/my_code/temp"
    m.setInPath(path)

    m.setBern(100)

test_from_python()

编译脚本 compile.sh:

c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) arr.cpp -o arr$(python3-config --extension-suffix)

结果

(test) luyanfeng@amax:~/my_code/temp$ ls
arr.cpp  compile.sh  test_arr.py
(test) luyanfeng@amax:~/my_code/temp$ bash compile.sh 
/usr/bin/ld: warning: /usr/lib/gcc/x86_64-linux-gnu/7/libstdc++.so: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
/usr/bin/ld: warning: /usr/lib/gcc/x86_64-linux-gnu/7/libstdc++.so: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
(test) luyanfeng@amax:~/my_code/temp$ python test_arr.py 
[0 0 0]
[0. 0. 0.]
[0 1 2]
[0. 1. 2.]
100000000
[0 0 0 ... 0 0 0]
cpus: 32
[ 0  1  1 ...  1 23 11]
Input Files Path : /home/luyanfeng/my_code/temp
Bern: 100
(test) luyanfeng@amax:~/my_code/temp$