From 17d9b1672dd3bac0e20c5cacce81f2be406c9862 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 16 Sep 2024 19:51:51 -0500 Subject: [PATCH 01/14] pin to numpy < 2.1 --- .github/environment.yml | 2 +- .github/workflows/build.yml | 2 +- src/pdal/io/NumpyReader.cpp | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/environment.yml b/.github/environment.yml index 26096eb..25365f5 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - scikit-build-core - - numpy + - numpy < 2.1 - pdal - compilers - python diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dad35b4..344a442 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: fail-fast: true matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - name: Check out diff --git a/src/pdal/io/NumpyReader.cpp b/src/pdal/io/NumpyReader.cpp index 199faaa..5a34b98 100644 --- a/src/pdal/io/NumpyReader.cpp +++ b/src/pdal/io/NumpyReader.cpp @@ -142,6 +142,8 @@ PyArrayObject* load_npy_file(std::string const& filename) { PyObject *py_filename = PyUnicode_FromString(filename.c_str()); + if (!py_filename) + throw pdal::pdal_error(plang::getTraceback()); PyObject *numpy_module = PyImport_ImportModule("numpy"); if (!numpy_module) throw pdal::pdal_error(plang::getTraceback()); From fff31941c83906fa865dd016e4feccbd6f1a2449 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 16 Sep 2024 19:56:17 -0500 Subject: [PATCH 02/14] no 3.13 yet --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 344a442..dad35b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: fail-fast: true matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Check out From 0cc494f76fcdfe4ddad7040f4aaefdf43b707dca Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 17 Sep 2024 10:16:34 -0500 Subject: [PATCH 03/14] set numpy 2.0 pin --- .github/environment.yml | 2 +- src/pdal/io/NumpyReader.cpp | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/environment.yml b/.github/environment.yml index 25365f5..998895c 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - scikit-build-core - - numpy < 2.1 + - numpy = 2.0 - pdal - compilers - python diff --git a/src/pdal/io/NumpyReader.cpp b/src/pdal/io/NumpyReader.cpp index 5a34b98..98e18b9 100644 --- a/src/pdal/io/NumpyReader.cpp +++ b/src/pdal/io/NumpyReader.cpp @@ -167,7 +167,10 @@ PyArrayObject* load_npy_file(std::string const& filename) if (!array) throw pdal::pdal_error(plang::getTraceback()); - return reinterpret_cast(array); + PyArrayObject* output = reinterpret_cast(array); + if (!output) + throw pdal_error(plang::getTraceback()); + return output; } PyArrayObject* load_npy_script(std::string const& source, @@ -198,7 +201,10 @@ PyArrayObject* load_npy_script(std::string const& source, Py_XDECREF(scriptArgs); - return reinterpret_cast(array); + PyArrayObject* output = reinterpret_cast(array); + if (!output) + throw pdal_error(plang::getTraceback()); + return output; } void NumpyReader::initialize() From c39e5933192b714c3cef36d09158fafd6f3aad9c Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 17 Sep 2024 11:59:28 -0500 Subject: [PATCH 04/14] numpy1 --- .github/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/environment.yml b/.github/environment.yml index 998895c..bb3fea1 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - scikit-build-core - - numpy = 2.0 + - numpy = 1 - pdal - compilers - python From 5ff864020263c52c99f7e93cbaf63ea0083eab08 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Sep 2024 17:56:16 -0500 Subject: [PATCH 05/14] restructure defines and undef NPY_API_SYMBOL_ATTRIBUTE --- src/pdal/io/NumpyReader.cpp | 20 +++++++++++--------- src/pdal/io/NumpyReader.hpp | 8 +++----- src/pdal/plang/Environment.cpp | 14 +++++++------- src/pdal/plang/Environment.hpp | 5 +++++ src/pdal/plang/Invocation.cpp | 4 +--- src/pdal/test/data/1.2-with-color.npy | Bin 36546 -> 36594 bytes 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/pdal/io/NumpyReader.cpp b/src/pdal/io/NumpyReader.cpp index 98e18b9..b30eb8a 100644 --- a/src/pdal/io/NumpyReader.cpp +++ b/src/pdal/io/NumpyReader.cpp @@ -43,7 +43,7 @@ #include #include -#include "../plang/Environment.hpp" + #if NPY_ABI_VERSION < 0x02000000 #define PyDataType_FIELDS(descr) ((descr)->fields) @@ -138,6 +138,7 @@ void NumpyReader::setArray(PyObject* array) Py_XINCREF(m_array); } + PyArrayObject* load_npy_file(std::string const& filename) { @@ -167,10 +168,11 @@ PyArrayObject* load_npy_file(std::string const& filename) if (!array) throw pdal::pdal_error(plang::getTraceback()); - PyArrayObject* output = reinterpret_cast(array); - if (!output) - throw pdal_error(plang::getTraceback()); - return output; + + PyArrayObject* nparray = reinterpret_cast(array); + if (!PyArray_Check(array)) + throw pdal_error("Numpy file did not return an array!"); + return nparray; } PyArrayObject* load_npy_script(std::string const& source, @@ -201,10 +203,10 @@ PyArrayObject* load_npy_script(std::string const& source, Py_XDECREF(scriptArgs); - PyArrayObject* output = reinterpret_cast(array); - if (!output) - throw pdal_error(plang::getTraceback()); - return output; + if (!PyArray_Check(array)) + throw pdal_error("Numpy script did not return an array!"); + + return reinterpret_cast(array); } void NumpyReader::initialize() diff --git a/src/pdal/io/NumpyReader.hpp b/src/pdal/io/NumpyReader.hpp index ebffbb6..e9f19a6 100644 --- a/src/pdal/io/NumpyReader.hpp +++ b/src/pdal/io/NumpyReader.hpp @@ -37,13 +37,11 @@ #include #include +#include "../plang/Environment.hpp" #include "../plang/Invocation.hpp" -#include -#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION -#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API -#define NO_IMPORT_ARRAY -#include +#define NO_IMPORT_ARRAY // Already have it from Environment.hpp +#include #include diff --git a/src/pdal/plang/Environment.cpp b/src/pdal/plang/Environment.cpp index 210aa64..85bb78c 100644 --- a/src/pdal/plang/Environment.cpp +++ b/src/pdal/plang/Environment.cpp @@ -39,12 +39,8 @@ #include "Environment.hpp" #include "Redirector.hpp" -#define NPY_TARGET_VERSION NPY_1_22_API_VERSION -#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION -#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API - -#include +#include #include #include @@ -147,7 +143,12 @@ Environment::Environment() { // #undef NUMPY_IMPORT_ARRAY_RETVAL // #define NUMPY_IMPORT_ARRAY_RETVAL VOID - _import_array(); + +#if NPY_ABI_VERSION < 0x02000000 + import_array(); +#else + PyArray_ImportNumPyAPI(); +#endif return ; }; @@ -165,7 +166,6 @@ Environment::Environment() throw pdal_error("unable to add redirector module!"); } - initNumpy(); PyImport_ImportModule("redirector"); diff --git a/src/pdal/plang/Environment.hpp b/src/pdal/plang/Environment.hpp index fed4fd6..c5ab5a1 100644 --- a/src/pdal/plang/Environment.hpp +++ b/src/pdal/plang/Environment.hpp @@ -49,6 +49,11 @@ #include "Script.hpp" #include "gil.hpp" +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_API_SYMBOL_ATTRIBUTE +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + namespace pdal { namespace plang diff --git a/src/pdal/plang/Invocation.cpp b/src/pdal/plang/Invocation.cpp index ee9342f..808fe5f 100644 --- a/src/pdal/plang/Invocation.cpp +++ b/src/pdal/plang/Invocation.cpp @@ -36,10 +36,8 @@ #include -#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION #define NO_IMPORT_ARRAY -#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API -#include +#include namespace { diff --git a/src/pdal/test/data/1.2-with-color.npy b/src/pdal/test/data/1.2-with-color.npy index ee9b0fd21f773847606d27bef3b0f719fbed36f5..5930c18ce06cf0c9b8c76c993fd3c67c60265367 100644 GIT binary patch delta 65 kcmX>!m+8}7CeF!zp}ql;j0|NPIo%l56^H|y-56_n0Rz4bssI20 delta 25 hcmew~m+8=4CeF!zp}ql;j0|oYIo%jH*E1IP0swhu2%`W1 From fa6e90c61f98a8ffe489da569d23ed6a0c993268 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Sep 2024 17:58:59 -0500 Subject: [PATCH 06/14] undo pin --- .github/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/environment.yml b/.github/environment.yml index bb3fea1..26096eb 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - scikit-build-core - - numpy = 1 + - numpy - pdal - compilers - python From 3fef210bda5b736de865164c0626463bbccdbc8f Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 24 Sep 2024 07:26:14 -0500 Subject: [PATCH 07/14] parameterize numpy --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dad35b4..cc80fde 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ jobs: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.9', '3.10', '3.11', '3.12'] + numpy-version: ['1.26', '2.0', '2.1'] steps: - name: Check out @@ -40,6 +41,10 @@ jobs: auto-update-conda: true environment-file: .github/environment.yml + - name: Install Numpy ${{ matrix.numpy-version }} + shell: bash -l {0} + run: | + mamba install python=${{ matrix.numpy-version }} - name: Install shell: bash -l {0} From 0905bc6dcd7cdd3eb9b76ff39b31928e93c1fb9f Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 24 Sep 2024 07:31:03 -0500 Subject: [PATCH 08/14] doh --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc80fde..f138977 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ defaults: jobs: build: - name: Build ${{ matrix.os }} py${{ matrix.python-version }} + name: ${{ matrix.os }} py${{ matrix.python-version }} numpy${{ matrix.numpy-version }} runs-on: ${{ matrix.os }} strategy: @@ -44,7 +44,7 @@ jobs: - name: Install Numpy ${{ matrix.numpy-version }} shell: bash -l {0} run: | - mamba install python=${{ matrix.numpy-version }} + mamba install numpy=${{ matrix.numpy-version }} - name: Install shell: bash -l {0} From 5b6ee0c7e4f94fbb09a4c4e22a911eb9f657784d Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 24 Sep 2024 07:44:33 -0500 Subject: [PATCH 09/14] exclude 2.1 for python 3.9 --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f138977..d354e28 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,9 @@ jobs: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.9', '3.10', '3.11', '3.12'] numpy-version: ['1.26', '2.0', '2.1'] + exclude: + - python-version: 3.9 + numpy-version: 2.1 steps: - name: Check out From 31e6462ccfe09ee796093bce7a8e30bc58177776 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 24 Sep 2024 08:59:23 -0500 Subject: [PATCH 10/14] use _import_array as before for numpy 1.x --- src/pdal/plang/Environment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pdal/plang/Environment.cpp b/src/pdal/plang/Environment.cpp index 85bb78c..0c91dc0 100644 --- a/src/pdal/plang/Environment.cpp +++ b/src/pdal/plang/Environment.cpp @@ -145,7 +145,7 @@ Environment::Environment() // #define NUMPY_IMPORT_ARRAY_RETVAL VOID #if NPY_ABI_VERSION < 0x02000000 - import_array(); + _import_array(); #else PyArray_ImportNumPyAPI(); #endif From 1d8403c3692ff7f15818812b7ce4f2790a1d448b Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 24 Sep 2024 10:40:47 -0500 Subject: [PATCH 11/14] cleanup and fix osx flat_namespace thing --- CMakeLists.txt | 2 +- src/pdal/io/NumpyReader.cpp | 38 ++++++++++++++-------------------- src/pdal/plang/Environment.cpp | 2 -- src/pdal/plang/Environment.hpp | 2 +- 4 files changed, 17 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 055c2bf..8d80763 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,7 +159,7 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") # executable. PDAL plugins need to link against libpython, but if a plugin # is loaded inside a python process, it must resolve symbols from the python # executable instead of libpython. Using flat namespace allows that. - set(PYTHON_LINK_LIBRARY ${PYTHON_LINK_LIBRARY} -Wl,-flat_namespace) + # set(PYTHON_LINK_LIBRARY ${PYTHON_LINK_LIBRARY} -Wl,-flat_namespace) endif() PDAL_PYTHON_ADD_PLUGIN(numpy_reader reader numpy diff --git a/src/pdal/io/NumpyReader.cpp b/src/pdal/io/NumpyReader.cpp index b30eb8a..299703c 100644 --- a/src/pdal/io/NumpyReader.cpp +++ b/src/pdal/io/NumpyReader.cpp @@ -360,32 +360,24 @@ Dimension::Id NumpyReader::registerDim(PointLayoutPtr layout, return id; } -namespace -{ - -Dimension::Type getType(PyArray_Descr *dtype, const std::string& name) +void NumpyReader::createFields(PointLayoutPtr layout) { - if (!dtype) - throw pdal_error("Can't fetch data type for numpy field."); - Dimension::Type pdalType = - plang::Environment::getPDALDataType(dtype->type_num); - if (pdalType == Dimension::Type::None) + auto getPDALType = [](int type_num, const std::string& name) { - std::ostringstream oss; - oss << "Unable to map dimension '" << name << "' because its " - "type '" << dtype->type_num <<"' is not mappable to PDAL"; - throw pdal_error(oss.str()); - } - return pdalType; -} - -} // unnamed namespace - + Dimension::Type pdalType = + plang::Environment::getPDALDataType(type_num); + if (pdalType == Dimension::Type::None) + { + std::ostringstream oss; + oss << "Unable to map dimension '" << name << "' because its " + "type '" << type_num <<"' is not mappable to PDAL"; + throw pdal_error(oss.str()); + } + return pdalType; + }; -void NumpyReader::createFields(PointLayoutPtr layout) -{ Dimension::Id id; Dimension::Type type; int offset; @@ -398,7 +390,7 @@ void NumpyReader::createFields(PointLayoutPtr layout) // Array isn't structured - just a bunch of data. if (m_numFields <= 0) { - type = getType(m_dtype, m_defaultDimension); + type = getPDALType(m_dtype->type_num, m_defaultDimension); id = registerDim(layout, m_defaultDimension, type); m_fields.push_back({id, type, 0}); } @@ -425,7 +417,7 @@ void NumpyReader::createFields(PointLayoutPtr layout) // Get type. PyArray_Descr* dt = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0); - type = getType(dt, name); + type = getPDALType(dt->type_num, name); char byteorder = dt->byteorder; int elsize = (int) PyDataType_ELSIZE(dt); diff --git a/src/pdal/plang/Environment.cpp b/src/pdal/plang/Environment.cpp index 0c91dc0..3aad88d 100644 --- a/src/pdal/plang/Environment.cpp +++ b/src/pdal/plang/Environment.cpp @@ -141,8 +141,6 @@ Environment::Environment() // the return. auto initNumpy = []() { -// #undef NUMPY_IMPORT_ARRAY_RETVAL -// #define NUMPY_IMPORT_ARRAY_RETVAL VOID #if NPY_ABI_VERSION < 0x02000000 _import_array(); diff --git a/src/pdal/plang/Environment.hpp b/src/pdal/plang/Environment.hpp index c5ab5a1..ab5d77a 100644 --- a/src/pdal/plang/Environment.hpp +++ b/src/pdal/plang/Environment.hpp @@ -51,7 +51,7 @@ #define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API #define NPY_TARGET_VERSION NPY_1_22_API_VERSION -#define NPY_API_SYMBOL_ATTRIBUTE +// #define NPY_API_SYMBOL_ATTRIBUTE #define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION namespace pdal From 3d7e7f3baf7ea0b81bd1b73e217ffd15061719e1 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 24 Sep 2024 17:14:29 -0500 Subject: [PATCH 12/14] put back NPY_API_SYMBOL_ATTRIBUTE --- .github/environment.yml | 4 ++++ CMakeLists.txt | 2 +- src/pdal/plang/Environment.hpp | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/environment.yml b/.github/environment.yml index 26096eb..23c4777 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -8,3 +8,7 @@ dependencies: - compilers - python - pip + - scikit-build-core + - pybind11 + - cmake + - ninja diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d80763..055c2bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,7 +159,7 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") # executable. PDAL plugins need to link against libpython, but if a plugin # is loaded inside a python process, it must resolve symbols from the python # executable instead of libpython. Using flat namespace allows that. - # set(PYTHON_LINK_LIBRARY ${PYTHON_LINK_LIBRARY} -Wl,-flat_namespace) + set(PYTHON_LINK_LIBRARY ${PYTHON_LINK_LIBRARY} -Wl,-flat_namespace) endif() PDAL_PYTHON_ADD_PLUGIN(numpy_reader reader numpy diff --git a/src/pdal/plang/Environment.hpp b/src/pdal/plang/Environment.hpp index ab5d77a..75f33c7 100644 --- a/src/pdal/plang/Environment.hpp +++ b/src/pdal/plang/Environment.hpp @@ -49,9 +49,9 @@ #include "Script.hpp" #include "gil.hpp" -#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_NPARRAY_API #define NPY_TARGET_VERSION NPY_1_22_API_VERSION -// #define NPY_API_SYMBOL_ATTRIBUTE +#define NPY_API_SYMBOL_ATTRIBUTE #define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION namespace pdal From 4417117405154c1bc7d8d9490b8c41b50295b10d Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 24 Sep 2024 17:37:35 -0500 Subject: [PATCH 13/14] set min CMake version to 3.13 --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 055c2bf..39e3448 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,4 @@ -cmake_minimum_required(VERSION 3.11.0) -project(pdal-python-plugins) +cmake_minimum_required(VERSION 3.13.0) project(pdal-python-plugins VERSION ${SKBUILD_PROJECT_VERSION} DESCRIPTION "PDAL Python Plugins" HOMEPAGE_URL "https://github.com/PDAL/python-plugins") From 4e960a39766dd85fa8b10c69cc4ad03a1e30ecb8 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 24 Sep 2024 17:37:40 -0500 Subject: [PATCH 14/14] bump ver to 1.6.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ad9157f..63215ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "numpy >= 1.22" ] -version="1.6.1" +version="1.6.2" [project.optional-dependencies] test = [ ]