diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml new file mode 100644 index 00000000000..54af32648f8 --- /dev/null +++ b/.github/workflows/black.yml @@ -0,0 +1,52 @@ +name: Black Formatting + +on: [push] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + Black: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Setup python version + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + # Install pip and packages + - name: Install pip + run: python -m pip install --upgrade pip + + - name: Install black + run: pip install black + + # Format with black + - name: Format with black + run: | + cd bindings/python + black . + + # Diff check + - name: Check git diff + # Output the diff in ${GITHUB_WORKSPACE} + run: git diff > black_formatting_diff.patch + + # Exit if diff + - name: Set job exit status + run: "[ ! -s black_formatting_diff.patch ]" + + # Artifacts + - name: Upload formatting diff + uses: actions/upload-artifact@v2 + with: + # We are in ${GITHUB_WORKSPACE} + # ${GITHUB_SHA} won't work: use ${{ github.sha }} + name: black_formatting_diff-${{ github.sha }} + path: black_formatting_diff.patch + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 00000000000..bb943f6d2be --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,65 @@ +name: Pytest + +on: [push] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + Pytest: + # The type of runner that the job will run on + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Setup python version + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Display python version + run: python -c "import sys; print(sys.version)" + + # Install pip and packages + - name: Install pip + run: python -m pip install --upgrade pip + + - name: Install pytest + run: pip install pytest + + # Install clang and it's python inteface via apt + - name: Add llvm keys + run: | + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' | sudo tee -a /etc/apt/sources.list + echo 'deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' | sudo tee -a /etc/apt/sources.list + - name: Install libclang and its python bindings + run: | + sudo apt-get update + sudo apt-get install -y libclang-11-dev python3-clang-11 + + # Add dist-package to path to enable apt installed python3-clang import + - name: Add dist-packages to PYTHONPATH + run: echo "::set-env name=PYTHONPATH::${PYTHON_PATH}:/usr/lib/python3/dist-packages" + - name: Display PYTHONPATH + run: python -c "import sys; print('\n'.join(sys.path))" + + # Test with pytest + - name: Test with pytest + run: cd ${_PYTHON_BINDINGS_PATH} && pytest --junitxml=${GITHUB_WORKSPACE}/result_${{ matrix.python-version }}.xml + env: + _PYTHON_BINDINGS_PATH: bindings/python + + # Artifacts + - name: Upload pytest test results + uses: actions/upload-artifact@v2 + with: + # We are in ${GITHUB_WORKSPACE} + # ${GITHUB_SHA} won't work: use ${{ github.sha }} + name: pytest_py${{ matrix.python-version }}-${{ github.sha }} + path: result_${{ matrix.python-version }}.xml + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} diff --git a/bindings/discussion.md b/bindings/discussion.md new file mode 100644 index 00000000000..18ec634a568 --- /dev/null +++ b/bindings/discussion.md @@ -0,0 +1,158 @@ +## Python bindings + +### Shared pointer usage +#### [05-06-2020] +- > When we want `cloud = pcl.PointCloud[pcl.PointXYZ]()` to be a shared_ptr by default, we set the holder class as shared_ptr. This is needed in some cases because the interface otherwise would be non-idomatic: + > ```py + > import pcl + > + > filter = pcl.filters.PassThrough[pcl.PointXYZ]() + > filter.setInput(cloud) + > ``` + > + > Here, cloud needs to be a shared_ptr. That can be done in 2 ways + > 1. `cloud = pcl.PointCloud[pcl.PointXYZ]()` and the holder class as shared_ptr + > 2. `cloud = pcl.make_shared[pcl.PointXYZ]()` and holder class as void + > + > The issue is the ease-of-use and expected semantics + > ```py + > cloud2 = cloud1 # Python user will assume this is a shallow copy + > ``` + > + > This will only be true for a variable held in a shared_ptr. This is the semantics in Python. + > + > However, wrapping everything in shared_ptr has downsides for C++ wrapper: + > ```py + > import pcl + > + > point = pcl.PointXYZ() + > cloud[100] = point + > ``` + > + > If PointXYZ is held in a shared_ptr... things go south. If not, things go south + +### Handling unions +#### [04-06-2020] +- > given `assert(&(union.r) == &(union.rgb));` does the following hold: + > `assert id(union_wrapper.r) == id(union_wrapper.rgb) ?` + Yes. Tested. +- Working example: + ```cpp + #include + namespace py = pybind11; + + union RGB { + int rgb; + struct { + int r; + int g; + int b; + }; + }; + + PYBIND11_MODULE(pcl, m) + { + py::class_(m, "RGB") + .def(py::init<>()) + .def_property( + "rgb", + [](RGB& self) -> int { return self.rgb; }, + [](RGB& self, int value) { self.rgb = value; }) + .def_property( + "r", + [](RGB& self) -> int { return self.r; }, + [](RGB& self, int value) { self.r = value; }) + .def_property( + "g", + [](RGB& self) -> int { return self.g; }, + [](RGB& self, int value) { self.g = value; }) + .def_property( + "b", + [](RGB& self) -> int { return self.b; }, + [](RGB& self, int value) { self.b = value; }); + } + ``` + +### General +#### [05-06-2020] +- MetaCPP relies on Clang's LibTooling to generate all the metadata: https://github.com/mlomb/MetaCPP + +#### [04-06-2020] +- > Was reading this yesterday: https://peerj.com/articles/cs-149.pdf + > + > Summary: + > - They too, automatically generate Python bindings for C++ code using Boost::Python. + > - The whole process is python based. + > - Same design as to what we have in mind i.e., parse, process, and generate. + > - Their data structure of choice for the whole process is Abstract Semantic Graph: https://github.com/StatisKit/AutoWIG/blob/master/src/py/autowig/asg.py + > - The architecture is plugin-based, and somebody added a pybind plugin some months back: https://github.com/StatisKit/AutoWIG/blob/master/src/py/autowig/pybind11_generator.py + > - They use libclang for frontend parsing(python API). The project was done some time back so they wrote their own py-libclang code: https://github.com/StatisKit/AutoWIG/blob/master/src/py/autowig/libclang_parser.py + > - Repo: https://github.com/StatisKit/AutoWIG + > + > I think it can act as a good reference for the project. Have a look at the pdf and the source if you wish. + > The libclang python part can be explored from their repo as of now (as python-libclang has no documentation whatsoever and the 1-2 example articles are outdated) +- Problems: + > Templates: + > * suffixing doesn't work well. Unless you're a fan of the pseudo-Hungarian notation espoused by older devs (and by MS). It's ok for 1 (or maybe 2) arguments. + > * Templates in Python represent an instantiation of C++ template. It should be easy to add/remove an instantiation without affecting needless code. It should also be visually easy to switch the template type without having to lookup the notation or count underscores + > * Python has a strong syntax for this: index lookup via __getitem__ and __setitem__ + > * Using strings as keys is bad because the editor can't help if spelling is wrong. Pandas MultiKey failed here. +- Use of a templating engine for pybind11 code gen (jinja2>mako) + +#### [03-06-2020] +- Ambiguity in the phrase: "full control over clang's AST" + +#### [02-06-2020] +- Use of python bindings of libclang, for faster prototyping: https://github.com/llvm/llvm-project/blob/master/clang/bindings/python/clang/cindex.py +- > protyping and exploring python bindings in which everything is runtime and can be done interactively would usually be my first approach + +#### [28-05-2020] +- > While reading lsst's documentation, came to find out they use a __str__ method: + > ```py + > cls.def("__str__", [](Class const& self) { + > std::ostringstream os; + > os << self; + > return os.str(); + > }); + > ``` +- > the << operator with ostreams is the most common way in C++ of extracting a string representation of a given object (I have no idea why there's no practice of implementing the cast to string method), +That being said I believe you can use << with std::stringstreams, effectively allowing you to fetch a string representation of PCL objects which have operator<< (std::ostream, ....) implemented. + +#### [15-05-2020] +- > You can create docstring from \brief part and copy the function signature via libtooling. + +#### [09-05-2020] +- Start with binding PointTypes. +- AST parsing helps in cases of convoluted code. +- > We can keep 2 approaches in parallel: + > 1. header parser on a limited number of files + > 2. libtooling to replace it + > 1st will allow the pipeline to be developed + > 2nd will replace that +- > We can make a prototype which works on manually provided API points +- > From my understanding: + > 1. Code -> AST -> JSON: use some tool for it first, then replace with libtooling + > 2. JSON -> cpp: Python tool, language dependent + > 3. CMake + compile_database.json: rest of toolchain + > 4. organize properly so usage in LANG is nice + +#### [05-05-2020] +- > I'd put PyPi integration immediately after we get 1 module working. That'd allow us to keep shipping improved bindings after GSoC (in case the timeline gets delayed) +The order in which modules are tackled should be the dependency order (because we don't have the popularity data from our users) + +*** + +## Javascript Bindings +#### [05-05-2020] +- Webassembly as an option: https://en.wikipedia.org/wiki/WebAssembly +- Emscripten as an option: https://emscripten.org/ +- > * Getting clang to compile to WebAsm will be the best "performance". + > * Using Emscripten on the other hand is a well-travelled road, but the performance will be similar to hand written JS (or worse). + > Both approaches need clang so that's a milestone we need to conquer first. + +*** + +## Jupyter Visualisation +#### [05-05-2020] +- > WebGL view straddles JS bindings and Python bindings. It should come before JS bindings in terms of priority keeping in mind the popularity of Python as a second language for the kind of C++ users PCL attracts (academia) +- > https://jupyter.org/widgets has pythreejs which is embedding js in python. .... That's 1 step above webgl, involves using JS in Py, but is faster to get up and running.... We can tackle this based on time when we reach some stage diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt new file mode 100644 index 00000000000..b27de1218d1 --- /dev/null +++ b/bindings/python/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.5) +project(bindings) + +# find_package(PCL REQUIRED) +find_package(PCL COMPONENTS common REQUIRED) + +# We can replace `find_package` with `add_subdirectory`, depending on usage. +# https://pybind11.readthedocs.io/en/stable/compiling.html#find-package-vs-add-subdirectory +find_package(pybind11) + +pybind11_add_module(pcl ${CMAKE_CURRENT_SOURCE_DIR}/pybind11-gen/common/include/pcl/impl/point_types.cpp) + +target_link_libraries(pcl PRIVATE ${PCL_LIBRARIES}) +# add_dependencies(pcl_demo some_other_target) diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 00000000000..e4d76f64486 --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,32 @@ +## Python bindings for PCL + +### common-archive +- `point_types.json`: contains JSON based meta of `point_types.h` and `point_types.hpp`. +- `py_point_types.cpp`: file generated by `generate_bindings.py` taking meta info from `point_types.json`. +- `union_test.cpp`: testing file for handling of unions. + +### generate_bindings.py +- Converts JSON data to pybind11 C++ code. +- Run: + ```py + python3 generate_bindings.py + ``` + +### CMakeLists.txt +- Finding PCL and pybind11, then adding the python module(s) via `pybind11_add_module` (which is a wrapper over `add_library`). + +### setup.py +- For using setuptools. Uses `CMakeLists.txt`. + +### libclang.py +- Using python libclang to parse C++ code, for generation of metadata by static analysis. +- Run: + ```py + python3 libclang.py + ``` + +### json/* +- JSON output generated by `libclang.py` + +### pybind11/* +- pybind11 C++ outupt generated by `generate_bindings.py` diff --git a/bindings/python/scripts/__init__.py b/bindings/python/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/bindings/python/scripts/context.py b/bindings/python/scripts/context.py new file mode 100644 index 00000000000..6955e9069be --- /dev/null +++ b/bindings/python/scripts/context.py @@ -0,0 +1,6 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +import scripts diff --git a/bindings/python/scripts/generate.py b/bindings/python/scripts/generate.py new file mode 100644 index 00000000000..368366a39b7 --- /dev/null +++ b/bindings/python/scripts/generate.py @@ -0,0 +1,419 @@ +from context import scripts +import scripts.utils as utils + + +class bind: + """ + Class containing functions for generating bindings from AST info. + + How to use: + - to_be_filled_after_updation + """ + + _initial_pybind_lines = [ + "#include ", + "#include ", + "#include ", + "namespace py = pybind11;", + "using namespace py::literals;", + ] # initial pybind lines to be written to binded file + + def __init__(self, root: dict, module_name: str) -> None: + self._module_name = module_name # main python module name + self._state_stack = [] # stack to keep track of the state (node kind) + self._linelist = [] # list of lines to be written to the binding file + self._skipped = [] # list of skipped items, to be used for debugging purposes + self._inclusion_list = [] # list of all inclusion directives (included files) + handled_by_pybind = self.skip # handled by pybind11 + handled_elsewhere = self.skip # handled in another kind's function + no_need_to_handle = self.skip # unnecessary kind + unsure = self.skip # unsure as to needed or not + self.kind_functions = { + "TRANSLATION_UNIT": no_need_to_handle, + "NAMESPACE": self.handle_namespace, + "CXX_BASE_SPECIFIER": handled_elsewhere, # in (handle_struct_decl) + "CXX_METHOD": handled_elsewhere, # in (handle_struct_decl) + "CONSTRUCTOR": self.handle_constructor, + "INCLUSION_DIRECTIVE": self.handle_inclusion_directive, + # DECLs: Declaration Kinds + "STRUCT_DECL": self.handle_struct_decl, + "CLASS_DECL": self.handle_struct_decl, + "VAR_DECL": handled_by_pybind, + "PARM_DECL": handled_elsewhere, # in (handle_constructor) + "FIELD_DECL": handled_elsewhere, # in (handle_struct_decl) + "ANONYMOUS_UNION_DECL": handled_elsewhere, # in (handle_struct_decl) via get_fields_from_anonymous + "ANONYMOUS_STRUCT_DECL": handled_elsewhere, # in (handle_struct_decl) via get_fields_from_anonymous + "FRIEND_DECL": unsure, + "FUNCTION_DECL": unsure, + # EXPRs: An expression that refers to a member of a struct, union, class, Objective-C class, etc. + "CALL_EXPR": handled_by_pybind, + "UNEXPOSED_EXPR": unsure, + "MEMBER_REF_EXPR": unsure, + "DECL_REF_EXPR": unsure, + "ARRAY_SUBSCRIPT_EXPR": handled_by_pybind, + "CXX_THROW_EXPR": handled_by_pybind, + "INIT_LIST_EXPR": no_need_to_handle, + "OBJ_BOOL_LITERAL_EXPR": unsure, + "CXX_NULL_PTR_LITERAL_EXPR": no_need_to_handle, + "CXX_STATIC_CAST_EXPR": no_need_to_handle, + "PAREN_EXPR": handled_by_pybind, + "CXX_DELETE_EXPR": handled_by_pybind, + # LITERALs + "INTEGER_LITERAL": unsure, + "FLOATING_LITERAL": unsure, + "STRING_LITERAL": no_need_to_handle, + "OBJC_STRING_LITERAL": no_need_to_handle, + "ALIGNED_ATTR": no_need_to_handle, + "BINARY_OPERATOR": no_need_to_handle, + "UNARY_OPERATOR": no_need_to_handle, + "MACRO_DEFINITION": unsure, + "MACRO_INSTANTIATION": unsure, + # REFs: A reference to a type declaration. + "NAMESPACE_REF": handled_elsewhere, # in (handle_constructor) + "TYPE_REF": handled_elsewhere, # in (handle_constructor) + "MEMBER_REF": handled_by_pybind, + "OVERLOADED_DECL_REF": unsure, + "TEMPLATE_REF": unsure, # check for usage in pcl_base.cpp; might need to add in cxx_methods + "VARIABLE_REF": handled_by_pybind, + # STMTs: A statement. + "COMPOUND_STMT": no_need_to_handle, + "RETURN_STMT": handled_by_pybind, + "IF_STMT": no_need_to_handle, + "FOR_STMT": handled_by_pybind, + "DECL_STMT": unsure, # handled_by_pybind + "SWITCH_STMT": handled_by_pybind, + "CASE_STMT": handled_by_pybind, + "DEFAULT_STMT": handled_by_pybind, + "CXX_TRY_STMT": handled_by_pybind, + "CXX_CATCH_STMT": handled_by_pybind, + # TEMPLATEs: A reference to a class template, function template, template parameter, or class template partial specialization. + "CLASS_TEMPLATE": no_need_to_handle, + "TEMPLATE_NON_TYPE_PARAMETER": no_need_to_handle, + "FUNCTION_TEMPLATE": no_need_to_handle, + } + + self.handle_node(item=root) + + def skip(self) -> None: + """ + Used to keep track of skipped elements, for debugging purposes. + + Skipped elements can be: + - elements which are not handled in their own function, or + - elements which are not handled at all (skipped). + """ + + self._skipped.append( + { + "line": self.item["line"], + "column": self.item["line"], + "kind": self.kind, + "name": self.name, + } + ) + + def end_scope(self) -> None: + """ + Used for adding ending characters (braces, semicolons, etc.) when state's scope ends. + """ + + kind = self._state_stack[-1]["kind"] + end_token = {} + + end_token["NAMESPACE"] = "}" + end_token["CLASS_TEMPLATE"] = ";" + end_token["STRUCT_DECL"] = ";" + end_token["CLASS_DECL"] = ";" + + self._linelist.append(end_token.get(kind, "")) + + @staticmethod + def get_fields_from_anonymous(item: dict) -> list: + """ + Helper function to extract fields from anonymous types. + + Parameters: + - item (dict): the anonymous type item from which to extract fields + + Returns: + - fields (list): A list of items of kind `CursorKind.FIELD_DECL` + """ + + # Nested types are not allowed inside anonymous types. + # See https://stackoverflow.com/questions/17637392/anonymous-union-can-only-have-non-static-data-members-gcc-c + + fields = [] + for sub_item in item["members"]: + # base condition + if sub_item["kind"] == "FIELD_DECL": + fields.append(sub_item) + # recurse + elif sub_item["kind"] in ("ANONYMOUS_UNION_DECL", "ANONYMOUS_STRUCT_DECL"): + fields += bind.get_fields_from_anonymous(item=sub_item) + return fields + + def handle_node(self, item: dict) -> None: + """ + Function for handling a node (any type). + + - Not to be called explicitly, it is called when a class' object is initialised. + - Begins with the root i.e., TRANSLATION_UNIT and then recursively works through the AST. + - Function pipeline: + >>> + | 1. Initialisations of member variables like item, kind, name, etc. + | 2. Push the item's info to the state stack. + | 3. Call the designated function for the item. + | 4. If the designated function was not to skip the item's handling, recursively call its members' functions. + <<< + 5. End the scope, if applicable. + 6. Pop the item's info from the stack. + """ + + self.item = item + self.kind = self.item["kind"] + self.name = self.item["name"] + self.members = self.item["members"] + self.depth = self.item["depth"] + + self._state_stack.append( + {"kind": self.kind, "name": self.name, "depth": self.depth} + ) + + self.kind_functions[self.kind]() + + if self.kind_functions[self.kind] is not self.skip: + for sub_item in self.members: + self.handle_node(item=sub_item) + + self.end_scope() + + self._state_stack.pop() + + def handle_namespace(self) -> None: + """ + Handles `CursorKind.NAMESPACE` + """ + + # TODO: Try `namespace::_` pattern 'cause this is not very robust + self._linelist.append(f"namespace {self.name}" + "{") + + def handle_struct_decl(self) -> None: + """ + Handles `CursorKind.STRUCT_DECL` and `CursorKind.CLASS_DECL` + + - Functions performed: + 1. Define struct/class declaration: + 1.1. Handles type references for templated classes. + 1.2. Handles base specifiers (parent classes). + 2. Handles anonymous field declarations (extract fields and declare as members). + 3. Handles field declarations. + 4. Handles class methods. + """ + + # TODO: Extract functions, too much nesting + + template_class_name = None + template_class_name_python = None + for sub_item in self.members: + if sub_item["kind"] == "TYPE_REF": + # TODO: Will this case only apply to templates? + # @TODO: Make more robust + type_ref = sub_item["name"].replace("struct ", "").replace("pcl::", "") + template_class_name = f"{self.name}<{type_ref}>" + template_class_name_python = f"{self.name}_{type_ref}" + + base_class_list = [ + sub_item["name"] + for sub_item in self.members + if sub_item["kind"] == "CXX_BASE_SPECIFIER" + ] + + base_class_list_string = [ + str(cls).replace("struct ", "").replace("pcl::", "") + for cls in base_class_list + ] + + if template_class_name: + struct_details = ",".join([template_class_name] + base_class_list_string) + self._linelist.append( + f'py::class_<{struct_details}>(m, "{template_class_name_python}")' + ) + else: + struct_details = ",".join([self.name] + base_class_list_string) + self._linelist.append(f'py::class_<{struct_details}>(m, "{self.name}")') + + # default constructor + self._linelist.append(".def(py::init<>())") + + # TODO: Merge this and next block via a design updation + # handle anonymous structs, etc. as field declarations + for sub_item in self.members: + fields = self.get_fields_from_anonymous(sub_item) + for field in fields: + if field["element_type"] == "ConstantArray": + # TODO: FIX: readwrite, not readonly + self._linelist.append( + f'.def_property_readonly("{field["name"]}", []({self.name}& obj) {{return obj.{field["name"]}; }})' # float[ ' + f'obj.{sub_item["name"]}' + '.size()];} )' + ) + else: + self._linelist.append( + f'.def_readwrite("{field["name"]}", &{self.name}::{field["name"]})' + ) + + for sub_item in self.members: + + # handle field declarations + if sub_item["kind"] == "FIELD_DECL": + if sub_item["element_type"] == "ConstantArray": + self._linelist.append( + f'.def_property_readonly("{sub_item["name"]}", []({self.name}& obj) {{return obj.{sub_item["name"]}; }})' # float[ ' + f'obj.{sub_item["name"]}' + '.size()];} )' + ) + else: + self._linelist.append( + f'.def_readwrite("{sub_item["name"]}", &{self.name}::{sub_item["name"]})' + ) + + # handle class methods + elif sub_item["kind"] == "CXX_METHOD": + # TODO: Add template args, currently blank + if sub_item["name"] not in ("PCL_DEPRECATED"): + self._linelist.append( + f'.def("{sub_item["name"]}", py::overload_cast<>(&{self.name}::{sub_item["name"]}))' + ) + + def handle_constructor(self) -> None: + """ + Handles `CursorKind.CONSTRUCTOR` + + - Bind the constructor by developing a parameter list. + """ + + # TODO: Extract functions, too much nesting + + parameter_type_list = [] + + # generate parameter type list + for sub_item in self.members: + if sub_item["kind"] == "PARM_DECL": + if sub_item["element_type"] == "LValueReference": + for sub_sub_item in sub_item["members"]: + if sub_sub_item["kind"] == "TYPE_REF": + # @TODO: Make more robust + type_ref = ( + sub_sub_item["name"] + .replace("struct ", "") + .replace("pcl::", "") + ) + parameter_type_list.append(f"{type_ref} &") + elif sub_item["element_type"] == "Elaborated": + namespace_ref = "" + for sub_sub_item in sub_item["members"]: + if sub_sub_item["kind"] == "NAMESPACE_REF": + namespace_ref += f'{sub_sub_item["name"]}::' + if sub_sub_item["kind"] == "TYPE_REF": + parameter_type_list.append( + f'{namespace_ref}{sub_sub_item["name"]}' + ) + elif sub_item["element_type"] in ("Float", "Int"): + parameter_type_list.append(f'{sub_item["element_type"].lower()}') + else: + parameter_type_list.append(f'{sub_item["element_type"]}') + parameter_type_list = ",".join(parameter_type_list) + + # default ctor `.def(py::init<>())` already inserted while handling struct/class decl + if parameter_type_list: + self._linelist.append(f".def(py::init<{parameter_type_list}>())") + + def handle_inclusion_directive(self) -> None: + """ + Handle `CursorKind.INCLUSION_DIRECTIVE` + """ + + # TODO: develop + pass + + # if self.name.startswith("pcl"): + # self._inclusion_list.append(self.name) + + +def generate(module_name: str, parsed_info: dict = None, source: str = None) -> str: + """ + The main function which handles generation of bindings. + + Parameters: + - module_name (str): Generated python module's name. + - parsed_info (dict): Parsed info about a C++ source file. + - source (str): File name + + Returns: + - lines_to_write (list): Lines to write in the binded file. + """ + + def combine_lines() -> list or Exception: + """ + Combine to-be-binded lines generated by different functions and class members. + + Returns: + - lines_to_write (list): Lines to write in the binded file. + """ + + lines_to_write = [f"#include <{filename}>"] + # TODO: Inclusion list path fix needed + # TODO: Currently commented, to be written later + # for inclusion in self._inclusion_list: + # lines_to_write.append(f"#include <{inclusion}>") + lines_to_write += bind_object._initial_pybind_lines + for i, _ in enumerate(bind_object._linelist): + if bind_object._linelist[i].startswith("namespace"): + continue + else: + bind_object._linelist[i] = "".join( + ( + f"PYBIND11_MODULE({bind_object._module_name}, m)", + "{", + bind_object._linelist[i], + ) + ) + break + for line in bind_object._linelist: + lines_to_write.append(line) + lines_to_write.append("}") + return lines_to_write + + # Argument checks and `parsed_info` value initialisation + if parsed_info and source: # Both args passed, choose parsed_info. + print("Both parsed_info and source arguments provided, choosing parsed_info.") + elif source: # If source passed, read JSON. + parsed_info = utils.read_json(filename=source) + elif parsed_info: # If parsed_info passed, just use that further on. + pass + else: # Both args are None. + raise Exception("Provide either parsed_info or source") + + # If parsed_info is not empty + if parsed_info: + bind_object = bind(root=parsed_info, module_name=module_name) + # Extract filename from parsed_info (TRANSLATION_UNIT's name contains the filepath) + filename = "pcl" + parsed_info["name"].rsplit("pcl")[-1] + return combine_lines() + else: + raise Exception("Empty dict: parsed_info") + + +def main(): + args = utils.parse_arguments(script="generate") + + for source in args.files: + source = utils.get_realpath(path=source) + lines_to_write = generate(module_name="pcl", source=source) + output_filepath = utils.get_output_path( + source=source, + output_dir=utils.join_path(args.pybind11_output_path, "pybind11-gen"), + split_from="json", + extension=".cpp", + ) + utils.write_to_file(filename=output_filepath, linelist=lines_to_write) + + +if __name__ == "__main__": + main() diff --git a/bindings/python/scripts/parse.py b/bindings/python/scripts/parse.py new file mode 100644 index 00000000000..b9158e44ff5 --- /dev/null +++ b/bindings/python/scripts/parse.py @@ -0,0 +1,279 @@ +import sys +import clang.cindex as clang + +from context import scripts +import scripts.utils as utils + + +def valid_children(node): + """ + A generator function yielding valid children nodes + + Parameters: + - node (dict): + - The node in the AST + - Keys: + - cursor: The cursor pointing to a node + - filename: + - The file's name to check if the node belongs to it + - Needed to ensure that only symbols belonging to the file gets parsed, not the included files' symbols + - depth: The depth of the node (root=0) + + Yields: + - child_node (dict): Same structure as the argument + """ + + cursor = node["cursor"] + filename = node["filename"] + depth = node["depth"] + + for child in cursor.get_children(): + child_node = {"cursor": child, "filename": filename, "depth": depth + 1} + # Check if the child belongs to the file + if child.location.file and child.location.file.name == filename: + yield (child_node) + + +def print_ast(node): + """ + Prints the AST by recursively traversing it + + Parameters: + - node (dict): + - The node in the AST + - Keys: + - cursor: The cursor pointing to a node + - filename: + - The file's name to check if the node belongs to it + - Needed to ensure that only symbols belonging to the file gets parsed, not the included files' symbols + - depth: The depth of the node (root=0) + + Returns: + - None + """ + + cursor = node["cursor"] + depth = node["depth"] + + print( + "-" * depth, + cursor.location.file, + f"L{cursor.location.line} C{cursor.location.column}", + cursor.kind.name, + cursor.spelling, + ) + + # Get cursor's children and recursively print + for child_node in valid_children(node): + print_ast(child_node) + + +def generate_parsed_info(node): + """ + Generates parsed information by recursively traversing the AST + + Parameters: + - node (dict): + - The node in the AST + - Keys: + - cursor: The cursor pointing to a node + - filename: + - The file's name to check if the node belongs to it + - Needed to ensure that only symbols belonging to the file gets parsed, not the included files' symbols + - depth: The depth of the node (root=0) + + Returns: + - parsed_info (dict): + - Contains key-value pairs of various traits of a node + - The key 'members' contains the node's children's `parsed_info` + """ + + parsed_info = dict() + + cursor = node["cursor"] + depth = node["depth"] + + parsed_info["depth"] = depth + parsed_info["line"] = cursor.location.line + parsed_info["column"] = cursor.location.column + parsed_info["kind"] = cursor.kind.name + if cursor.is_anonymous(): + parsed_info["kind"] = "ANONYMOUS_" + parsed_info["kind"] + parsed_info["name"] = cursor.spelling + if cursor.type.kind.spelling != "Invalid": + parsed_info["element_type"] = cursor.type.kind.spelling + if cursor.access_specifier.name != "INVALID": + parsed_info["access_specifier"] = cursor.access_specifier.name + if cursor.result_type.spelling != "": + parsed_info["result_type"] = cursor.result_type.spelling + if cursor.brief_comment: + parsed_info["brief_comment"] = cursor.brief_comment + if cursor.raw_comment: + parsed_info["raw_comment"] = cursor.raw_comment + + # add result of various kinds of checks available in cindex.py + + cursorkind_checks = { + "kind_is_declaration": cursor.kind.is_declaration, + "kind_is_reference": cursor.kind.is_reference, + "kind_is_expression": cursor.kind.is_expression, + "kind_is_statement": cursor.kind.is_statement, + "kind_is_attribute": cursor.kind.is_attribute, + "kind_is_invalid": cursor.kind.is_invalid, + "kind_is_translation_unit": cursor.kind.is_translation_unit, + "kind_is_preprocessing": cursor.kind.is_preprocessing, + "kind_is_unexposed": cursor.kind.is_unexposed, + } + + # check for deleted ctor analogous to `is_default_constructor` unavailable + cursor_checks = { + "is_definition": cursor.is_definition, + "is_const_method": cursor.is_const_method, + "is_converting_constructor": cursor.is_converting_constructor, + "is_copy_constructor": cursor.is_copy_constructor, + "is_default_constructor": cursor.is_default_constructor, + "is_move_constructor": cursor.is_move_constructor, + "is_default_method": cursor.is_default_method, + "is_mutable_field": cursor.is_mutable_field, + "is_pure_virtual_method": cursor.is_pure_virtual_method, + "is_static_method": cursor.is_static_method, + "is_virtual_method": cursor.is_virtual_method, + "is_abstract_record": cursor.is_abstract_record, + "is_scoped_enum": cursor.is_scoped_enum, + "is_anonymous": cursor.is_anonymous, + "is_bitfield": cursor.is_bitfield, + } + + type_checks = { + "type_is_const_qualified": cursor.type.is_const_qualified, + "type_is_volatile_qualified": cursor.type.is_volatile_qualified, + "type_is_restrict_qualified": cursor.type.is_restrict_qualified, + "type_is_pod": cursor.type.is_pod, + } + + for checks in (cursorkind_checks, cursor_checks, type_checks): + for check, check_call in checks.items(): + parsed_info[check] = check_call() + + # special case handling for `cursor.type.is_function_variadic()` + if cursor.type.kind.spelling == "FunctionProto": + parsed_info["type_is_function_variadic"] = cursor.type.is_function_variadic() + + parsed_info["members"] = [] + + # Get cursor's children and recursively add their info to a dictionary, as members of the parent + for child_node in valid_children(node): + child_parsed_info = generate_parsed_info(child_node) + parsed_info["members"].append(child_parsed_info) + + return parsed_info + + +def get_compilation_commands(compilation_database_path, filename): + """ + Returns the compilation commands extracted from the compilation database + + Parameters: + - compilation_database_path: The path to `compile_commands.json` + - filename: The file's name to get its compilation commands + + Returns: + - compilation commands (list): The arguments passed to the compiler + """ + + # Build a compilation database found in the given directory + compilation_database = clang.CompilationDatabase.fromDirectory( + buildDir=compilation_database_path + ) + + # Get compiler arguments from the compilation database for the given file + compilation_commands = compilation_database.getCompileCommands(filename=filename) + + """ + - compilation_commands: + - An iterable object providing all the compilation commands available to build filename. + - type: + - compilation_commands[0]: + - Since we have only one command per filename in the compile_commands.json, extract 0th element + - type: + - compilation_commands[0].arguments: + - Get compiler arguments from the CompileCommand object + - type: + - list(compilation_commands[0].arguments)[1:-1]: + - Convert the generator object to list, and extract compiler arguments + - 0th element is the compiler name + - nth element is the filename + """ + + return list(compilation_commands[0].arguments)[1:-1] + + +def parse_file(source, compilation_database_path=None): + """ + Returns the parsed_info for a file + + Parameters: + - source: Source to parse + - compilation_database_path: The path to `compile_commands.json` + + Returns: + - parsed_info (dict) + """ + + # Create a new index to start parsing + index = clang.Index.create() + + # Get compiler arguments + compilation_commands = get_compilation_commands( + compilation_database_path=compilation_database_path, filename=source, + ) + + """ + - Parse the given source code file by running clang and generating the AST before loading + - option `PARSE_DETAILED_PROCESSING_RECORD`: + - Indicates that the parser should construct a detailed preprocessing record, + including all macro definitions and instantiations. + - Required to get the `INCLUSION_DIRECTIVE`s. + """ + source_ast = index.parse( + path=source, + args=compilation_commands, + options=clang.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD, + ) + + # Dictionary to hold a node's information + root_node = { + "cursor": source_ast.cursor, + "filename": source_ast.spelling, + "depth": 0, + } + + # For testing purposes + # print_ast(root_node) + + return generate_parsed_info(root_node) + + +def main(): + # Get command line arguments + args = utils.parse_arguments(script="parse") + for source in args.files: + source = utils.get_realpath(path=source) + + # Parse the source file + parsed_info = parse_file(source, args.compilation_database_path) + + # Output path for dumping the parsed info into a json file + output_filepath = utils.get_output_path( + source=source, + output_dir=utils.join_path(args.json_output_path, "json"), + split_from="pcl", + extension=".json", + ) + + # Dump the parsed info at output path + utils.dump_json(filepath=output_filepath, info=parsed_info) + + +if __name__ == "__main__": + main() diff --git a/bindings/python/scripts/utils.py b/bindings/python/scripts/utils.py new file mode 100644 index 00000000000..9724dddf836 --- /dev/null +++ b/bindings/python/scripts/utils.py @@ -0,0 +1,114 @@ +import os +import json +import argparse + + +def get_realpath(path): + return os.path.realpath(path) + + +def ensure_dir_exists(dir): + if not os.path.exists(dir): + os.makedirs(dir) + + +def get_parent_directory(file): + return os.path.dirname(os.path.dirname(file)) + + +def join_path(*args): + return os.path.join(*args) + + +def get_output_path(source, output_dir, split_from, extension): + """ + Returns json output path after manipulation of the source file's path + + Arguments: + - source: The source's file name + - output_dir: The output directory to write the json output + - split_from: The folder from which to split the path + - extension: Output extension + + Returns: + - output_path: The output's realpath + """ + + # split_path: contains the path after splitting. For split_path = pcl, contains the path as seen in the pcl directory + _, split_path = source.split(f"{split_from}{os.sep}", 1) + + # relative_dir: contains the relative output path for the json file + # source_filename: contains the source's file name + relative_dir, source_filename = os.path.split(split_path) + + # filename: contains the output json's file name + filename, _ = source_filename.split(".") + filename = f"{filename}{extension}" + + # dir: final output path + dir = join_path(output_dir, relative_dir) + + # make the output directory if it doesn't exist + ensure_dir_exists(dir) + + output_path = get_realpath(join_path(dir, filename)) + + return output_path + + +def dump_json(filepath, info, indent=2, separators=None): + with open(filepath, "w") as f: + json.dump(info, f, indent=indent, separators=separators) + + +def read_json(filename): + with open(filename, "r") as f: + return json.load(f) + + +def write_to_file(filename, linelist): + with open(filename, "w") as f: + for line in linelist: + f.writelines(line) + f.writelines("\n") + + +def parse_arguments(script): + """ + Returns parsed command line arguments for a given script + + Arguments: + - script: The python script for which the custom command line arguments should be parsed + + Return: + - args: Parsed command line arguments + """ + + if script == "parse": + parser = argparse.ArgumentParser(description="C++ libclang parser") + parser.add_argument( + "--compilation_database_path", + default=get_parent_directory(file=__file__), + help="Path to compilation database (json)", + ) + parser.add_argument( + "--json_output_path", + default=get_parent_directory(file=__file__), + help="Output path for generated json", + ) + parser.add_argument("files", nargs="+", help="The source files to parse") + + if script == "generate": + parser = argparse.ArgumentParser(description="JSON to pybind11 generation") + parser.add_argument("files", nargs="+", help="JSON input") + parser.add_argument( + "--pybind11_output_path", + default=get_parent_directory(file=__file__), + help="Output path for generated cpp", + ) + + else: + args = None + + args = parser.parse_args() + return args diff --git a/bindings/python/setup.py b/bindings/python/setup.py new file mode 100644 index 00000000000..e20324e1486 --- /dev/null +++ b/bindings/python/setup.py @@ -0,0 +1,87 @@ +import os +import re +import sys +import platform +import subprocess + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +from distutils.version import LooseVersion + + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=""): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + + +class CMakeBuild(build_ext): + def run(self): + try: + out = subprocess.check_output(["cmake", "--version"]) + except OSError: + raise RuntimeError( + "CMake must be installed to build the following extensions: " + + ", ".join(e.name for e in self.extensions) + ) + + if platform.system() == "Windows": + cmake_version = LooseVersion( + re.search(r"version\s*([\d.]+)", out.decode()).group(1) + ) + if cmake_version < "3.1.0": + raise RuntimeError("CMake >= 3.1.0 is required on Windows") + + for ext in self.extensions: + self.build_extension(ext) + + def build_extension(self, ext): + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + # required for auto-detection of auxiliary "native" libs + if not extdir.endswith(os.path.sep): + extdir += os.path.sep + + cmake_args = [ + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + extdir, + "-DPYTHON_EXECUTABLE=" + sys.executable, + ] + + cfg = "Debug" if self.debug else "Release" + build_args = ["--config", cfg] + + if platform.system() == "Windows": + cmake_args += [ + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) + ] + if sys.maxsize > 2 ** 32: + cmake_args += ["-A", "x64"] + build_args += ["--", "/m"] + else: + cmake_args += ["-DCMAKE_BUILD_TYPE=" + cfg] + build_args += ["--", "-j2"] + + env = os.environ.copy() + env["CXXFLAGS"] = '{} -DVERSION_INFO=\\"{}\\"'.format( + env.get("CXXFLAGS", ""), self.distribution.get_version() + ) + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + subprocess.check_call( + ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env + ) + subprocess.check_call( + ["cmake", "--build", "."] + build_args, cwd=self.build_temp + ) + + +setup( + name="bindings", + version="0.0.1", + author="Divyanshu Madan", + author_email="divyanshumadan99@gmail.com", + description="Python bindings for PCL", + long_description="", + ext_modules=[CMakeExtension("bindings")], + cmdclass=dict(build_ext=CMakeBuild), + zip_safe=False, +) diff --git a/bindings/python/tests/compile_commands.json b/bindings/python/tests/compile_commands.json new file mode 100644 index 00000000000..4eefaf4b70e --- /dev/null +++ b/bindings/python/tests/compile_commands.json @@ -0,0 +1,7 @@ +[ + { + "directory": "/home/divyanshu/Projects/active/pcl/bindings/python/tests", + "command": "/usr/bin/clang++ -std=c++14 /home/divyanshu/Projects/active/pcl/common/include/pcl/impl/point_types.hpp", + "file": "/home/divyanshu/Projects/active/pcl/common/include/pcl/impl/point_types.hpp" + } +] diff --git a/bindings/python/tests/context.py b/bindings/python/tests/context.py new file mode 100644 index 00000000000..6955e9069be --- /dev/null +++ b/bindings/python/tests/context.py @@ -0,0 +1,6 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +import scripts diff --git a/bindings/python/tests/test_generate.py b/bindings/python/tests/test_generate.py new file mode 100644 index 00000000000..7c5a9d447fd --- /dev/null +++ b/bindings/python/tests/test_generate.py @@ -0,0 +1,116 @@ +from context import scripts +import scripts.generate as generate +import test_parse + + +""" +TODO +Big areas of missing tests: +- function +- templated and instantiated function +- templated and instantiated class +- anonymous struct +- enum + +""" + + +def remove_whitespace(string): + """ + Removes whitespace from the string. + + Parameters: + - string (str) + + Returns: The stripped off string. + """ + + return "".join([x for x in string if not x.isspace()]) + + +def generate_bindings(cpp_code_block, module_name, tmp_path): + """ + Returns binded code for a cpp code block + - Steps: + 1. Get parsed info for the cpp code block (via get_parsed_info in test_parse.py). + 2. Generate bindings for the parsed_info (via generate in generate.py). + 3. Convert the list to a string and then return the stripped off string. + + Parameters: + - tmp_path (pathlib.PosixPath): The tmp_path for test folder + - cpp_code_block (str): The cpp code block to generate bindings for + + Returns: + - binded_code (str): The generated binded code + """ + + # Get parsed info for the cpp code block + parsed_info = test_parse.get_parsed_info( + tmp_path=tmp_path, file_contents=cpp_code_block + ) + # Get the binded code + binded_code = generate.generate(module_name=module_name, parsed_info=parsed_info) + # List to string + binded_code = "".join(binded_code) + + return remove_whitespace(binded_code) + + +def get_expected_string(expected_module_code): + """ + Returns expected output string after combining inclusions and pybind11's initial lines. + + Parameters: + - expected_module_code (str): Module code to be combined. + + Returns: + - expected_output (str): The stripped off, combined code. + """ + + file_include = "#include " + + # Get pybind11's intial lines in the form of a string + initial_pybind_lines = "".join(generate.bind._initial_pybind_lines) + + expected_output = remove_whitespace( + file_include + initial_pybind_lines + expected_module_code + ) + + return expected_output + + +def test_struct_without_members(tmp_path): + cpp_code_block = "struct AStruct {};" + output = generate_bindings( + tmp_path=tmp_path, cpp_code_block=cpp_code_block, module_name="pcl" + ) + + expected_module_code = """ + PYBIND11_MODULE(pcl, m){ + py::class_(m, "AStruct") + .def(py::init<>()); + } + """ + + assert output == get_expected_string(expected_module_code=expected_module_code) + + +def test_struct_with_members(tmp_path): + cpp_code_block = """ + struct AStruct { + int aMember; + }; + """ + output = generate_bindings( + tmp_path=tmp_path, cpp_code_block=cpp_code_block, module_name="pcl" + ) + + expected_module_code = """ + PYBIND11_MODULE(pcl, m){ + py::class_(m, "AStruct") + .def(py::init<>()) + .def_readwrite("aMember", &AStruct::aMember); + } + """ + + assert output == get_expected_string(expected_module_code=expected_module_code) diff --git a/bindings/python/tests/test_parse.py b/bindings/python/tests/test_parse.py new file mode 100644 index 00000000000..a2c797d3c43 --- /dev/null +++ b/bindings/python/tests/test_parse.py @@ -0,0 +1,414 @@ +from context import scripts +import scripts.parse as parse + + +def create_compilation_database(tmp_path, filepath): + input = tmp_path / "compile_commands.json" + x = [ + { + "directory": f"{tmp_path}", + "command": f"/usr/bin/clang++ -std=c++14 {filepath}", + "file": f"{filepath}", + } + ] + + with open(input, "w") as f: + f.write(str(x)) + + return str(tmp_path) + + +def get_parsed_info(tmp_path, file_contents): + source_path = tmp_path / "file.cpp" + + with open(source_path, "w") as f: + f.write(str(file_contents)) + + parsed_info = parse.parse_file( + source=str(source_path), + compilation_database_path=create_compilation_database( + tmp_path=tmp_path, filepath=source_path + ), + ) + + return parsed_info + + +def test_anonymous_decls(tmp_path): + file_contents = """ + union { + struct { + enum {}; + }; + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + union_decl = parsed_info["members"][0] + + assert union_decl["kind"] == "ANONYMOUS_UNION_DECL" + assert union_decl["name"] == "" + + struct_decl = union_decl["members"][0] + + assert struct_decl["kind"] == "ANONYMOUS_STRUCT_DECL" + assert struct_decl["name"] == "" + + enum_decl = struct_decl["members"][0] + + assert enum_decl["kind"] == "ANONYMOUS_ENUM_DECL" + assert enum_decl["name"] == "" + + +def test_translation_unit(tmp_path): + file_contents = "" + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + assert parsed_info["kind"] == "TRANSLATION_UNIT" + assert parsed_info["depth"] == 0 + assert parsed_info["name"] == str(tmp_path / "file.cpp") + + +def test_namespace(tmp_path): + file_contents = "namespace a_namespace {}" + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + namespace = parsed_info["members"][0] + + assert namespace["kind"] == "NAMESPACE" + assert namespace["name"] == "a_namespace" + + +def test_namespace_ref(tmp_path): + file_contents = """ + #include + std::ostream anOstream; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + inclusion_directive = parsed_info["members"][0] + + assert inclusion_directive["kind"] == "INCLUSION_DIRECTIVE" + assert inclusion_directive["name"] == "ostream" + + var_decl = parsed_info["members"][1] + namespace_ref = var_decl["members"][0] + + assert namespace_ref["kind"] == "NAMESPACE_REF" + assert namespace_ref["name"] == "std" + + +def test_var_decl(tmp_path): + file_contents = "int anInt = 1;" + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + var_decl = parsed_info["members"][0] + + assert var_decl["kind"] == "VAR_DECL" + assert var_decl["element_type"] == "Int" + assert var_decl["name"] == "anInt" + + +def test_field_decl(tmp_path): + file_contents = """ + struct AStruct { + int aClassMember; + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + struct_decl = parsed_info["members"][0] + field_decl = struct_decl["members"][0] + + assert field_decl["kind"] == "FIELD_DECL" + assert field_decl["element_type"] == "Int" + assert field_decl["name"] == "aClassMember" + + +def test_parsed_info_structure(tmp_path): + file_contents = "" + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + assert type(parsed_info) is dict + assert type(parsed_info["members"]) is list + assert len(parsed_info["members"]) == 0 + + +def test_call_expr(tmp_path): + file_contents = """ + int aFunction() { + return 1; + } + int anInt = aFunction(); + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + var_decl = parsed_info["members"][1] + call_expr = var_decl["members"][0] + + assert call_expr["kind"] == "CALL_EXPR" + assert call_expr["name"] == "aFunction" + + assert var_decl["name"] == "anInt" + + +def test_struct_decl(tmp_path): + file_contents = "struct AStruct {};" + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + struct_decl = parsed_info["members"][0] + + assert struct_decl["kind"] == "STRUCT_DECL" + assert struct_decl["name"] == "AStruct" + + +def test_public_inheritance(tmp_path): + file_contents = """ + struct BaseStruct {}; + struct DerivedStruct: public BaseStruct {}; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + child_struct_decl = parsed_info["members"][1] + cxx_base_specifier = child_struct_decl["members"][0] + + assert cxx_base_specifier["kind"] == "CXX_BASE_SPECIFIER" + assert cxx_base_specifier["access_specifier"] == "PUBLIC" + assert cxx_base_specifier["name"] == "struct BaseStruct" + + +def test_member_function(tmp_path): + file_contents = """ + struct AStruct { + void aMethod() {} + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + struct_decl = parsed_info["members"][0] + cxx_method = struct_decl["members"][0] + + assert cxx_method["kind"] == "CXX_METHOD" + assert cxx_method["result_type"] == "void" + assert cxx_method["name"] == "aMethod" + + +def test_type_ref(tmp_path): + file_contents = """ + struct SomeUsefulType {}; + + class AClass { + void aMethod(SomeUsefulType aParameter) {}; + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + class_decl = parsed_info["members"][1] + cxx_method = class_decl["members"][0] + parm_decl = cxx_method["members"][0] + + assert parm_decl["name"] == "aParameter" + + type_ref = parm_decl["members"][0] + + assert type_ref["kind"] == "TYPE_REF" + assert type_ref["name"] == "struct SomeUsefulType" + + +def test_simple_constructor(tmp_path): + file_contents = """ + struct AStruct { + AStruct() {} + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + struct_decl = parsed_info["members"][0] + constructor = struct_decl["members"][0] + + assert constructor["kind"] == "CONSTRUCTOR" + assert constructor["access_specifier"] == "PUBLIC" + assert constructor["name"] == "AStruct" + + +def test_unexposed_expr(tmp_path): + file_contents = """ + class SimpleClassWithConstructor { + int aClassMember; + SimpleClassWithConstructor(int aConstructorParameter) : aClassMember(aConstructorParameter) {}; + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + struct_decl = parsed_info["members"][0] + constructor = struct_decl["members"][1] + member_ref = constructor["members"][1] + + assert member_ref["name"] == "aClassMember" + + unexposed_expr = constructor["members"][2] + + assert unexposed_expr["kind"] == "UNEXPOSED_EXPR" + assert unexposed_expr["name"] == "aConstructorParameter" + + +# @TODO: Not sure how to reproduce. Maybe later. +# def test_member_ref_expr(tmp_path): + + +def test_decl_ref_expr(tmp_path): + file_contents = """ + struct AStruct { + int firstMember, secondMember; + AStruct(int firstFunctionParameter, int secondFunctionParameter) + : firstMember(secondFunctionParameter), secondMember(firstFunctionParameter) + {} + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + struct_decl = parsed_info["members"][0] + constructor = struct_decl["members"][2] + unexposed_expr_1 = constructor["members"][3] + unexposed_expr_2 = constructor["members"][5] + decl_ref_expr_1 = unexposed_expr_1["members"][0] + decl_ref_expr_2 = unexposed_expr_2["members"][0] + + assert decl_ref_expr_1["kind"] == "DECL_REF_EXPR" + assert decl_ref_expr_2["kind"] == "DECL_REF_EXPR" + assert decl_ref_expr_1["name"] == "secondFunctionParameter" + assert decl_ref_expr_2["name"] == "firstFunctionParameter" + + +def test_member_ref(tmp_path): + file_contents = """ + struct AStruct { + int firstMember, secondMember; + AStruct(int firstFunctionParameter, int secondFunctionParameter) + : firstMember(secondFunctionParameter), secondMember(firstFunctionParameter) + {} + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + struct_decl = parsed_info["members"][0] + constructor = struct_decl["members"][2] + member_ref_1 = constructor["members"][2] + member_ref_2 = constructor["members"][4] + + assert member_ref_1["kind"] == "MEMBER_REF" + assert member_ref_2["kind"] == "MEMBER_REF" + assert member_ref_1["element_type"] == "Int" + assert member_ref_2["element_type"] == "Int" + assert member_ref_1["name"] == "firstMember" + assert member_ref_2["name"] == "secondMember" + + +def test_class_template(tmp_path): + file_contents = """ + template + struct AStruct {}; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + class_template = parsed_info["members"][0] + + assert class_template["kind"] == "CLASS_TEMPLATE" + assert class_template["name"] == "AStruct" + + template_type_parameter = class_template["members"][0] + + assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["name"] == "T" + assert template_type_parameter["access_specifier"] == "PUBLIC" + + +def test_template_non_type_parameter(tmp_path): + file_contents = """ + template + struct AStruct {}; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + class_template = parsed_info["members"][0] + + assert class_template["kind"] == "CLASS_TEMPLATE" + assert class_template["name"] == "AStruct" + + template_non_type_parameter = class_template["members"][0] + + assert template_non_type_parameter["kind"] == "TEMPLATE_NON_TYPE_PARAMETER" + assert template_non_type_parameter["element_type"] == "Int" + assert template_non_type_parameter["name"] == "N" + + +def test_function_template(tmp_path): + file_contents = """ + template + void aFunction() {} + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + function_template = parsed_info["members"][0] + + assert function_template["kind"] == "FUNCTION_TEMPLATE" + assert function_template["result_type"] == "void" + assert function_template["name"] == "aFunction" + + template_type_parameter = function_template["members"][0] + + assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["name"] == "T" + assert template_type_parameter["access_specifier"] == "PUBLIC" + + +def test_template_type_parameter(tmp_path): + file_contents = """ + template + struct AStruct {}; + + template + void aFunction() {} + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + class_template = parsed_info["members"][0] + template_type_parameter = class_template["members"][0] + + assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["element_type"] == "Unexposed" + assert template_type_parameter["name"] == "T" + + function_template = parsed_info["members"][1] + template_type_parameter = function_template["members"][0] + + assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["element_type"] == "Unexposed" + assert template_type_parameter["name"] == "P" + + +def test_default_delete_constructor(tmp_path): + file_contents = """ + class aClass { + aClass() = default; + + // disable the copy constructor + aClass(double) = delete; + }; + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + + class_decl = parsed_info["members"][0] + + default_constructor = class_decl["members"][0] + + assert default_constructor["kind"] == "CONSTRUCTOR" + assert default_constructor["name"] == "aClass" + assert default_constructor["result_type"] == "void" + assert default_constructor["is_default_constructor"] + + delete_constructor = class_decl["members"][1] + + assert delete_constructor["kind"] == "CONSTRUCTOR" + assert delete_constructor["name"] == "aClass" + assert delete_constructor["result_type"] == "void" + # no check available for deleted ctor analogous to `is_default_constructor`