Add comparative benchmarks

.github/workflows/benchmarks.yml

@@ -23,115 +23,121 @@ jobs:
         date-polyfill: [OFF]
         shared: [OFF]
         cpp_standard: [20]
-           
+
     runs-on: ${{ matrix.os }}
     name: Benchmark / ${{matrix.os}} / ${{ matrix.compiler }} / ${{ matrix.version }} / ${{ matrix.arch }} / ${{ matrix.config }} / date-polyfill ${{ matrix.date-polyfill}} / shared ${{ matrix.shared }} / cpp${{ matrix.cpp_standard }}
     env:
       SCCACHE_GHA_ENABLED: "true"
- 
+
     steps:
+      - name: Run sccache-cache
+        uses: mozilla-actions/[email protected]
+
+      - name: Install pthread
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libpthread-stubs0-dev libboost-thread-dev
+
+      - uses: rui314/setup-mold@v1
+        with:
+          mold-version: 2.40.1
+          make-default: true
+
+      - name: Install GCC
+        if: matrix.compiler == 'gcc'
+        uses: egor-tensin/setup-gcc@v1
+        with:
+          version: ${{matrix.version}}
+          platform: x64
+
+      - name: Install LLVM and Clang
+        if: matrix.compiler == 'clang'
+        run: |
+          wget https://apt.llvm.org/llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh ${{matrix.version}}
+          sudo apt-get install -y clang-tools-${{matrix.version}} 
+          sudo update-alternatives --install /usr/bin/clang++				    clang++					    /usr/bin/clang++-${{matrix.version}} 200
+          sudo update-alternatives --install /usr/bin/clang					    clang					      /usr/bin/clang-${{matrix.version}} 200
+          sudo update-alternatives --install /usr/bin/clang-scan-deps   clang-scan-deps     /usr/bin/clang-scan-deps-${{matrix.version}} 200 
+          sudo update-alternatives --set clang /usr/bin/clang-${{matrix.version}}
+          sudo update-alternatives --set clang++ /usr/bin/clang++-${{matrix.version}}
+          sudo update-alternatives --set clang-scan-deps /usr/bin/clang-scan-deps-${{matrix.version}}
+
+      - name: Install specific version of tzdata
+        if: matrix.date-polyfill == 'OFF'
+        run: sudo apt-get install tzdata=2024a-2ubuntu1 -y --allow-downgrades
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set conda environment
+        uses: mamba-org/setup-micromamba@main
+        with:
+          environment-name: myenv
+          environment-file: environment-dev.yml
+          init-shell: bash
+          cache-downloads: true
+
+      - name: Configure using CMake
+        run: |
+          if [[ "${{matrix.compiler}}" = "gcc" ]]; then export CC=gcc-${{matrix.version}}; export CXX=g++-${{matrix.version}}; else export CC=clang; export CXX=clang++; fi
+          cmake -G Ninja \
+            -Bbuild \
+            -DCMAKE_BUILD_TYPE:STRING=${{matrix.config}} \
+            -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+            -DCMAKE_CXX_STANDARD=${{matrix.cpp_standard}} \
+            -DUSE_DATE_POLYFILL=${{matrix.date-polyfill}} \
+            -DBUILD_TESTS=OFF \
+            -DENABLE_INTEGRATION_TEST=OFF \
+            -DBUILD_EXAMPLES=OFF \
+            -DBUILD_BENCHMARKS=ON \
+            -DBUILD_COMPARATIVE_BENCHMARKS=ON \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+            -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING \
+            -DSPARROW_BUILD_SHARED=${{matrix.shared}} \
+            -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=mold" \
+            -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=mold"
+
+      - name: Build the benchmark target(s)
+        working-directory: build
+        run: |
+          cmake --build . --config ${{matrix.config}} --target sparrow_benchmarks
+          cmake --build . --config ${{matrix.config}} --target sparrow_benchmarks_comparative
+
+      - name: Run benchmarks
+        working-directory: build
+        run: |
+          cmake --build . --config ${{matrix.config}} --target run_benchmarks_json
+          cmake --build . --config ${{matrix.config}} --target run_comparative_benchmarks_json
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: |
+            ./**/sparrow_benchmarks.json
+            ./**/sparrow_benchmarks_comparative.json
+
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmarks
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: "googlecpp"
+          # Where the output from the benchmark tool is stored
+          output-file-path: build/benchmarks/sparrow_benchmarks.json
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          # Workflow will fail when an alert happens
+          fail-on-alert: true
 
-    - name: Run sccache-cache
-      uses: mozilla-actions/[email protected]
-
-    - name: Install pthread
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libpthread-stubs0-dev libboost-thread-dev
-
-    - uses: rui314/setup-mold@v1
-      with:
-        mold-version: 2.40.1
-        make-default: true
-
-    - name: Install GCC
-      if: matrix.compiler == 'gcc'
-      uses: egor-tensin/setup-gcc@v1
-      with:
-        version: ${{matrix.version}}
-        platform: x64
-
-    - name: Install LLVM and Clang
-      if: matrix.compiler == 'clang'
-      run: |
-        wget https://apt.llvm.org/llvm.sh
-        chmod +x llvm.sh
-        sudo ./llvm.sh ${{matrix.version}}
-        sudo apt-get install -y clang-tools-${{matrix.version}} 
-        sudo update-alternatives --install /usr/bin/clang++				    clang++					    /usr/bin/clang++-${{matrix.version}} 200
-        sudo update-alternatives --install /usr/bin/clang					    clang					      /usr/bin/clang-${{matrix.version}} 200
-        sudo update-alternatives --install /usr/bin/clang-scan-deps   clang-scan-deps     /usr/bin/clang-scan-deps-${{matrix.version}} 200 
-        sudo update-alternatives --set clang /usr/bin/clang-${{matrix.version}}
-        sudo update-alternatives --set clang++ /usr/bin/clang++-${{matrix.version}}
-        sudo update-alternatives --set clang-scan-deps /usr/bin/clang-scan-deps-${{matrix.version}}
-
-    - name: Install specific version of tzdata
-      if: matrix.date-polyfill == 'OFF'
-      run: sudo apt-get install tzdata=2024a-2ubuntu1 -y --allow-downgrades
-
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Set conda environment
-      uses: mamba-org/setup-micromamba@main
-      with:
-        environment-name: myenv
-        environment-file: environment-dev.yml
-        init-shell: bash
-        cache-downloads: true
-
-    - name: Configure using CMake
-      run: |
-        if [[ "${{matrix.compiler}}" = "gcc" ]]; then export CC=gcc-${{matrix.version}}; export CXX=g++-${{matrix.version}}; else export CC=clang; export CXX=clang++; fi
-        cmake -G Ninja \
-          -Bbuild \
-          -DCMAKE_BUILD_TYPE:STRING=${{matrix.config}} \
-          -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
-          -DCMAKE_CXX_STANDARD=${{matrix.cpp_standard}} \
-          -DUSE_DATE_POLYFILL=${{matrix.date-polyfill}} \
-          -DBUILD_TESTS=OFF \
-          -DENABLE_INTEGRATION_TEST=OFF \
-          -DBUILD_EXAMPLES=OFF \
-          -DBUILD_BENCHMARKS=ON \
-          -DCMAKE_C_COMPILER_LAUNCHER=sccache \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-          -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING \
-          -DSPARROW_BUILD_SHARED=${{matrix.shared}} \
-          -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=mold" \
-          -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=mold"
-
-    - name: Build the benchmark target(s)
-      working-directory: build
-      run: cmake --build . --config ${{matrix.config}} --target sparrow_benchmarks
-
-    - name: Run benchmarks
-      working-directory: build
-      run: cmake --build . --config ${{matrix.config}} --target run_benchmarks_json
-
-    - name: Upload benchmark results
-      uses: actions/upload-artifact@v4
-      with:
-        name: benchmark-results
-        path: ./**/sparrow_benchmarks.json
-
-    - name: Download previous benchmark data
-      uses: actions/cache@v4
-      with:
-        path: ./cache
-        key: ${{ runner.os }}-benchmarks
-
-    - name: Store benchmark result
-      uses: benchmark-action/github-action-benchmark@v1
-      with:
-        # What benchmark tool the output.txt came from
-        tool: 'googlecpp'
-        # Where the output from the benchmark tool is stored
-        output-file-path: build/benchmarks/sparrow_benchmarks.json
-        # Where the previous data file is stored
-        external-data-json-path: ./cache/benchmark-data.json
-        # Workflow will fail when an alert happens
-        fail-on-alert: true
-
-    - name: Run sccache stat for check
-      shell: bash
-      run: ${SCCACHE_PATH} --show-stats
+      - name: Run sccache stat for check
+        shell: bash
+        run: ${SCCACHE_PATH} --show-stats

CMakeLists.txt

@@ -92,6 +92,8 @@ OPTION(BUILD_TESTS "Build sparrow test suite" OFF)
 MESSAGE(STATUS "🔧 Build tests: ${BUILD_TESTS}")
 OPTION(BUILD_BENCHMARKS "Build sparrow benchmark suite" OFF)
 MESSAGE(STATUS "🔧 Build benchmarks: ${BUILD_BENCHMARKS}")
+OPTION(BUILD_COMPARATIVE_BENCHMARKS "Build comparative benchmarks" OFF)
+MESSAGE(STATUS "🔧 Build comparative benchmarks: ${BUILD_COMPARATIVE_BENCHMARKS}")
 OPTION(BUILD_DOCS "Build sparrow documentation" OFF)
 MESSAGE(STATUS "🔧 Build docs: ${BUILD_DOCS}")
 OPTION(BUILD_EXAMPLES "Build sparrow examples" OFF)

benchmarks/CMakeLists.txt

@@ -20,13 +20,6 @@ if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
     set(SPARROW_INCLUDE_DIR ${sparrow_INCLUDE_DIRS})
 endif()
 
-if(NOT CMAKE_BUILD_TYPE)
-    message(STATUS "Setting benchmarks build type to Release")
-    set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
-else()
-    message(STATUS "Benchmarks build type is ${CMAKE_BUILD_TYPE}")
-endif()
-
 set(SPARROW_BENCHMARK_SOURCES
     main.cpp
     bench_dynamic_bitset.cpp
@@ -74,3 +67,7 @@ set_target_properties(run_benchmarks_json
     PROPERTIES
         FOLDER "Benchmarks"
 )
+
+if(BUILD_COMPARATIVE_BENCHMARKS)
+    add_subdirectory(comparative)
+endif()

benchmarks/comparative/CMakeLists.txt

@@ -0,0 +1,42 @@
+# Comparative benchmarks between sparrow and Apache Arrow
+
+message(STATUS "Configuring comparative benchmarks...")
+
+add_executable(sparrow_benchmarks_comparative
+    benchmark_primitive.cpp
+)
+
+target_link_libraries(sparrow_benchmarks_comparative
+    PRIVATE
+        sparrow
+        arrow_static
+        benchmark::benchmark
+        benchmark::benchmark_main
+)
+
+set(ARROW_INCLUDE_DIRS  ${CMAKE_BINARY_DIR}/_deps/arrow-src/cpp/src ${CMAKE_BINARY_DIR}/_deps/arrow-build/src)
+
+target_include_directories(sparrow_benchmarks_comparative
+    PRIVATE
+        ${ARROW_INCLUDE_DIRS}
+)
+
+message("Arrow include dir: ${ARROW_INCLUDE_DIR}")
+
+set_target_properties(sparrow_benchmarks_comparative PROPERTIES FOLDER "Benchmarks")
+
+add_custom_target(run_comparative_benchmarks
+    COMMAND sparrow_benchmarks_comparative
+    DEPENDS sparrow_benchmarks_comparative
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Running comparative benchmarks (sparrow vs Arrow)"
+)
+set_target_properties(run_comparative_benchmarks PROPERTIES FOLDER "Benchmarks")
+
+add_custom_target(run_comparative_benchmarks_json
+    COMMAND sparrow_benchmarks_comparative --benchmark_format=json --benchmark_out=sparrow_benchmarks_comparative.json
+    DEPENDS sparrow_benchmarks_comparative
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Running comparative benchmarks (JSON output)"
+)
+set_target_properties(run_comparative_benchmarks_json PROPERTIES FOLDER "Benchmarks")