Skip to content

ci: use self-hosted runners #1258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 0 additions & 141 deletions .buildkite/pipeline.yml

This file was deleted.

108 changes: 31 additions & 77 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,8 @@ concurrency:
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}

jobs:
test:
timeout-minutes: 90
test-cpu:
name: Julia ${{ matrix.version }} - ${{ matrix.test_group }} - ${{ matrix.os }} - ${{ matrix.runtime }} - assertions=${{ matrix.assertions }} - ${{ github.event_name }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -86,77 +84,33 @@ jobs:
# libReactant: packaged
# version: '1.10'
# test_group: integration
env:
TMPDIR: ${{ github.workspace }}/tmp
steps:
- uses: actions/checkout@v4
- name: Create TMPDIR
run: |
mkdir -p ${{ env.TMPDIR }}
- uses: julia-actions/setup-julia@v2
if: ${{ ! matrix.assertions }}
with:
version: ${{ matrix.version }}
- uses: julia-actions/cache@v2
- uses: actions/checkout@v4
if: ${{ matrix.assertions }}
with:
repository: "JuliaLang/julia"
ref: release-${{ matrix.version }}
path: "julia"
- name: Compile Julia
if: ${{ matrix.assertions }}
run: |
sed -i.bak 's/exit 2/exit 0/g' julia/deps/tools/jlchecksum
make -C julia -j $(nproc) FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1 JULIA_PRECOMPILE=0
echo $PWD/julia/usr/bin >> $GITHUB_PATH
- name: "Setup Runtime Preferences"
uses: "DamianReeves/write-file-action@master"
with:
path: "LocalPreferences.toml"
write-mode: "overwrite"
contents: |
[Reactant]
xla_runtime = "${{ matrix.runtime }}"
- name: "Install Dependencies"
run: |
import Pkg
Pkg.Registry.update()
# Install packages present in subdirectories
dev_pks = Pkg.PackageSpec[]
for path in ("lib/ReactantCore",)
push!(dev_pks, Pkg.PackageSpec(; path))
end
Pkg.develop(dev_pks)
shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0}
# Only in Julia v1.10 we need to install `ReactantCore` manually.
if: ${{ matrix.version == '1.10' }}
env:
JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
- name: "Run Tests"
timeout-minutes: 60
run: |
import Pkg
Pkg.Registry.update()
Pkg.test(; coverage="user")
shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0}
id: run_tests
env:
JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
ENABLE_PJRT_COMPATIBILITY: 1
REACTANT_TEST_GROUP: ${{ matrix.test_group }}
XLA_FLAGS: "--xla_force_host_platform_device_count=12"
JULIA_DEBUG: "Reactant,Reactant_jll"
- name: Upload MLIR modules
uses: actions/upload-artifact@v4
timeout-minutes: 10
if: ${{ always() }}
with:
name: "mlir-${{ matrix.version }}-${{ matrix.test_group }}-${{ matrix.os }}-${{ matrix.runtime }}-assertions=${{ matrix.assertions }}-${{ github.event_name }}"
path: "**/*.mlir"
retention-days: 90
overwrite: false
- uses: julia-actions/julia-processcoverage@v1
- uses: codecov/codecov-action@v5
with:
files: lcov.info
uses: ./.github/workflows/CICommon.yml
with:
julia_version: ${{ matrix.version }}
runner: ${{ matrix.os }}
xla_runtime: ${{ matrix.runtime }}
assertions: ${{ matrix.assertions }}
test_group: ${{ matrix.test_group }}

test-cuda:
name: Julia ${{ matrix.version }} - ${{ matrix.test_group }} - linux - ${{ matrix.runtime }} - cuda - ${{ github.event_name }}
strategy:
fail-fast: false
matrix:
version:
- "1.10"
- "1.11"
test_group:
- core
- neural_networks
- integration
runtime:
- "PJRT"
- "IFRT"
uses: ./.github/workflows/CICommon.yml
with:
julia_version: ${{ matrix.version }}
runner: cuda
xla_runtime: ${{ matrix.runtime }}
assertions: false
test_group: ${{ matrix.test_group }}
100 changes: 100 additions & 0 deletions .github/workflows/CICommon.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
name: "CI Common"

on:
workflow_call:
inputs:
julia_version:
description: 'Julia version'
required: true
type: string
runner:
description: 'OS/Runner'
required: true
type: string
xla_runtime:
description: 'The XLA runtime'
required: true
type: string
assertions:
description: 'Enable assertions'
required: false
default: false
type: boolean
test_group:
description: 'Test group'
required: true
type: string

jobs:
test:
name: Julia ${{ inputs.julia_version }} - ${{ inputs.runner }} - ${{ inputs.xla_runtime }} - ${{ github.event_name }}
runs-on: ${{ inputs.runner }}
env:
TMPDIR: ${{ github.workspace }}/tmp
JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
JULIA_DEBUG: "Reactant,Reactant_jll"
REACTANT_TEST_GROUP: ${{ inputs.test_group }}
XLA_FLAGS: "--xla_force_host_platform_device_count=12"
LD_LIBRARY_PATH: "" # use binaries from our JLLs
steps:
- uses: actions/checkout@v4
- name: Create TMPDIR
run: |
mkdir -p ${{ env.TMPDIR }}
- uses: julia-actions/setup-julia@v2
with:
version: ${{ inputs.julia_version }}
- uses: julia-actions/cache@v2
- uses: actions/checkout@v4
if: ${{ inputs.assertions }}
with:
repository: "JuliaLang/julia"
ref: release-${{ inputs.julia_version }}
path: "julia"
- name: Compile Julia
if: ${{ inputs.assertions }}
run: |
sed -i.bak 's/exit 2/exit 0/g' julia/deps/tools/jlchecksum
make -C julia -j $(nproc) FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1 JULIA_PRECOMPILE=0
echo $PWD/julia/usr/bin >> $GITHUB_PATH
- name: "Setup Runtime Preferences"
uses: "DamianReeves/write-file-action@master"
with:
path: "LocalPreferences.toml"
write-mode: "overwrite"
contents: |
[Reactant]
xla_runtime = "${{ inputs.xla_runtime }}"
- name: "Install Dependencies"
run: |
import Pkg
Pkg.Registry.update()
# Install packages present in subdirectories
dev_pks = Pkg.PackageSpec[]
for path in ("lib/ReactantCore",)
push!(dev_pks, Pkg.PackageSpec(; path))
end
Pkg.develop(dev_pks)
shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0}
# Only in Julia v1.10 we need to install `ReactantCore` manually.
if: ${{ inputs.julia_version == '1.10' }}
- name: "Run Tests"
timeout-minutes: 60
run: |
import Pkg
Pkg.Registry.update()
Pkg.test(; coverage="user")
shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0}
id: run_tests
- name: Upload MLIR modules
uses: actions/upload-artifact@v4
timeout-minutes: 10
with:
name: "mlir-${{ inputs.julia_version }}-${{ inputs.test_group }}-${{ inputs.os }}-${{ inputs.xla_runtime }}-assertions=${{ inputs.assertions }}-${{ github.event_name }}"
path: "**/*.mlir"
retention-days: 90
overwrite: false
- uses: julia-actions/julia-processcoverage@v1
- uses: codecov/codecov-action@v5
with:
files: lcov.info
Loading