Skip to content
CI

Enable verbose logging for template analyzer #296

Sign in to view logs

Enable verbose logging for template analyzer

Enable verbose logging for template analyzer #296

Workflow file for this run

.github/workflows/ci.yml at 3d6d3d3
name: CI
on:
pull_request:
branches: [ main ]
push:
branches: [ main ]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions: read-all
env:
REGISTRY_NAME: ${{ secrets.ACR_NAME }}.azurecr.io
TAG: gha${{ github.run_number }}
jobs:
lint:
name: Linting
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.11'
- name: Install ruff
run: pip install ruff
- name: Run ruff check
run: ruff check
dependency-review:
name: Dependency Review
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Dependency Review
uses: actions/dependency-review-action@v4
with:
fail-on-severity: low
allow-licenses: MIT, Apache-2.0, BSD-3-Clause
comment-summary-in-pr: on-failure
test:
name: Test ${{ matrix.component.name }} (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.10', '3.11', '3.12']
# python-version: ['3.10', '3.11', '3.12', '3.13'] # Temporarily dropped 3.13 until resolution of https://github.com/pytorch/pytorch/issues/165448
component:
- name: 'Analyzer'
path: 'presidio-analyzer'
extras: '--all-extras'
spacy-models: 'en_core_web_lg en_core_web_sm'
- name: 'Anonymizer'
path: 'presidio-anonymizer'
extras: ''
spacy-models: ''
- name: 'Image Redactor'
path: 'presidio-image-redactor'
extras: ''
spacy-models: 'en_core_web_lg'
- name: 'CLI'
path: 'presidio-cli'
extras: ''
spacy-models: ''
- name: 'Structured'
path: 'presidio-structured'
extras: ''
spacy-models: ''
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: ${{ matrix.component.path }}/pyproject.toml
- name: Install Poetry
run: |
python -m pip install --upgrade pip
python -m pip install poetry
- name: Install system dependencies for Image Redactor
if: matrix.component.name == 'Image Redactor'
run: |
sudo apt-get update
sudo apt-get install tesseract-ocr -y
- name: Install dependencies
working-directory: ${{ matrix.component.path }}
run: |
poetry install ${{ matrix.component.extras }}
- name: Install presidio-analyzer for Image Redactor
if: matrix.component.name == 'Image Redactor'
working-directory: ${{ matrix.component.path }}
run: |
poetry run pip install -e ../presidio-analyzer/.
- name: Download spaCy models
if: matrix.component.spacy-models != ''
working-directory: ${{ matrix.component.path }}
run: |
for model in ${{ matrix.component.spacy-models }}; do
poetry run python -m spacy download $model
done
- name: Run unit tests
working-directory: ${{ matrix.component.path }}
run: |
poetry run pytest -vv --tb=short
- name: Build wheel package
working-directory: ${{ matrix.component.path }}
run: |
pip install build
python -m build --wheel
build-platform-images:
name: Build ${{ matrix.image }} (${{ matrix.platform }})
runs-on: ${{ matrix.runner }}
needs: [lint]
if: github.ref == 'refs/heads/main'
permissions:
id-token: write
contents: read
strategy:
matrix:
include:
- image: presidio-anonymizer
platform: linux/amd64
runner: ubuntu-latest
- image: presidio-analyzer
platform: linux/amd64
runner: ubuntu-latest
- image: presidio-image-redactor
platform: linux/amd64
runner: ubuntu-latest
- image: presidio-anonymizer
platform: linux/arm64
runner: ubuntu-24.04-arm
- image: presidio-analyzer
platform: linux/arm64
runner: ubuntu-24.04-arm
- image: presidio-image-redactor
platform: linux/arm64
runner: ubuntu-24.04-arm
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Azure Login using OIDC
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Log in to Azure Container Registry
run: az acr login --name ${{ secrets.ACR_NAME }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build and Push ${{ matrix.image }} for ${{ matrix.platform }}
run: |
# Create platform-specific tag
PLATFORM_TAG=$(echo "${{ matrix.platform }}" | sed 's/\//-/g')
docker buildx build \
--platform ${{ matrix.platform }} \
--push \
--tag ${{ env.REGISTRY_NAME }}/${{ matrix.image }}:${{ env.TAG }}-${PLATFORM_TAG} \
--cache-from type=registry,ref=${{ env.REGISTRY_NAME }}/public/${{ matrix.image }}:latest \
--cache-to type=inline \
--attest type=sbom \
--attest type=provenance,mode=max \
./${{ matrix.image }}
env:
REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
TAG: ${{ env.TAG }}
create-manifests:
name: Create Multi-Platform Manifests
runs-on: ubuntu-latest
needs: build-platform-images
if: github.ref == 'refs/heads/main'
permissions:
id-token: write
contents: read
steps:
- name: Azure Login using OIDC
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Log in to Azure Container Registry
run: az acr login --name ${{ secrets.ACR_NAME }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Create all multi-platform manifests
run: |
IMAGES=("presidio-anonymizer" "presidio-analyzer" "presidio-image-redactor")
for image in "${IMAGES[@]}"; do
echo "Creating manifest for $image"
docker buildx imagetools create \
--tag ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }} \
${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-amd64 \
${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-arm64
done
env:
REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
TAG: ${{ env.TAG }}
e2e-tests:
name: E2E Tests (${{ matrix.platform }})
runs-on: ${{ matrix.runner }}
needs: create-manifests
if: github.ref == 'refs/heads/main'
permissions:
id-token: write
contents: read
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
- platform: linux/arm64
runner: ubuntu-24.04-arm
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Azure Login using OIDC
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Log in to Azure Container Registry
run: az acr login --name ${{ secrets.ACR_NAME }}
- name: Pull Presidio images from ACR
run: |
export DOCKER_DEFAULT_PLATFORM=${{ matrix.platform }}
docker compose -f docker-compose.yml pull
env:
REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
TAG: :${{ env.TAG }}
- name: Start Presidio cluster
run: |
export DOCKER_DEFAULT_PLATFORM=${{ matrix.platform }}
docker compose -f docker-compose.yml up -d
env:
REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
TAG: :${{ env.TAG }}
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
- name: Cache E2E dependencies
uses: actions/cache@v4
with:
path: |
~/.cache/pip
e2e-tests/env
key: ${{ runner.os }}-${{ matrix.platform }}-e2e-${{ hashFiles('e2e-tests/requirements.txt') }}
restore-keys: |
${{ runner.os }}-${{ matrix.platform }}-e2e-
- name: Install E2E test dependencies
working-directory: e2e-tests
run: |
python -m venv env
source env/bin/activate
python -m pip install --upgrade pip
pip install -r requirements.txt
python -m spacy download en_core_web_lg
- name: Wait for services to be ready
run: sleep 60
- name: Run E2E tests
working-directory: e2e-tests
run: |
source env/bin/activate
pytest -vv --tb=short
- name: Capture Docker logs on failure (${{ matrix.platform }})
if: failure()
run: |
echo "Capturing Docker logs for platform: ${{ matrix.platform }}"
docker compose -f docker-compose.yml logs
- name: Stop Docker containers
if: always()
run: |
docker compose -f docker-compose.yml down
# Local build and test jobs for external contributors (no secrets required)
local-build-and-test:
name: Local Build and E2E Tests (${{ matrix.platform }})
runs-on: ${{ matrix.runner }}
needs: [lint]
if: github.ref != 'refs/heads/main'
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
- platform: linux/arm64
runner: ubuntu-24.04-arm
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Presidio images locally
run: |
# Build all images locally without pushing
docker compose -f docker-compose.yml build
env:
REGISTRY_NAME: 'mcr.microsoft.com'
IMAGE_PREFIX: ''
TAG: gha${{ github.run_number }}
- name: Start Presidio cluster locally
run: |
docker compose -f docker-compose.yml up -d
env:
REGISTRY_NAME: 'mcr.microsoft.com'
IMAGE_PREFIX: ''
TAG: gha${{ github.run_number }}
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
- name: Cache E2E dependencies
uses: actions/cache@v4
with:
path: |
~/.cache/pip
e2e-tests/env
key: ${{ runner.os }}-${{ matrix.platform }}-local-e2e-${{ hashFiles('e2e-tests/requirements.txt') }}
restore-keys: |
${{ runner.os }}-${{ matrix.platform }}-local-e2e-
- name: Install E2E test dependencies
working-directory: e2e-tests
run: |
python -m venv env
source env/bin/activate
python -m pip install --upgrade pip
pip install -r requirements.txt
python -m spacy download en_core_web_lg
- name: Wait for services to be ready
run: sleep 60
- name: Run E2E tests
working-directory: e2e-tests
run: |
source env/bin/activate
pytest -vv --tb=short
- name: Capture Docker logs on failure
if: failure()
run: |
echo "Capturing Docker logs for local build"
docker compose -f docker-compose.yml logs
- name: Stop Docker containers
if: always()
run: |
docker compose -f docker-compose.yml down