diff --git a/.github/actions/log-storage/action.yml b/.github/actions/log-storage/action.yml new file mode 100644 index 0000000000..6dfe33c3b7 --- /dev/null +++ b/.github/actions/log-storage/action.yml @@ -0,0 +1,63 @@ +name: 'Log Storage Utilization' +description: 'Log disk space, Docker storage, and optional details' +inputs: + label: + description: 'Label for this storage check (e.g., "After Build", "Initial")' + required: false + default: 'Storage Utilization' + show-docker-images: + description: 'Whether to show Docker images table' + required: false + default: 'false' + show-build-dirs: + description: 'Whether to show build directory sizes' + required: false + default: 'false' + show-packages: + description: 'Whether to show package sizes' + required: false + default: 'false' + show-top-dirs: + description: 'Whether to show top directories in home' + required: false + default: 'false' + +runs: + using: 'composite' + steps: + - name: Log storage utilization + shell: bash + run: | + echo "📊 ${{ inputs.label }}" + echo "$(printf '=%.0s' {1..50})" + echo "" + echo "💾 Disk Space:" + df -h + echo "" + echo "đŸ“Ļ Docker Storage:" + docker system df || true + + if [[ "${{ inputs.show-docker-images }}" == "true" ]]; then + echo "" + echo "đŸŗ Docker Images:" + docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" || true + fi + + if [[ "${{ inputs.show-build-dirs }}" == "true" ]]; then + echo "" + echo "🏗ī¸ Build Directory Sizes:" + du -sh src/*/build 2>/dev/null | sort -hr | head -20 || true + fi + + if [[ "${{ inputs.show-packages }}" == "true" ]]; then + echo "" + echo "đŸ“Ļ Package Sizes:" + du -sh deployment/native-packages/build/* 2>/dev/null | sort -hr || true + fi + + if [[ "${{ inputs.show-top-dirs }}" == "true" ]]; then + echo "" + echo "📁 Top Directories:" + du -sh /home/runner/* 2>/dev/null | sort -hr | head -10 || true + fi + diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml new file mode 100644 index 0000000000..e43e623f73 --- /dev/null +++ b/.github/actions/setup-build-env/action.yml @@ -0,0 +1,75 @@ +name: 'Setup Build Environment' +description: 'Set up Java, Gradle, Docker and other tools for X-Road builds' +inputs: + java-version: + description: 'Java version to set up' + required: false + default: '21' + gradle-cache-read-only: + description: 'Whether Gradle cache should be read-only' + required: false + default: 'false' + setup-docker-buildx: + description: 'Whether to set up Docker Buildx for multi-platform image building' + required: false + default: 'false' + setup-packaging-tools: + description: 'Whether to install packaging tools (debhelper, devscripts)' + required: false + default: 'false' + free-disk-space: + description: 'Whether to free up disk space' + required: false + default: 'false' + +runs: + using: 'composite' + steps: + - name: Log initial storage + uses: ./.github/actions/log-storage + with: + label: 'Initial Storage Utilization' + show-top-dirs: 'true' + + - name: Install packaging tools + if: inputs.setup-packaging-tools == 'true' && runner.os == 'Linux' + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + run: sudo apt-get update && sudo apt-get install -y curl software-properties-common build-essential unzip debhelper devscripts + + - name: Free Disk Space (Ubuntu) + if: inputs.free-disk-space == 'true' && runner.os == 'Linux' + uses: jlumbroso/free-disk-space@main + with: + android: true + dotnet: true + haskell: true + + - name: Set up JDK ${{ inputs.java-version }} + uses: actions/setup-java@v5 + with: + java-version: ${{ inputs.java-version }} + distribution: 'temurin' + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + with: + cache-read-only: ${{ inputs.gradle-cache-read-only }} + gradle-home-cache-cleanup: true + dependency-graph: generate-and-submit + add-job-summary-as-pr-comment: always + build-scan-publish: true + build-scan-terms-of-use-url: "https://gradle.com/terms-of-service" + build-scan-terms-of-use-agree: "yes" + + - name: Set up Docker Buildx + if: inputs.setup-docker-buildx == 'true' + uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host + + - name: Log post-setup storage + uses: ./.github/actions/log-storage + with: + label: 'Post-Setup Storage Utilization' diff --git a/.github/workflows/_build-and-package.yml b/.github/workflows/_build-and-package.yml new file mode 100644 index 0000000000..2a898bb87c --- /dev/null +++ b/.github/workflows/_build-and-package.yml @@ -0,0 +1,266 @@ +name: Build and Package + +on: + workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: true + type: string + sha: + description: 'Git SHA for the build' + required: true + type: string + outputs: + image_registry: + description: 'Image registry URL used for builds' + value: ${{ jobs.build-and-package.outputs.image_registry }} + service_image_tag: + description: 'Service image tag used for builds' + value: ${{ jobs.build-and-package.outputs.service_image_tag }} + +jobs: + build-and-package: + name: Build, test and package code + runs-on: ubuntu-24.04 + outputs: + image_registry: ${{ steps.build-config.outputs.image_registry }} + service_image_tag: ${{ steps.build-config.outputs.service_image_tag }} + steps: + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: start-measurement + label: 'Build, unit tests and packaging' + + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 # SonarCloud: Shallow clones should be disabled for a better relevancy of analysis + + - name: Setup build environment + uses: ./.github/actions/setup-build-env + with: + gradle-cache-read-only: ${{ github.ref != 'refs/heads/develop' }} + setup-docker-buildx: true + setup-packaging-tools: false + free-disk-space: false + + - name: Cache SonarCloud packages + uses: actions/cache@v4 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + + - name: Build, test and package code setup measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: get-measurement + label: 'Build environment setup' + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup build configuration + id: build-config + working-directory: ./src + run: | + # Setup GHCR registry URL (unified for both base images and application images) + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + IMAGE_REGISTRY="ghcr.io/${REPO_LOWERCASE}" + + # Read version and build type from gradle.properties for service image tagging + XROAD_VERSION=$(grep "^xroadVersion=" gradle.properties | cut -d'=' -f2 | tr -d ' ') + XROAD_BUILD_TYPE=$(grep "^xroadBuildType=" gradle.properties | cut -d'=' -f2 | tr -d ' ') + + # Function to get branch name for service image tagging + get_branch_name() { + local branch_name="" + # For pull requests, prioritize head_ref (actual source branch) over ref_name (merge branch) + if [[ -n "${{ github.head_ref }}" ]]; then + branch_name="${{ github.head_ref }}" + echo "🔍 Using PR source branch: ${branch_name}" >&2 + elif [[ -n "${{ github.ref_name }}" ]]; then + branch_name="${{ github.ref_name }}" + echo "🔍 Using ref branch: ${branch_name}" >&2 + else + branch_name="unknown" + echo "⚠ī¸ No branch name found, using: ${branch_name}" >&2 + fi + # Clean up branch name for Docker tag compatibility + echo "$branch_name" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-zA-Z0-9._-]/-/g' | sed 's/--*/-/g' | sed 's/^-\|-$//g' + } + + # Determine service image suffix for branch-aware tagging + BRANCH_NAME=$(get_branch_name) + + # Use GitHub run number for build uniqueness (handles re-runs of same commit) + BUILD_NUMBER="${{ github.run_number }}" + + if [[ "${{ github.ref_type }}" == "tag" || "$XROAD_BUILD_TYPE" == "RELEASE" ]]; then + # For releases, use simple version + SERVICE_IMAGE_TAG="${XROAD_VERSION}" + elif [[ "$BRANCH_NAME" == "develop-8.x" ]]; then + # For develop branch, include build number for uniqueness + SERVICE_IMAGE_TAG="${XROAD_VERSION}-SNAPSHOT-${BUILD_NUMBER}" + else + # For feature branches, include branch name and build number + SERVICE_IMAGE_TAG="${XROAD_VERSION}-SNAPSHOT-${BRANCH_NAME}-${BUILD_NUMBER}" + fi + + # Export to GITHUB_ENV for subsequent steps + echo "IMAGE_REGISTRY=${IMAGE_REGISTRY}" >> $GITHUB_ENV + echo "SERVICE_IMAGE_TAG=${SERVICE_IMAGE_TAG}" >> $GITHUB_ENV + echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_ENV + echo "XROAD_VERSION=${XROAD_VERSION}" >> $GITHUB_ENV + echo "XROAD_BUILD_TYPE=${XROAD_BUILD_TYPE}" >> $GITHUB_ENV + echo "PACKAGE_BUILDER_IMAGE_TAG=${XROAD_VERSION}-${XROAD_BUILD_TYPE}" >> $GITHUB_ENV + + # Export to GITHUB_OUTPUT for job outputs + echo "image_registry=${IMAGE_REGISTRY}" >> $GITHUB_OUTPUT + echo "service_image_tag=${SERVICE_IMAGE_TAG}" >> $GITHUB_OUTPUT + + echo "🏗ī¸ Build configuration:" + echo " Registry: ${IMAGE_REGISTRY}" + echo " Service Image Tag: ${SERVICE_IMAGE_TAG}" + echo " Package Builder Image Tag: ${PACKAGE_BUILDER_IMAGE_TAG}" + echo " Branch: ${BRANCH_NAME}" + echo " X-Road Version: ${XROAD_VERSION}" + echo " Build Type: ${XROAD_BUILD_TYPE}" + echo " Build Number: ${BUILD_NUMBER}" + echo " Commit SHA: ${{ inputs.sha }}" + + - name: Build and test source + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + working-directory: ./src + run: | + ./gradlew -Dorg.gradle.jvmargs=-Xmx6g \ + -PsonarqubeHost=https://sonarcloud.io -PsonarqubeProjectKey=nordic-institute_X-Road -PsonarqubeOrganization=nordic-institute \ + build sonar test jacocoTestReport -Pfrontend-npm-audit + + - name: Build Security Server images + working-directory: ./deployment/security-server/images + env: + IMAGE_REGISTRY: ${{ env.IMAGE_REGISTRY }} + IMAGE_TAG: ${{ env.SERVICE_IMAGE_TAG }} + run: | + ./build-images.sh --platforms linux/amd64,linux/arm64 --push + + - name: Log storage after build + if: always() + uses: ./.github/actions/log-storage + with: + label: 'Storage After Build' + show-build-dirs: 'true' + + - name: Test report + env: + NODE_OPTIONS: '--max-old-space-size=6144' + uses: dorny/test-reporter@v2 + if: success() || failure() + with: + name: Unit and integration tests + path: src/**/build/test-results/**/TEST-*.xml + reporter: java-junit + list-suites: 'failed' + list-tests: 'failed' + + - name: Upload intTest reports + uses: actions/upload-artifact@v4 + if: failure() + with: + name: Integration Test reports + path: | + src/service/signer/signer-int-test/build/allure-report/ + src/service/signer/signer-int-test/build/container-logs/ + src/service/op-monitor/op-monitor-int-test/build/allure-report/ + src/service/op-monitor/op-monitor-int-test/build/container-logs/ + + - name: Build, test and package code execution measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: get-measurement + label: 'Build, unit tests and packaging' + + - name: Build RHEL8 packages + working-directory: ./src + env: + IMAGE_REGISTRY: ${{ env.IMAGE_REGISTRY }} + IMAGE_TAG: ${{ env.PACKAGE_BUILDER_IMAGE_TAG }} + run: | + ./build_packages.sh -d -r rpm-el8 --package-only + echo "Packages built successfully, removing packages.." + rm -rf ${{ github.workspace }}/deployment/native-packages/build/rhel/8 + + - name: Build RHEL9 packages + working-directory: ./src + env: + IMAGE_REGISTRY: ${{ env.IMAGE_REGISTRY }} + IMAGE_TAG: ${{ env.PACKAGE_BUILDER_IMAGE_TAG }} + run: | + ./build_packages.sh -d -r rpm-el9 --package-only + echo "Packages built successfully, removing packages.." + rm -rf ${{ github.workspace }}/deployment/native-packages/build/rhel/9 + + - name: Build Ubuntu 22.04 (Jammy) packages + working-directory: ./src + env: + IMAGE_REGISTRY: ${{ env.IMAGE_REGISTRY }} + IMAGE_TAG: ${{ env.PACKAGE_BUILDER_IMAGE_TAG }} + run: | + ./build_packages.sh -d -r jammy --package-only + echo "Packages built successfully, removing packages.." + rm -rf ${{ github.workspace }}/deployment/native-packages/build/ubuntu22.04 + + - name: Build Ubuntu 24.04 (Noble) packages + working-directory: ./src + env: + IMAGE_REGISTRY: ${{ env.IMAGE_REGISTRY }} + IMAGE_TAG: ${{ env.PACKAGE_BUILDER_IMAGE_TAG }} + run: ./build_packages.sh -d -r noble --package-only + + - name: Build and push Central Server dev image + working-directory: ./development/docker/central-server + run: | + echo "🏗ī¸ Building Central Server dev image:" + echo " Registry: ${IMAGE_REGISTRY}" + echo " Version: ${SERVICE_IMAGE_TAG}" + echo " Branch: ${BRANCH_NAME}" + + ./build-cs-dev-image.sh \ + --environment ci \ + --registry ${{ env.IMAGE_REGISTRY }} \ + --version ${{ env.SERVICE_IMAGE_TAG }} \ + --packages-path ${{ github.workspace }}/deployment/native-packages/build/ubuntu24.04 \ + --platforms linux/amd64 + + - name: Log storage after all packaging + if: always() + uses: ./.github/actions/log-storage + with: + label: 'Storage After All Packaging' + show-packages: 'true' + + - name: Packaging and upload artifacts measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: get-measurement + label: 'Packaging and upload artifacts' + + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: display-results + pr-comment: true + + - name: Log final storage + if: always() + uses: ./.github/actions/log-storage + with: + label: 'Final Storage Utilization' diff --git a/.github/workflows/_test-central-server.yml b/.github/workflows/_test-central-server.yml new file mode 100644 index 0000000000..c29151f7b6 --- /dev/null +++ b/.github/workflows/_test-central-server.yml @@ -0,0 +1,113 @@ +name: Central Server System Tests + +on: + workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: true + type: string + sha: + description: 'Git SHA for the build' + required: true + type: string + image_registry: + description: 'Image registry URL' + required: true + type: string + service_image_tag: + description: 'Service image tag to use' + required: true + type: string + +jobs: + test-central-server: + name: Run Central Server system tests + runs-on: ubuntu-22.04 + steps: + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: start-measurement + label: 'Central Server system tests' + + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup build environment + uses: ./.github/actions/setup-build-env + with: + gradle-cache-read-only: true + free-disk-space: false + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Central Server test setup measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: get-measurement + label: 'Set up Central Server tests' + + - name: Run Central Server system tests + working-directory: ./src + run: | + ./gradlew -Dorg.gradle.jvmargs=-Xmx1g \ + :central-server:admin-service:ui-system-test:systemTest \ + -PsystemTestCsImageName=${{ inputs.image_registry }}/central-server-dev:${{ inputs.service_image_tag }} + + - name: Log storage after tests + if: always() + uses: ./.github/actions/log-storage + with: + label: 'Storage After Central Server Tests' + + - name: Test Central Server measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: get-measurement + label: 'Test Central Server' + + - name: Test report + env: + NODE_OPTIONS: '--max-old-space-size=6144' + uses: dorny/test-reporter@v2 + if: success() || failure() + with: + name: Central Server system test + path: src/central-server/admin-service/ui-system-test/build/test-results/**/TEST-*.xml + reporter: java-junit + + - name: Upload CS screenshots + if: failure() + uses: actions/upload-artifact@v4 + with: + name: CS System Test screenshots + path: src/central-server/admin-service/ui-system-test/build/reports/test-automation/selenide-failures/*.png + + - name: Upload CS report + uses: actions/upload-artifact@v4 + if: failure() + with: + name: CS System Test report + path: | + src/central-server/admin-service/ui-system-test/build/allure-report/ + + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: display-results + pr-comment: true + + - name: Upload GMT jsons + uses: actions/upload-artifact@v4 + with: + name: CS System Test GMT jsons + path: | + /tmp/eco-ci/lap-data.json + /tmp/eco-ci/total-data.json diff --git a/.github/workflows/_test-e2e.yml b/.github/workflows/_test-e2e.yml new file mode 100644 index 0000000000..a8a0a0eab6 --- /dev/null +++ b/.github/workflows/_test-e2e.yml @@ -0,0 +1,126 @@ +name: End-to-End Tests + +on: + workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: true + type: string + sha: + description: 'Git SHA for the build' + required: true + type: string + image_registry: + description: 'Image registry URL' + required: true + type: string + service_image_tag: + description: 'Service image tag to use' + required: true + type: string + +jobs: + test-e2e: + name: Run E2E tests + runs-on: ubuntu-22.04 + steps: + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 + with: + task: start-measurement + label: 'E2E tests' + + # Setup + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup build environment + uses: ./.github/actions/setup-build-env + with: + gradle-cache-read-only: true + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create .env file for compose + working-directory: ./development/docker/security-server + run: | + # Create .env file with GHCR image references for Security Server components + REGISTRY="${{ inputs.image_registry }}" + TAG="${{ inputs.service_image_tag }}" + + cat > .env < .env <> $GITHUB_OUTPUT + echo "Using tag: ${IMAGE_TAG}" + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push AMD64 builder images + working-directory: ./deployment/native-packages/docker + env: + IMAGE_REGISTRY: ${{ env.REGISTRY }}/${{ github.repository }} + IMAGE_TAG: ${{ steps.version.outputs.tag }}-amd64 + BUILD_PLATFORMS: linux/amd64 + run: | + export IMAGE_REGISTRY=$(echo "$IMAGE_REGISTRY" | tr '[:upper:]' '[:lower:]') + ./prepare-builder-image.sh all + + build-arm64: + name: Build ARM64 images + runs-on: ubuntu-24.04-arm + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Read version from gradle.properties + id: version + working-directory: ./src + run: | + XROAD_VERSION=$(grep "^xroadVersion=" gradle.properties | cut -d'=' -f2 | tr -d ' ') + XROAD_BUILD_TYPE=$(grep "^xroadBuildType=" gradle.properties | cut -d'=' -f2 | tr -d ' ') + IMAGE_TAG="${XROAD_VERSION}-${XROAD_BUILD_TYPE}" + echo "tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + echo "Using tag: ${IMAGE_TAG}" + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push ARM64 builder images + working-directory: ./deployment/native-packages/docker + env: + IMAGE_REGISTRY: ${{ env.REGISTRY }}/${{ github.repository }} + IMAGE_TAG: ${{ steps.version.outputs.tag }}-arm64 + BUILD_PLATFORMS: linux/arm64 + run: | + export IMAGE_REGISTRY=$(echo "$IMAGE_REGISTRY" | tr '[:upper:]' '[:lower:]') + ./prepare-builder-image.sh all + + create-manifests: + name: Create multi-arch manifests + runs-on: ubuntu-24.04 + needs: [build-amd64, build-arm64] + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create multi-arch manifests + env: + TAG: ${{ needs.build-amd64.outputs.tag }} + run: | + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + REGISTRY="${{ env.REGISTRY }}/${REPO_LOWERCASE}" + + IMAGES=(deb-jammy deb-noble rpm-el8 rpm-el9) + + for image in "${IMAGES[@]}"; do + echo "Creating manifest for package-builder-${image}:${TAG}" + + docker buildx imagetools create \ + --tag "${REGISTRY}/package-builder-${image}:${TAG}" \ + --tag "${REGISTRY}/package-builder-${image}:latest" \ + "${REGISTRY}/package-builder-${image}:${TAG}-amd64" \ + "${REGISTRY}/package-builder-${image}:${TAG}-arm64" + + echo "✅ Created manifest for ${image}" + done + + summary: + name: Build summary + runs-on: ubuntu-24.04 + needs: [build-amd64, build-arm64, create-manifests] + if: always() + + steps: + - name: Add build summary + env: + TAG: ${{ needs.build-amd64.outputs.tag }} + run: | + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + + AMD64_STATUS="${{ needs.build-amd64.result }}" + ARM64_STATUS="${{ needs.build-arm64.result }}" + MANIFEST_STATUS="${{ needs.create-manifests.result }}" + + if [[ "$AMD64_STATUS" == "success" ]]; then AMD64_ICON="✅"; else AMD64_ICON="❌"; fi + if [[ "$ARM64_STATUS" == "success" ]]; then ARM64_ICON="✅"; else ARM64_ICON="❌"; fi + if [[ "$MANIFEST_STATUS" == "success" ]]; then MANIFEST_ICON="✅"; else MANIFEST_ICON="❌"; fi + + cat >> $GITHUB_STEP_SUMMARY << EOF + # Package Builder Images Build + + **Trigger:** ${{ github.event_name }} + **Branch/Tag:** ${{ github.ref_name }} + **Version Tag:** \`${TAG}\` + **Registry:** \`${{ env.REGISTRY }}/${REPO_LOWERCASE}\` + + ## Build Status + - ${AMD64_ICON} **AMD64** (ubuntu-24.04): $AMD64_STATUS + - ${ARM64_ICON} **ARM64** (ubuntu-24.04-arm): $ARM64_STATUS + - ${MANIFEST_ICON} **Multi-arch manifests**: $MANIFEST_STATUS + + ## Built Images + Each image is available for both amd64 and arm64: + - \`package-builder-deb-jammy:${TAG}\` + - \`package-builder-deb-noble:${TAG}\` + - \`package-builder-rpm-el8:${TAG}\` + - \`package-builder-rpm-el9:${TAG}\` + + ### Architecture-Specific Tags (for debugging) + - \`package-builder-deb-jammy:${TAG}-amd64\` + - \`package-builder-deb-jammy:${TAG}-arm64\` + - (same pattern for all images) + EOF diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 00c2f4e5c5..041d306265 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,4 +1,4 @@ -name: Build and test +name: Build and Test on: # Capture this event so that gradle caches are updated when a PR is merged to develop # More information on why: https://github.com/gradle/gradle-build-action#using-the-caches-read-only @@ -15,602 +15,88 @@ on: - 'src/**' - '.github/**' - 'development/**' + permissions: contents: write # Required for https://github.com/gradle/actions/tree/main/setup-gradle#github-dependency-graph-support pull-requests: write # https://github.com/gradle/actions/tree/main/setup-gradle#adding-job-summary-as-a-pull-request-comment actions: read # Required for https://github.com/dorny/test-reporter checks: write # Required for https://github.com/dorny/test-reporter + packages: write # Required to pull base images from GHCR and push application images + # Cancels previous workflow run on PR if a new one is started (does not affect push to develop). # This is because github.head_ref is empty on push events so defaults to the unique github.run_id. # More info: https://docs.github.com/en/actions/using-jobs/using-concurrency concurrency: group: ${{ github.head_ref || github.run_id }} cancel-in-progress: true + jobs: - BuildAndPackageWithUnitTests: + build-and-package: name: Build, test and package code - runs-on: ubuntu-22.04 - services: - registry: - image: registry:2 - ports: - - 5555:5000 - steps: - - name: Start Measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: start-measurement - label: 'Build, unit tests and packaging' - - uses: actions/checkout@v5 - with: - fetch-depth: 0 # SonarCloud: Shallow clones should be disabled for a better relevancy of analysis - - name: Ensure required packages - env: - DEBIAN_FRONTEND: noninteractive # Less output to log - run: sudo apt-get update && sudo apt-get install -y curl software-properties-common build-essential unzip debhelper devscripts - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - android: true - tool-cache: false - dotnet: false - haskell: false - large-packages: false - docker-images: false - swap-storage: false - - name: Set up JDK 21 - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'temurin' - - name: Cache SonarCloud packages - uses: actions/cache@v4 - with: - path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - name: Set up Gradle - uses: gradle/actions/setup-gradle@v4 - with: - cache-read-only: ${{ github.ref != 'refs/heads/develop' }} - gradle-home-cache-cleanup: true - dependency-graph: generate-and-submit - add-job-summary-as-pr-comment: always - build-scan-publish: true - build-scan-terms-of-use-url: "https://gradle.com/terms-of-service" - build-scan-terms-of-use-agree: "yes" - - name: Build, test and package code setup measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'Build environment setup' - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver-opts: network=host - - name: Build base docker images - working-directory: ./deployment/security-server/base-images - run: | - chmod +x build-base-images.sh - ./build-base-images.sh localhost:5555 - - name: Build and test source - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - working-directory: ./src - run: | - echo "Available disk space pre-build:" - df -h - docker system df - docker builder prune -f - df -h - ./gradlew -Dorg.gradle.jvmargs=-Xmx6g -PsonarqubeHost=https://sonarcloud.io -PsonarqubeProjectKey=nordic-institute_X-Road -PsonarqubeOrganization=nordic-institute -PxroadBuildType=RELEASE build sonar test intTest jacocoTestReport -Pfrontend-npm-audit -PbuildImages=true -PxroadImageRegistry=localhost:5555 - echo "Available disk space post-build:" - df -h - echo "Available disk space post-prune:" - docker system prune -f --volumes - df -h - echo "Deleting Quarkus-Build folders..." - find . -type d -name 'quarkus-build' -exec rm -rf {} + - df -h - - name: Test report - env: - NODE_OPTIONS: '--max-old-space-size=6144' - uses: dorny/test-reporter@v2 - if: success() || failure() - with: - name: Unit and integration tests - path: src/**/build/test-results/**/TEST-*.xml - reporter: java-junit - list-suites: 'failed' - list-tests: 'failed' - - name: Upload intTest reports - uses: actions/upload-artifact@v4 - if: failure() - with: - name: Integration Test reports - path: | - src/service/signer/signer-int-test/build/allure-report/ - src/service/signer/signer-int-test/build/container-logs/ - src/service/op-monitor/op-monitor-int-test/build/allure-report/ - src/service/op-monitor/op-monitor-int-test/build/container-logs/ - - name: Build, test and package code execution measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'Build, unit tests and packaging' -# - name: Store files for arm packaging -# uses: pyTooling/upload-artifact@v4 # https://github.com/actions/upload-artifact/issues/38 -# with: -# name: workspace-for-arm -# include-hidden-files: true -# path: | -# ** -# !**/src/lib/** -# !**/src/test/** -# !**/build/classes/** -# !**/build/tmp/** -# !**/build/allure-** -# !**/build/test-results/** -# !**/build/reports/** -# !**/.git/** -# !**/.pnpm-store/** -# !**/pnpm-node/** -# !**/node_modules/** -# !**/executionHistory/** - - name: Build RHEL8 packages - run: | - docker build -t rhel8 ${{ github.workspace }}/deployment/native-packages/docker/rpm-el8/ && docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v ${{ github.workspace }}:/workspace rhel8 ./deployment/native-packages/build-rpm.sh - echo "Packages built successfully, removing packages.." - rm -rf ${{ github.workspace }}/deployment/native-packages/build/rhel/8 - - name: Build RHEL9 packages - run: | - docker build -t rhel9 ${{ github.workspace }}/deployment/native-packages/docker/rpm-el9/ && docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v ${{ github.workspace }}:/workspace rhel9 ./deployment/native-packages/build-rpm.sh - echo "Packages built successfully, removing packages.." - rm -rf ${{ github.workspace }}/deployment/native-packages/build/rhel/9 - - name: Build 22.04 (Jammy) packages - env: - DEBEMAIL: 'info@niis.org' - DEBFULLNAME: 'NIIS' - run: ./deployment/native-packages/build-deb.sh jammy -release - - name: Build 24.04 (Noble) packages - env: - DEBEMAIL: 'info@niis.org' - DEBFULLNAME: 'NIIS' - run: | - ./deployment/native-packages/build-deb.sh noble -release - echo "Packages built successfully, removing packages.." - rm -rf ${{ github.workspace }}/deployment/native-packages/build/ubuntu24.04 - - name: Store deb files for system tests - uses: actions/upload-artifact@v4 - with: - name: debian-packages - path: deployment/native-packages/build/ubuntu22.04/*.deb - compression-level: 0 #No point in compressing these - - name: Packaging and upload artifacts measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'Packaging and upload artifacts' - - name: Show Energy Results - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: display-results - pr-comment: true - - name: Clean up workspace #otherwise might run out of disk space in the next steps - working-directory: ./src - run: ./gradlew clean - - name: Save docker images from registry - run: | - mkdir -p ${{ runner.temp }}/docker-images - docker pull localhost:5555/ss-backup-manager:latest - docker pull localhost:5555/ss-configuration-client:latest - docker pull localhost:5555/ss-proxy:latest - docker pull localhost:5555/ss-signer:latest - docker pull localhost:5555/ss-monitor:latest - docker pull localhost:5555/ss-op-monitor:latest - docker pull localhost:5555/ss-proxy-ui-api:latest - docker pull localhost:5555/ss-db-serverconf-init:latest - docker pull localhost:5555/ss-db-messagelog-init:latest - docker pull localhost:5555/ss-db-opmonitor-init:latest + uses: ./.github/workflows/_build-and-package.yml + secrets: inherit + with: + ref: ${{ github.ref }} + sha: ${{ github.sha }} - docker save -o ${{ runner.temp }}/docker-images/ss-backup-manager.tar localhost:5555/ss-backup-manager:latest - docker save -o ${{ runner.temp }}/docker-images/ss-configuration-client.tar localhost:5555/ss-configuration-client:latest - docker save -o ${{ runner.temp }}/docker-images/ss-proxy.tar localhost:5555/ss-proxy:latest - docker save -o ${{ runner.temp }}/docker-images/ss-signer.tar localhost:5555/ss-signer:latest - docker save -o ${{ runner.temp }}/docker-images/ss-monitor.tar localhost:5555/ss-monitor:latest - docker save -o ${{ runner.temp }}/docker-images/ss-op-monitor.tar localhost:5555/ss-op-monitor:latest - docker save -o ${{ runner.temp }}/docker-images/ss-proxy-ui-api.tar localhost:5555/ss-proxy-ui-api:latest - docker save -o ${{ runner.temp }}/docker-images/ss-db-serverconf-init.tar localhost:5555/ss-db-serverconf-init:latest - docker save -o ${{ runner.temp }}/docker-images/ss-db-messagelog-init.tar localhost:5555/ss-db-messagelog-init:latest - docker save -o ${{ runner.temp }}/docker-images/ss-db-opmonitor-init.tar localhost:5555/ss-db-opmonitor-init:latest - - name: Store docker images for other jobs - uses: actions/upload-artifact@v4 - with: - name: docker-images - path: ${{ runner.temp }}/docker-images -# BuildAndPackageOnArm: -# name: Build and package code for arm architecture -# needs: BuildAndPackageWithUnitTests -# -# runs-on: ubuntu-22.04-arm -# steps: -# - name: Start Measurement -# uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 -# with: -# task: start-measurement -# label: 'Packaging for arm architecture' -# - name: Ensure required packages -# env: -# DEBIAN_FRONTEND: noninteractive # Less output to log -# run: sudo apt-get update && sudo apt-get install -y curl software-properties-common build-essential unzip debhelper devscripts -# - name: Set up JDK 21 -# uses: actions/setup-java@v5 -# with: -# java-version: '21' -# distribution: 'temurin' -# - name: Set up Gradle -# uses: gradle/actions/setup-gradle@v4 -# with: -# cache-read-only: ${{ github.ref != 'refs/heads/develop' }} -# gradle-home-cache-cleanup: true -# dependency-graph: generate-and-submit -# add-job-summary-as-pr-comment: always -# - name: Download workspace -# uses: pyTooling/download-artifact@v5 # https://github.com/actions/upload-artifact/issues/38 -# with: -# name: workspace-for-arm -# - name: Set up measurement for arm packaging -# uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 -# with: -# task: get-measurement -# label: 'Set up arm packaging' -# - name: Build RHEL8 packages -# run: | -# docker build -t rhel8 ${{ github.workspace }}/deployment/native-packages/docker/rpm-el8/ && docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v ${{ github.workspace }}:/workspace rhel8 ./deployment/native-packages/build-rpm.sh -# echo "Packages built successfully, removing packages.." -# rm -rf ${{ github.workspace }}/deployment/native-packages/build/rhel/8 -# - name: Build RHEL9 packages -# run: | -# docker build -t rhel9 ${{ github.workspace }}/deployment/native-packages/docker/rpm-el9/ && docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v ${{ github.workspace }}:/workspace rhel9 ./deployment/native-packages/build-rpm.sh -# echo "Packages built successfully, removing packages.." -# rm -rf ${{ github.workspace }}/deployment/native-packages/build/rhel/9 -# - name: Build 22.04 (Jammy) packages -# env: -# DEBEMAIL: 'info@niis.org' -# DEBFULLNAME: 'NIIS' -# run: ./deployment/native-packages/build-deb.sh jammy -release -# - name: Build 24.04 (Noble) packages -# env: -# DEBEMAIL: 'info@niis.org' -# DEBFULLNAME: 'NIIS' -# run: | -# ./deployment/native-packages/build-deb.sh noble -release -# echo "Packages built successfully, removing packages.." -# rm -rf ${{ github.workspace }}/deployment/native-packages/build/ubuntu24.04 -# - name: Store deb files for Docker dev images -# uses: actions/upload-artifact@v4 -# with: -# name: debian-packages-arm -# path: deployment/native-packages/build/ubuntu22.04/*.deb -# compression-level: 0 #No point in compressing these -# - name: Measurement for arm packaging -# uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 -# with: -# task: get-measurement -# label: 'Packaging for arm architecture' -# - name: Show Energy Results -# uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 -# with: -# task: display-results -# pr-comment: true - RunCSSystemTests: + test-central-server: name: Run Central Server system tests - needs: BuildAndPackageWithUnitTests - runs-on: ubuntu-22.04 - services: - registry: - image: registry:2 - ports: - - 5555:5000 - steps: - - name: Start Measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: start-measurement - label: 'Central Server system tests' - - uses: actions/checkout@v5 - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - android: true - tool-cache: false - dotnet: false - haskell: false - large-packages: false - docker-images: false - swap-storage: false - - name: Set up JDK 21 - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'temurin' - - name: Set up Gradle - uses: gradle/actions/setup-gradle@v4 - with: - cache-read-only: true - - name: Initialize docker setup - working-directory: ./development/docker/central-server - run: ./init_context.sh - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver-opts: network=host - - name: Download debian packages - uses: actions/download-artifact@v5 - with: - name: debian-packages - path: ./development/docker/central-server/build/packages/ - - name: List docker build files - run: ls -lah ./development/docker/central-server/build - - name: Build CS docker image - uses: docker/build-push-action@v6 - with: - context: ./development/docker/central-server/ - push: true - build-args: | - PACKAGE_SOURCE=internal - tags: localhost:5555/xrd-centralserver:${{ github.sha }} - - name: Central Server test setup measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'Set up Central Server tests' - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - android: true - tool-cache: false - dotnet: false - haskell: false - large-packages: false - docker-images: false - swap-storage: false - - name: Run Central Server system tests - working-directory: ./src - run: ./gradlew -Dorg.gradle.jvmargs=-Xmx1g :central-server:admin-service:ui-system-test:systemTest -PsystemTestCsImageName=localhost:5555/xrd-centralserver:${{ github.sha }} - - name: Test Central Server measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'Test Central Server' - - name: Test report - env: - NODE_OPTIONS: '--max-old-space-size=6144' - uses: dorny/test-reporter@v2 - if: success() || failure() - with: - name: Central Server system test - path: src/central-server/admin-service/ui-system-test/build/test-results/**/TEST-*.xml - reporter: java-junit - - name: Upload CS screenshots - if: failure() - uses: actions/upload-artifact@v4 - with: - name: CS System Test screenshots - path: src/central-server/admin-service/ui-system-test/build/reports/test-automation/selenide-failures/*.png - - name: Upload CS report - uses: actions/upload-artifact@v4 - if: failure() - with: - name: CS System Test report - path: | - src/central-server/admin-service/ui-system-test/build/allure-report/ - - name: Show Energy Results - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: display-results - pr-comment: true - - name: Upload GMT jsons - uses: actions/upload-artifact@v4 - with: - name: CS System Test GMT jsons - path: | - /tmp/eco-ci/lap-data.json - /tmp/eco-ci/total-data.json - RunSSSystemTests: - name: Run Security Server system tests - needs: BuildAndPackageWithUnitTests - runs-on: ubuntu-22.04 - services: - registry: - image: registry:2 - ports: - - 5555:5000 - steps: - - name: Start Measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: start-measurement - label: 'Security Server system tests' - - uses: actions/checkout@v5 - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - android: true - tool-cache: false - dotnet: false - haskell: false - large-packages: false - docker-images: false - swap-storage: false - - name: Set up JDK 21 - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'temurin' - - name: Set up Gradle - uses: gradle/actions/setup-gradle@v4 - with: - cache-read-only: true - - name: Download SS docker images - uses: actions/download-artifact@v5 - with: - name: docker-images - path: ${{ runner.temp }}/docker-images - - name: Load SS docker images into registry - run: | - for tar in ${{ runner.temp }}/docker-images/*.tar; do - echo "Processing $tar..." - IMAGE_ID=$(docker load -i "$tar" | awk '/Loaded image:/ {print $NF}') - docker push "$IMAGE_ID" - done - - name: Security Server test setup measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'Set up Security Server tests' - - name: Run Security Server system tests - working-directory: ./src - run: | - echo "Available disk space pre-build:" - df -h - docker system df - docker images - ./gradlew -Dorg.gradle.jvmargs=-Xmx1g :security-server:system-test:systemTest - - name: Test Security Server measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'Test Security Server' - - name: Test report - env: - NODE_OPTIONS: '--max-old-space-size=6144' - uses: dorny/test-reporter@v2 - if: success() || failure() - with: - name: Security Server system tests - path: src/security-server/system-test/build/test-results/**/TEST-*.xml - reporter: java-junit - - name: Upload SS report - uses: actions/upload-artifact@v4 - if: failure() - with: - name: SS System Test report - path: | - src/security-server/system-test/build/a2c_logs/ - src/security-server/system-test/build/allure-report/ - src/security-server/system-test/build/container-logs/ - src/security-server/system-test/build/reports/test-automation/selenide-failures/*.png + needs: build-and-package + uses: ./.github/workflows/_test-central-server.yml + secrets: inherit + with: + ref: ${{ github.ref }} + sha: ${{ github.sha }} + image_registry: ${{ needs.build-and-package.outputs.image_registry }} + service_image_tag: ${{ needs.build-and-package.outputs.service_image_tag }} + + test-security-server: + name: Run Security Server system tests + needs: build-and-package + uses: ./.github/workflows/_test-security-server.yml + secrets: inherit + with: + ref: ${{ github.ref }} + sha: ${{ github.sha }} + image_registry: ${{ needs.build-and-package.outputs.image_registry }} + service_image_tag: ${{ needs.build-and-package.outputs.service_image_tag }} - - name: Show Energy Results - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: display-results - pr-comment: true - RunE2ETests: + test-e2e: name: Run E2E tests - needs: BuildAndPackageWithUnitTests + needs: build-and-package + uses: ./.github/workflows/_test-e2e.yml + secrets: inherit + with: + ref: ${{ github.ref }} + sha: ${{ github.sha }} + image_registry: ${{ needs.build-and-package.outputs.image_registry }} + service_image_tag: ${{ needs.build-and-package.outputs.service_image_tag }} + + build-summary: + name: Build Summary + needs: [build-and-package, test-central-server, test-security-server, test-e2e] + if: always() runs-on: ubuntu-22.04 - services: - registry: - image: registry:2 - ports: - - 5555:5000 steps: - - name: Start Measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: start-measurement - label: 'E2E tests' - # Setup - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - android: true - tool-cache: false - dotnet: false - haskell: false - large-packages: false - docker-images: false - swap-storage: false - - uses: actions/checkout@v5 - - name: Set up JDK 21 - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'temurin' - - name: Set up Gradle - uses: gradle/actions/setup-gradle@v4 - with: - cache-read-only: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver-opts: network=host - # Build Central Server - - name: Initialize CS docker setup - working-directory: ./development/docker/central-server - run: ./init_context.sh - - name: Download debian packages for CS - uses: actions/download-artifact@v5 - with: - name: debian-packages - path: ./development/docker/central-server/build/packages/ - - name: Build CS docker image - uses: docker/build-push-action@v6 - with: - context: ./development/docker/central-server/ - push: true - build-args: | - PACKAGE_SOURCE=internal - tags: localhost:5555/xrd-centralserver:${{ github.sha }} - - name: Download SS docker images - uses: actions/download-artifact@v4 - with: - name: docker-images - path: ${{ runner.temp }}/docker-images - - name: Load SS docker images into registry - run: | - for tar in ${{ runner.temp }}/docker-images/*.tar; do - echo "Processing $tar..." - IMAGE_ID=$(docker load -i "$tar" | awk '/Loaded image:/ {print $NF}') - docker push "$IMAGE_ID" - done - - name: E2E test setup measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'Set up E2E tests' - # Execute and report - - name: Run E2E tests - working-directory: ./src + - name: Generate build summary run: | - docker network create xroad-network - ./gradlew -Dorg.gradle.jvmargs=-Xmx1g :security-server:e2e-test:e2eTest -Pe2eTestCSImage=localhost:5555/xrd-centralserver:${{ github.sha }} - - name: E2E test measurement - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: get-measurement - label: 'E2E tests' - - name: Test report - env: - NODE_OPTIONS: '--max-old-space-size=6144' - uses: dorny/test-reporter@v2 - if: success() || failure() - with: - name: E2E tests - path: src/security-server/e2e-test/build/test-results/**/TEST-*.xml - reporter: java-junit - - name: Upload E2E report - uses: actions/upload-artifact@v4 - if: failure() - with: - name: E2E report - path: | - src/security-server/e2e-test/build/allure-report/ - - name: Show Energy Results - uses: green-coding-solutions/eco-ci-energy-estimation@v5.0 - with: - task: display-results - pr-comment: true + cat >> $GITHUB_STEP_SUMMARY << EOF + # Build and Test Summary + + | Job | Status | Duration | + |-----|--------|----------| + | Build and Package | ${{ needs.build-and-package.result == 'success' && '✅ Success' || needs.build-and-package.result == 'failure' && '❌ Failed' || needs.build-and-package.result == 'cancelled' && '⏚ī¸ Cancelled' || 'âŗ Skipped' }} | - | + | Central Server Tests | ${{ needs.test-central-server.result == 'success' && '✅ Success' || needs.test-central-server.result == 'failure' && '❌ Failed' || needs.test-central-server.result == 'cancelled' && '⏚ī¸ Cancelled' || 'âŗ Skipped' }} | - | + | Security Server Tests | ${{ needs.test-security-server.result == 'success' && '✅ Success' || needs.test-security-server.result == 'failure' && '❌ Failed' || needs.test-security-server.result == 'cancelled' && '⏚ī¸ Cancelled' || 'âŗ Skipped' }} | - | + | E2E Tests | ${{ needs.test-e2e.result == 'success' && '✅ Success' || needs.test-e2e.result == 'failure' && '❌ Failed' || needs.test-e2e.result == 'cancelled' && '⏚ī¸ Cancelled' || 'âŗ Skipped' }} | - | + + ## Overall Result + + ${{ (needs.build-and-package.result == 'success' && needs.test-central-server.result == 'success' && needs.test-security-server.result == 'success' && needs.test-e2e.result == 'success') && '🎉 All tests passed!' || '⚠ī¸ Some tests failed or were skipped' }} + + ## Quick Actions + + - 🔄 [Re-run failed jobs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) + - 🐛 [View detailed logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) + - 📊 [Build trends](https://github.com/${{ github.repository }}/actions) + EOF diff --git a/.github/workflows/cleanup-snapshots.yaml b/.github/workflows/cleanup-snapshots.yaml new file mode 100644 index 0000000000..8000bdc5ed --- /dev/null +++ b/.github/workflows/cleanup-snapshots.yaml @@ -0,0 +1,141 @@ +name: Cleanup Old Snapshot Images + +on: + # Run daily at 03:00 UTC (after daily base image builds at 02:00) + schedule: + - cron: '0 3 * * *' + + # Allow manual triggering + workflow_dispatch: + inputs: + dry_run: + description: 'Dry run (only show what would be deleted)' + required: false + type: boolean + default: true + cut_off: + description: 'Age cutoff (e.g., "7d", "2w")' + required: false + type: string + default: '7d' + +permissions: + packages: write + contents: read + +env: + # Set to 'true' to enable dry-run for scheduled runs (safe for testing) + # Set to 'false' to actually delete images on scheduled runs + SCHEDULED_DRY_RUN: 'true' + +jobs: + cleanup-snapshots: + name: Cleanup old snapshot images + runs-on: ubuntu-24.04 + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Fetch multi-arch digests to protect + id: fetch-digests + continue-on-error: true + run: | + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + + echo "🔍 Fetching multi-arch digests from recent SNAPSHOT images..." + + # Get list of all container packages + PACKAGES=$(curl -sL \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/orgs/${{ github.repository_owner }}/packages?package_type=container&per_page=100" \ + | jq -r '.[].name' || echo "") + + all_digests="" + + # For each package, get recent SNAPSHOT versions and extract their digests + for package in $PACKAGES; do + echo "đŸ“Ļ Checking package: ${package}" + + # Get package versions from last 14 days (to protect recent multi-arch images) + versions=$(curl -sL \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/${package}/versions?per_page=50" \ + | jq -r --arg cutoff "$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-14d +%Y-%m-%dT%H:%M:%SZ)" \ + '.[] | select(.created_at > $cutoff) | select(.metadata.container.tags[] | contains("SNAPSHOT")) | .name' \ + 2>/dev/null || echo "") + + for version in $versions; do + # Try to inspect the manifest to get platform-specific digests + digests=$(docker manifest inspect "ghcr.io/${REPO_LOWERCASE}/${package}:${version}" 2>/dev/null \ + | jq -r '.manifests[]?.digest' 2>/dev/null || echo "") + + if [[ -n "$digests" ]]; then + echo " ✅ Found digests for ${package}:${version}" + all_digests="$all_digests $digests" + fi + done + done + + # Clean up and format for output + clean_digests=$(echo "$all_digests" | tr ' ' ',' | sed 's/,\+/,/g' | sed 's/^,//' | sed 's/,$//') + + echo "skip-digests=${clean_digests}" >> $GITHUB_OUTPUT + echo "✅ Total digests to protect: $(echo "$clean_digests" | tr ',' '\n' | grep -c 'sha256' || echo 0)" + + - name: Cleanup old snapshot images + uses: snok/container-retention-policy@v3.0.1 + with: + account: ${{ github.repository_owner }} + token: ${{ secrets.GITHUB_TOKEN }} + image-names: "*" + image-tags: "*SNAPSHOT*" + cut-off: ${{ inputs.cut_off || '7d' }} + skip-shas: ${{ steps.fetch-digests.outputs.skip-digests }} + # For manual runs: use input (default true) + # For scheduled runs: use SCHEDULED_DRY_RUN env var + dry-run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || github.event_name == 'schedule' && env.SCHEDULED_DRY_RUN == 'true' }} + + - name: Generate summary + if: always() + run: | + # Determine dry-run status + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + DRY_RUN="${{ inputs.dry_run }}" + else + DRY_RUN="${{ env.SCHEDULED_DRY_RUN }}" + fi + + cat >> $GITHUB_STEP_SUMMARY << EOF + # 🧹 Container Image Cleanup Summary + + ## Configuration + - **Trigger**: ${{ github.event_name }} + - **Cut-off Age**: ${{ inputs.cut_off || '7d' }} + - **Dry Run**: ${DRY_RUN} + - **Target**: All images with SNAPSHOT tags + - **Images Cleaned**: Base images, service images, and all other SNAPSHOT tagged images + + ## How It Works + The workflow: + - 🔍 Queries GitHub API for all packages and recent SNAPSHOT versions + - 🛡ī¸ Extracts multi-arch digests from last 14 days to prevent partial deletion + - 🗑ī¸ Deletes SNAPSHOT images older than the cut-off age + - ✅ Keeps all non-SNAPSHOT images (releases) + - ✅ Protects images newer than cut-off age (from all branches) + + ## Next Steps + $(if [[ "${DRY_RUN}" == "true" ]]; then echo "💡 **This was a dry run**. No images were deleted."; else echo "✨ Images have been deleted. Check logs for details."; fi) + + $(if [[ "${{ github.event_name }}" == "schedule" && "${DRY_RUN}" == "true" ]]; then echo "ℹī¸ **Scheduled runs are in dry-run mode**. To enable actual deletion, change \`SCHEDULED_DRY_RUN: 'false'\` in the workflow file."; fi) + + --- + + 📝 **Note**: GitHub has a 30-day grace period before permanently deleting images. + You can [restore accidentally deleted images](https://docs.github.com/en/packages/learn-github-packages/deleting-and-restoring-a-package). + EOF diff --git a/.scripts/base-script.sh b/.scripts/base-script.sh new file mode 100755 index 0000000000..6f4f432195 --- /dev/null +++ b/.scripts/base-script.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# X-Road Base Script - Common utilities and functions +# This script should be sourced by other scripts to provide common functionality + +# Check if we're running with Bash +if [[ -z "${BASH_VERSION}" ]]; then + echo "Error: This script requires Bash. Please run with bash." + exit 1 +fi + +# Detect color support +isTextColoringEnabled=$(command -v tput >/dev/null && tput setaf 1 &>/dev/null && echo true || echo false) + +# Set XROAD_HOME to repository root using the script's location +if [ -z "$XROAD_HOME" ]; then + # Use the script's location instead of pwd to find repo root + # This script is now at .scripts/base-script.sh, so go up one level + XROAD_HOME=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..") + echo "XROAD_HOME is not set. Setting it to $XROAD_HOME" +fi + +# Color codes for enhanced logging (ANSI escape sequences) +if [ "$isTextColoringEnabled" = true ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + NC='\033[0m' # No Color +else + RED='' + GREEN='' + YELLOW='' + BLUE='' + NC='' +fi + +# Legacy function - kept for backward compatibility +errorExit() { + if $isTextColoringEnabled; then + echo "$(tput setaf 1)*** $*(tput sgr0)" 1>&2 + else + echo "*** $*" 1>&2 + fi + exit 1 +} + +# Enhanced logging functions with consistent formatting +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +# Key-value logging function +function log_kv() { + # Validate input parameters + if [ $# -ne 4 ]; then + echo "Usage: log_kv " + echo "Colors (0-7): black red green yellow blue magenta cyan white" + return 1 + fi + + local key="$1" + local value="$2" + local key_color="$3" + local value_color="$4" + + if [ "${isTextColoringEnabled}" = true ] && [ -t 1 ]; then + # Validate color numbers + if ! [[ "$key_color" =~ ^[0-7]$ ]] || ! [[ "$value_color" =~ ^[0-7]$ ]]; then + echo "Error: Colors must be numbers 0-7" + return 1 + fi + + # Print with colors + tput setaf "$key_color" + echo -n "$key" + tput sgr0 + echo -n ": " + tput setaf "$value_color" + echo "$value" + tput sgr0 + else + # Fallback to plain text if colors not supported + echo "$key: $value" + fi +} + +# Format duration in seconds to human-readable format +format_duration() { + local duration=$1 + local minutes=$((duration / 60)) + local seconds=$((duration % 60)) + printf "%dm %ds" $minutes $seconds +} + +# Read property from gradle.properties or similar files +read_gradle_property() { + local property_name="$1" + local property_file="$2" + + if [[ ! -f "$property_file" ]]; then + log_error "Properties file not found: $property_file" + return 1 + fi + + # Read property value, handling comments and empty lines + grep "^${property_name}=" "$property_file" | cut -d'=' -f2- | tr -d ' \t' +} + diff --git a/deployment/native-packages/docker/prepare-builder-image.sh b/deployment/native-packages/docker/prepare-builder-image.sh new file mode 100755 index 0000000000..d5b472e5ca --- /dev/null +++ b/deployment/native-packages/docker/prepare-builder-image.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# Unified script to prepare package builder images +# - Pull from registry if available +# - Build locally if pull fails +# - Push to registry after building +# Used by both build_packages.sh and CI workflows + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Configuration from environment variables +IMAGE_REGISTRY="${IMAGE_REGISTRY:-localhost:5555}" +IMAGE_TAG="${IMAGE_TAG:-latest}" +BUILD_PLATFORMS="${BUILD_PLATFORMS:-}" # Empty = host platform only +RELEASE="${1:-}" + +# Available releases +ALL_RELEASES=(deb-jammy deb-noble rpm-el8 rpm-el9) + +# Color codes +isTextColoringEnabled=$(command -v tput >/dev/null && tput setaf 1 &>/dev/null && echo true || echo false) + +log_info() { + if $isTextColoringEnabled; then + echo "$(tput setaf 4)[INFO]$(tput sgr0) $1" + else + echo "[INFO] $1" + fi +} + +log_warn() { + if $isTextColoringEnabled; then + echo "$(tput setaf 3)[WARN]$(tput sgr0) $1" + else + echo "[WARN] $1" + fi +} + +log_error() { + if $isTextColoringEnabled; then + echo "$(tput setaf 1)[ERROR]$(tput sgr0) $1" >&2 + else + echo "[ERROR] $1" >&2 + fi +} + +log_success() { + if $isTextColoringEnabled; then + echo "$(tput setaf 2)[SUCCESS]$(tput sgr0) $1" + else + echo "[SUCCESS] $1" + fi +} + +# Validate arguments +if [[ -z "$RELEASE" ]]; then + echo "Usage: $0 " >&2 + echo "" >&2 + echo "Examples:" >&2 + echo " $0 deb-noble # Prepare single image" >&2 + echo " $0 all # Prepare all images" >&2 + echo "" >&2 + echo "Available releases: ${ALL_RELEASES[*]}" >&2 + exit 1 +fi + +# Determine which releases to process +if [[ "$RELEASE" == "all" ]]; then + RELEASES_TO_PROCESS=("${ALL_RELEASES[@]}") +else + RELEASES_TO_PROCESS=("$RELEASE") +fi + +# Set up Docker Buildx if building for multiple platforms +if [[ -n "$BUILD_PLATFORMS" ]] && [[ "$BUILD_PLATFORMS" == *","* ]]; then + log_info "Setting up Docker Buildx for multi-platform build..." + if ! docker buildx inspect xroad-builder &>/dev/null; then + docker buildx create --name xroad-builder --driver docker-container --driver-opt network=host --bootstrap --use >/dev/null + else + docker buildx use xroad-builder + fi +fi + +# Process each release +FAILED_BUILDS=() +for release in "${RELEASES_TO_PROCESS[@]}"; do + IMAGE_NAME="${IMAGE_REGISTRY}/package-builder-${release}:${IMAGE_TAG}" + + log_info "Processing ${release}..." + log_info " Image: ${IMAGE_NAME}" + + # Try to pull from registry + log_warn "Attempting to pull from registry..." + if docker pull "$IMAGE_NAME" 2>/dev/null; then + log_success "Pulled ${release} from registry" + continue + fi + + # Build if pull failed + log_warn "Could not pull from registry, building..." + + if [[ ! -d "$SCRIPT_DIR/$release" ]]; then + log_error "Dockerfile directory not found: $SCRIPT_DIR/$release" + FAILED_BUILDS+=("$release") + continue + fi + + BUILD_START=$(date +%s) + + if [[ -n "$BUILD_PLATFORMS" ]] && [[ "$BUILD_PLATFORMS" == *","* ]]; then + # Multi-platform build with buildx + log_info " Platforms: ${BUILD_PLATFORMS}" + if docker buildx build \ + --platform "$BUILD_PLATFORMS" \ + --file "$SCRIPT_DIR/$release/Dockerfile" \ + --tag "$IMAGE_NAME" \ + --push \ + "$SCRIPT_DIR/$release/" >/dev/null; then + + BUILD_END=$(date +%s) + DURATION=$((BUILD_END - BUILD_START)) + log_success "Built and pushed ${release} (${DURATION}s)" + else + log_error "Failed to build ${release}" + FAILED_BUILDS+=("$release") + fi + else + # Single platform build (host architecture) + if [[ -n "$BUILD_PLATFORMS" ]]; then + log_info " Platform: ${BUILD_PLATFORMS}" + else + log_info " Platform: host" + fi + + if docker build -q -t "$IMAGE_NAME" "$SCRIPT_DIR/$release/"; then + log_info " Pushing to registry..." + if docker push "$IMAGE_NAME" >/dev/null 2>&1; then + BUILD_END=$(date +%s) + DURATION=$((BUILD_END - BUILD_START)) + log_success "Built and pushed ${release} (${DURATION}s)" + else + log_warn "Built ${release} but push failed (image available locally)" + fi + else + log_error "Failed to build ${release}" + FAILED_BUILDS+=("$release") + fi + fi + echo +done + +# Summary +if [[ ${#FAILED_BUILDS[@]} -gt 0 ]]; then + log_error "Failed: ${FAILED_BUILDS[*]}" + exit 1 +else + if [[ ${#RELEASES_TO_PROCESS[@]} -gt 1 ]]; then + log_success "✅ All ${#RELEASES_TO_PROCESS[@]} images prepared successfully" + fi +fi + diff --git a/deployment/security-server/base-images/Dockerfile-signer-baseline b/deployment/security-server/base-images/Dockerfile-signer-baseline deleted file mode 100644 index 77bbe36e5a..0000000000 --- a/deployment/security-server/base-images/Dockerfile-signer-baseline +++ /dev/null @@ -1,5 +0,0 @@ -ARG REGISTRY_URL=localhost:5555 -FROM ${REGISTRY_URL}/ss-baseline-runtime:latest - -ARG TARGETARCH -COPY --from=pkcs11driver ${TARGETARCH}/libpkcs11wrapper.so /usr/share/xroad/lib/libpkcs11wrapper.so diff --git a/deployment/security-server/base-images/build-base-images.sh b/deployment/security-server/base-images/build-base-images.sh deleted file mode 100755 index e524a10c0d..0000000000 --- a/deployment/security-server/base-images/build-base-images.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Set registry URL, default to localhost:5555 if not provided -REGISTRY_URL=${1:-localhost:5555} - -echo "Preparing LICENSE.txt and 3RD-PARTY-NOTICES.txt files" -rm -rf build/ -mkdir build -cp ../../../src/LICENSE.txt build/ -cp ../../../src/3RD-PARTY-NOTICES.txt build/ - -echo "Preparing buildx.." -if ! docker buildx inspect multiarch-builder &>/dev/null; then - docker buildx create --name multiarch-builder --driver docker-container --driver-opt network=host --bootstrap --use -else - docker buildx use multiarch-builder -fi - -echo "Building and pushing multi-arch images to $REGISTRY_URL .." -docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag "$REGISTRY_URL"/ss-baseline-runtime \ - --file Dockerfile-baseline \ - --push \ - . - -docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag "$REGISTRY_URL"/ss-baseline-backup-manager-runtime \ - --file Dockerfile-backup-manager-baseline \ - --build-arg REGISTRY_URL="$REGISTRY_URL" \ - --push \ - . - -docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag "$REGISTRY_URL"/ss-baseline-signer-runtime \ - --file Dockerfile-signer-baseline \ - --build-arg REGISTRY_URL="$REGISTRY_URL" \ - --build-context pkcs11driver=../../../src/libs/pkcs11wrapper \ - --push \ - . diff --git a/deployment/security-server/images/admin-service/Dockerfile b/deployment/security-server/images/admin-service/Dockerfile new file mode 100644 index 0000000000..a72d0ea04b --- /dev/null +++ b/deployment/security-server/images/admin-service/Dockerfile @@ -0,0 +1,13 @@ +ARG REGISTRY=localhost:5555 +ARG IMAGE_TAG=latest +ARG BASE_IMAGE=base-images/ss-baseline-runtime +FROM ${REGISTRY}/${BASE_IMAGE}:${IMAGE_TAG} + +COPY --from=build-artifacts --chown=xroad:xroad /proxy-ui-api-1.0.jar /opt/app/proxy-ui-api.jar +COPY --from=entrypoint --chmod=755 /entrypoint.sh /opt/app/entrypoint.sh + +WORKDIR /opt/app +USER xroad + +ENTRYPOINT ["/bin/bash", "/opt/app/entrypoint.sh"] + diff --git a/deployment/security-server/images/admin-service/entrypoint.sh b/deployment/security-server/images/admin-service/entrypoint.sh new file mode 100644 index 0000000000..0df1ab0c8c --- /dev/null +++ b/deployment/security-server/images/admin-service/entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +log() { echo "$(date --utc -Iseconds) INFO [entrypoint] $*"; } + +DEBUG_OPTS="" +if [[ "${DEBUG:-false}" == "true" ]]; then + log "DEBUG mode enabled - starting with JMX and debug agent" + JMX_COMMON_PARAMS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.local.only=false \ + -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost" + JMX_OPTS="$JMX_COMMON_PARAMS -Dcom.sun.management.jmxremote.port=9990 -Dcom.sun.management.jmxremote.rmi.port=9990" + + # DEBUG_SUSPEND controls whether the JVM suspends until debugger attaches (y/n, default: n) + DEBUG_SUSPEND="${DEBUG_SUSPEND:-n}" + log "Debug suspend mode: ${DEBUG_SUSPEND}" + DEBUG_AGENT="-Xdebug -agentlib:jdwp=transport=dt_socket,address=*:9999,server=y,suspend=${DEBUG_SUSPEND}" + DEBUG_OPTS="$DEBUG_AGENT $JMX_OPTS" +fi + +exec java \ + $DEBUG_OPTS \ + -Dspring.profiles.include=containerized \ + -jar /opt/app/proxy-ui-api.jar + diff --git a/deployment/security-server/base-images/Dockerfile-backup-manager-baseline b/deployment/security-server/images/base-images/backup-manager/Dockerfile similarity index 60% rename from deployment/security-server/base-images/Dockerfile-backup-manager-baseline rename to deployment/security-server/images/base-images/backup-manager/Dockerfile index b8f0601471..aebb708dfe 100644 --- a/deployment/security-server/base-images/Dockerfile-backup-manager-baseline +++ b/deployment/security-server/images/base-images/backup-manager/Dockerfile @@ -1,14 +1,18 @@ -ARG REGISTRY_URL=localhost:5555 - -FROM ${REGISTRY_URL}/ss-baseline-runtime:latest +# Base image for backup-manager with backup/restore tools +ARG REGISTRY=localhost:5555 +ARG IMAGE_TAG=latest +ARG BASE_IMAGE=base-images/ss-baseline-runtime +FROM ${REGISTRY}/${BASE_IMAGE}:${IMAGE_TAG} ENV DEBIAN_FRONTEND=noninteractive +# Install backup/restore dependencies RUN apt-get -qq update && \ apt-get -qq install --no-install-recommends curl postgresql-client && \ apt-get -qq clean && \ rm -rf /var/lib/apt/lists/* +# Install kubectl for Kubernetes backup operations RUN ARCH=$(dpkg --print-architecture) && \ KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) && \ curl -sSLo /tmp/kubectl https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl && \ @@ -17,5 +21,13 @@ RUN ARCH=$(dpkg --print-architecture) && \ install -m 0755 /tmp/kubectl /usr/local/bin/kubectl && \ rm /tmp/kubectl /tmp/kubectl.sha256 +# Create backup directory RUN mkdir -p /var/lib/xroad/backup && \ chown -R xroad:xroad /var/lib/xroad + +# Copy backup/restore scripts and configs +# files/ directory mirrors the container filesystem structure +COPY --chown=xroad:xroad files/ / + +RUN chmod 755 /usr/share/xroad/scripts/generate_gpg_keypair.sh + diff --git a/src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/autobackup.sh b/deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/autobackup.sh similarity index 100% rename from src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/autobackup.sh rename to deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/autobackup.sh diff --git a/src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/backup_openbao_db.sh b/deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/backup_openbao_db.sh similarity index 100% rename from src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/backup_openbao_db.sh rename to deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/backup_openbao_db.sh diff --git a/src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/backup_serverconf_db.sh b/deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/backup_serverconf_db.sh similarity index 100% rename from src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/backup_serverconf_db.sh rename to deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/backup_serverconf_db.sh diff --git a/src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/create_backup.sh b/deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/create_backup.sh similarity index 100% rename from src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/create_backup.sh rename to deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/create_backup.sh diff --git a/src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/restore_backup.sh b/deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/restore_backup.sh similarity index 100% rename from src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/restore_backup.sh rename to deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/restore_backup.sh diff --git a/src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/restore_openbao_db.sh b/deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/restore_openbao_db.sh similarity index 100% rename from src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/restore_openbao_db.sh rename to deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/restore_openbao_db.sh diff --git a/src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/restore_serverconf_db.sh b/deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/restore_serverconf_db.sh similarity index 100% rename from src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/containerised/restore_serverconf_db.sh rename to deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/containerised/restore_serverconf_db.sh diff --git a/src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/generate_gpg_keypair.sh b/deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/generate_gpg_keypair.sh similarity index 100% rename from src/service/backup-manager/backup-manager-application/src/main/jib/usr/share/xroad/scripts/generate_gpg_keypair.sh rename to deployment/security-server/images/base-images/backup-manager/files/usr/share/xroad/scripts/generate_gpg_keypair.sh diff --git a/deployment/security-server/base-images/Dockerfile-baseline b/deployment/security-server/images/base-images/baseline/Dockerfile similarity index 73% rename from deployment/security-server/base-images/Dockerfile-baseline rename to deployment/security-server/images/base-images/baseline/Dockerfile index 4d915ee4d8..f55989899f 100644 --- a/deployment/security-server/base-images/Dockerfile-baseline +++ b/deployment/security-server/images/base-images/baseline/Dockerfile @@ -1,3 +1,4 @@ +ARG REGISTRY=localhost:5555 FROM eclipse-temurin:21-jre RUN groupadd --system --gid 999 xroad && \ @@ -10,14 +11,13 @@ RUN groupadd --system --gid 999 xroad && \ RUN mkdir /var/log/xroad && \ mkdir /var/tmp/xroad && \ mkdir /var/cache/xroad && \ -# mkdir /opt/app/cache && \ mkdir -p /etc/xroad/ssl && \ mkdir -p /etc/xroad/globalconf && \ -# chown -R xroad:xroad /optapp && \ chown -R xroad:xroad /var/log/xroad && \ chown -R xroad:xroad /var/tmp/xroad && \ chown -R xroad:xroad /var/cache/xroad && \ chown -R xroad:xroad /etc/xroad -COPY build/LICENSE.txt /opt/app/LICENSE.txt -COPY build/3RD-PARTY-NOTICES.txt /opt/app/3RD-PARTY-NOTICES.txt \ No newline at end of file +# Copy license files from build context +COPY --from=build /LICENSE.txt /opt/app/LICENSE.txt +COPY --from=build /3RD-PARTY-NOTICES.txt /opt/app/3RD-PARTY-NOTICES.txt \ No newline at end of file diff --git a/deployment/security-server/images/base-images/signer/Dockerfile b/deployment/security-server/images/base-images/signer/Dockerfile new file mode 100644 index 0000000000..d0476f1d30 --- /dev/null +++ b/deployment/security-server/images/base-images/signer/Dockerfile @@ -0,0 +1,10 @@ +ARG REGISTRY=localhost:5555 +ARG IMAGE_TAG=latest +ARG BASE_IMAGE=base-images/ss-baseline-runtime + +FROM ${REGISTRY}/${BASE_IMAGE}:${IMAGE_TAG} + +# Copy PKCS#11 wrapper library for hardware security module support +ARG TARGETARCH +COPY --from=pkcs11driver ${TARGETARCH}/libpkcs11wrapper.so /usr/share/xroad/lib/libpkcs11wrapper.so + diff --git a/deployment/security-server/images/build-images.sh b/deployment/security-server/images/build-images.sh new file mode 100755 index 0000000000..5dc9711603 --- /dev/null +++ b/deployment/security-server/images/build-images.sh @@ -0,0 +1,294 @@ +#!/usr/bin/env bash + +set -e + +# Source base script for common utilities +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/../../../" && pwd)" +source "${ROOT_DIR}/.scripts/base-script.sh" + +# Additional paths +SRC_DIR="${ROOT_DIR}/src" +GRADLE_PROPERTIES="${SRC_DIR}/gradle.properties" +SERVICE_CONFIG_CSV="${SCRIPT_DIR}/service-config.csv" + +# Show help +show_help() { + cat < Service name from service-config.csv + (no arguments) Build all services (default) + +OPTIONS: + --platforms PLATFORMS Build platforms (e.g., linux/amd64,linux/arm64) + Default: host platform only + --push Push images to registry + Default: true in CI, false locally + --help Show this help + +ENVIRONMENT VARIABLES: + IMAGE_REGISTRY Docker registry URL (default: localhost:5555) + IMAGE_TAG Image tag to use (default: xroadVersion-xroadBuildType) + +EXAMPLES: + # Build all services for host platform (local dev) + ./build-images.sh all + + # Build specific services + ./build-images.sh proxy signer + + # Build and push multi-platform images (CI) + IMAGE_REGISTRY=ghcr.io/niis/x-road \\ + IMAGE_TAG=7.8.0-123 \\ + ./build-images.sh all --platforms linux/amd64,linux/arm64 --push + +EOF + exit 0 +} + +# Parse arguments +SERVICES=() +PLATFORMS="" +PUSH="" + +while [[ $# -gt 0 ]]; do + case $1 in + --help) + show_help + ;; + --platforms) + PLATFORMS="$2" + shift 2 + ;; + --push) + PUSH="true" + shift + ;; + -*) + log_error "Unknown option: $1" + show_help + ;; + *) + # Assume it's a service name - will validate against CSV later + SERVICES+=("$1") + shift + ;; + esac +done + +# If no services specified, read all from CSV +if [[ ${#SERVICES[@]} -eq 0 ]]; then + line_number=0 + while IFS= read -r line || [[ -n "$line" ]]; do + line_number=$((line_number + 1)) + [[ $line_number -eq 1 ]] && continue # Skip header + [[ -z "${line// /}" ]] && continue + IFS=',' read -r svc_name _ _ _ _ _ _ <<<"$line" + SERVICES+=("$svc_name") + done <"$SERVICE_CONFIG_CSV" +fi + +# Determine environment and defaults +REGISTRY="${IMAGE_REGISTRY:-localhost:5555}" + +if [[ -z "$PUSH" ]]; then + if [[ "$REGISTRY" == "localhost:"* ]]; then + PUSH="false" + else + PUSH="true" + fi +fi + +# Read version and build type from gradle.properties to construct image tags +if [[ ! -f "$GRADLE_PROPERTIES" ]]; then + log_error "gradle.properties not found at: $GRADLE_PROPERTIES" + exit 1 +fi + +XROAD_VERSION=$(read_gradle_property "xroadVersion" "$GRADLE_PROPERTIES") +XROAD_BUILD_TYPE=$(read_gradle_property "xroadBuildType" "$GRADLE_PROPERTIES") + +if [[ -z "$XROAD_VERSION" ]]; then + log_error "xroadVersion not found in gradle.properties" + exit 1 +fi + +if [[ -z "$XROAD_BUILD_TYPE" ]]; then + log_error "xroadBuildType not found in gradle.properties" + exit 1 +fi + +# Construct image tags from version and build type (can be overridden via environment) +IMAGE_TAG="${IMAGE_TAG:-${XROAD_VERSION}-${XROAD_BUILD_TYPE}}" + +log_info "=== X-Road Security Server Images Build ===" +log_info "Registry: $REGISTRY" +log_info "Image Tag: $IMAGE_TAG" +log_info "Platforms: ${PLATFORMS:-host platform}" +log_info "Push: $PUSH" +log_info "Services: ${SERVICES[*]}" +echo + +# Set up Docker Buildx +log_info "Setting up Docker Buildx..." +if [[ -n "$PLATFORMS" ]]; then + # Multi-platform build - use docker-container driver + if ! docker buildx inspect xroad-builder &>/dev/null; then + docker buildx create --name xroad-builder --driver docker-container --driver-opt network=host --bootstrap --use + else + docker buildx use xroad-builder + fi +else + # Local build - use default docker driver for better image sharing + docker buildx use default 2>/dev/null || docker buildx use orbstack +fi + +# Service definitions loaded from service-config.csv (defined at top of script) +get_service_def() { + local service=$1 + local line_number=0 + + # Find and return the requested service + while IFS= read -r line || [[ -n "$line" ]]; do + line_number=$((line_number + 1)) + + # Skip header row (first line) + [[ $line_number -eq 1 ]] && continue + [[ -z "${line// /}" ]] && continue + + # Parse CSV: service_name,dockerfile,gradle_path,image_name,build_artifact,base_image,extra_build_arg + IFS=',' read -r svc_name dockerfile gradle_path img_name build_artifact base_img extra_arg <<<"$line" + + if [[ "$svc_name" == "$service" ]]; then + # Return in expected order: image_name gradle_path dockerfile build_artifact base_image extra_build_arg + echo "$img_name $gradle_path $dockerfile $build_artifact $base_img $extra_arg" + return 0 + fi + done <"$SERVICE_CONFIG_CSV" + + # Service not found + return 1 +} + +BUILD_START_TIME=$(date +%s) + +# Prepare all build contexts upfront (unused contexts are silently ignored by Docker) +BUILD_DIR="${SCRIPT_DIR}/build" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +# License files (for base and DB images) +cp "${ROOT_DIR}/LICENSE" "$BUILD_DIR/LICENSE.txt" +cp "${ROOT_DIR}/src/3RD-PARTY-NOTICES.txt" "$BUILD_DIR/" + +# PKCS#11 wrapper (for signer) +PKCS11_DIR="${SRC_DIR}/libs/pkcs11wrapper" + +log_info "Build contexts prepared" + +# Build each service +for service in "${SERVICES[@]}"; do + service_def=$(get_service_def "$service") + if [[ -z "$service_def" ]]; then + log_error "Unknown service: $service" + continue + fi + + read -r image_name gradle_path dockerfile build_artifact base_image extra_build_arg <<<"$service_def" + + log_info "Building $service ($image_name)..." + build_start=$(date +%s) + + # Image name from CSV already includes full path (e.g., ss-proxy or base-images/ss-baseline-runtime) + full_image_name="${REGISTRY}/${image_name}" + + # Prepare build artifact context if specified in CSV + artifact_context_args=() + if [[ "$build_artifact" != "-" && -n "$build_artifact" ]]; then + artifact_path="${SRC_DIR}/${gradle_path}/${build_artifact}" + + # Validate artifact exists + if [[ ! -d "$artifact_path" ]]; then + log_error "Build artifact not found: $artifact_path" + log_error "Ensure the project is built first" + exit 1 + fi + + # Mount artifact directory directly as build-artifacts context + artifact_context_args+=(--build-context "build-artifacts=${artifact_path}") + fi + + # Build context directory is always the dockerfile's directory + context_dir="${SCRIPT_DIR}/$(dirname "$dockerfile")" + + # Build command with ALL contexts + build_cmd=( + docker buildx build + --file "${SCRIPT_DIR}/${dockerfile}" + --build-arg "REGISTRY=${REGISTRY}" + --build-arg "IMAGE_TAG=${IMAGE_TAG}" + --build-context "build=${BUILD_DIR}" + --build-context "license=${BUILD_DIR}" + --build-context "pkcs11driver=${PKCS11_DIR}" + --build-context "entrypoint=${context_dir}" + ) + + # Add BASE_IMAGE build arg if specified in CSV (pass as-is, Dockerfile constructs full path) + if [[ "$base_image" != "-" && -n "$base_image" ]]; then + build_cmd+=(--build-arg "BASE_IMAGE=${base_image}") + fi + + # Add extra build arg if specified in CSV (e.g., CHANGELOG_FILE=messagelog-changelog.xml) + if [[ "$extra_build_arg" != "-" && -n "$extra_build_arg" ]]; then + build_cmd+=(--build-arg "$extra_build_arg") + fi + + # Add platform if specified + [[ -n "$PLATFORMS" ]] && build_cmd+=(--platform "$PLATFORMS") + + # Add service-specific artifact context (from CSV config) + [[ ${#artifact_context_args[@]} -gt 0 ]] && build_cmd+=("${artifact_context_args[@]}") + + # Add tag and push/load + build_cmd+=( + --tag "${full_image_name}:${IMAGE_TAG}" + ) + + if [[ "$PUSH" == "true" ]]; then + build_cmd+=(--push) + else + build_cmd+=(--load) + fi + + # Add build context directory + build_cmd+=("$context_dir") + + # Execute build + if "${build_cmd[@]}"; then + build_end=$(date +%s) + build_duration=$((build_end - build_start)) + log_success "Built $service in $(format_duration $build_duration)" + else + log_error "Failed to build $service" + exit 1 + fi +done + +# Summary +BUILD_END_TIME=$(date +%s) +TOTAL_BUILD_TIME=$((BUILD_END_TIME - BUILD_START_TIME)) + +echo +log_success "=== Build Complete ===" +log_success "Total time: $(format_duration $TOTAL_BUILD_TIME)" +log_success "Built ${#SERVICES[@]} service(s)" +if [[ "$PUSH" == "true" ]]; then + log_success "Images pushed to $REGISTRY" +else + log_success "Images loaded to local Docker" +fi diff --git a/deployment/security-server/images/init-db/Dockerfile b/deployment/security-server/images/init-db/Dockerfile new file mode 100644 index 0000000000..951806cebe --- /dev/null +++ b/deployment/security-server/images/init-db/Dockerfile @@ -0,0 +1,18 @@ +FROM liquibase:4.33.0 + +# Build arg for the changelog file name (e.g., "serverconf-changelog.xml") +ARG CHANGELOG_FILE + +# Liquibase configuration +ENV LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog/${CHANGELOG_FILE}" \ + LIQUIBASE_COMMAND_DRIVER="org.postgresql.Driver" + +COPY --from=build-artifacts --chown=liquibase:liquibase / /liquibase/changelog/ + +COPY --from=license --chown=liquibase:liquibase /LICENSE.txt /liquibase/changelog/LICENSE.txt +COPY --from=license --chown=liquibase:liquibase /3RD-PARTY-NOTICES.txt /liquibase/changelog/3RD-PARTY-NOTICES.txt + +USER liquibase +WORKDIR /liquibase + +CMD ["--log-level=debug", "update"] diff --git a/deployment/security-server/images/quarkus/Dockerfile b/deployment/security-server/images/quarkus/Dockerfile new file mode 100644 index 0000000000..99fed66c25 --- /dev/null +++ b/deployment/security-server/images/quarkus/Dockerfile @@ -0,0 +1,14 @@ +ARG REGISTRY=localhost:5555 +ARG IMAGE_TAG=latest +ARG BASE_IMAGE=base-images/ss-baseline-runtime + +FROM ${REGISTRY}/${BASE_IMAGE}:${IMAGE_TAG} + +COPY --from=build-artifacts --chown=xroad:xroad / /opt/app/ +COPY --from=entrypoint --chmod=755 /entrypoint.sh /opt/app/entrypoint.sh + +WORKDIR /opt/app +USER xroad + +ENTRYPOINT ["/bin/sh", "/opt/app/entrypoint.sh"] + diff --git a/deployment/security-server/images/quarkus/entrypoint.sh b/deployment/security-server/images/quarkus/entrypoint.sh new file mode 100644 index 0000000000..2215b5246a --- /dev/null +++ b/deployment/security-server/images/quarkus/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Common entrypoint for Quarkus services + +log() { echo "$(date --utc -Iseconds) INFO [entrypoint] $*"; } + +DEBUG_OPTS="" +if [ "${DEBUG:-false}" = "true" ]; then + log "DEBUG mode enabled - starting with JMX and debug agent" + JMX_COMMON_PARAMS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.local.only=false \ + -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost" + JMX_OPTS="$JMX_COMMON_PARAMS -Dcom.sun.management.jmxremote.port=9990 -Dcom.sun.management.jmxremote.rmi.port=9990" + + # DEBUG_SUSPEND controls whether the JVM suspends until debugger attaches (y/n, default: n) + DEBUG_SUSPEND="${DEBUG_SUSPEND:-n}" + log "Debug suspend mode: ${DEBUG_SUSPEND}" + DEBUG_AGENT="-Xdebug -agentlib:jdwp=transport=dt_socket,address=*:9999,server=y,suspend=${DEBUG_SUSPEND}" + DEBUG_OPTS="$DEBUG_AGENT $JMX_OPTS" +fi + +exec java \ + -Djava.util.logging.manager=org.jboss.logmanager.LogManager \ + -Dquarkus.profile=containerized \ + $DEBUG_OPTS \ + -jar /opt/app/quarkus-run.jar diff --git a/deployment/security-server/images/service-config.csv b/deployment/security-server/images/service-config.csv new file mode 100644 index 0000000000..8bfd0e5983 --- /dev/null +++ b/deployment/security-server/images/service-config.csv @@ -0,0 +1,15 @@ +service_name,dockerfile,gradle_path,image_name,build_artifact,base_image,extra_build_arg +baseline,base-images/baseline/Dockerfile,-,base-images/ss-baseline-runtime,-,-,- +backup-manager-base,base-images/backup-manager/Dockerfile,-,base-images/ss-baseline-backup-manager-runtime,-,base-images/ss-baseline-runtime,- +signer-base,base-images/signer/Dockerfile,-,base-images/ss-baseline-signer-runtime,-,base-images/ss-baseline-runtime,- +proxy,quarkus/Dockerfile,service/proxy/proxy-application,ss-proxy,build/quarkus-app,base-images/ss-baseline-runtime,- +monitor,quarkus/Dockerfile,service/monitor/monitor-application,ss-monitor,build/quarkus-app,base-images/ss-baseline-runtime,- +op-monitor,quarkus/Dockerfile,service/op-monitor/op-monitor-application,ss-op-monitor,build/quarkus-app,base-images/ss-baseline-runtime,- +configuration-client,quarkus/Dockerfile,service/configuration-client/configuration-client-application,ss-configuration-client,build/quarkus-app,base-images/ss-baseline-runtime,- +message-log-archiver,quarkus/Dockerfile,service/message-log-archiver/message-log-archiver-application,ss-message-log-archiver,build/quarkus-app,base-images/ss-baseline-runtime,- +signer,quarkus/Dockerfile,service/signer/signer-application,ss-signer,build/quarkus-app,base-images/ss-baseline-signer-runtime,- +backup-manager,quarkus/Dockerfile,service/backup-manager/backup-manager-application,ss-backup-manager,build/quarkus-app,base-images/ss-baseline-backup-manager-runtime,- +admin-service,admin-service/Dockerfile,security-server/admin-service/application,ss-proxy-ui-api,build/libs,base-images/ss-baseline-runtime,- +db-messagelog-init,init-db/Dockerfile,addons/messagelog/messagelog-db,ss-db-messagelog-init,build/resources/main/liquibase,-,CHANGELOG_FILE=messagelog-changelog.xml +db-opmonitor-init,init-db/Dockerfile,service/op-monitor/op-monitor-db,ss-db-opmonitor-init,build/resources/main/liquibase,-,CHANGELOG_FILE=op-monitor-changelog.xml +db-serverconf-init,init-db/Dockerfile,security-server/admin-service/infra-jpa,ss-db-serverconf-init,build/resources/main/liquibase,-,CHANGELOG_FILE=serverconf-changelog.xml diff --git a/development/.scripts/base-script.sh b/development/.scripts/base-script.sh deleted file mode 100755 index 333d9c8b76..0000000000 --- a/development/.scripts/base-script.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -isTextColoringEnabled=$(command -v tput >/dev/null && tput setaf 1 &>/dev/null && echo true || echo false) - -if [ -z "$XROAD_HOME" ]; then - XROAD_HOME=$(realpath "$(pwd)/../..") - echo "XROAD_HOME is not set. Setting it to $XROAD_HOME" -fi - -errorExit() { - if $isTextColoringEnabled; then - echo "$(tput setaf 1)*** $*(tput sgr0)" 1>&2 - else - echo "*** $*" 1>&2 - fi - exit 1 -} - -log_warn() { - if $isTextColoringEnabled; then - echo "$(tput setaf 3)*** $*$(tput sgr0)" - else - echo "*** $*" - fi -} - -log_info() { - if $isTextColoringEnabled; then - echo "$(tput setaf 2)$*$(tput sgr0)" - else - echo "$*" - fi -} - -function log_kv() { - # Validate input parameters - if [ $# -ne 4 ]; then - echo "Usage: log_kv " - echo "Colors (0-7): black red green yellow blue magenta cyan white" - return 1 - fi - - local key="$1" - local value="$2" - local key_color="$3" - local value_color="$4" - - if [ "${isTextColoringEnabled}" = true ] && [ -t 1 ]; then - # Validate color numbers - if ! [[ "$key_color" =~ ^[0-7]$ ]] || ! [[ "$value_color" =~ ^[0-7]$ ]]; then - echo "Error: Colors must be numbers 0-7" - return 1 - fi - - # Print with colors - tput setaf "$key_color" - echo -n "$key" - tput sgr0 - echo -n ": " - tput setaf "$value_color" - echo "$value" - tput sgr0 - else - # Fallback to plain text if colors not supported - echo "$key: $value" - fi -} \ No newline at end of file diff --git a/development/docker/central-server/Dockerfile b/development/docker/central-server/Dockerfile index 53a596cd0b..fe0de580b4 100644 --- a/development/docker/central-server/Dockerfile +++ b/development/docker/central-server/Dockerfile @@ -28,7 +28,7 @@ ARG COMPONENT=main # Add X-Road dependencies repo ADD ["https://artifactory.niis.org/api/gpg/key/public","/tmp/deps-repokey.gpg"] RUN echo "deb https://artifactory.niis.org/xroad-dependencies-deb xroad external" >/etc/apt/sources.list.d/xroad-deps.list \ - && && apt-key add '/tmp/deps-repokey.gpg' \ + && apt-key add '/tmp/deps-repokey.gpg' ADD ["$REPO_KEY","/tmp/repokey.gpg"] ADD ["${REPO}/dists/${DIST}/Release","/tmp/Release"] @@ -54,7 +54,7 @@ ADD ["https://artifactory.niis.org/api/gpg/key/public","/tmp/deps-repokey.gpg"] RUN echo "deb https://artifactory.niis.org/xroad-dependencies-deb xroad external" >/etc/apt/sources.list.d/xroad-deps.list \ && apt-key add '/tmp/deps-repokey.gpg' -RUN --mount=type=bind,source=build/packages,target=/tmp/packages \ +RUN --mount=type=bind,from=packages,target=/tmp/packages \ cp -r /tmp/packages /tmp/repo \ && cd /tmp/repo && dpkg-scanpackages -m . > Packages \ && echo "deb [trusted=yes] file:/tmp/repo /" >/etc/apt/sources.list.d/xroad.list \ diff --git a/development/docker/central-server/build-cs-dev-image.sh b/development/docker/central-server/build-cs-dev-image.sh new file mode 100755 index 0000000000..5d2ed29727 --- /dev/null +++ b/development/docker/central-server/build-cs-dev-image.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash + +# Central Server Development Image Build Script +# Supports both local development and CI environments +# Usage: ./build-cs-dev-image.sh [options] +# +# Options: +# --registry REGISTRY Registry URL (default: localhost:5555 for local, ghcr.io for CI) +# --environment ENV Environment: local|ci (default: local) +# --version VERSION Image version tag (default: read from gradle.properties) +# --packages-path PATH Path to Ubuntu 24.04 packages (default: auto-detect) +# --platforms PLATFORMS Build platforms (default: linux/amd64,linux/arm64) +# --help Show this help + +set -e + +# Source base script for common utilities +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../../../.scripts/base-script.sh" + +# Default configuration +GRADLE_PROPERTIES="${XROAD_HOME}/src/gradle.properties" + +# Default values +ENVIRONMENT="local" +REGISTRY="" +VERSION="" +PACKAGES_PATH="" +PLATFORMS="" # Empty by default = build for host platform only +BUILD_START_TIME=$(date +%s) + +# Help function +show_help() { + cat </dev/null; then + docker buildx create --name xroad-builder --driver docker-container --driver-opt network=host --bootstrap --use +else + docker buildx use xroad-builder +fi + +# Build the image +log_info "Building central-server-dev image..." +build_start=$(date +%s) + +build_cmd=( + docker buildx build + --file "$SCRIPT_DIR/Dockerfile" + --build-arg PACKAGE_SOURCE=internal + --build-context "packages=$PACKAGES_PATH" +) + +# Add platform flag only if specified +if [[ -n "$PLATFORMS" ]]; then + build_cmd+=(--platform "$PLATFORMS") +fi + +build_cmd+=( + --tag "${IMAGE_NAME}:${VERSION}" + --push + "$SCRIPT_DIR" +) + +# Execute build + +if "${build_cmd[@]}"; then + build_end=$(date +%s) + build_duration=$((build_end - build_start)) + + log_success "Built central-server-dev in $(format_duration $build_duration)" +else + log_error "Failed to build central-server-dev image" + exit 1 +fi + +# Calculate total build time +BUILD_END_TIME=$(date +%s) +TOTAL_BUILD_TIME=$((BUILD_END_TIME - BUILD_START_TIME)) + +# Output summary +log_success "=== Build Complete ===" +log_success "Total time: $(format_duration $TOTAL_BUILD_TIME)" +log_success "Image: ${IMAGE_NAME}:${VERSION}" +echo +log_success "✅ Central Server dev image built successfully!" diff --git a/development/docker/central-server/init_context.sh b/development/docker/central-server/init_context.sh deleted file mode 100755 index 6ab00308da..0000000000 --- a/development/docker/central-server/init_context.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -rm -rf ./build -mkdir -p ./build/packages \ No newline at end of file diff --git a/development/docker/security-server/.env b/development/docker/security-server/.env deleted file mode 100644 index bb8863cea2..0000000000 --- a/development/docker/security-server/.env +++ /dev/null @@ -1,13 +0,0 @@ -CONFIGURATION_CLIENT_IMG=localhost:5555/ss-configuration-client:latest -PROXY_IMG=localhost:5555/ss-proxy:latest -PROXY_UI_IMG=localhost:5555/ss-proxy-ui-api:latest -SIGNER_IMG=localhost:5555/ss-signer:latest -BACKUP_MANAGER_IMG=localhost:5555/ss-backup-manager:latest -MONITOR_IMG=localhost:5555/ss-monitor:latest -OP_MONITOR_IMG=localhost:5555/ss-op-monitor:latest - -SERVERCONF_INIT_IMG=localhost:5555/ss-db-serverconf-init:latest -MESSAGELOG_INIT_IMG=localhost:5555/ss-db-messagelog-init:latest -OP_MONITOR_INIT_IMG=localhost:5555/ss-db-opmonitor-init:latest - -XROAD_SECRET_STORE_TOKEN=bao-root-token \ No newline at end of file diff --git a/development/docker/security-server/.gitignore b/development/docker/security-server/.gitignore new file mode 100644 index 0000000000..f275bf3c59 --- /dev/null +++ b/development/docker/security-server/.gitignore @@ -0,0 +1,3 @@ +# CI-generated environment file - contains image registry configuration +.env + diff --git a/development/docker/security-server/compose.yaml b/development/docker/security-server/compose.yaml index 1eb78981fb..9387e8b98f 100644 --- a/development/docker/security-server/compose.yaml +++ b/development/docker/security-server/compose.yaml @@ -142,6 +142,7 @@ services: image: ${CONFIGURATION_CLIENT_IMG} pull_policy: always environment: + - DEBUG=true - XROAD_HOST=configuration-client - XROAD_SECRET_STORE_HOST=openbao - XROAD_SECRET_STORE_TOKEN=${XROAD_SECRET_STORE_TOKEN} @@ -174,6 +175,7 @@ services: image: ${MONITOR_IMG} pull_policy: always environment: + - DEBUG=true - XROAD_HOST=monitor - XROAD_SECRET_STORE_HOST=openbao - XROAD_SECRET_STORE_TOKEN=${XROAD_SECRET_STORE_TOKEN} @@ -201,6 +203,7 @@ services: image: ${SIGNER_IMG} pull_policy: always environment: + - DEBUG=true - XROAD_HOST=signer - XROAD_SECRET_STORE_HOST=openbao - XROAD_SECRET_STORE_TOKEN=${XROAD_SECRET_STORE_TOKEN} @@ -232,6 +235,7 @@ services: image: ${PROXY_IMG} pull_policy: always environment: + - DEBUG=true - XROAD_HOST=proxy - XROAD_SECRET_STORE_HOST=openbao - XROAD_SECRET_STORE_TOKEN=${XROAD_SECRET_STORE_TOKEN} @@ -269,6 +273,7 @@ services: image: ${PROXY_UI_IMG} pull_policy: always environment: + - DEBUG=true - XROAD_HOST=ui - XROAD_SECRET_STORE_HOST=openbao - XROAD_SECRET_STORE_TOKEN=${XROAD_SECRET_STORE_TOKEN} @@ -311,6 +316,7 @@ services: image: ${BACKUP_MANAGER_IMG} pull_policy: always environment: + - DEBUG=true - XROAD_HOST=backup-manager - XROAD_SECRET_STORE_HOST=openbao - XROAD_SECRET_STORE_TOKEN=${XROAD_SECRET_STORE_TOKEN} diff --git a/development/docker/security-server/env.template b/development/docker/security-server/env.template new file mode 100644 index 0000000000..ade9328d7a --- /dev/null +++ b/development/docker/security-server/env.template @@ -0,0 +1,22 @@ +# Template for .env file used by docker-compose +# Copy this to .env and customize for your environment +# +# For local development, use localhost:5555 (requires local registry) +# For CI, this file is auto-generated with GHCR URLs + +# Secret store token for OpenBao +XROAD_SECRET_STORE_TOKEN=root-token + +# Security Server component images +# Format: REGISTRY/image-name:tag +SERVERCONF_INIT_IMG=localhost:5555/ss-db-serverconf-init:7.8.0-SNAPSHOT +MESSAGELOG_INIT_IMG=localhost:5555/ss-db-messagelog-init:7.8.0-SNAPSHOT +OP_MONITOR_INIT_IMG=localhost:5555/ss-db-opmonitor-init:7.8.0-SNAPSHOT +CONFIGURATION_CLIENT_IMG=localhost:5555/ss-configuration-client:7.8.0-SNAPSHOT +MONITOR_IMG=localhost:5555/ss-monitor:7.8.0-SNAPSHOT +SIGNER_IMG=localhost:5555/ss-signer:7.8.0-SNAPSHOT +PROXY_IMG=localhost:5555/ss-proxy:7.8.0-SNAPSHOT +PROXY_UI_IMG=localhost:5555/ss-proxy-ui-api:7.8.0-SNAPSHOT +BACKUP_MANAGER_IMG=localhost:5555/ss-backup-manager:7.8.0-SNAPSHOT +OP_MONITOR_IMG=localhost:5555/ss-op-monitor:7.8.0-SNAPSHOT + diff --git a/development/docker/security-server/signer-with-hsm/Dockerfile b/development/docker/security-server/signer-with-hsm/Dockerfile index 7615e584d1..5e6e0a9590 100644 --- a/development/docker/security-server/signer-with-hsm/Dockerfile +++ b/development/docker/security-server/signer-with-hsm/Dockerfile @@ -1,5 +1,6 @@ -ARG REGISTRY_URL=localhost:5555 -FROM ${REGISTRY_URL}/ss-signer +ARG BASE_SIGNER_IMAGE + +FROM ${BASE_SIGNER_IMAGE} USER root diff --git a/development/k8s/build-images.sh b/development/k8s/build-images.sh index 333d884a42..25d1ec12da 100755 --- a/development/k8s/build-images.sh +++ b/development/k8s/build-images.sh @@ -1,14 +1,10 @@ #!/bin/bash -source ./../.scripts/base-script.sh +source "${BASH_SOURCE%/*}/../../.scripts/base-script.sh" REGISTRY_URL=${1:-localhost:5555} - -echo "Building baseline images.." -(cd $XROAD_HOME/deployment/security-server/base-images && ./build-base-images.sh $REGISTRY_URL) - -echo "Building Security Server images.." -(cd $XROAD_HOME/src && ./gradlew assemble -PxroadImageRegistry=$REGISTRY_URL -PbuildImages=true) +echo "Building Security Server images (including base).." +(cd $XROAD_HOME/deployment/security-server/images && IMAGE_REGISTRY=$REGISTRY_URL ./build-images.sh all --push) echo "Building OpenBao init-runner" docker buildx build \ diff --git a/development/k8s/init-ss2.sh b/development/k8s/init-ss2.sh index 1d46c5d14a..9b6e8c6096 100755 --- a/development/k8s/init-ss2.sh +++ b/development/k8s/init-ss2.sh @@ -1,6 +1,6 @@ #!/bin/bash -source ./../.scripts/base-script.sh +source "${BASH_SOURCE%/*}/../../.scripts/base-script.sh" hurl --insecure \ --variables-file $XROAD_HOME/development/hurl/scenarios/k8-ss2/vars.env \ diff --git a/development/native-lxd-stack/shutdown-env.sh b/development/native-lxd-stack/shutdown-env.sh index aa9abcb042..23d4b92814 100755 --- a/development/native-lxd-stack/shutdown-env.sh +++ b/development/native-lxd-stack/shutdown-env.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -source ./../.scripts/base-script.sh +source "${BASH_SOURCE%/*}/../../.scripts/base-script.sh" if limactl list | grep -q '^xroad-lxd'; then # Check current status diff --git a/development/native-lxd-stack/start-env.sh b/development/native-lxd-stack/start-env.sh index 14542d723a..b4316591c6 100755 --- a/development/native-lxd-stack/start-env.sh +++ b/development/native-lxd-stack/start-env.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -source ./../.scripts/base-script.sh +source "${BASH_SOURCE%/*}/../../.scripts/base-script.sh" RECREATE=false SKIP_COMPILE=false diff --git a/development/native-lxd-stack/stop-all-lxd-containers.sh b/development/native-lxd-stack/stop-all-lxd-containers.sh index 3db90aa619..42439db061 100755 --- a/development/native-lxd-stack/stop-all-lxd-containers.sh +++ b/development/native-lxd-stack/stop-all-lxd-containers.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -source ./../.scripts/base-script.sh +source "${BASH_SOURCE%/*}/../../.scripts/base-script.sh" echo "Listing all LXD containers..." lxc list diff --git a/development/native-lxd-stack/stop-env.sh b/development/native-lxd-stack/stop-env.sh index aa9abcb042..23d4b92814 100644 --- a/development/native-lxd-stack/stop-env.sh +++ b/development/native-lxd-stack/stop-env.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -source ./../.scripts/base-script.sh +source "${BASH_SOURCE%/*}/../../.scripts/base-script.sh" if limactl list | grep -q '^xroad-lxd'; then # Check current status diff --git a/src/addons/messagelog/messagelog-db/build.gradle.kts b/src/addons/messagelog/messagelog-db/build.gradle.kts index 5c52856014..c7659a47b7 100644 --- a/src/addons/messagelog/messagelog-db/build.gradle.kts +++ b/src/addons/messagelog/messagelog-db/build.gradle.kts @@ -1,6 +1,5 @@ plugins { id("xroad.java-conventions") - id("xroad.jib-conventions") alias(libs.plugins.jandex) } @@ -15,51 +14,3 @@ dependencies { implementation(libs.slf4j.api) implementation(libs.mapstruct) } - -val libsRef = project.extensions.getByType().named("libs") - -tasks.named("jib") { - dependsOn("prepareLicenseFiles") -} - -jib { - from { - image = "liquibase:${libsRef.findVersion("liquibase").get()}" - } - to { - image = "${project.property("xroadImageRegistry")}/ss-db-messagelog-init" - tags = setOf("latest") - } - container { - entrypoint = listOf("/liquibase/docker-entrypoint.sh") - workingDirectory = "/liquibase" - user = "liquibase" - args = listOf( - "--log-level=debug", - "update" - ) - environment = mapOf( - "LIQUIBASE_COMMAND_CHANGELOG_FILE" to "changelog/messagelog-changelog.xml", - "LIQUIBASE_COMMAND_DRIVER" to "org.postgresql.Driver", - ) - - } - extraDirectories { - paths { - path { - setFrom(project.file("src/main/resources/liquibase/").toPath()) - into = "/liquibase/changelog" - } - path { - setFrom(layout.buildDirectory.dir("jib-extra/license")) - into = "/liquibase/changelog" - } - } - } -} - -tasks { - named("assemble") { - dependsOn("jib") - } -} diff --git a/src/buildSrc/build.gradle.kts b/src/buildSrc/build.gradle.kts index dd9de996d1..23954e7c28 100644 --- a/src/buildSrc/build.gradle.kts +++ b/src/buildSrc/build.gradle.kts @@ -16,7 +16,6 @@ dependencies { implementation(libs.licenseGradlePlugin) implementation(libs.archUnitGradlePlugin) implementation(libs.protobufGradlePlugin) - implementation(libs.jibGradlePlugin) implementation(libs.quarkusGradlePlugin) testImplementation(libs.junit.jupiterEngine) } diff --git a/src/buildSrc/src/main/kotlin/xroad.jib-conventions.gradle.kts b/src/buildSrc/src/main/kotlin/xroad.jib-conventions.gradle.kts deleted file mode 100644 index 2fd4b1782d..0000000000 --- a/src/buildSrc/src/main/kotlin/xroad.jib-conventions.gradle.kts +++ /dev/null @@ -1,50 +0,0 @@ -plugins { - java - id("com.google.cloud.tools.jib") -} - -jib { - setAllowInsecureRegistries(true) - from { - image = "${project.property("xroadImageRegistry")}/ss-baseline-runtime:latest" - platforms { - platform { - architecture = "arm64" - os = "linux" - } - platform { - architecture = "amd64" - os = "linux" - } - } - } - - to { - tags = setOf("latest") - } - - container { - appRoot = "/opt/app" - creationTime.set("USE_CURRENT_TIMESTAMP") - user = "xroad" - } -} - -val buildImages: String = project.findProperty("buildImages")?.toString() ?: "false" -if (!buildImages.toBoolean()) { - tasks.withType().configureEach { - enabled = false - } -} else { - tasks.withType().configureEach { - doFirst { - System.setProperty("jib.console", "plain") - } - } -} - -tasks.register("prepareLicenseFiles") { - into(layout.buildDirectory.dir("jib-extra/license")) - from(rootProject.file("LICENSE.txt")) - from(rootProject.file("3RD-PARTY-NOTICES.txt")) -} diff --git a/src/buildSrc/src/main/kotlin/xroad.quarkus-application-conventions.gradle.kts b/src/buildSrc/src/main/kotlin/xroad.quarkus-application-conventions.gradle.kts index 947c9b1292..6c03f45d7b 100644 --- a/src/buildSrc/src/main/kotlin/xroad.quarkus-application-conventions.gradle.kts +++ b/src/buildSrc/src/main/kotlin/xroad.quarkus-application-conventions.gradle.kts @@ -1,5 +1,4 @@ plugins { - id("xroad.jib-conventions") id("io.quarkus") } @@ -9,48 +8,10 @@ dependencies { implementation(platform(libs.findLibrary("quarkus-bom").get())) } -val buildEnv: String = project.findProperty("buildEnv")?.toString() ?: "dev" //TODO default to prod later -val buildImages: String = project.findProperty("buildImages")?.toString() ?: "false" - quarkus { quarkusBuildProperties.putAll( buildMap { put("quarkus.package.jar.type", "fast-jar") - put("quarkus.container-image.build", buildImages) - put("quarkus.container-image.registry", "${project.property("xroadImageRegistry")}") - put("quarkus.container-image.insecure", "true") - put("quarkus.container-image.push", buildImages) - put("quarkus.container-image.builder", "jib") - put("quarkus.jib.working-directory", "/opt/app") - - put("quarkus.jib.base-jvm-image", "${project.property("xroadImageRegistry")}/ss-baseline-runtime:latest") - put("quarkus.jib.platforms", "linux/amd64,linux/arm64/v8") - put("quarkus.jib.user", "xroad") - - val jvmArgs = mutableListOf( - "-Dquarkus.profile=containerized", - "-Djava.library.path=/usr/share/xroad/lib" - ) - - if (buildEnv == "dev") { - // Add debug parameters - each as a separate list item - jvmArgs.add("-Xdebug") - jvmArgs.add("-agentlib:jdwp=transport=dt_socket,address=*:9999,server=y,suspend=n") - - // Add JMX parameters - each as a separate list item - jvmArgs.add("-Dcom.sun.management.jmxremote=true") - jvmArgs.add("-Dcom.sun.management.jmxremote.local.only=false") - jvmArgs.add("-Dcom.sun.management.jmxremote.authenticate=false") - jvmArgs.add("-Dcom.sun.management.jmxremote.ssl=false") - jvmArgs.add("-Djava.rmi.server.hostname=localhost") - jvmArgs.add("-Dcom.sun.management.jmxremote.port=9990") - jvmArgs.add("-Dcom.sun.management.jmxremote.rmi.port=9990") - } - -// Set the JVM arguments - jvmArgs.forEachIndexed { index, arg -> - put("quarkus.jib.jvm-additional-arguments[$index]", arg) - } } ) } diff --git a/src/build_packages.sh b/src/build_packages.sh index 710aaa5979..60ca3186af 100755 --- a/src/build_packages.sh +++ b/src/build_packages.sh @@ -95,15 +95,19 @@ buildLocally() { buildBuilderImage() { local release="$1" test -n "$release" || errorExit "Error, release not specified." - warn "Preparing $release image..." - docker build -q -t "xroad-$release" "$XROAD/../deployment/native-packages/docker/$release" || errorExit "Error building $release image." + + "$XROAD/../deployment/native-packages/docker/prepare-builder-image.sh" "$release" || errorExit "Error preparing $release image." } runInBuilderImage() { local release="$1" shift test -n "$release" || errorExit "Error, release not specified." - local image="xroad-$release" + + # Use same image name as prepare-builder-image.sh + local registry="${IMAGE_REGISTRY:-localhost:5555}" + local tag="${IMAGE_TAG:-latest}" + local image="${registry}/package-builder-${release}:${tag}" OPTS=("--rm" "-v" "$XROAD/..:/workspace" "-u" "$(id -u):$(id -g)" "-e" "HOME=/workspace/deployment/native-packages") # check if running attached to terminal diff --git a/src/central-server/admin-service/int-test/build.gradle.kts b/src/central-server/admin-service/int-test/build.gradle.kts index 5494dd52b7..81f626b39e 100644 --- a/src/central-server/admin-service/int-test/build.gradle.kts +++ b/src/central-server/admin-service/int-test/build.gradle.kts @@ -65,10 +65,6 @@ tasks.register("intTest") { shouldRunAfter(tasks.test) } -tasks.named("check") { - dependsOn(tasks.named("intTest")) -} - archUnit { setSkip(true) } diff --git a/src/central-server/management-service/int-test/build.gradle.kts b/src/central-server/management-service/int-test/build.gradle.kts index cb7fc1c65f..b84c9157a1 100644 --- a/src/central-server/management-service/int-test/build.gradle.kts +++ b/src/central-server/management-service/int-test/build.gradle.kts @@ -51,10 +51,6 @@ tasks.register("intTest") { shouldRunAfter(tasks.test) } -tasks.named("check") { - dependsOn(tasks.named("intTest")) -} - archUnit { setSkip(true) } diff --git a/src/gradle.properties b/src/gradle.properties index 3ffe125cc6..4cbe037087 100644 --- a/src/gradle.properties +++ b/src/gradle.properties @@ -13,6 +13,3 @@ sonarqubeOrganization= #Frontend defaults frontendNodeVersion=22.17.1 - -#Image creation defaults -xroadImageRegistry=localhost:5555 diff --git a/src/gradle/libs.versions.toml b/src/gradle/libs.versions.toml index 2253c3769c..2375d04cde 100644 --- a/src/gradle/libs.versions.toml +++ b/src/gradle/libs.versions.toml @@ -61,8 +61,6 @@ quarkus-rest = { group = "io.quarkus", name = "quarkus-rest", version.ref = "qua quarkus-kubernetes = { group = "io.quarkus", name = "quarkus-kubernetes", version.ref = "quarkus" } quarkus-kubernetesConfig = { group = "io.quarkus", name = "quarkus-kubernetes-config", version.ref = "quarkus" } quarkus-health = { group = "io.quarkus", name = "quarkus-smallrye-health", version.ref = "quarkus" } -quarkus-containerImage-docker = { group = "io.quarkus", name = "quarkus-container-image-docker", version.ref = "quarkus" } #TODO remove one once finalized which to use -quarkus-containerImage-jib = { group = "io.quarkus", name = "quarkus-container-image-jib", version.ref = "quarkus" } quarkus-config-yaml = { group = "io.quarkus", name = "quarkus-config-yaml", version.ref = "quarkus" } quarkus-scheduler = { group = "io.quarkus", name = "quarkus-scheduler", version.ref = "quarkus" } quarkus-quartz = { group = "io.quarkus", name = "quarkus-quartz", version.ref = "quarkus" } @@ -203,7 +201,6 @@ archUnit-plugin-core = { module = "com.societegenerale.commons:arch-unit-build-p licenseGradlePlugin = { module = "gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin", version = "0.16.1" } archUnitGradlePlugin = { module = "com.societegenerale.commons:arch-unit-gradle-plugin", version.ref = "archUnitPlugin" } protobufGradlePlugin = { module = "com.google.protobuf:com.google.protobuf.gradle.plugin", version.ref = "protobufPlugin" } -jibGradlePlugin = { module = "com.google.cloud.tools.jib:com.google.cloud.tools.jib.gradle.plugin", version = "3.4.5" } quarkusGradlePlugin = { module = "io.quarkus:gradle-application-plugin", version.ref = "quarkus" } [bundles] @@ -219,7 +216,6 @@ quarkus-core = [ quarkus-containerized = [ "quarkus-kubernetes", "quarkus-health", - "quarkus-containerImage-jib" ] [plugins] diff --git a/src/security-server/admin-service/application/build.gradle.kts b/src/security-server/admin-service/application/build.gradle.kts index aadafb1734..8881910c6c 100644 --- a/src/security-server/admin-service/application/build.gradle.kts +++ b/src/security-server/admin-service/application/build.gradle.kts @@ -1,6 +1,5 @@ plugins { id("xroad.java-conventions") - id("xroad.jib-conventions") alias(libs.plugins.springBoot) } @@ -123,37 +122,6 @@ tasks.register("copyDeps") { tasks.assemble { dependsOn(tasks.named("copyDeps")) - dependsOn(tasks.named("jib")) -} - -tasks.named("jib") { - dependsOn("bootJar") -} - -jib { - from { - image = "${project.property("xroadImageRegistry")}/ss-baseline-runtime" - } - to { - image = "${project.property("xroadImageRegistry")}/ss-proxy-ui-api" - tags = setOf("latest") - } - container { - entrypoint = listOf("/bin/bash", "/opt/app/entrypoint.sh") - workingDirectory = "/opt/app" - user = "xroad" - } - extraDirectories { - paths { - path { - setFrom(project.file("src/main/jib").toPath()) - into = "/" - } - } - permissions = mapOf( - "/usr/share/xroad/scripts/generate_gpg_keypair.sh" to "755" - ) - } } tasks.test { diff --git a/src/security-server/admin-service/application/src/main/jib/opt/app/entrypoint.sh b/src/security-server/admin-service/application/src/main/jib/opt/app/entrypoint.sh deleted file mode 100755 index 58e2f3f384..0000000000 --- a/src/security-server/admin-service/application/src/main/jib/opt/app/entrypoint.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -log() { echo "$(date --utc -Iseconds) INFO [entrypoint] $*"; } - -XROAD_SCRIPT_LOCATION=/opt/app/scripts - -JMX_COMMON_PARAMS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.local.only=false \ - -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost" -JMX_OPTS="$JMX_COMMON_PARAMS -Dcom.sun.management.jmxremote.port=9990 -Dcom.sun.management.jmxremote.rmi.port=9990" - -exec java \ - -Xdebug -agentlib:jdwp=transport=dt_socket,address=*:9999,server=y,suspend=n \ - $JMX_OPTS \ - -Dspring.profiles.include=containerized \ - -cp @jib-classpath-file @jib-main-class-file diff --git a/src/security-server/admin-service/infra-jpa/build.gradle.kts b/src/security-server/admin-service/infra-jpa/build.gradle.kts index ba96b63676..aeec0d8d14 100644 --- a/src/security-server/admin-service/infra-jpa/build.gradle.kts +++ b/src/security-server/admin-service/infra-jpa/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - id("xroad.jib-conventions") + java } sourceSets { @@ -22,51 +22,3 @@ tasks.register("changelogJar") { artifacts { add("changelogJar", tasks.named("changelogJar")) } - -val libs = project.extensions.getByType().named("libs") - -tasks.named("jib") { - dependsOn("prepareLicenseFiles") -} - -jib { - from { - image = "liquibase:${libs.findVersion("liquibase").get()}" - } - to { - image = "${project.property("xroadImageRegistry")}/ss-db-serverconf-init" - tags = setOf("latest") - } - container { - entrypoint = listOf("/liquibase/docker-entrypoint.sh") - workingDirectory = "/liquibase" - user = "liquibase" - args = listOf( - "--log-level=debug", - "update" - ) - environment = mapOf( - "LIQUIBASE_COMMAND_CHANGELOG_FILE" to "changelog/serverconf-changelog.xml", - "LIQUIBASE_COMMAND_DRIVER" to "org.postgresql.Driver", - ) - - } - extraDirectories { - paths { - path { - setFrom(project.file("build/resources/main/liquibase/").toPath()) - into = "/liquibase/changelog" - } - path { - setFrom(layout.buildDirectory.dir("jib-extra/license")) - into = "/liquibase/changelog" - } - } - } -} - -tasks { - named("assemble") { - dependsOn("jib") - } -} diff --git a/src/security-server/admin-service/int-test/build.gradle.kts b/src/security-server/admin-service/int-test/build.gradle.kts index b6f501c2d9..f6c71e4120 100644 --- a/src/security-server/admin-service/int-test/build.gradle.kts +++ b/src/security-server/admin-service/int-test/build.gradle.kts @@ -59,10 +59,6 @@ tasks.register("intTest") { shouldRunAfter(tasks.test) } -tasks.named("check") { - dependsOn(tasks.named("intTest")) -} - archUnit { setSkip(true) } diff --git a/src/security-server/e2e-test/build.gradle.kts b/src/security-server/e2e-test/build.gradle.kts index 868dbaef9a..12781692ce 100644 --- a/src/security-server/e2e-test/build.gradle.kts +++ b/src/security-server/e2e-test/build.gradle.kts @@ -27,9 +27,6 @@ tasks.register("e2eTest") { if (project.hasProperty("e2eTestCSImage")) { systemTestArgs += "-Dtest-automation.custom.cs-image=${project.property("e2eTestCSImage")}" } - if (project.hasProperty("e2eTestSSImage")) { - systemTestArgs += "-Dtest-automation.custom.ss-image=${project.property("e2eTestSSImage")}" - } if (project.hasProperty("e2eTestCAImage")) { systemTestArgs += "-Dtest-automation.custom.cs-image=${project.property("e2eTestCAImage")}" } diff --git a/src/security-server/e2e-test/src/intTest/resources/compose.ss-hsm.e2e.yaml b/src/security-server/e2e-test/src/intTest/resources/compose.ss-hsm.e2e.yaml index af4f112740..142bc6946d 100644 --- a/src/security-server/e2e-test/src/intTest/resources/compose.ss-hsm.e2e.yaml +++ b/src/security-server/e2e-test/src/intTest/resources/compose.ss-hsm.e2e.yaml @@ -1,8 +1,8 @@ services: signer: - image: "" + image: ss-signer-with-hsm:latest # this is the name of image that is built from context below build: - context: ../../../development/docker/security-server/signer-with-hsm - args: - - REGISTRY_URL=localhost:5555 - dockerfile: Dockerfile + context: ../../../development/docker/security-server/signer-with-hsm + args: + - BASE_SIGNER_IMAGE=${SIGNER_IMG:-localhost:5555/ss-signer:latest} + dockerfile: Dockerfile diff --git a/src/security-server/system-test/src/intTest/resources/compose.systemtest.yaml b/src/security-server/system-test/src/intTest/resources/compose.systemtest.yaml index 2662ce0cad..64e5c24755 100644 --- a/src/security-server/system-test/src/intTest/resources/compose.systemtest.yaml +++ b/src/security-server/system-test/src/intTest/resources/compose.systemtest.yaml @@ -62,7 +62,7 @@ services: start_period: 5s interval: 5s retries: 40 - test: [ "CMD", "sh", "-c", "pg_isready -U postgres -h localhost" ] + test: [ "CMD", "sh", "-c", "pg_isready -U opmonitor -d op-monitor -h localhost" ] deploy: resources: reservations: @@ -95,6 +95,7 @@ services: image: ${OP_MONITOR_IMG} pull_policy: always environment: + - QUARKUS_LOG_LEVEL=INFO - XROAD_HOST=op-monitor - XROAD_OP_MONITOR_LISTEN_ADDRESS=0.0.0.0 - XROAD_OP_MONITOR_SCHEME=https @@ -103,12 +104,12 @@ services: - XROAD_SECRET_STORE_TOKEN=${XROAD_SECRET_STORE_TOKEN} - XROAD_SECRET_STORE_SCHEME=http - XROAD_COMMON_RPC_CHANNEL_CONFIGURATION_CLIENT_HOST=configuration-client - - xroad.db.op-monitor.hibernate.connection.url=jdbc:postgresql://db-opmonitor:5432/op-monitor - - xroad.db.op-monitor.hibernate.connection.driver_class=org.postgresql.Driver - - xroad.db.op-monitor.hibernate.connection.username=opmonitor - - xroad.db.op-monitor.hibernate.connection.password=secret + - XROAD_DB_OP_MONITOR_HIBERNATE_CONNECTION_URL=jdbc:postgresql://db-opmonitor:5432/op-monitor + - XROAD_DB_OP_MONITOR_HIBERNATE_CONNECTION_DRIVER_CLASS=org.postgresql.Driver + - XROAD_DB_OP_MONITOR_HIBERNATE_CONNECTION_USERNAME=opmonitor + - XROAD_DB_OP_MONITOR_HIBERNATE_CONNECTION_PASSWORD=secret # we shorten here messages processing from 1 min to 1 second - - xroad.op-monitor.records-available-timestamp-offset-seconds=1 + - XROAD_OP_MONITOR_RECORDS_AVAILABLE_TIMESTAMP_OFFSET_SECONDS=1 healthcheck: interval: 5s retries: 40 diff --git a/src/service/backup-manager/backup-manager-application/build.gradle.kts b/src/service/backup-manager/backup-manager-application/build.gradle.kts index 89d51c58ae..8f4ae5b95e 100644 --- a/src/service/backup-manager/backup-manager-application/build.gradle.kts +++ b/src/service/backup-manager/backup-manager-application/build.gradle.kts @@ -4,15 +4,6 @@ plugins { id("xroad.quarkus-application-conventions") } -quarkus { - quarkusBuildProperties.putAll( - buildMap { - put("quarkus.jib.base-jvm-image", "${project.property("xroadImageRegistry")}/ss-baseline-backup-manager-runtime:latest") - put("quarkus.container-image.image", "${project.property("xroadImageRegistry")}/ss-backup-manager") - } - ) -} - dependencies { implementation(platform(libs.quarkus.bom)) diff --git a/src/service/configuration-client/configuration-client-application/build.gradle.kts b/src/service/configuration-client/configuration-client-application/build.gradle.kts index 8db3f2e673..92bf8168e1 100644 --- a/src/service/configuration-client/configuration-client-application/build.gradle.kts +++ b/src/service/configuration-client/configuration-client-application/build.gradle.kts @@ -4,13 +4,6 @@ plugins { id("maven-publish") } -quarkus { - quarkusBuildProperties.putAll( - buildMap { - put("quarkus.container-image.image", "${project.property("xroadImageRegistry")}/ss-configuration-client") - } - ) -} publishing { publications { diff --git a/src/service/message-log-archiver/message-log-archiver-application/build.gradle.kts b/src/service/message-log-archiver/message-log-archiver-application/build.gradle.kts index dd7b88c7b4..3d0a8cf569 100644 --- a/src/service/message-log-archiver/message-log-archiver-application/build.gradle.kts +++ b/src/service/message-log-archiver/message-log-archiver-application/build.gradle.kts @@ -3,14 +3,6 @@ plugins { id("xroad.quarkus-application-conventions") } -quarkus { - quarkusBuildProperties.putAll( - buildMap { - put("quarkus.container-image.image", "${project.property("xroadImageRegistry")}/ss-message-log-archiver") - } - ) -} - dependencies { implementation(platform(libs.quarkus.bom)) diff --git a/src/service/monitor/monitor-application/build.gradle.kts b/src/service/monitor/monitor-application/build.gradle.kts index 2e80215c9a..b74c9ed19b 100644 --- a/src/service/monitor/monitor-application/build.gradle.kts +++ b/src/service/monitor/monitor-application/build.gradle.kts @@ -3,14 +3,6 @@ plugins { id("xroad.quarkus-application-conventions") } -quarkus { - quarkusBuildProperties.putAll( - buildMap { - put("quarkus.container-image.image", "${project.property("xroadImageRegistry")}/ss-monitor") - } - ) -} - dependencies { implementation(platform(libs.quarkus.bom)) implementation(project(":lib:bootstrap-quarkus")) diff --git a/src/service/op-monitor/op-monitor-application/build.gradle.kts b/src/service/op-monitor/op-monitor-application/build.gradle.kts index 00afea6157..edecccc9b1 100644 --- a/src/service/op-monitor/op-monitor-application/build.gradle.kts +++ b/src/service/op-monitor/op-monitor-application/build.gradle.kts @@ -3,14 +3,6 @@ plugins { id("xroad.quarkus-application-conventions") } -quarkus { - quarkusBuildProperties.putAll( - buildMap { - put("quarkus.container-image.image", "${project.property("xroadImageRegistry")}/ss-op-monitor") - } - ) -} - dependencies { implementation(platform(libs.quarkus.bom)) diff --git a/src/service/op-monitor/op-monitor-core/src/main/java/org/niis/xroad/opmonitor/core/config/OpMonitorDbProperties.java b/src/service/op-monitor/op-monitor-core/src/main/java/org/niis/xroad/opmonitor/core/config/OpMonitorDbProperties.java index cf19427261..2eb709f640 100644 --- a/src/service/op-monitor/op-monitor-core/src/main/java/org/niis/xroad/opmonitor/core/config/OpMonitorDbProperties.java +++ b/src/service/op-monitor/op-monitor-core/src/main/java/org/niis/xroad/opmonitor/core/config/OpMonitorDbProperties.java @@ -28,13 +28,15 @@ package org.niis.xroad.opmonitor.core.config; import io.smallrye.config.ConfigMapping; -import io.smallrye.config.WithName; +import io.smallrye.config.WithParentName; +import io.smallrye.config.WithUnnamedKey; import java.util.Map; -@ConfigMapping(prefix = "xroad.db.op-monitor") +@ConfigMapping(prefix = "xroad.db.op-monitor.hibernate") public interface OpMonitorDbProperties { - @WithName("hibernate") + @WithParentName + @WithUnnamedKey Map hibernate(); } diff --git a/src/service/op-monitor/op-monitor-db/build.gradle.kts b/src/service/op-monitor/op-monitor-db/build.gradle.kts index c2366b8c52..07daa8f27e 100644 --- a/src/service/op-monitor/op-monitor-db/build.gradle.kts +++ b/src/service/op-monitor/op-monitor-db/build.gradle.kts @@ -1,8 +1,5 @@ plugins { - id("xroad.jib-conventions") -} - -dependencies { + java } configurations { @@ -17,51 +14,3 @@ tasks.register("changelogJar") { artifacts { add("changelogJar", tasks.named("changelogJar")) } - -val libs = project.extensions.getByType().named("libs") - -tasks.named("jib") { - dependsOn("prepareLicenseFiles") -} - -jib { - from { - image = "liquibase:${libs.findVersion("liquibase").get()}" - } - to { - image = "${project.property("xroadImageRegistry")}/ss-db-opmonitor-init" - tags = setOf("latest") - } - container { - entrypoint = listOf("/liquibase/docker-entrypoint.sh") - workingDirectory = "/liquibase" - user = "liquibase" - args = listOf( - "--log-level=debug", - "update" - ) - environment = mapOf( - "LIQUIBASE_COMMAND_CHANGELOG_FILE" to "changelog/op-monitor-changelog.xml", - "LIQUIBASE_COMMAND_DRIVER" to "org.postgresql.Driver", - ) - - } - extraDirectories { - paths { - path { - setFrom(project.file("src/main/resources/liquibase/").toPath()) - into = "/liquibase/changelog" - } - path { - setFrom(layout.buildDirectory.dir("jib-extra/license")) - into = "/liquibase/changelog" - } - } - } -} - -tasks { - named("assemble") { - dependsOn("jib") - } -} diff --git a/src/service/op-monitor/op-monitor-int-test/build.gradle.kts b/src/service/op-monitor/op-monitor-int-test/build.gradle.kts index 3e34bd145b..b8f0e321a3 100644 --- a/src/service/op-monitor/op-monitor-int-test/build.gradle.kts +++ b/src/service/op-monitor/op-monitor-int-test/build.gradle.kts @@ -26,7 +26,8 @@ tasks.register("intTest") { val intTestArgs = mutableListOf() - intTestArgs += "-Dtest-automation.custom.image-name=${project.property("xroadImageRegistry")}/ss-op-monitor:latest" + // TODO: make it configurable when fixing int tests + intTestArgs += "-Dtest-automation.custom.image-name=localhost:5555/ss-op-monitor:latest" if (project.hasProperty("intTestProfilesInclude")) { intTestArgs += "-Dspring.profiles.include=${project.property("intTestProfilesInclude")}" @@ -46,10 +47,6 @@ tasks.register("intTest") { } } -tasks.named("check") { - dependsOn(tasks.named("intTest")) -} - archUnit { setSkip(true) } diff --git a/src/service/op-monitor/op-monitor-int-test/src/intTest/resources/compose.intTest.yaml b/src/service/op-monitor/op-monitor-int-test/src/intTest/resources/compose.intTest.yaml index 6c8abaefae..92a66bb532 100644 --- a/src/service/op-monitor/op-monitor-int-test/src/intTest/resources/compose.intTest.yaml +++ b/src/service/op-monitor/op-monitor-int-test/src/intTest/resources/compose.intTest.yaml @@ -7,10 +7,10 @@ services: - XROAD_COMMON_RPC_USE_TLS=false - XROAD_COMMON_GLOBAL_CONF_SOURCE=FILESYSTEM - XROAD_OP_MONITOR_RPC_LISTEN_ADDRESS=0.0.0.0 - - xroad.db.op-monitor.hibernate.connection.url=jdbc:postgresql://db-opmonitor:5432/op-monitor - - xroad.db.op-monitor.hibernate.connection.username=opmonitor - - xroad.db.op-monitor.hibernate.connection.password=secret - - xroad.db.op-monitor.hibernate.connection.driver_class=org.postgresql.Driver + - XROAD_DB_OP_MONITOR_HIBERNATE_CONNECTION_URL=jdbc:postgresql://db-opmonitor:5432/op-monitor + - XROAD_DB_OP_MONITOR_HIBERNATE_CONNECTION_USERNAME=opmonitor + - XROAD_DB_OP_MONITOR_HIBERNATE_CONNECTION_PASSWORD=secret + - XROAD_DB_OP_MONITOR_HIBERNATE_CONNECTION_DRIVER_CLASS=org.postgresql.Driver ports: - "0:9999" # debug port healthcheck: diff --git a/src/service/proxy/proxy-application/build.gradle.kts b/src/service/proxy/proxy-application/build.gradle.kts index c265dd86ec..c25d1b4fcf 100644 --- a/src/service/proxy/proxy-application/build.gradle.kts +++ b/src/service/proxy/proxy-application/build.gradle.kts @@ -4,14 +4,6 @@ plugins { id("xroad.quarkus-application-conventions") } -quarkus { - quarkusBuildProperties.putAll( - buildMap { - put("quarkus.container-image.image", "${project.property("xroadImageRegistry")}/ss-proxy") - put("quarkus.jib.jvm-entrypoint", "/bin/sh,/opt/app/entrypoint.sh") - } - ) -} configurations.named("runtimeClasspath") { exclude(group = "xml-apis", module = "xml-apis") diff --git a/src/service/proxy/proxy-application/src/main/jib/opt/app/entrypoint.sh b/src/service/proxy/proxy-application/src/main/jib/opt/app/entrypoint.sh deleted file mode 100644 index 94a3fbde1e..0000000000 --- a/src/service/proxy/proxy-application/src/main/jib/opt/app/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -log() { echo "$(date --utc -Iseconds) INFO [entrypoint] $*"; } - -XROAD_SCRIPT_LOCATION=/opt/app/scripts - -JMX_COMMON_PARAMS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.local.only=false \ - -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost" -JMX_OPTS="$JMX_COMMON_PARAMS -Dcom.sun.management.jmxremote.port=9990 -Dcom.sun.management.jmxremote.rmi.port=9990" - -exec java \ - -Xdebug -agentlib:jdwp=transport=dt_socket,address=*:9999,server=y,suspend=n \ - -Djava.util.logging.manager=org.jboss.logmanager.LogManager \ - -Dquarkus.profile=containerized \ - $JMX_OPTS \ - -jar /opt/app/quarkus-run.jar diff --git a/src/service/signer/signer-application/build.gradle.kts b/src/service/signer/signer-application/build.gradle.kts index 4464bb8b18..8e7d36f407 100644 --- a/src/service/signer/signer-application/build.gradle.kts +++ b/src/service/signer/signer-application/build.gradle.kts @@ -3,14 +3,6 @@ plugins { id("xroad.quarkus-application-conventions") } -quarkus { - quarkusBuildProperties.putAll( - buildMap { - put("quarkus.container-image.image", "${project.property("xroadImageRegistry")}/ss-signer") - put("quarkus.jib.base-jvm-image", "${project.property("xroadImageRegistry")}/ss-baseline-signer-runtime:latest") - } - ) -} dependencies { implementation(platform(libs.quarkus.bom)) diff --git a/src/service/signer/signer-int-test/build.gradle.kts b/src/service/signer/signer-int-test/build.gradle.kts index 1749832fae..573c0e3cfb 100644 --- a/src/service/signer/signer-int-test/build.gradle.kts +++ b/src/service/signer/signer-int-test/build.gradle.kts @@ -48,10 +48,6 @@ tasks.register("intTest") { } } -tasks.named("check") { - dependsOn(tasks.named("intTest")) -} - archUnit { setSkip(true) } diff --git a/src/service/signer/signer-int-test/src/intTest/resources/compose.intTest.yaml b/src/service/signer/signer-int-test/src/intTest/resources/compose.intTest.yaml index d2ed0afb21..87697d8556 100644 --- a/src/service/signer/signer-int-test/src/intTest/resources/compose.intTest.yaml +++ b/src/service/signer/signer-int-test/src/intTest/resources/compose.intTest.yaml @@ -109,7 +109,7 @@ services: start_period: 5s interval: 5s retries: 40 - test: [ "CMD", "pg_isready", "-U", "postgres", "-h", "localhost" ] + test: [ "CMD", "pg_isready", "-U", "serverconf", "-h", "localhost" ] deploy: resources: reservations: