feat: migrate to traefik

.devcontainer/devcontainer.json

@@ -5,13 +5,18 @@
     },
     "workspaceFolder": "${localWorkspaceFolder}",
     "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
+    "initializeCommand": "mkdir -p ${localEnv:HOME}/.local/spark/http-proxy/certs",
     "mounts": [
         "source=${localEnv:HOME}/.config/gcloud,target=/home/vscode/.config/gcloud,type=bind",
+        "source=${localEnv:HOME}/.local/spark/http-proxy/certs,target=${localEnv:HOME}/.local/spark/http-proxy/certs,type=bind",
         "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind",
         "source=sparkci-bashhistory,target=/commandhistory,type=volume"
     ],
+    "containerEnv": {
+        "LOCAL_HOME": "${localEnv:HOME}"
+    },
     "runArgs": [
-        "--add-host=host.docker.internal:host-gateway",
+        "--add-host=host.docker.internal:host-gateway"
     ],
     "features": {
         "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
@@ -58,4 +63,4 @@
             }
         }
     }
-}
+}

.github/copilot-instructions.md

@@ -0,0 +1,45 @@
+---
+applyTo: "*.go"
+---
+
+You are a Golang and Devops expert. Your task is to write high-quality, idiomatic Go code that adheres to best practices in software development.
+Focus on clarity, maintainability, and performance. Do not write code that is overly complex or difficult to understand.
+Do not try to reach a result if it requires writing code that is not idiomatic or that does not follow best practices.
+When in doubt, do not write code. Instead, ask for clarification or more information about the requirements.
+
+Please adhere to the principles of "Effective Go" to ensure the code is clear, idiomatic, and maintainable. Pay close attention to the following conventions:
+
+**1. Formatting:**
+All code should be formatted with `gofmt`. Ensure that the output is consistent with the standard Go formatting.
+
+**2. Naming Conventions:**
+* **Packages:** Use short, concise, and all-lowercase names. Avoid camelCase or snake_case.
+* **Getters:** Method names for getters should not have a "Get" prefix. For a variable `owner`, the getter should be named `owner()`, not `Owner()`.
+* **Interfaces:** Interfaces that are satisfied by a single method should be named by the method name plus the "-er" suffix (e.g., `Reader`, `Writer`).
+
+**3. Control Structures:**
+* **For Loops:** Utilize the generalized `for` loop. Use the `for...range` clause for iterating over arrays, slices, strings, and maps.
+* **Switch Statements:** Use the flexible and powerful `switch` statement. Remember that `switch` cases in Go do not fall through by default.
+
+**4. Data Handling:**
+* **Allocation:**
+    * Use `new(T)` to allocate memory for a new zero-value of type T and return a pointer to it.
+    * Use `make(T, args)` to create slices, maps, and channels, and return an initialized (not zeroed) value of type T.
+* **Composite Literals:** Use composite literals to create instances of structs, arrays, slices, and maps. Omit the type name from the elements of the literal when it is redundant.
+
+**5. General Principles:**
+* Write idiomatic Go code. Do not simply translate code from other languages like C++, Java, or Python.
+* Strive for simplicity and clarity.
+* Keep comments concise and informative, explaining what the code *does*, not *how* it does it.
+
+---
+applyTo: "*.*"
+---
+
+When you want to create new documentation files, follow these steps:
+
+1. Create a new Markdown file in the appropriate directory, that is docs/.
+2. Use the existing documentation files as a reference for structure and formatting.
+3. Include relevant information, code snippets, and examples to illustrate the topic.
+4. Follow the established naming conventions and directory structure.
+5. Update any necessary configuration files (e.g., `mkdocs.yml`, `README.md`) to include the new documentation.

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Docker
+name: CI/CD Pipeline
 
 on:
   push:
@@ -10,7 +10,7 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  BASE_IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   test:
@@ -25,22 +25,206 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build image for testing
+      - name: Build services image for testing
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./Dockerfile
+          file: ./build/Dockerfile
           push: false
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          tags: ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}-services:test
+          cache-from: type=gha,scope=services
+          cache-to: type=gha,mode=max,scope=services
+
+      - name: Build Traefik image for testing
+        uses: docker/build-push-action@v5
+        with:
+          context: ./build/traefik
+          file: ./build/traefik/Dockerfile
+          push: false
+          tags: ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}-traefik:test
+          cache-from: type=gha,scope=traefik
+          cache-to: type=gha,mode=max,scope=traefik
+
+      - name: Build Prometheus image for testing
+        uses: docker/build-push-action@v5
+        with:
+          context: ./build/prometheus
+          file: ./build/prometheus/Dockerfile
+          push: false
+          tags: ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}-prometheus:test
+          cache-from: type=gha,scope=prometheus
+          cache-to: type=gha,mode=max,scope=prometheus
+
+      - name: Build Grafana image for testing
+        uses: docker/build-push-action@v5
+        with:
+          context: ./build/grafana
+          file: ./build/grafana/Dockerfile
+          push: false
+          tags: ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}-grafana:test
+          cache-from: type=gha,scope=grafana
+          cache-to: type=gha,mode=max,scope=grafana
+
+      - name: Install test dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y dnsutils curl
+
+      - name: Run integration tests
+        run: |
+          chmod +x test/test.sh
+          make test
+
+      - name: Validate compose configuration
+        run: |
+          docker compose config
+          docker compose build --dry-run || echo "Dry run not supported, skipping"
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-logs-${{ github.run_id }}
+          path: |
+            test-logs/
+          retention-days: 7
+
+  security-scan:
+    if: github.ref != 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: "fs"
+          scan-ref: "."
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: "trivy-results.sarif"
+
+  dev-deploy:
+    runs-on: ubuntu-latest
+    # Deploy dev images from any branch that's not main (for testing)
+    if: github.ref != 'refs/heads/main' && github.event_name == 'push'
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: traefik
+            context: ./build/traefik
+            dockerfile: ./build/traefik/Dockerfile
+            image_name: ${{ github.repository }}-traefik
+            cache_scope: traefik
+          - name: services
+            context: .
+            dockerfile: ./build/Dockerfile
+            image_name: ${{ github.repository }}-services
+            cache_scope: services
+          - name: prometheus
+            context: ./build/prometheus
+            dockerfile: ./build/prometheus/Dockerfile
+            image_name: ${{ github.repository }}-prometheus
+            cache_scope: prometheus
+          - name: grafana
+            context: ./build/grafana
+            dockerfile: ./build/grafana/Dockerfile
+            image_name: ${{ github.repository }}-grafana
+            cache_scope: grafana
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract branch name
+        id: branch
+        run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+
+      - name: Generate dev tags
+        id: dev-tags
+        run: |
+          BRANCH_NAME="${{ steps.branch.outputs.branch }}"
+          # Sanitize branch name for Docker tag (replace special chars with hyphens)
+          SAFE_BRANCH=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9._-]/-/g' | sed 's/--*/-/g' | sed 's/^-\|-$//g')
+          SHORT_SHA="${{ github.sha }}"
+          SHORT_SHA="${SHORT_SHA:0:7}"
+
+          echo "dev_tag=${SAFE_BRANCH}" >> $GITHUB_OUTPUT
+          echo "dev_sha_tag=${SAFE_BRANCH}-${SHORT_SHA}" >> $GITHUB_OUTPUT
+
+      - name: Build and push dev image (${{ matrix.name }})
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ matrix.image_name }}:${{ steps.dev-tags.outputs.dev_tag }}
+            ${{ env.REGISTRY }}/${{ matrix.image_name }}:${{ steps.dev-tags.outputs.dev_sha_tag }}
+          labels: |
+            org.opencontainers.image.title=${{ matrix.name }} HTTP Proxy (Dev)
+            org.opencontainers.image.description=Spark HTTP Proxy - ${{ matrix.name }} component (Development build from ${{ steps.branch.outputs.branch }})
+            org.opencontainers.image.vendor=SparkFabrik
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.ref.name=${{ steps.branch.outputs.branch }}
+          cache-from: type=gha,scope=${{ matrix.cache_scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
 
   deploy:
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main'
     permissions:
       contents: read
       packages: write
+      security-events: write # For security scanning
+    strategy:
+      fail-fast: false # Continue building other images if one fails
+      matrix:
+        include:
+          - name: traefik
+            context: ./build/traefik
+            dockerfile: ./build/traefik/Dockerfile
+            image_name: ${{ github.repository }}-traefik
+            cache_scope: traefik
+          - name: services
+            context: .
+            dockerfile: ./build/Dockerfile
+            image_name: ${{ github.repository }}-services
+            cache_scope: services
+          - name: prometheus
+            context: ./build/prometheus
+            dockerfile: ./build/prometheus/Dockerfile
+            image_name: ${{ github.repository }}-prometheus
+            cache_scope: prometheus
+          - name: grafana
+            context: ./build/grafana
+            dockerfile: ./build/grafana/Dockerfile
+            image_name: ${{ github.repository }}-grafana
+            cache_scope: grafana
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -59,20 +243,36 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ matrix.image_name }}
           tags: |
             type=ref,event=branch
             type=sha,prefix={{branch}}-
             type=raw,value=latest,enable={{is_default_branch}}
+          labels: |
+            org.opencontainers.image.title=${{ matrix.name }} HTTP Proxy
+            org.opencontainers.image.description=Spark HTTP Proxy - ${{ matrix.name }} component
+            org.opencontainers.image.vendor=SparkFabrik
 
-      - name: Build and push Docker image
+      - name: Build and push Docker image (${{ matrix.name }})
         uses: docker/build-push-action@v5
         with:
-          context: .
-          file: ./Dockerfile
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=${{ matrix.cache_scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          # Security and optimization
+          provenance: true
+          sbom: true
+
+      - name: Run security scan
+        uses: docker/scout-action@v1
+        if: github.event_name != 'pull_request'
+        with:
+          command: cves
+          image: ${{ env.REGISTRY }}/${{ matrix.image_name }}:latest
+          only-severities: critical,high
+          exit-code: true

.gitignore

@@ -1 +1,2 @@
 join-networks.tar.gz
+.env