feat(DoutD): enable worker to spawn containers on host (ADAnalise-container)

.github/workflows/build.yaml

@@ -64,16 +64,14 @@ jobs:
       with:
         convention-name: conventionalcommits
 
-    - name: Install Conda Dependencies
-      uses: conda-incubator/setup-miniconda@v3
+    - uses: conda-incubator/setup-miniconda@v3
       with:
-        miniconda-version: "latest"
-        mamba-version: "*"
+        miniforge-version: latest
         environment-file: conda/env.yaml
         channels: conda-forge,nodefaults
         activate-environment: alertflow
-        use-mamba: true
-        miniforge-variant: Mambaforge
+        auto-update-conda: true
+        conda-solver: libmamba
 
     - name: Create environment variables file
       run: |
@@ -93,11 +91,11 @@ jobs:
 
     - name: Build Base Image
       run: |
-        sugar build
+        sugar compose build
 
     - name: Start Airflow
       run: |
-        sugar up --options -d
+        sugar compose up --options -d
 
     - name: Wait Containers
       run: |

.sugar.yaml

@@ -1,13 +1,11 @@
-version: 1.0.0
-compose-app: docker-compose
+backend: compose
 env-file: .env
 defaults:
-  group: {{ env.ENV }}
-
+  group: dev
 groups:
   dev:
     project-name: alertflow
-    compose-path:
+    config-path:
       - docker/compose.yaml
       - docker/compose-dev.yaml
     env-file: .env
@@ -21,7 +19,7 @@ groups:
 
   prod:
     project-name: alertflow
-    compose-path: 
+    config-path:
       - docker/compose.yaml
       - docker/compose-prod.yaml
     env-file: .env

alertflow/dags/satellite-weather/brasil.py

@@ -16,9 +16,9 @@
 """
 
 import os
-from pathlib import Path
 from datetime import date, timedelta
 from itertools import chain
+from pathlib import Path
 
 import pendulum
 from airflow import DAG
@@ -45,7 +45,7 @@
     dag_id="COPERNICUS_BRASIL",
     description="ETL of weather data for Brazil",
     tags=["Brasil", "Copernicus"],
-    schedule="@monthly",
+    schedule="@daily",
     default_args=DEFAULT_ARGS,
     start_date=pendulum.datetime(2000, 1, 1),
     catchup=True,
@@ -70,7 +70,9 @@ def fetch_ds(locale, dt, uri, api_key):
             )
             table_geocodes = set(chain(*cur.fetchall()))
 
-        all_geocodes = set([adm.code for adm in ADM2.filter(adm0=locale)])
+        all_geocodes = set(
+            map(int, [adm.code for adm in ADM2.filter(adm0=locale)])
+        )
         geocodes = all_geocodes.difference(table_geocodes)
         print("TABLE_GEO ", f"[{len(table_geocodes)}]: ", table_geocodes)
         print("DIFF_GEO: ", f"[{len(geocodes)}]: ", geocodes)
@@ -83,7 +85,7 @@ def fetch_ds(locale, dt, uri, api_key):
             locale=locale,
         ) as ds:
             for geocode in geocodes:
-                adm = ADM2.get(code=geocode):
+                adm = ADM2.get(code=geocode)
                 with engine.connect() as conn:
                     ds.cope.to_sql(adm, conn, tablename, "weather")
             file = Path(f"{basename}.zip")

docker/Dockerfile

@@ -44,12 +44,14 @@ RUN addgroup --gid ${HOST_GID} airflow \
   && chmod 0440 /etc/sudoers.d/airflow \
   && chown -R ${HOST_UID}:${HOST_GID} ${AIRFLOW_HOME}/ /opt/airflow/
 
+RUN groupadd docker && usermod -aG docker airflow && newgrp docker
+
 ENV PATH "$PATH:/home/airflow/.local/bin"
 ENV PATH "$PATH:/usr/bin/dirname"
 
 COPY --chown=airflow alertflow/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
 COPY --chown=airflow docker/scripts/entrypoint.sh /entrypoint.sh
-COPY --chown=airflow pyproject.toml ${AIRFLOW_HOME}
+COPY --chown=airflow pyproject.toml README.md ${AIRFLOW_HOME}
 RUN chmod +x /entrypoint.sh
 
 USER airflow
@@ -59,6 +61,6 @@ RUN curl -sSL https://install.python-poetry.org | python3
 WORKDIR ${AIRFLOW_HOME}
 
 RUN poetry config virtualenvs.create false \
-  && poetry install --only main
+  && poetry install --only main --no-root
 
 ENTRYPOINT [ "/entrypoint.sh" ]

docker/compose.yaml

@@ -189,6 +189,8 @@ services:
         condition: service_healthy
       startup:
         condition: service_completed_successfully
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - alertflow
 

pyproject.toml

@@ -15,8 +15,7 @@ sqlalchemy = "<2.0"
 flake8 = "^7.0.0"
 black = "^24.2.0"
 isort = "^5.13.2"
-containers-sugar = "1.11.1"
-compose-go = "^2.24.6"
+containers-sugar = "^1.16.1"
 apache-airflow = "^2.10.2"
 
 [build-system]