Merge branch 'main' into ODSC-64654-register-model-artifact-reference

Author: mrDzurb

.github/workflows/run-forecast-explainer-tests.yml

@@ -0,0 +1,59 @@
+name: "Forecast Explainer Tests"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ "main", "operators/**" ]
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test:
+    name: python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
+            "test-requirements-operators.txt"
+
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
+
+      - name: "Run Forecast Explainer Tests"
+        timeout-minutes: 180
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          $CONDA/bin/conda init
+          source /home/runner/.bashrc
+          pip install -r test-requirements-operators.txt
+          pip install "oracle-automlx[forecasting]>=25.1.1"
+          pip install pandas>=2.2.0
+          python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast/test_explainers.py

.github/workflows/run-forecast-unit-tests.yml

@@ -56,6 +56,6 @@ jobs:
           $CONDA/bin/conda init
           source /home/runner/.bashrc
           pip install -r test-requirements-operators.txt
-          pip install "oracle-automlx[forecasting]>=24.4.0"
+          pip install "oracle-automlx[forecasting]>=25.1.1"
           pip install pandas>=2.2.0
-          python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast
+          python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast --ignore=tests/operators/forecast/test_explainers.py

.github/workflows/run-unittests-py39-py310.yml renamed to .github/workflows/run-unittests-py310-py311.yml

@@ -1,4 +1,4 @@
-name: "[Py3.9-3.11] - All Unit Tests"
+name: "[Py3.10-3.11] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -33,15 +33,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
         name: ["unitary", "slow_tests"]
         include:
           - name: "unitary"
             test-path: "tests/unitary"
             # `model` tests running in "slow_tests",
             # `feature_store` tests has its own test suite
-            # `forecast` tests not supported in python 3.9,3.10 (automlx dependency). Tests are running in python3.8 test env, see run-unittests-py38-cov-report.yml
-            # 'pii' tests run only with py3.8, 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
+            # `forecast` tests not run in this suite
             # 'hpo' tests hangs if run together with all unitary tests. Tests running in separate command before running all unitary
             ignore-path: |
               --ignore tests/unitary/with_extras/model \

.github/workflows/run-unittests-py38-cov-report.yml renamed to .github/workflows/run-unittests-py39-cov-report.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][COV REPORT] - All Unit Tests"
+name: "[Py3.9][COV REPORT] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -26,7 +26,7 @@ env:
 
 jobs:
   test:
-    name: python 3.8, ${{ matrix.name }}
+    name: python 3.9, ${{ matrix.name }}
     runs-on: ubuntu-latest
     timeout-minutes: 90
 
@@ -58,7 +58,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.9"
           cache: "pip"
           cache-dependency-path: |
             pyproject.toml
@@ -71,7 +71,7 @@ jobs:
         name: "Test env setup"
         timeout-minutes: 30
 
-      # Installing pii deps for python3.8 test setup only, it will not work with python3.9/3.10, because
+      # Installing pii deps for python3.9 test setup only, it will not work with python3.9/3.10, because
       # 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
       - name: "Install PII dependencies"
         run: |

.gitignore

@@ -86,7 +86,8 @@ celerybeat-schedule
 *.sage.py
 
 # dotenv
-.env
+.env*
+run_ads.sh
 
 # virtualenv
 .venv

README-development.md

@@ -1,15 +1,41 @@
+<!-- TOC -->
 # Summary
 
 The Oracle Accelerated Data Science (ADS) SDK used by data scientists and analysts for
 data exploration and experimental machine learning to democratize machine learning and
-analytics by providing easy-to-use, performant, and user friendly tools that
+analytics by providing easy-to-use, 
+performant, and user friendly tools that
 brings together the best of data science practices.
 
 The ADS SDK helps you connect to different data sources, perform exploratory data analysis,
 data visualization, feature engineering, model training, model evaluation, and
 model interpretation. ADS also allows you to connect to the model catalog to save and load
 models to and from the catalog.
 
+- [Summary](#summary)
+  - [Documentation](#documentation)
+  - [Get Support](#get-support)
+  - [Getting started](#getting-started)
+    - [Step 1: Create a conda environment](#step-1-create-a-conda-environment)
+    - [Step 2: Activate your environment](#step-2-activate-your-environment)
+    - [Step 3: Clone ADS and install dependencies](#step-3-clone-ads-and-install-dependencies)
+    - [Step 4: Setup configuration files](#step-4-setup-configuration-files)
+    - [Step 5: Versioning and generation the wheel](#step-5-versioning-and-generation-the-wheel)
+  - [Running tests](#running-tests)
+    - [Running default setup tests](#running-default-setup-tests)
+    - [Running all unit tests](#running-all-unit-tests)
+    - [Running integration tests](#running-integration-tests)
+    - [Running opctl integration tests](#running-opctl-integration-tests)
+  - [Local Setup of AQUA API JupyterLab Server](#local-setup-of-aqua-api-jupyterlab-server)
+    - [Step 1: Requirements](#step-1-requirements)
+    - [Step 2: Create local .env files](#step-2-create-local-env-files)
+    - [Step 3: Add the run\_ads.sh script in the ADS Repository](#step-3-add-the-run_adssh-script-in-the-ads-repository)
+    - [Step 4: Run the JupyterLab Server](#step-4-run-the-jupyterlab-server)
+    - [Step 5: Run the unit tests for the AQUA API](#step-5-run-the-unit-tests-for-the-aqua-api)
+  - [Security](#security)
+  - [License](#license)
+
+
 ## Documentation
 
  - [ads-documentation](https://docs.oracle.com/en-us/iaas/tools/ads-sdk/latest/index.html)
@@ -137,6 +163,100 @@ To build development container, see the [Build Development Container Image](http
   python3 -m pytest tests/integration/opctl
 ```
 
+## Local Setup of AQUA API JupyterLab Server
+These are the steps to run the AQUA (AI Quick Actions) API Server for development and testing purposes. The source code for the AQUA API Server is [here](https://github.com/oracle/accelerated-data-science/tree/21ba00b95aef8581991fee6c7d558e2f2b1680ac/ads/aqua) within this repository.
+
+### Step 1: Requirements
++ Complete the [Getting Started](#getting-started) Section above, create a conda environment with python >3.9 or 3.10
++ install any Rest API Client in your IDE (Thunder Client on [vscode](https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client) or Postman) 
++ Activate the conda environment from the Getting Started Section and run
+
+```
+pip install -r test-requirements.txt
+```
+
+### Step 2: Create local .env files 
+Running the local JuypterLab server requires setting OCI authentication, proxy, and OCI namespace parameters. Adapt this .env file with your specific OCI profile and OCIDs to set these variables.
+
+```
+CONDA_BUCKET_NS="your_conda_bucket"
+http_proxy=""
+https_proxy=""
+HTTP_PROXY=""
+HTTPS_PROXY=""
+OCI_ODSC_SERVICE_ENDPOINT="your_service_endpoint"
+AQUA_SERVICE_MODELS_BUCKET="service-managed-models"
+AQUA_TELEMETRY_BUCKET_NS="" 
+PROJECT_COMPARTMENT_OCID="ocid1.compartment.oc1.<your_ocid>" 
+OCI_CONFIG_PROFILE="your_oci_profile_name"
+OCI_IAM_TYPE="security_token" # no modification needed if using token-based auth
+TENANCY_OCID="ocid1.tenancy.oc1.<your_ocid>"
+AQUA_JOB_SUBNET_ID="ocid1.subnet.oc1.<your_ocid>"
+ODSC_MODEL_COMPARTMENT_OCID="ocid1.compartment.oc1.<your_ocid>" 
+PROJECT_OCID="ocid1.datascienceproject.oc1.<your_ocid>"
+```
+
+### Step 3: Add the run_ads.sh script in the ADS Repository 
++ add the shell script below and .env file from step 2 to your local directory of the cloned ADS Repository
++ Run ```chmox +x run_ads.sh``` after you create this script.
+```
+#!/bin/bash
+
+#### Check if a CLI command is provided
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 <cli command>"
+  exit 1
+fi
+
+#### Load environment variables from .env file
+if [ -f .env ]; then
+  export $(grep -v '^#' .env.int | xargs)
+else
+  echo "Error: .env.int file not found!"
+  exit 1
+fi
+
+# Execute the CLI command
+"$@"
+```
+
+### Step 4: Run the JupyterLab Server 
+We can start the JupyterLab server using the following command
+
+``` 
+./run_ads.sh jupyter lab --no-browser --ServerApp.disable_check_xsrf=True
+```
++ run ```pkill jupyter-lab``` to kill the JupyterLab server and re-run server to reflect changes made locally to the AQUA API
++ to test if server is running via CLI, run this in terminal
+
+```
+./run_ads.sh ads aqua model list
+```
+
+To make calls to the API, use the link http://localhost:8888/aqua/insert_handler_here with a REST API Client like Thunder Client/ Postman.
+
+Examples of handlers
+```
+GET http://localhost:8888/aqua/model # calling the model_handler.py
+
+GET http://localhost:8888/aqua/deployments # calling the deployment_handler.py
+```
+Handlers can be found [here](https://github.com/oracle/accelerated-data-science/tree/21ba00b95aef8581991fee6c7d558e2f2b1680ac/ads/aqua/extension).
+
+### Step 5: Run the unit tests for the AQUA API
+All the unit tests can be found [here](https://github.com/oracle/accelerated-data-science/tree/main/tests/unitary/with_extras/aqua). 
+The following commands detail how the unit tests can be run.
+```
+# Run all tests in AQUA project
+python -m pytest -q tests/unitary/with_extras/aqua/test_deployment.py
+
+# Run all tests specific to a module within in AQUA project (ex. test_deployment.py, test_model.py, etc.)
+python -m pytest -q tests/unitary/with_extras/aqua/test_deployment.py
+
+# Run specific test method within the module (replace test_get_deployment_default_params with targeted test method)
+python -m pytest tests/unitary/with_extras/aqua/test_deployment.py -k "test_get_deployment_default_params"
+```
+
 ## Security
 
 Consult the [security guide](./SECURITY.md) for our responsible security

THIRD_PARTY_LICENSES.txt

@@ -96,6 +96,12 @@ fsspec
 * Source code: https://github.com/intake/filesystem_spec
 * Project home: https://github.com/intake/filesystem_spec
 
+httpx
+* Copyright (c) 2021 ProjectDiscovery, Inc.
+* License: MIT License
+* Source code: https://github.com/projectdiscovery/httpx
+* Project home: https://github.com/projectdiscovery/httpx
+
 geopandas
 * Copyright  (c) 2013-2016, GeoPandas developers
 * License: BSD 3-Clause "New" or "Revised" License
@@ -391,6 +397,12 @@ tabulate
 * Source code: https://github.com/astanin/python-tabulate
 * Project home: https://github.com/astanin/python-tabulate
 
+tenacity
+* No listed copyright holder
+* License: Apache-2.0 license
+* Source code: https://github.com/jd/tenacity
+* Project home: https://github.com/jd/tenacity
+
 tensorflow
 * Copyright (c) 2023 Google, Inc.
 * License: Apache-2.0 license

ads/aqua/__init__.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
 import os
+from logging import getLogger
 
 from ads import logger, set_auth
+from ads.aqua.client.client import AsyncClient, Client
 from ads.aqua.common.utils import fetch_service_compartment
 from ads.config import OCI_RESOURCE_PRINCIPAL_VERSION
 
@@ -19,6 +20,7 @@ def get_logger_level():
     return level
 
 
+logger = getLogger(__name__)
 logger.setLevel(get_logger_level())
 
 
@@ -27,7 +29,6 @@ def set_log_level(log_level: str):
 
     log_level = log_level.upper()
     logger.setLevel(log_level.upper())
-    logger.handlers[0].setLevel(log_level)
 
 
 if OCI_RESOURCE_PRINCIPAL_VERSION: