Merge pull request #29 from synccomputingcode/graham/configure

[PROD-1091] [Databricks Conference '23] Updates to sync-cli configure command

Author: gseyffert-sync

sync/__init__.py

@@ -1,4 +1,4 @@
 """Library for leveraging the power of Sync"""
-__version__ = "0.0.14"
+__version__ = "0.0.15"
 
 TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

sync/api/projects.py

@@ -2,7 +2,6 @@
 """
 
 import logging
-
 from typing import List
 
 from sync.api.predictions import get_predictions
@@ -42,7 +41,7 @@ def create_project(
     app_id: str,
     description: str = None,
     s3_url: str = None,
-    prediction_preference: Preference = Preference.BALANCED,
+    prediction_preference: Preference = Preference.ECONOMY,
     prediction_params: dict = None,
 ) -> Response[dict]:
     """Creates a Sync project for tracking and optimizing Apache Spark applications
@@ -53,7 +52,7 @@ def create_project(
     :type description: str, optional
     :param s3_url: S3 URL under which to store project configurations and logs, defaults to None
     :type s3_url: str, optional
-    :param prediction_preference: preferred prediction solution, defaults to Preference.BALANCED
+    :param prediction_preference: preferred prediction solution, defaults to `Preference.ECONOMY`
     :type prediction_preference: Preference, optional
     :param prediction_params: dictionary of prediction parameters, defaults to None. Valid options are documented here - https://developers.synccomputing.com/reference/create_project_v1_projects_post
     :type prediction_preference: dict, optional
@@ -67,7 +66,7 @@ def create_project(
                 "description": description,
                 "s3_url": s3_url,
                 "prediction_preference": prediction_preference,
-                "prediction_params": prediction_params
+                "prediction_params": prediction_params,
             }
         )
     )

sync/cli/__init__.py

@@ -43,13 +43,10 @@ def configure():
         show_default=False,
     )
 
-    project_url = click.prompt(
-        "Default project S3 URL", default=CONFIG.default_project_url or OPTIONAL_DEFAULT
-    )
     prediction_preference = click.prompt(
         "Default prediction preference",
         type=click.Choice([p.value for p in Preference]),
-        default=(CONFIG.default_prediction_preference or Preference.BALANCED).value,
+        default=(CONFIG.default_prediction_preference or Preference.ECONOMY).value,
     )
 
     dbx_host = OPTIONAL_DEFAULT
@@ -74,7 +71,6 @@ def configure():
     init(
         APIKey(api_key_id=api_key_id, api_key_secret=api_key_secret),
         Configuration(
-            default_project_url=project_url if project_url != OPTIONAL_DEFAULT else None,
             default_prediction_preference=prediction_preference,
         ),
         DatabricksConf(host=dbx_host, token=dbx_token, aws_region_name=dbx_region)

sync/config.py

@@ -8,7 +8,7 @@
 from urllib.parse import urlparse
 
 import boto3 as boto
-from pydantic import BaseSettings, Field, validator
+from pydantic import BaseSettings, Field, validator, Extra
 
 from .models import Preference
 
@@ -39,25 +39,12 @@ def customise_sources(cls, init_settings, env_settings, file_secret_settings):
 
 
 class Configuration(BaseSettings):
-    default_project_url: Union[str, None] = Field(
-        description="default location for Sync project data"
-    )
-    default_prediction_preference: Union[Preference, None] = Preference.BALANCED
+    default_prediction_preference: Union[Preference, None] = Preference.ECONOMY
     api_url: str = Field("https://api.synccomputing.com", env="SYNC_API_URL")
 
-    @validator("default_project_url")
-    def validate_url(cls, url):
-        if url:
-            # There are valid S3 URLs (e.g. with spaces) not supported by Pydantic URL types: https://docs.pydantic.dev/usage/types/#urls
-            # Hence the manual validation here
-            parsed_url = urlparse(url.rstrip("/"))
-
-            if parsed_url.scheme != "s3":
-                raise ValueError("Only S3 URLs please!")
-
-            return parsed_url.geturl()
-
     class Config:
+        extra = Extra.ignore
+
         @classmethod
         def customise_sources(cls, init_settings, env_settings, file_secret_settings):
             return (init_settings, env_settings, json_config_settings_source(CONFIG_FILE))
@@ -124,8 +111,11 @@ def get_api_key() -> APIKey:
     :rtype: APIKey
     """
     global _api_key
-    if not _api_key:
-        _api_key = APIKey()
+    if _api_key is None:
+        try:
+            _api_key = APIKey()
+        except ValueError:
+            pass
     return _api_key
 
 
@@ -136,11 +126,21 @@ def get_config() -> Configuration:
     :rtype: Configuration
     """
     global _config
-    if not _config:
+    if _config is None:
         _config = Configuration()
     return _config
 
 
+def get_databricks_config() -> DatabricksConf:
+    global _db_config
+    if _db_config is None:
+        try:
+            _db_config = DatabricksConf()
+        except ValueError:
+            pass
+    return _db_config
+
+
 CONFIG: Configuration
 _config = None
 API_KEY: APIKey
@@ -151,26 +151,11 @@ def get_config() -> Configuration:
 
 def __getattr__(name):
     if name == "CONFIG":
-        global _config
-        if _config is None:
-            _config = Configuration()
-        return _config
+        return get_config()
     elif name == "API_KEY":
-        global _api_key
-        if _api_key is None:
-            try:
-                _api_key = APIKey()
-            except ValueError:
-                pass
-        return _api_key
+        return get_api_key()
     elif name == "DB_CONFIG":
-        global _db_config
-        if _db_config is None:
-            try:
-                _db_config = DatabricksConf()
-            except ValueError:
-                pass
-        return _db_config
+        return get_databricks_config()
     else:
         raise AttributeError(f"module '{__name__}' has no attribute '{name}'")