Services documentation

mlos_bench/mlos_bench/__init__.py

@@ -150,18 +150,20 @@
 The entry point for these configs can be found `here
 <https://github.com/microsoft/MLOS/blob/main/mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc>`_.
 
->>> from subprocess import run
 >>> # Note: we show the command wrapped in python here for testing purposes.
 >>> # Alternatively replace test-cli-local-env-bench.jsonc with
 >>> # test-cli-local-env-opt.jsonc for one that does an optimization loop.
 >>> cmd = "mlos_bench \
 ... --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \
 ... --globals experiment_test_local.jsonc \
 ... --tunable_values tunable-values/tunable-values-local.jsonc"
+
 >>> print(f"Here's the shell command you'd actually run:\n# {cmd}")
 Here's the shell command you'd actually run:
 # mlos_bench --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc --globals experiment_test_local.jsonc --tunable_values tunable-values/tunable-values-local.jsonc
+
 >>> # Now we run the command and check the output.
+>>> from subprocess import run
 >>> result = run(cmd, shell=True, capture_output=True, text=True, check=True)
 >>> assert result.returncode == 0
 >>> lines = result.stderr.splitlines()

mlos_bench/mlos_bench/config/__init__.py

@@ -178,7 +178,7 @@
   for some examples of CLI configs.
 
 Globals and Variable Substitution
-+++++++++++++++++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 :py:attr:`Globals <mlos_bench.config.schemas.config_schemas.ConfigSchema.GLOBALS>`
 are basically just key-value variables that can be used in other configs using
@@ -208,14 +208,15 @@
 Here is a list of some well known variables that are provided or required by the
 system and may be used in the config files:
 
-- ``$experiment_id``: A unique identifier for the ``Experiment``.
+- ``$experiment_id`` : A unique identifier for the ``Experiment``.
     Typically provided in globals.
-- ``$trial_id``: A unique identifier for the ``Trial`` currently being executed.
+- ``$trial_id`` : A unique identifier for the ``Trial`` currently being executed.
     This can be useful in the configs for :py:mod:`mlos_bench.environments` for
     instance (e.g., when writing scripts).
-- ``$trial_runner_id``: A unique identifier for the ``TrialRunner``.
-    This can be useful when running multiple trials in parallel (e.g., to
-    provision a numbered VM per worker).
+- ``$trial_runner_id`` : A unique identifier for the
+    :py:class:`~mlos_bench.schedulers.trial_runner.TrialRunner`. This can be
+    useful when running multiple trials in parallel (e.g., to provision a
+    numbered VM per worker).
 
 Tunable Configs
 ^^^^^^^^^^^^^^^
@@ -311,7 +312,7 @@
 expected structure of the ``config`` section.
 
 In certain cases (e.g., script command execution) the variable substitution rules
-take on slightly different behavior
+take on slightly different behavior.
 See various documentation in :py:mod:`mlos_bench.environments` for more details.
 
 Config Processing

mlos_bench/mlos_bench/services/__init__.py

@@ -6,6 +6,18 @@
 Services for implementing Environments for mlos_bench.
 
 TODO: Improve documentation here.
+
+Overview
+--------
+TODO: Explain Service mix-ins and how they get used with Environments.
+
+Config
+------
+TODO: Explain how to configure Services.
+
+See Also
+--------
+TODO: Provide references to different related classes.
 """
 
 from mlos_bench.services.base_fileshare import FileShareService

mlos_bench/mlos_bench/services/config_persistence.py

@@ -7,9 +7,51 @@
 benchmark :py:class:`.Environment`, :py:mod:`~mlos_bench.tunables`, :py:class:`.Service`
 functions, etc from JSON configuration files and strings.
 
+Typically the :py:class:`.ConfigPersistenceService` is provided automatically by
+the :py:mod:`mlos_bench.launcher`.
+
+It's ``config_path`` parameter is a list of directories to search for the
+configuration files referenced in other JSON config files or
+:py:mod:`mlos_bench.run` ``--cli-options``.
+
+That value can itself be adjusted with the ``--config-path`` CLI option or set
+in a ``--config`` CLI options file.
+
+Regardless of the values there, the service will always search the included
+config files from the :py:mod:`mlos_bench` package.
+
 See Also
 --------
 mlos_bench.config : Overview of the configuration system.
+mlos_bench.run : CLI options for the ``mlos_bench`` command.
+
+Examples
+--------
+>>> # Create a new instance of the ConfigPersistenceService.
+>>> # This will search for config files in the current directory's config subdir.
+>>> # It will also search in the built-in config files that come with the package.
+>>> # And the current working directory.
+>>> service = ConfigPersistenceService(config={"config_path": ["./config"]})
+
+>>> # One of the things the ConfigLoaderType does is find and load config files
+>>> # referenced in other JSON files.
+>>> # For instance:
+>>> config_file_path = "optimizers/mlos_core_default_opt.jsonc"
+>>> resolved_file = service.resolve_path(config_file_path)
+
+>>> # The resolved file should be the same as the expected file.
+>>> # That is, the resolved file should be the same as the built-in file.
+>>> from importlib.resources import files
+>>> from os.path import abspath, samefile
+>>> expected_file = abspath(files("mlos_bench.config").joinpath(config_file_path))
+>>> assert samefile(resolved_file, expected_file)
+
+>>> # Create an Optimizer from that file.
+>>> from mlos_bench.config.schemas.config_schemas import ConfigSchema
+>>> config = service.load_config(config_file_path, schema_type=ConfigSchema.OPTIMIZER)
+>>> optimizer = service.build_optimizer(tunables=TunableGroups(), service=service, config=config)
+>>> from mlos_bench.optimizers.base_optimizer import Optimizer
+>>> assert isinstance(optimizer, Optimizer)
 """
 
 import logging

mlos_bench/mlos_bench/services/types/__init__.py

@@ -2,8 +2,53 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 #
-"""Service types for implementing declaring Service behavior for Environments to use in
-mlos_bench.
+"""
+Service types (i.e., :external:py:class:`~typing.Protocol`) for declaring implementation
+:py:class:`~mlos_bench.services.base_service.Service` behavior for
+:py:mod:`~mlos_bench.environments` to use in :py:mod:`mlos_bench`.
+
+Overview
+--------
+Service loading in ``mlos_bench`` uses a
+`mix-in <https://en.wikipedia.org/wiki/Mixin#In_Python>`_ approach to combine the
+functionality of multiple classes specified at runtime through config files into
+a single class that the :py:mod:`~mlos_bench.environments` can use to invoke the
+actions that they were configured to perform (e.g., provisioning a VM, deploying
+a network, running a script, etc.).
+
+Since Services are loaded at runtime and can be swapped out by referencing a
+different set of config files via the ``--services`` :py:mod:`mlos_bench.run`
+CLI option, this can make it difficult to do type and config checking.
+
+To address this we define :external:py:func:`~typing.runtime_checkable` decorated
+`Protocols <https://peps.python.org/pep-0544/>`_ (e.g., *interfaces* in other
+languages) to declare the expected behavior of the ``Services`` that are loaded
+at runtime in the ``Environments`` that use them.
+
+For example, the :py:class:`.SupportsFileShareOps` Protocol declares the
+expected behavior of a Service that can
+:py:meth:`~.SupportsFileShareOps.download` and
+:py:meth:`~.SupportsFileShareOps.upload` files to and from a remote file share.
+
+But we can have more than one Service that implements that Protocol (e.g., one
+for Azure, one for AWS, one for a remote SSH server, etc.).
+
+This allows us to define the expected behavior of the Service that the
+Environment will need, but not the specific implementation details.
+
+It also allows users to define Environment configs that are more reusable so
+that we can swap out the Service implementations at runtime without having to
+change the Environment config.
+
+That way we can run Experiments on more than one platform rather easily.
+
+See the classes below for an overview of the types of Services that are
+currently available for Environments.
+
+Notes
+-----
+If you find that there are missing types or that you need to add a new Service
+type, please `submit a PR <https://github.com/microsoft/MLOS>`_ to add it here.
 """
 
 from mlos_bench.services.types.authenticator_type import SupportsAuth

mlos_bench/mlos_bench/services/types/bound_method.py

@@ -2,7 +2,14 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 #
-"""Protocol representing a bound method."""
+"""
+Protocol representing a bound method.
+
+Notes
+-----
+Mostly just used for type checking of
+:py:class:`~mlos_bench.services.base_service.Service` mix-ins.
+"""
 
 from typing import Any, Protocol, runtime_checkable
 

mlos_bench/mlos_bench/services/types/config_loader_type.py

@@ -2,7 +2,13 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 #
-"""Protocol interface for helper functions to lookup and load configs."""
+"""
+Protocol interface for helper functions to lookup and load configs.
+
+See Also
+--------
+:py:class:`~mlos_bench.services.config_persistence.ConfigPersistenceService`
+"""
 
 from __future__ import annotations
 

mlos_bench/mlos_bench/services/types/remote_config_type.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 #
-"""Protocol interface for configuring cloud services."""
+"""Protocol interface for configuring cloud (Saas) services."""
 
 from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable