From b7b0c2e731544f8972efecdd633e947a5e5c8287 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 29 Apr 2025 16:35:09 +0100 Subject: [PATCH 01/22] chore(ebrains): copy over necessary files --- .../models/EbrainsRepositoryResource.ts | 96 +++++ .../models/ebrains_repository_resource.py | 236 +++++++++++++ .../osbrepository/adapters/ebrainsadapter.py | 147 ++++++++ .../workspaces/tasks/ebrains-copy/Dockerfile | 15 + .../workspaces/tasks/ebrains-copy/README.md | 16 + .../tasks/ebrains-copy/docker-compose.yaml | 6 + .../workspaces/tasks/ebrains-copy/run.sh | 38 ++ .../model/ebrains_repository_resource.py | 327 ++++++++++++++++++ 8 files changed, 881 insertions(+) create mode 100644 applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts create mode 100644 applications/workspaces/server/workspaces/models/ebrains_repository_resource.py create mode 100644 applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py create mode 100644 applications/workspaces/tasks/ebrains-copy/Dockerfile create mode 100644 applications/workspaces/tasks/ebrains-copy/README.md create mode 100644 applications/workspaces/tasks/ebrains-copy/docker-compose.yaml create mode 100644 applications/workspaces/tasks/ebrains-copy/run.sh create mode 100644 libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py diff --git a/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts b/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts new file mode 100644 index 000000000..32ff4c469 --- /dev/null +++ b/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts @@ -0,0 +1,96 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * OSB Workspaces manager API + * Opensource Brain Platform - Reference Workspaces manager API + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +import { + RepositoryResourceBase, + RepositoryResourceBaseFromJSON, + RepositoryResourceBaseFromJSONTyped, + RepositoryResourceBaseToJSON, +} from './'; + +/** + * figshare repository resource + * @export + * @interface BiomodelsRepositoryResource + */ +export interface BiomodelsRepositoryResource { + /** + * file name + * @type {string} + * @memberof BiomodelsRepositoryResource + */ + name?: string; + /** + * Download URL of the Resource + * @type {string} + * @memberof BiomodelsRepositoryResource + */ + path?: string; + /** + * OSB Repository id + * @type {number} + * @memberof BiomodelsRepositoryResource + */ + osbrepositoryId?: number; + /** + * File size in bytes of the RepositoryResource + * @type {number} + * @memberof BiomodelsRepositoryResource + */ + size?: number; + /** + * Date/time the ReposityResource is last modified + * @type {Date} + * @memberof BiomodelsRepositoryResource + */ + timestampModified?: Date; +} + +export function BiomodelsRepositoryResourceFromJSON(json: any): BiomodelsRepositoryResource { + return BiomodelsRepositoryResourceFromJSONTyped(json, false); +} + +export function BiomodelsRepositoryResourceFromJSONTyped(json: any, ignoreDiscriminator: boolean): BiomodelsRepositoryResource { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'name': !exists(json, 'name') ? undefined : json['name'], + 'path': !exists(json, 'path') ? undefined : json['path'], + 'osbrepositoryId': !exists(json, 'osbrepository_id') ? undefined : json['osbrepository_id'], + 'size': !exists(json, 'size') ? undefined : json['size'], + 'timestampModified': !exists(json, 'timestamp_modified') ? undefined : (new Date(json['timestamp_modified'])), + }; +} + +export function BiomodelsRepositoryResourceToJSON(value?: BiomodelsRepositoryResource | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'name': value.name, + 'path': value.path, + 'osbrepository_id': value.osbrepositoryId, + 'size': value.size, + 'timestamp_modified': value.timestampModified === undefined ? undefined : (value.timestampModified.toISOString()), + }; +} + + diff --git a/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py b/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py new file mode 100644 index 000000000..087d3313b --- /dev/null +++ b/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py @@ -0,0 +1,236 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from workspaces.models.base_model_ import Model +from workspaces.models.repository_resource_base import RepositoryResourceBase +from workspaces import util + +from workspaces.models.repository_resource_base import RepositoryResourceBase # noqa: E501 + +class BiomodelsRepositoryResource(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, name=None, path=None, osbrepository_id=None, size=None, timestamp_modified=None, ref=None, sha=None): # noqa: E501 + """BiomodelsRepositoryResource - a model defined in OpenAPI + + :param name: The name of this BiomodelsRepositoryResource. # noqa: E501 + :type name: str + :param path: The path of this BiomodelsRepositoryResource. # noqa: E501 + :type path: str + :param osbrepository_id: The osbrepository_id of this BiomodelsRepositoryResource. # noqa: E501 + :type osbrepository_id: int + :param size: The size of this BiomodelsRepositoryResource. # noqa: E501 + :type size: int + :param timestamp_modified: The timestamp_modified of this BiomodelsRepositoryResource. # noqa: E501 + :type timestamp_modified: datetime + :param ref: The ref of this BiomodelsRepositoryResource. # noqa: E501 + :type ref: str + :param sha: The sha of this BiomodelsRepositoryResource. # noqa: E501 + :type sha: str + """ + self.openapi_types = { + 'name': str, + 'path': str, + 'osbrepository_id': int, + 'size': int, + 'timestamp_modified': datetime, + 'ref': str, + 'sha': str + } + + self.attribute_map = { + 'name': 'name', + 'path': 'path', + 'osbrepository_id': 'osbrepository_id', + 'size': 'size', + 'timestamp_modified': 'timestamp_modified', + 'ref': 'ref', + 'sha': 'sha' + } + + self._name = name + self._path = path + self._osbrepository_id = osbrepository_id + self._size = size + self._timestamp_modified = timestamp_modified + self._ref = ref + self._sha = sha + + @classmethod + def from_dict(cls, dikt) -> 'BiomodelsRepositoryResource': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The BiomodelsRepositoryResource of this BiomodelsRepositoryResource. # noqa: E501 + :rtype: BiomodelsRepositoryResource + """ + return util.deserialize_model(dikt, cls) + + @property + def name(self): + """Gets the name of this BiomodelsRepositoryResource. + + file name # noqa: E501 + + :return: The name of this BiomodelsRepositoryResource. + :rtype: str + """ + return self._name + + @name.setter + def name(self, name): + """Sets the name of this BiomodelsRepositoryResource. + + file name # noqa: E501 + + :param name: The name of this BiomodelsRepositoryResource. + :type name: str + """ + + self._name = name + + @property + def path(self): + """Gets the path of this BiomodelsRepositoryResource. + + Download URL of the Resource # noqa: E501 + + :return: The path of this BiomodelsRepositoryResource. + :rtype: str + """ + return self._path + + @path.setter + def path(self, path): + """Sets the path of this BiomodelsRepositoryResource. + + Download URL of the Resource # noqa: E501 + + :param path: The path of this BiomodelsRepositoryResource. + :type path: str + """ + + self._path = path + + @property + def osbrepository_id(self): + """Gets the osbrepository_id of this BiomodelsRepositoryResource. + + OSB Repository id # noqa: E501 + + :return: The osbrepository_id of this BiomodelsRepositoryResource. + :rtype: int + """ + return self._osbrepository_id + + @osbrepository_id.setter + def osbrepository_id(self, osbrepository_id): + """Sets the osbrepository_id of this BiomodelsRepositoryResource. + + OSB Repository id # noqa: E501 + + :param osbrepository_id: The osbrepository_id of this BiomodelsRepositoryResource. + :type osbrepository_id: int + """ + + self._osbrepository_id = osbrepository_id + + @property + def size(self): + """Gets the size of this BiomodelsRepositoryResource. + + File size in bytes of the RepositoryResource # noqa: E501 + + :return: The size of this BiomodelsRepositoryResource. + :rtype: int + """ + return self._size + + @size.setter + def size(self, size): + """Sets the size of this BiomodelsRepositoryResource. + + File size in bytes of the RepositoryResource # noqa: E501 + + :param size: The size of this BiomodelsRepositoryResource. + :type size: int + """ + + self._size = size + + @property + def timestamp_modified(self): + """Gets the timestamp_modified of this BiomodelsRepositoryResource. + + Date/time the ReposityResource is last modified # noqa: E501 + + :return: The timestamp_modified of this BiomodelsRepositoryResource. + :rtype: datetime + """ + return self._timestamp_modified + + @timestamp_modified.setter + def timestamp_modified(self, timestamp_modified): + """Sets the timestamp_modified of this BiomodelsRepositoryResource. + + Date/time the ReposityResource is last modified # noqa: E501 + + :param timestamp_modified: The timestamp_modified of this BiomodelsRepositoryResource. + :type timestamp_modified: datetime + """ + + self._timestamp_modified = timestamp_modified + + @property + def ref(self): + """Gets the ref of this BiomodelsRepositoryResource. + + The GIT ref # noqa: E501 + + :return: The ref of this BiomodelsRepositoryResource. + :rtype: str + """ + return self._ref + + @ref.setter + def ref(self, ref): + """Sets the ref of this BiomodelsRepositoryResource. + + The GIT ref # noqa: E501 + + :param ref: The ref of this BiomodelsRepositoryResource. + :type ref: str + """ + + self._ref = ref + + @property + def sha(self): + """Gets the sha of this BiomodelsRepositoryResource. + + The GIT sha of the resource # noqa: E501 + + :return: The sha of this BiomodelsRepositoryResource. + :rtype: str + """ + return self._sha + + @sha.setter + def sha(self, sha): + """Sets the sha of this BiomodelsRepositoryResource. + + The GIT sha of the resource # noqa: E501 + + :param sha: The sha of this BiomodelsRepositoryResource. + :type sha: str + """ + + self._sha = sha diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py new file mode 100644 index 000000000..7bcc1ff3f --- /dev/null +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -0,0 +1,147 @@ +import re +import sys +from typing import List +import requests + +from cloudharness import log as logger +from workspaces.models import RepositoryResourceNode, RepositoryInfo +from workspaces.models.resource_origin import ResourceOrigin +from workspaces.models.biomodels_repository_resource import BiomodelsRepositoryResource + +from .utils import add_to_tree + + +class BiomodelsException(Exception): + pass + + +class BiomodelsAdapter: + """ + Adapter for Biomodels + + https://www.ebi.ac.uk/biomodels/ + """ + + def __init__(self, osbrepository, uri=None): + self.osbrepository = osbrepository + self.uri = uri if uri else osbrepository.uri + self.api_url = "https://www.ebi.ac.uk/biomodels" + + try: + self.model_id = re.search( + f"{self.api_url}/(\\w+)", + self.uri.strip("/")).group(1) + + except AttributeError: + raise BiomodelsException(f"{uri} is not a valid Biomodels URL") + + def get_json(self, uri): + logger.debug(f"Getting: {uri}") + try: + r = requests.get( + uri, + params={"format": "json"} + ) + if r.status_code == 200: + return r.json() + else: + raise BiomodelsException( + f"Unexpected requests status code: {r.status_code}") + except Exception as e: + raise BiomodelsException("Unexpected error:", sys.exc_info()[0]) + + def get_base_uri(self): + return self.uri + + def get_info(self) -> RepositoryInfo: + info = self.get_json( + f"{self.api_url}/{self.model_id}") + return RepositoryInfo(name=info["name"], contexts=self.get_contexts(), tags=info["format"]["name"], summary=info.get("description", "")) + + def get_contexts(self): + result = self.get_json(f"{self.api_url}/{self.model_id}") + revisions = result["history"]["revisions"] + return [str(v["version"]) for v in revisions] + + def _get_filelist(self, context): + logger.debug(f"Getting filelist: {context}") + contents = self.get_json(f"{self.api_url}/model/files/{self.model_id}.{context}") + files = (contents.get("additional", []) + contents.get("main", [])) + return files + + def get_resources(self, context): + logger.debug(f"Getting resources: {context}") + files = self._get_filelist(context) + + tree = RepositoryResourceNode( + resource=BiomodelsRepositoryResource( + name="/", + path="/", + osbrepository_id=self.osbrepository.id, + ref=context, + ), + children=[], + ) + + for afile in files: + download_url = f"{self.api_url}/model/download/{self.model_id}.{context}?filename={afile['name']}" + add_to_tree( + tree=tree, + tree_path=[afile["name"]], + path=download_url, + size=int(afile["fileSize"]), + osbrepository_id=self.osbrepository.id, + ) + + return tree + + def get_description(self, context): + logger.debug(f"Getting description: {context}") + try: + result = self.get_json(f"{self.api_url}/{self.model_id}.{context}") + return result["description"] + except Exception as e: + logger.debug( + "unable to get the description from biomodels, %", str(e)) + return "" + + def get_tags(self, context): + # using the format name for the moment, since they don't do explict + # tags/keywords + logger.debug(f"Getting tags: {context}") + result = self.get_json(f"{self.api_url}/{self.model_id}.{context}") + return result["format"]["name"] + + # biomodels files are usually small, so one task is enough + def create_copy_task(self, workspace_id, origins: List[ResourceOrigin]): + import workspaces.service.workflow as workflow + + # no file tree in Biomodels from the looks of it + folder = self.osbrepository.name + + # if nothing is selected, origins has one entry with path "/" + # we get the file list and download individual files + # Biomodels does allow downloading the archive, but that is generated + # on the fly and can require us to wait for an unspecified amount of + # time + if len(origins) == 1 and origins[0].path == "/": + """ + # to use the archive method, just set paths to "" + paths = "" + """ + files = self._get_filelist(self.osbrepository.default_context) + download_url_prefix = f"{self.api_url}/model/download/{self.model_id}.{self.osbrepository.default_context}?filename=" + paths = "\\".join(f"{download_url_prefix}{file['name']}" for file in files) + else: + paths = "\\".join(o.path for o in origins) + + # username / password are not currently used + return workflow.create_copy_task( + image_name="workspaces-biomodels-copy", + workspace_id=workspace_id, + folder=folder, + url=f"{self.model_id}.{self.osbrepository.default_context}", + paths=paths, + username="", + password="", + ) diff --git a/applications/workspaces/tasks/ebrains-copy/Dockerfile b/applications/workspaces/tasks/ebrains-copy/Dockerfile new file mode 100644 index 000000000..f59d0fb53 --- /dev/null +++ b/applications/workspaces/tasks/ebrains-copy/Dockerfile @@ -0,0 +1,15 @@ +ARG CLOUDHARNESS_BASE +FROM $CLOUDHARNESS_BASE + +# much faster than curl/wget +# https://pkgs.alpinelinux.org/packages?name=aria2&branch=edge +RUN apk add aria2 unzip + + +ADD . / + +ENV shared_directory / +ENV workspace_id 1 + +RUN chmod +x ./run.sh +CMD ./run.sh diff --git a/applications/workspaces/tasks/ebrains-copy/README.md b/applications/workspaces/tasks/ebrains-copy/README.md new file mode 100644 index 000000000..feec14f05 --- /dev/null +++ b/applications/workspaces/tasks/ebrains-copy/README.md @@ -0,0 +1,16 @@ +# Biomodels copy task + + +How to test + +``` +shared_directory=/tmp folder=osbv2/develop url=BIOMD0000000998.9 ./run.sh +``` + +The above should checkout the file README.md and the full directory applications/workspaces inside /tmp/osbv2/develop + + +``` +shared_directory=/tmp folder=osbv2/develop url=https://github.com/OpenSourceBrain/OSBv2 branch=develop paths= ./run.sh +``` +This should checkout the whole repo diff --git a/applications/workspaces/tasks/ebrains-copy/docker-compose.yaml b/applications/workspaces/tasks/ebrains-copy/docker-compose.yaml new file mode 100644 index 000000000..e099cf379 --- /dev/null +++ b/applications/workspaces/tasks/ebrains-copy/docker-compose.yaml @@ -0,0 +1,6 @@ +version: "3.7" +services: + copy: + image: osb/workspaces-ebrains-copy:latest + environment: + - url=https://www.ebi.ac.uk/ebrains/MODEL2311220001s diff --git a/applications/workspaces/tasks/ebrains-copy/run.sh b/applications/workspaces/tasks/ebrains-copy/run.sh new file mode 100644 index 000000000..9f2de1914 --- /dev/null +++ b/applications/workspaces/tasks/ebrains-copy/run.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +# remove the pvc from the path (if it has one) +# and append the folder +export download_path=`echo $shared_directory | cut -d ":" -f 2`/"${folder}" + +timestamp="$(date +"%Y%m%d%H%M%S-biomodels")" + +mkdir -p "${download_path}" +cd "${download_path}" + +# check is paths has a value, otherwise download the archive and unzip it +# note: we don't use the archive system because the archive is generated on the +# fly and can make us wait for an unspecified amount of time, which tools can't +# work with +# -> left here for completeness +if [ -z "$paths" ]; then + echo Biomodels downloading archive of "${url}" to "${download_path}" + # use ..="true" and ..="false" here, otherwise aria2c gets confused + aria2c --retry-wait=2 --max-tries=5 --timeout=300 --max-concurrent-downloads=5 --max-connection-per-server=5 --allow-overwrite="true" --auto-file-renaming="false" --out="$timestamp.omex" "https://www.ebi.ac.uk/biomodels/model/download/${url}" + unzip -o "$timestamp.omex" && rm -vf "$timestamp.omex" +else + touch filelist + # Split paths by ## and checkout each path + IFS='\' + for path in $paths; do + echo Biomodels copy "${path}" to "${download_path}" + echo "${path}" >> filelist + done + echo Biomodels downloading files + aria2c --retry-wait=2 --max-tries=5 --input-file=filelist --max-concurrent-downloads=5 --max-connection-per-server=5 --allow-overwrite "true" --auto-file-renaming "false" + rm filelist -f +fi + +# fix permissions +chown -R 1000:100 "${download_path}" diff --git a/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py b/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py new file mode 100644 index 000000000..9af539478 --- /dev/null +++ b/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py @@ -0,0 +1,327 @@ +""" + Workspaces manager API + + Opensource Brain Platform - Reference Workspaces manager API # noqa: E501 + + The version of the OpenAPI document: 0.2.0 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from workspaces_cli.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, +) +from ..model_utils import OpenApiModel +from workspaces_cli.exceptions import ApiAttributeError + + +def lazy_import(): + from workspaces_cli.model.repository_resource_base import RepositoryResourceBase + globals()['RepositoryResourceBase'] = RepositoryResourceBase + + +class BiomodelsRepositoryResource(ModelComposed): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + discriminator_value_class_map (dict): A dict to go from the discriminator + variable value to the discriminator class name. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + } + + validations = { + } + + @cached_property + def additional_properties_type(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + """ + lazy_import() + return (bool, date, datetime, dict, float, int, list, str, none_type,) # noqa: E501 + + _nullable = False + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + lazy_import() + return { + 'name': (str,), # noqa: E501 + 'path': (str,), # noqa: E501 + 'osbrepository_id': (int,), # noqa: E501 + 'size': (int,), # noqa: E501 + 'timestamp_modified': (datetime,), # noqa: E501 + } + + @cached_property + def discriminator(): + return None + + + attribute_map = { + 'name': 'name', # noqa: E501 + 'path': 'path', # noqa: E501 + 'osbrepository_id': 'osbrepository_id', # noqa: E501 + 'size': 'size', # noqa: E501 + 'timestamp_modified': 'timestamp_modified', # noqa: E501 + } + + read_only_vars = { + } + + @classmethod + @convert_js_args_to_python_args + def _from_openapi_data(cls, *args, **kwargs): # noqa: E501 + """BiomodelsRepositoryResource - a model defined in OpenAPI + + Keyword Args: + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + name (str): file name. [optional] # noqa: E501 + path (str): Download URL of the Resource. [optional] # noqa: E501 + osbrepository_id (int): OSB Repository id. [optional] # noqa: E501 + size (int): File size in bytes of the RepositoryResource. [optional] # noqa: E501 + timestamp_modified (datetime): Date/time the ReposityResource is last modified. [optional] # noqa: E501 + """ + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _path_to_item = kwargs.pop('_path_to_item', ()) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + self = super(OpenApiModel, cls).__new__(cls) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + + constant_args = { + '_check_type': _check_type, + '_path_to_item': _path_to_item, + '_spec_property_naming': _spec_property_naming, + '_configuration': _configuration, + '_visited_composed_classes': self._visited_composed_classes, + } + composed_info = validate_get_composed_info( + constant_args, kwargs, self) + self._composed_instances = composed_info[0] + self._var_name_to_model_instances = composed_info[1] + self._additional_properties_model_instances = composed_info[2] + discarded_args = composed_info[3] + + for var_name, var_value in kwargs.items(): + if var_name in discarded_args and \ + self._configuration is not None and \ + self._configuration.discard_unknown_keys and \ + self._additional_properties_model_instances: + # discard variable. + continue + setattr(self, var_name, var_value) + + return self + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + '_composed_instances', + '_var_name_to_model_instances', + '_additional_properties_model_instances', + ]) + + @convert_js_args_to_python_args + def __init__(self, *args, **kwargs): # noqa: E501 + """BiomodelsRepositoryResource - a model defined in OpenAPI + + Keyword Args: + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + name (str): file name. [optional] # noqa: E501 + path (str): Download URL of the Resource. [optional] # noqa: E501 + osbrepository_id (int): OSB Repository id. [optional] # noqa: E501 + size (int): File size in bytes of the RepositoryResource. [optional] # noqa: E501 + timestamp_modified (datetime): Date/time the ReposityResource is last modified. [optional] # noqa: E501 + """ + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _path_to_item = kwargs.pop('_path_to_item', ()) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + + constant_args = { + '_check_type': _check_type, + '_path_to_item': _path_to_item, + '_spec_property_naming': _spec_property_naming, + '_configuration': _configuration, + '_visited_composed_classes': self._visited_composed_classes, + } + composed_info = validate_get_composed_info( + constant_args, kwargs, self) + self._composed_instances = composed_info[0] + self._var_name_to_model_instances = composed_info[1] + self._additional_properties_model_instances = composed_info[2] + discarded_args = composed_info[3] + + for var_name, var_value in kwargs.items(): + if var_name in discarded_args and \ + self._configuration is not None and \ + self._configuration.discard_unknown_keys and \ + self._additional_properties_model_instances: + # discard variable. + continue + setattr(self, var_name, var_value) + if var_name in self.read_only_vars: + raise ApiAttributeError(f"`{var_name}` is a read-only attribute. Use `from_openapi_data` to instantiate " + f"class with read only attributes.") + + @cached_property + def _composed_schemas(): + # we need this here to make our import statements work + # we must store _composed_schemas in here so the code is only run + # when we invoke this method. If we kept this at the class + # level we would get an error beause the class level + # code would be run when this module is imported, and these composed + # classes don't exist yet because their module has not finished + # loading + lazy_import() + return { + 'anyOf': [ + ], + 'allOf': [ + RepositoryResourceBase, + ], + 'oneOf': [ + ], + } From 235d11bec18495e517e59bebcb29341055ccb133 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 29 Apr 2025 16:37:28 +0100 Subject: [PATCH 02/22] chore(ebrains): update repository resource --- .../models/ebrains_repository_resource.py | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py b/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py index 087d3313b..299bc1570 100644 --- a/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py +++ b/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py @@ -11,28 +11,28 @@ from workspaces.models.repository_resource_base import RepositoryResourceBase # noqa: E501 -class BiomodelsRepositoryResource(Model): +class EbrainsRepositoryResource(Model): """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). Do not edit the class manually. """ def __init__(self, name=None, path=None, osbrepository_id=None, size=None, timestamp_modified=None, ref=None, sha=None): # noqa: E501 - """BiomodelsRepositoryResource - a model defined in OpenAPI + """EbrainsRepositoryResource - a model defined in OpenAPI - :param name: The name of this BiomodelsRepositoryResource. # noqa: E501 + :param name: The name of this EbrainsRepositoryResource. # noqa: E501 :type name: str - :param path: The path of this BiomodelsRepositoryResource. # noqa: E501 + :param path: The path of this EbrainsRepositoryResource. # noqa: E501 :type path: str - :param osbrepository_id: The osbrepository_id of this BiomodelsRepositoryResource. # noqa: E501 + :param osbrepository_id: The osbrepository_id of this EbrainsRepositoryResource. # noqa: E501 :type osbrepository_id: int - :param size: The size of this BiomodelsRepositoryResource. # noqa: E501 + :param size: The size of this EbrainsRepositoryResource. # noqa: E501 :type size: int - :param timestamp_modified: The timestamp_modified of this BiomodelsRepositoryResource. # noqa: E501 + :param timestamp_modified: The timestamp_modified of this EbrainsRepositoryResource. # noqa: E501 :type timestamp_modified: datetime - :param ref: The ref of this BiomodelsRepositoryResource. # noqa: E501 + :param ref: The ref of this EbrainsRepositoryResource. # noqa: E501 :type ref: str - :param sha: The sha of this BiomodelsRepositoryResource. # noqa: E501 + :param sha: The sha of this EbrainsRepositoryResource. # noqa: E501 :type sha: str """ self.openapi_types = { @@ -64,34 +64,34 @@ def __init__(self, name=None, path=None, osbrepository_id=None, size=None, times self._sha = sha @classmethod - def from_dict(cls, dikt) -> 'BiomodelsRepositoryResource': + def from_dict(cls, dikt) -> 'EbrainsRepositoryResource': """Returns the dict as a model :param dikt: A dict. :type: dict - :return: The BiomodelsRepositoryResource of this BiomodelsRepositoryResource. # noqa: E501 - :rtype: BiomodelsRepositoryResource + :return: The EbrainsRepositoryResource of this EbrainsRepositoryResource. # noqa: E501 + :rtype: EbrainsRepositoryResource """ return util.deserialize_model(dikt, cls) @property def name(self): - """Gets the name of this BiomodelsRepositoryResource. + """Gets the name of this EbrainsRepositoryResource. file name # noqa: E501 - :return: The name of this BiomodelsRepositoryResource. + :return: The name of this EbrainsRepositoryResource. :rtype: str """ return self._name @name.setter def name(self, name): - """Sets the name of this BiomodelsRepositoryResource. + """Sets the name of this EbrainsRepositoryResource. file name # noqa: E501 - :param name: The name of this BiomodelsRepositoryResource. + :param name: The name of this EbrainsRepositoryResource. :type name: str """ @@ -99,22 +99,22 @@ def name(self, name): @property def path(self): - """Gets the path of this BiomodelsRepositoryResource. + """Gets the path of this EbrainsRepositoryResource. Download URL of the Resource # noqa: E501 - :return: The path of this BiomodelsRepositoryResource. + :return: The path of this EbrainsRepositoryResource. :rtype: str """ return self._path @path.setter def path(self, path): - """Sets the path of this BiomodelsRepositoryResource. + """Sets the path of this EbrainsRepositoryResource. Download URL of the Resource # noqa: E501 - :param path: The path of this BiomodelsRepositoryResource. + :param path: The path of this EbrainsRepositoryResource. :type path: str """ @@ -122,22 +122,22 @@ def path(self, path): @property def osbrepository_id(self): - """Gets the osbrepository_id of this BiomodelsRepositoryResource. + """Gets the osbrepository_id of this EbrainsRepositoryResource. OSB Repository id # noqa: E501 - :return: The osbrepository_id of this BiomodelsRepositoryResource. + :return: The osbrepository_id of this EbrainsRepositoryResource. :rtype: int """ return self._osbrepository_id @osbrepository_id.setter def osbrepository_id(self, osbrepository_id): - """Sets the osbrepository_id of this BiomodelsRepositoryResource. + """Sets the osbrepository_id of this EbrainsRepositoryResource. OSB Repository id # noqa: E501 - :param osbrepository_id: The osbrepository_id of this BiomodelsRepositoryResource. + :param osbrepository_id: The osbrepository_id of this EbrainsRepositoryResource. :type osbrepository_id: int """ @@ -145,22 +145,22 @@ def osbrepository_id(self, osbrepository_id): @property def size(self): - """Gets the size of this BiomodelsRepositoryResource. + """Gets the size of this EbrainsRepositoryResource. File size in bytes of the RepositoryResource # noqa: E501 - :return: The size of this BiomodelsRepositoryResource. + :return: The size of this EbrainsRepositoryResource. :rtype: int """ return self._size @size.setter def size(self, size): - """Sets the size of this BiomodelsRepositoryResource. + """Sets the size of this EbrainsRepositoryResource. File size in bytes of the RepositoryResource # noqa: E501 - :param size: The size of this BiomodelsRepositoryResource. + :param size: The size of this EbrainsRepositoryResource. :type size: int """ @@ -168,22 +168,22 @@ def size(self, size): @property def timestamp_modified(self): - """Gets the timestamp_modified of this BiomodelsRepositoryResource. + """Gets the timestamp_modified of this EbrainsRepositoryResource. Date/time the ReposityResource is last modified # noqa: E501 - :return: The timestamp_modified of this BiomodelsRepositoryResource. + :return: The timestamp_modified of this EbrainsRepositoryResource. :rtype: datetime """ return self._timestamp_modified @timestamp_modified.setter def timestamp_modified(self, timestamp_modified): - """Sets the timestamp_modified of this BiomodelsRepositoryResource. + """Sets the timestamp_modified of this EbrainsRepositoryResource. Date/time the ReposityResource is last modified # noqa: E501 - :param timestamp_modified: The timestamp_modified of this BiomodelsRepositoryResource. + :param timestamp_modified: The timestamp_modified of this EbrainsRepositoryResource. :type timestamp_modified: datetime """ @@ -191,22 +191,22 @@ def timestamp_modified(self, timestamp_modified): @property def ref(self): - """Gets the ref of this BiomodelsRepositoryResource. + """Gets the ref of this EbrainsRepositoryResource. The GIT ref # noqa: E501 - :return: The ref of this BiomodelsRepositoryResource. + :return: The ref of this EbrainsRepositoryResource. :rtype: str """ return self._ref @ref.setter def ref(self, ref): - """Sets the ref of this BiomodelsRepositoryResource. + """Sets the ref of this EbrainsRepositoryResource. The GIT ref # noqa: E501 - :param ref: The ref of this BiomodelsRepositoryResource. + :param ref: The ref of this EbrainsRepositoryResource. :type ref: str """ @@ -214,22 +214,22 @@ def ref(self, ref): @property def sha(self): - """Gets the sha of this BiomodelsRepositoryResource. + """Gets the sha of this EbrainsRepositoryResource. The GIT sha of the resource # noqa: E501 - :return: The sha of this BiomodelsRepositoryResource. + :return: The sha of this EbrainsRepositoryResource. :rtype: str """ return self._sha @sha.setter def sha(self, sha): - """Sets the sha of this BiomodelsRepositoryResource. + """Sets the sha of this EbrainsRepositoryResource. The GIT sha of the resource # noqa: E501 - :param sha: The sha of this BiomodelsRepositoryResource. + :param sha: The sha of this EbrainsRepositoryResource. :type sha: str """ From 2b338dbb72fdaed9c0ffa6956a4a5281f6e3e96d Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 29 Apr 2025 16:40:03 +0100 Subject: [PATCH 03/22] feat(ebrains): add openapi entry --- applications/workspaces/api/openapi.yaml | 16 +++++++++++----- .../server/workspaces/openapi/openapi.yaml | 16 +++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/applications/workspaces/api/openapi.yaml b/applications/workspaces/api/openapi.yaml index 8b2037ce0..d2be29c9f 100644 --- a/applications/workspaces/api/openapi.yaml +++ b/applications/workspaces/api/openapi.yaml @@ -994,6 +994,7 @@ components: - figshare - github - biomodels + - ebrains type: string RepositoryContentType: description: | @@ -1293,6 +1294,16 @@ components: sha: description: The GIT sha of the resource type: string + BiomodelsRepositoryResource: + description: biomodels repository resource + allOf: + - + $ref: '#/components/schemas/RepositoryResourceBase' + EbrainsRepositoryResource: + description: ebrains repository resource + allOf: + - + $ref: '#/components/schemas/RepositoryResourceBase' DownloadResource: description: Download Resource (files/folders) allOf: @@ -1506,11 +1517,6 @@ components: $ref: '#/components/schemas/Tag' - x-secondary: osbrepository_tag - BiomodelsRepositoryResource: - description: figshare repository resource - allOf: - - - $ref: '#/components/schemas/RepositoryResourceBase' securitySchemes: bearerAuth: scheme: bearer diff --git a/applications/workspaces/server/workspaces/openapi/openapi.yaml b/applications/workspaces/server/workspaces/openapi/openapi.yaml index 8b2037ce0..d2be29c9f 100644 --- a/applications/workspaces/server/workspaces/openapi/openapi.yaml +++ b/applications/workspaces/server/workspaces/openapi/openapi.yaml @@ -994,6 +994,7 @@ components: - figshare - github - biomodels + - ebrains type: string RepositoryContentType: description: | @@ -1293,6 +1294,16 @@ components: sha: description: The GIT sha of the resource type: string + BiomodelsRepositoryResource: + description: biomodels repository resource + allOf: + - + $ref: '#/components/schemas/RepositoryResourceBase' + EbrainsRepositoryResource: + description: ebrains repository resource + allOf: + - + $ref: '#/components/schemas/RepositoryResourceBase' DownloadResource: description: Download Resource (files/folders) allOf: @@ -1506,11 +1517,6 @@ components: $ref: '#/components/schemas/Tag' - x-secondary: osbrepository_tag - BiomodelsRepositoryResource: - description: figshare repository resource - allOf: - - - $ref: '#/components/schemas/RepositoryResourceBase' securitySchemes: bearerAuth: scheme: bearer From 748a92d95538e3657a108d2dec53f34c7b1c7846 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 29 Apr 2025 16:40:46 +0100 Subject: [PATCH 04/22] feat(ebrains): update repository resource --- .../workspaces_cli/model/ebrains_repository_resource.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py b/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py index 9af539478..39273f7d5 100644 --- a/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py +++ b/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py @@ -34,7 +34,7 @@ def lazy_import(): globals()['RepositoryResourceBase'] = RepositoryResourceBase -class BiomodelsRepositoryResource(ModelComposed): +class EbrainsRepositoryResource(ModelComposed): """NOTE: This class is auto generated by OpenAPI Generator. Ref: https://openapi-generator.tech @@ -113,7 +113,7 @@ def discriminator(): @classmethod @convert_js_args_to_python_args def _from_openapi_data(cls, *args, **kwargs): # noqa: E501 - """BiomodelsRepositoryResource - a model defined in OpenAPI + """EbrainsRepositoryResource - a model defined in OpenAPI Keyword Args: _check_type (bool): if True, values for parameters in openapi_types @@ -217,7 +217,7 @@ def _from_openapi_data(cls, *args, **kwargs): # noqa: E501 @convert_js_args_to_python_args def __init__(self, *args, **kwargs): # noqa: E501 - """BiomodelsRepositoryResource - a model defined in OpenAPI + """EbrainsRepositoryResource - a model defined in OpenAPI Keyword Args: _check_type (bool): if True, values for parameters in openapi_types From 2d7b4746b85d60220f9d5be9741829d4917f6ea9 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 29 Apr 2025 16:41:37 +0100 Subject: [PATCH 05/22] feat(ebrains): update repository resource --- .../models/EbrainsRepositoryResource.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts b/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts index 32ff4c469..8bad9089d 100644 --- a/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts +++ b/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts @@ -23,46 +23,46 @@ import { /** * figshare repository resource * @export - * @interface BiomodelsRepositoryResource + * @interface EbrainsRepositoryResource */ -export interface BiomodelsRepositoryResource { +export interface EbrainsRepositoryResource { /** * file name * @type {string} - * @memberof BiomodelsRepositoryResource + * @memberof EbrainsRepositoryResource */ name?: string; /** * Download URL of the Resource * @type {string} - * @memberof BiomodelsRepositoryResource + * @memberof EbrainsRepositoryResource */ path?: string; /** * OSB Repository id * @type {number} - * @memberof BiomodelsRepositoryResource + * @memberof EbrainsRepositoryResource */ osbrepositoryId?: number; /** * File size in bytes of the RepositoryResource * @type {number} - * @memberof BiomodelsRepositoryResource + * @memberof EbrainsRepositoryResource */ size?: number; /** * Date/time the ReposityResource is last modified * @type {Date} - * @memberof BiomodelsRepositoryResource + * @memberof EbrainsRepositoryResource */ timestampModified?: Date; } -export function BiomodelsRepositoryResourceFromJSON(json: any): BiomodelsRepositoryResource { - return BiomodelsRepositoryResourceFromJSONTyped(json, false); +export function EbrainsRepositoryResourceFromJSON(json: any): EbrainsRepositoryResource { + return EbrainsRepositoryResourceFromJSONTyped(json, false); } -export function BiomodelsRepositoryResourceFromJSONTyped(json: any, ignoreDiscriminator: boolean): BiomodelsRepositoryResource { +export function EbrainsRepositoryResourceFromJSONTyped(json: any, ignoreDiscriminator: boolean): EbrainsRepositoryResource { if ((json === undefined) || (json === null)) { return json; } @@ -76,7 +76,7 @@ export function BiomodelsRepositoryResourceFromJSONTyped(json: any, ignoreDiscri }; } -export function BiomodelsRepositoryResourceToJSON(value?: BiomodelsRepositoryResource | null): any { +export function EbrainsRepositoryResourceToJSON(value?: EbrainsRepositoryResource | null): any { if (value === undefined) { return undefined; } From 9f105ec9282394fdc5692bb70d815e1eae071f59 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 29 Apr 2025 18:37:58 +0100 Subject: [PATCH 06/22] wip(ebrains) --- .../osbrepository/adapters/ebrainsadapter.py | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index 7bcc1ff3f..38df8e08f 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -3,69 +3,69 @@ from typing import List import requests + +from fairgraph import KGClient, KGProxy +from fairgraph.errors import ResolutionFailure +from fairgraph.openminds.core import FileRepository, Model, ModelVersion from cloudharness import log as logger from workspaces.models import RepositoryResourceNode, RepositoryInfo from workspaces.models.resource_origin import ResourceOrigin -from workspaces.models.biomodels_repository_resource import BiomodelsRepositoryResource +from workspaces.models.biomodels_repository_resource import EbrainsRepositoryResource from .utils import add_to_tree -class BiomodelsException(Exception): +class EbrainsException(Exception): pass -class BiomodelsAdapter: +class EbrainsAdapter: """ - Adapter for Biomodels + Adapter for Ebrains - https://www.ebi.ac.uk/biomodels/ + https://search.kg.ebrains.eu/ """ def __init__(self, osbrepository, uri=None): self.osbrepository = osbrepository self.uri = uri if uri else osbrepository.uri - self.api_url = "https://www.ebi.ac.uk/biomodels" + self.api_url = "https://search.kg.ebrains.eu/" + # TODO: get permanent application auth token from EBRAINS + self.kg_client = KGClient(client_id="SOME ID", client_secret="SOME SECRET", host="core.kg.ebrains.eu") try: self.model_id = re.search( - f"{self.api_url}/(\\w+)", + f"{self.api_url}/instances/(\\w+)", self.uri.strip("/")).group(1) except AttributeError: - raise BiomodelsException(f"{uri} is not a valid Biomodels URL") + raise EbrainsException(f"{uri} is not a valid Ebrains URL") + + def get_json(self, uri=None): + logger.debug(f"Getting: {self.model_id}") + + model = Model.from_id(id=self.model_id, client=self.client) + return model - def get_json(self, uri): - logger.debug(f"Getting: {uri}") - try: - r = requests.get( - uri, - params={"format": "json"} - ) - if r.status_code == 200: - return r.json() - else: - raise BiomodelsException( - f"Unexpected requests status code: {r.status_code}") - except Exception as e: - raise BiomodelsException("Unexpected error:", sys.exc_info()[0]) def get_base_uri(self): return self.uri def get_info(self) -> RepositoryInfo: - info = self.get_json( - f"{self.api_url}/{self.model_id}") + info = self.get_json() return RepositoryInfo(name=info["name"], contexts=self.get_contexts(), tags=info["format"]["name"], summary=info.get("description", "")) def get_contexts(self): - result = self.get_json(f"{self.api_url}/{self.model_id}") - revisions = result["history"]["revisions"] - return [str(v["version"]) for v in revisions] + result = self.get_json() + if isinstance(result.versions, list): + revisions = result.versions + else: + revisions = [result.versions] + return revisions def _get_filelist(self, context): logger.debug(f"Getting filelist: {context}") - contents = self.get_json(f"{self.api_url}/model/files/{self.model_id}.{context}") + contents = self.get_json() files = (contents.get("additional", []) + contents.get("main", [])) return files @@ -74,7 +74,7 @@ def get_resources(self, context): files = self._get_filelist(context) tree = RepositoryResourceNode( - resource=BiomodelsRepositoryResource( + resource=EbrainsRepositoryResource( name="/", path="/", osbrepository_id=self.osbrepository.id, @@ -98,7 +98,7 @@ def get_resources(self, context): def get_description(self, context): logger.debug(f"Getting description: {context}") try: - result = self.get_json(f"{self.api_url}/{self.model_id}.{context}") + result = self.get_json() return result["description"] except Exception as e: logger.debug( @@ -109,19 +109,19 @@ def get_tags(self, context): # using the format name for the moment, since they don't do explict # tags/keywords logger.debug(f"Getting tags: {context}") - result = self.get_json(f"{self.api_url}/{self.model_id}.{context}") + result = self.get_json() return result["format"]["name"] # biomodels files are usually small, so one task is enough def create_copy_task(self, workspace_id, origins: List[ResourceOrigin]): import workspaces.service.workflow as workflow - # no file tree in Biomodels from the looks of it + # no file tree in Ebrains from the looks of it folder = self.osbrepository.name # if nothing is selected, origins has one entry with path "/" # we get the file list and download individual files - # Biomodels does allow downloading the archive, but that is generated + # Ebrains does allow downloading the archive, but that is generated # on the fly and can require us to wait for an unspecified amount of # time if len(origins) == 1 and origins[0].path == "/": From 32996a654aca70fbd09d40a9f5a9ca4486642cd3 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 30 Apr 2025 12:17:30 +0100 Subject: [PATCH 07/22] chore(ebrains): register adapter --- .../server/workspaces/service/osbrepository/adapters/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py index 332728c21..daffc3242 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py @@ -2,3 +2,4 @@ from workspaces.service.osbrepository.adapters.figshareadapter import FigShareAdapter from workspaces.service.osbrepository.adapters.githubadapter import GitHubAdapter from workspaces.service.osbrepository.adapters.biomodelsadapter import BiomodelsAdapter +from workspaces.service.osbrepository.adapters.ebrainsadapter import EbrainsAdapter From f7a72de29cc53ec8a5ea665ebf238a647863d95c Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 30 Apr 2025 12:19:54 +0100 Subject: [PATCH 08/22] feat(ebrains): register adapter in service --- .../server/workspaces/service/osbrepository/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/__init__.py b/applications/workspaces/server/workspaces/service/osbrepository/__init__.py index 57ee30952..8a559632c 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/__init__.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/__init__.py @@ -7,7 +7,7 @@ -from workspaces.service.osbrepository.adapters import DandiAdapter, FigShareAdapter, GitHubAdapter, BiomodelsAdapter +from workspaces.service.osbrepository.adapters import DandiAdapter, FigShareAdapter, GitHubAdapter, BiomodelsAdapter, EbrainsAdapter def get_repository_adapter(osbrepository: OSBRepository=None, repository_type=None, uri=None, *args, **kwargs): @@ -22,6 +22,8 @@ def get_repository_adapter(osbrepository: OSBRepository=None, repository_type=No return FigShareAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs) elif repository_type == "biomodels": return BiomodelsAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs) + elif repository_type == "ebrains": + return EbrainsAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs) return None From e28d361f491839cb41c40660bd2fb08cc825f9a8 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 30 Apr 2025 14:04:31 +0100 Subject: [PATCH 09/22] feat(ebrains): implement getting resources in adapter --- .../osbrepository/adapters/ebrainsadapter.py | 310 ++++++++++++++++-- 1 file changed, 282 insertions(+), 28 deletions(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index 38df8e08f..6921022db 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -1,7 +1,8 @@ import re import sys -from typing import List +from typing import List, Optional import requests +from functools import cache, cached_property from fairgraph import KGClient, KGProxy @@ -13,6 +14,7 @@ from workspaces.models.biomodels_repository_resource import EbrainsRepositoryResource from .utils import add_to_tree +from .githubadapter import GitHubAdapter class EbrainsException(Exception): @@ -31,6 +33,7 @@ def __init__(self, osbrepository, uri=None): self.uri = uri if uri else osbrepository.uri self.api_url = "https://search.kg.ebrains.eu/" # TODO: get permanent application auth token from EBRAINS + # self.kg_client = KGClient(token="", host="core.kg.ebrains.eu") self.kg_client = KGClient(client_id="SOME ID", client_secret="SOME SECRET", host="core.kg.ebrains.eu") try: @@ -41,37 +44,291 @@ def __init__(self, osbrepository, uri=None): except AttributeError: raise EbrainsException(f"{uri} is not a valid Ebrains URL") - def get_json(self, uri=None): + @cache + def get_model(self, uri: Optional[str] = None) -> Model: + """Get model object using FairGraph""" logger.debug(f"Getting: {self.model_id}") + model = Model.from_id(id=self.model_id, client=self.kg_client) - model = Model.from_id(id=self.model_id, client=self.client) return model - def get_base_uri(self): return self.uri + @cache def get_info(self) -> RepositoryInfo: - info = self.get_json() - return RepositoryInfo(name=info["name"], contexts=self.get_contexts(), tags=info["format"]["name"], summary=info.get("description", "")) + """Get repository metadata from model object""" + model = self.get_model() + return RepositoryInfo(name=model.name, contexts=self.get_contexts(), tags=self._get_keywords(model), summary=model.description) + + def _get_keywords(self, model: Model) -> list[str]: + """Get keywords from model + + :param model: model object + :returns: list of keywords + + """ + keywords: list[str] = [] + if model.study_targets: + if isinstance(model.study_targets, KGProxy): + keyws = model.study_targets.resolve(self.kg_client) + else: + keyws = model.study_targets + + if isinstance(keyws, list): + for k in keyws: + if isinstance(k, KGProxy): + try: + keywords.append(k.resolve(self.kg_client).name) + except ResolutionFailure: + pass + else: + keywords.append(k.name) + else: + if isinstance(keyws, KGProxy): + keywords.append(keyws.resolve().name) + else: + keywords.append(keyws.name) + + if model.abstraction_level: + if isinstance(model.abstraction_level, KGProxy): + abs_l = model.abstraction_level.resolve(self.kg_client) + else: + abs_l = model.abstraction_level + keywords.append(abs_l.name) + + return keywords + + @cache + def get_contexts(self) -> list[str]: + model = self.get_model() + if isinstance(model.versions, list): + versions = model.versions + else: + versions = [model.versions] + + contexts = [] + + for v in versions: + v_r: Optional[ModelVersion] = None + if isinstance(v, KGProxy): + try: + v_r = v.resolve(self.kg_client) + except ResolutionFailure: + logger.error(f"ERROR: Could not resolve {v.id}") + continue + else: + v_r = v + + if v_r: + contexts.append(v_r.version_identifier) + + return contexts + + def _get_file_storage_url(self, context: str) -> Optional[str]: + """Get the URL of the file storage for the provided context + + :param context: TODO + :returns: TODO + + """ + model = self.get_model() + if isinstance(model.versions, list): + versions = model.versions + else: + versions = [model.versions] + + for v in versions: + v_r: Optional[ModelVersion] = None + if isinstance(v, KGProxy): + try: + v_r = v.resolve(self.kg_client) + except ResolutionFailure: + logger.error(f"ERROR: Could not resolve {v.id}") + continue + else: + v_r = v + if v_r: + if context == v_r.version_identifier: + repository: FileRepository = v_r.repository + try: + repository_r = repository.resolve(self.kg_client) + return repository_r.name + except ResolutionFailure: + logger.error(f"Could not resolve {repository.id}") + + return None + + + def _get_ebrains_data_proxy_file_list(self, url: str) -> list[str]: + """Get the list of files from an ebrains data proxy URL. + + The complete url will be of this form: + + .. code-block:: + + https://data-proxy.ebrains.eu/api/v1/buckets/m-0ffae3c2-443c-44fd-919f-70a4b01506a4?prefix=CA1_pyr_mpg150211_A_idA_080220241322/ + + The API documentation is here: + https://data-proxy.ebrains.eu/api/docs + + This URL returns a JSON response with all the objects listed. + So we can get the file list from there. To get the download URL, we need + this end point for each object in the list: + + .. code-block:: + + /v1/buckets/{bucket_name}/{object_name} + + :param url: url of repository + :returns: dict of files and their download URLs + + """ + file_list: dict[str, str] = {} + top_level_url: str = url.split("?prefix=")[0] + + r = requests.get(url) + if r.status_code == 200: + logger.debug("data-proxy: response is") + logger.debug(r) + + json_r = r.json() + object_list = json_r["objects"] + for anobject in object_list: + object_url = top_level_url + "/" + anobject["name"] + file_list[anobject["name"]] = object_url - def get_contexts(self): - result = self.get_json() - if isinstance(result.versions, list): - revisions = result.versions else: - revisions = [result.versions] - return revisions + logger.error(f"Something went wrong: {r.response_code}") + + if len(file_list.items()) == 0: + logger.warn("No files found for this: check kg.ebrains.eu to verify") + + return file_list + + + def _get_cscs_file_list(self, url: str) -> dict[str, str]: + """Get the list of files from a CSCS repository URL. + + The complete url will be of this form: + + .. code-block: + + https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/hippocampus_optimization/rat/CA1/v4.0.5/optimizations_Python3/CA1_pyr_cACpyr_mpg141208_B_idA_20190328144006/CA1_pyr_cACpyr_mpg141208_B_idA_20190328144006.zip?use_cell=cell_seed3_0.hoc&bluenaas=true + + To get the file list, we only need the top level: + + .. code-block:: + + https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/hippocampus_optimization + + We then need to limit the file list to the bits we want, because the top + level container contains all the files and all the versions: + + .. code-block:: + + rat/CA1/v4.0.5/optimizations_Python3/CA1_pyr_cACpyr_mpg141208_B_idA_20190328144006/CA1_pyr_cACpyr_mpg141208_B_idA_20190328144006 + + Note that even if the url is wrong (eg, in the shown example, the file list + does not include a folder called `optimizations_Python3` at all), the cscs + server still returns a zip. However, manually checking search.kg.ebrains.eu + shows that the corresponding entry does not have a file list. It simply + says "no files available". + + Also note that the url may include a `prefix=` parameter which specifies + the file directory structure. + + Most of these directories also include a zipped version. For the moment, we + include this in the file list. + + :param url: url of repository + :returns: dict of files and their download URLs + + """ + file_list: dict[str, str] = {} + file_list_url: str = "" + file_list_string: str = "" + + special_suffixes = [ + "py", + "hoc", + "xz", + "zip", + "pkl", + "json", + "pdf", + "mod", + "txt", + "png", + "zip", + "ipynb", + ] + + logger.debug(f"Getting file list for {url}") + if ".zip?" in url: + url_portions: list[str] = url.split(".zip")[0].split("/") + file_list_url = "/".join(url_portions[:6]) + file_list_string = "/".join(url_portions[6:]) + # assume it's with prefix + elif "?prefix=" in url: + file_list_url = url.split("?prefix=")[0] + file_list_string = url.split("?prefix=")[1] + else: + logger.warning(f"Other cscs url format: {url}") + # test if it's one of the file types + for suf in special_suffixes: + if url.endswith(suf): + file_list = {url: url} + break + + return file_list + + r = requests.get(file_list_url) + if r.status_code == 200: + for line in r.text.split(): + if ( + line.startswith(file_list_string) + and not line.endswith("/") + and line != file_list_string + ): + file_list[line] = file_list_url + "/" + line + else: + logger.error(f"Something went wrong: {r.response_code}") + + if len(file_list) == 0: + logger.warn("No files found for this: check kg.ebrains.eu to verify") + return file_list - def _get_filelist(self, context): - logger.debug(f"Getting filelist: {context}") - contents = self.get_json() - files = (contents.get("additional", []) + contents.get("main", [])) - return files def get_resources(self, context): logger.debug(f"Getting resources: {context}") - files = self._get_filelist(context) + + download_url = self._get_file_storage_url(context) + if "github" in download_url: + gh_adapter = GitHubAdapter(self.osbrepository, download_url) + return gh_adapter.get_resources(context) + + elif "modeldb" in download_url.lower(): + model_id = "" + # urls with model id after the last slash + # https://modeldb.science/249408?tab=7 + if "modeldb.science" in download_url or "modeldb.yale.edu" in download_url: + model_id = download_url.split("/")[-1].split('?')[0] + # legacy urls with model id as a parameter + # https://senselab.med.yale.edu/ModelDB/showmodel.cshtml?model=249408#tabs-1 + else: + model_id = download_url.split("?model=")[-1].split('#')[0] + + modeldb_url = f"https://github.com/ModelDBRepository/{model_id}" + gh_adapter = GitHubAdapter(self.osbrepository, modeldb_url) + return gh_adapter.get_resources(context) + + elif "cscs.ch" in download_url: + files = self._get_cscs_file_list(download_url) + elif "data-proxy.ebrains.eu" in download_url: + files = self._get_ebrains_data_proxy_file_list(download_url) + else: + files = ["TODO: handle other special cases"] tree = RepositoryResourceNode( resource=EbrainsRepositoryResource( @@ -83,13 +340,11 @@ def get_resources(self, context): children=[], ) - for afile in files: - download_url = f"{self.api_url}/model/download/{self.model_id}.{context}?filename={afile['name']}" + for afile, url in files.items(): add_to_tree( tree=tree, - tree_path=[afile["name"]], - path=download_url, - size=int(afile["fileSize"]), + tree_path=afile, + path=url, osbrepository_id=self.osbrepository.id, ) @@ -98,8 +353,8 @@ def get_resources(self, context): def get_description(self, context): logger.debug(f"Getting description: {context}") try: - result = self.get_json() - return result["description"] + result = self.get_model() + return result.description except Exception as e: logger.debug( "unable to get the description from biomodels, %", str(e)) @@ -109,10 +364,9 @@ def get_tags(self, context): # using the format name for the moment, since they don't do explict # tags/keywords logger.debug(f"Getting tags: {context}") - result = self.get_json() - return result["format"]["name"] + model = self.get_model() + return self._get_keywords(model) - # biomodels files are usually small, so one task is enough def create_copy_task(self, workspace_id, origins: List[ResourceOrigin]): import workspaces.service.workflow as workflow From 3bebe6998409f681efc6cef5455bf9c56e76e3e3 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 30 Apr 2025 14:57:24 +0100 Subject: [PATCH 10/22] feat(ebrains): implement copy task --- .../osbrepository/adapters/ebrainsadapter.py | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index 6921022db..4f9a7a1c1 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -373,25 +373,42 @@ def create_copy_task(self, workspace_id, origins: List[ResourceOrigin]): # no file tree in Ebrains from the looks of it folder = self.osbrepository.name + download_url = self._get_file_storage_url(self.osbrepository.default_context) + if "github" in download_url: + gh_adapter = GitHubAdapter(self.osbrepository, download_url) + return gh_adapter.create_copy_task(workspace_id, origins) + + elif "modeldb" in download_url.lower(): + model_id = "" + # urls with model id after the last slash + # https://modeldb.science/249408?tab=7 + if "modeldb.science" in download_url or "modeldb.yale.edu" in download_url: + model_id = download_url.split("/")[-1].split('?')[0] + # legacy urls with model id as a parameter + # https://senselab.med.yale.edu/ModelDB/showmodel.cshtml?model=249408#tabs-1 + else: + model_id = download_url.split("?model=")[-1].split('#')[0] + + modeldb_url = f"https://github.com/ModelDBRepository/{model_id}" + gh_adapter = GitHubAdapter(self.osbrepository, modeldb_url) + return gh_adapter.create_copy_task(workspace_id, origins) + # if nothing is selected, origins has one entry with path "/" # we get the file list and download individual files - # Ebrains does allow downloading the archive, but that is generated - # on the fly and can require us to wait for an unspecified amount of - # time if len(origins) == 1 and origins[0].path == "/": - """ - # to use the archive method, just set paths to "" - paths = "" - """ - files = self._get_filelist(self.osbrepository.default_context) - download_url_prefix = f"{self.api_url}/model/download/{self.model_id}.{self.osbrepository.default_context}?filename=" - paths = "\\".join(f"{download_url_prefix}{file['name']}" for file in files) + files: dict[str, str] = {} + if "cscs.ch" in download_url: + files = self._get_cscs_file_list(download_url) + elif "data-proxy.ebrains.eu" in download_url: + files = self._get_ebrains_data_proxy_file_list(download_url) + + paths = "\\".join(list(files.values())) else: paths = "\\".join(o.path for o in origins) # username / password are not currently used return workflow.create_copy_task( - image_name="workspaces-biomodels-copy", + image_name="workspaces-ebrains-copy", workspace_id=workspace_id, folder=folder, url=f"{self.model_id}.{self.osbrepository.default_context}", From 55687aac23c7b84b1f41309ec2dd2a33ef2ae51a Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 30 Apr 2025 15:34:43 +0100 Subject: [PATCH 11/22] feat: add script for local deployment We'll note this in the documentation also. --- local-scripts/osbv2-local.sh | 228 +++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100755 local-scripts/osbv2-local.sh diff --git a/local-scripts/osbv2-local.sh b/local-scripts/osbv2-local.sh new file mode 100755 index 000000000..491d182ce --- /dev/null +++ b/local-scripts/osbv2-local.sh @@ -0,0 +1,228 @@ +#!/bin/bash + +# Copyright 2025 OSBv2 contributors +# Author: Ankur Sinha +# File : osbv2-local.sh +# +# Script to help with local deployments of OSBv2 +# To be executed from the root folder of the OSBv2 repository. + +# depends on how you install it, by default in the parent folder from where +# this script is called +CLOUD_HARNESS_URL="https://github.com/MetaCell/cloud-harness.git" +CLOUD_HARNESS_DIR_LOCATION="../" +CLOUD_HARNESS_DIR="${CLOUD_HARNESS_DIR_LOCATION}/cloud-harness" +CLOUD_HARNESS_DEFAULT="release/2.5.0" +CLOUD_HARNESS_BRANCH="" +SKAFFOLD="skaffold" + +# Application to deploy +DEPLOYMENT_APP="" +DEFAULT_DEPLOYMENT_APP="osb-portal" + +# Py version +# Cloud harness doesn't always work on newer versions +PY_VERSION="python3.12" +# +# if not, specify location of virtualenv here +OSB_DIR="./" +VENV_DIR="${OSB_DIR}/.venv" + +deploy () { + if ! command -v helm >/dev/null || ! command -v $SKAFFOLD >/dev/null || ! command -v harness-deployment >/dev/null ; then + echo "helm, skaffold, and cloud-harness are required but were not found." + echo + echo "Please install helm and skaffold as noted in their documentation:" + echo "- https://helm.sh/docs/intro/install/" + echo "- https://skaffold.dev/docs/install/" + echo + echo "To install cloud-harness, please see the -u/-U options" + exit 1 + fi + + pushd $OSB_DIR + echo "-> deploying" + echo "-> checking (and starting) docker daemon" + systemctl is-active docker --quiet || sudo systemctl start docker.service + echo "-> starting minkube" + minikube start --memory="10000mb" --cpus=8 --disk-size="60000mb" --kubernetes-version=v1.32 --driver=docker || notify_fail "Failed: minikube start" + echo "-> enabling ingress addon" + minikube addons enable ingress || notify_fail "Failed: ingress add on" + echo "-> setting up osblocal namespace" + kubectl get ns osblocal || kubectl create ns osblocal || notify_fail "Failed: ns set up" + echo "-> setting up minikube docker env" + eval $(minikube docker-env) || notify_fail "Failed: env setup" + echo "-> harnessing deployment" + harness_deployment + echo "-> running skaffold" + $SKAFFOLD dev --cleanup=false || { notify_fail "Failed: skaffold" ; minikube stop; } + #$SKAFFOLD dev || notify_fail "Failed: skaffold" + popd +} + +function harness_deployment() { + # `-e local` does not build nwbexplorer/netpyne + # use -e dev for that, but that will send e-mails to Filippo and Zoraan + # suggested: create a new file in deploy/values-ankur.yaml where you use + # your e-mail address, and then use `-e ankur` to use these values. + pushd $OSB_DIR + harness-deployment ../cloud-harness . -l -n osblocal -d osb.local -dtls -m build -e local -i $DEPLOYMENT_APP || notify_fail "Failed: harness-deployment" + #harness-deployment ../cloud-harness . -l -n osblocal -d osb.local -u -dtls -m build -e local -i workspaces || notify_fail "Failed: harness-deployment" + popd +} + +notify_fail () { + if ! command -v notify-send >/dev/null + then + echo "-> $1" + else + notify-send -t 1000 -i "org.gnome.Terminal" -a "Terminal" "OSBv2 deployment" "$1" + fi + exit 1 +} + +function update_cloud_harness() { + echo "Updating cloud harness" + CLOUD_HARNESS_PACKAGES=$(pip list | grep cloud | tr -s " " | cut -d " " -f1 | tr '\n' ' ') + pip uninstall ${CLOUD_HARNESS_PACKAGES} -y || echo "No cloud harness packages installed" + if ! [ -d "${CLOUD_HARNESS_DIR}" ] + then + echo "Cloud harness folder does not exist. Cloning" + pushd "${CLOUD_HARNESS_DIR_LOCATION}" && git clone "${CLOUD_HARNESS_URL}" && popd + fi + pushd "$CLOUD_HARNESS_DIR" && git clean -dfx && git fetch && git checkout ${CLOUD_HARNESS_BRANCH} && git pull && pip install -r requirements.txt && popd +} + +function activate_venv() { + if [ -f "${VENV_DIR}/bin/activate" ] + then + source "${VENV_DIR}/bin/activate" + else + echo "No virtual environment found at ${VENV_DIR}. Creating" + ${PY_VERSION} -m venv "${VENV_DIR}" && source "${VENV_DIR}/bin/activate" + fi +} + +# don't actually need this because when the script exists, the environment is +# lost anyway +function deactivate_venv() { + deactivate +} + +function print_versions() { + echo "** docker **" + docker version + echo "\n** minikube **" + minikube version + echo "\n** cloud harness **" + pushd "${CLOUD_HARNESS_DIR}" && git log --oneline | head -1 && popd + echo "\n** helm **" + helm version + echo "\n** skaffold **" + $SKAFFOLD version + echo "\n** python **" + python --version + echo "\n** git **" + git --version +} + +clean () { + pushd $OSB_DIR + echo "-> Cleaning up all images." + docker image prune --all + docker builder prune --all + $SKAFFOLD delete + minikube stop + minikube delete + docker image prune --all + docker builder prune --all + popd +} + +usage () { + echo "Script for automating local deployments of OSBv2" + echo + echo "USAGE $0 -[dDbBvuUch]" + echo + echo "-d: deploy" + echo "-D: deploy " + echo "-b: run 'harness-deployment': required when you have made changes and want to refresh the deployment" + echo "-B: run 'harness-deployment ': required when you have made changes and want to refresh the deployment" + echo "-v: print version information" + echo "-u branch: update and install provided cloud_harness branch $CLOUD_HARNESS_DEFAULT" + echo "-U branch: update and install specified cloud_harness branch ($CLOUD_HARNESS_DEFAULT)" + echo "-c: clean up minikube and docker: sometimes needed with an outdated cache" + echo "-h: print this and exit" +} + +if [ $# -lt 1 ] +then + usage + exit 1 +fi + + +# parse options +while getopts ":vdD:uU:hbB:c" OPTION +do + case $OPTION in + v) + activate_venv + print_versions + deactivate_venv + exit 0 + ;; + b) + DEPLOYMENT_APP="${DEFAULT_DEPLOYMENT_APP}" + activate_venv + harness_deployment + deactivate_venv + exit 0 + ;; + B) + DEPLOYMENT_APP="${OPTARG}" + activate_venv + harness_deployment + deactivate_venv + exit 0 + ;; + d) + DEPLOYMENT_APP="${DEFAULT_DEPLOYMENT_APP}" + activate_venv + deploy + exit 0 + ;; + D) + DEPLOYMENT_APP="${OPTARG}" + activate_venv + deploy + exit 0 + ;; + c) + clean + exit 0 + ;; + u) + CLOUD_HARNESS_BRANCH="${CLOUD_HARNESS_DEFAULT}" + activate_venv + update_cloud_harness + deactivate_venv + exit 0 + ;; + U) + CLOUD_HARNESS_BRANCH="${OPTARG}" + activate_venv + update_cloud_harness + deactivate_venv + exit 0 + ;; + h) + usage + exit 0 + ;; + ?) + usage + exit 1 + ;; + esac +done From 1b5fc11104233f148f2acee0c9b1a8f0b5414819 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Thu, 1 May 2025 10:37:00 +0100 Subject: [PATCH 12/22] fix: include fairgraph in reqs --- applications/workspaces/server/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/workspaces/server/requirements.txt b/applications/workspaces/server/requirements.txt index 1c23952c9..67f0cd543 100644 --- a/applications/workspaces/server/requirements.txt +++ b/applications/workspaces/server/requirements.txt @@ -15,6 +15,7 @@ cryptography==43.0.3 debugpy==1.8.9 deprecation==2.1.0 durationpy==0.9 +fairgraph==0.12.2 Flask==2.2.5 Flask-Cors==5.0.0 Flask-SQLAlchemy==3.0.2 @@ -75,4 +76,4 @@ types-toml==0.10.8.20240310 typing_extensions==4.12.2 urllib3==2.0.7 websocket-client==1.8.0 -Werkzeug==2.2.3 \ No newline at end of file +Werkzeug==2.2.3 From 684a714a518a8298c27b8307293d8e98182958a1 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Thu, 1 May 2025 11:09:22 +0100 Subject: [PATCH 13/22] fix(ebrains): correct and add missing imports --- applications/workspaces/server/workspaces/models/__init__.py | 1 + .../workspaces/service/osbrepository/adapters/ebrainsadapter.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/workspaces/server/workspaces/models/__init__.py b/applications/workspaces/server/workspaces/models/__init__.py index 9a1bd7f5d..8904300e7 100644 --- a/applications/workspaces/server/workspaces/models/__init__.py +++ b/applications/workspaces/server/workspaces/models/__init__.py @@ -46,3 +46,4 @@ from workspaces.models.workspace_resource_entity_all_of import WorkspaceResourceEntityAllOf from workspaces.models.repository_info import RepositoryInfo from workspaces.models.biomodels_repository_resource import BiomodelsRepositoryResource +from workspaces.models.ebrains_repository_resource import EbrainsRepositoryResource diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index 4f9a7a1c1..266d2eb7d 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -11,7 +11,7 @@ from cloudharness import log as logger from workspaces.models import RepositoryResourceNode, RepositoryInfo from workspaces.models.resource_origin import ResourceOrigin -from workspaces.models.biomodels_repository_resource import EbrainsRepositoryResource +from workspaces.models.ebrains_repository_resource import EbrainsRepositoryResource from .utils import add_to_tree from .githubadapter import GitHubAdapter From 9954c6666e57158147b8d210116208eb7fac8bf6 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Thu, 1 May 2025 13:20:37 +0100 Subject: [PATCH 14/22] feat(ebrains): add to repo add box dialog --- .../src/apiclient/workspaces/models/RepositoryType.ts | 5 ++++- .../osb-portal/src/apiclient/workspaces/models/index.ts | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/applications/osb-portal/src/apiclient/workspaces/models/RepositoryType.ts b/applications/osb-portal/src/apiclient/workspaces/models/RepositoryType.ts index f46285a84..76f3602f5 100644 --- a/applications/osb-portal/src/apiclient/workspaces/models/RepositoryType.ts +++ b/applications/osb-portal/src/apiclient/workspaces/models/RepositoryType.ts @@ -17,6 +17,8 @@ * * dandi - DANDI repository * * figshare - FigShare repository * * github - Github repository + * * biomodels - Biomodels repository + * * ebrains - Ebrains repository * @export * @enum {string} */ @@ -24,7 +26,8 @@ export enum RepositoryType { Dandi = 'dandi', Figshare = 'figshare', Github = 'github', - Biomodels = 'biomodels' + Biomodels = 'biomodels', + Ebrains = 'ebrains' } export function RepositoryTypeFromJSON(json: any): RepositoryType { diff --git a/applications/osb-portal/src/apiclient/workspaces/models/index.ts b/applications/osb-portal/src/apiclient/workspaces/models/index.ts index d0d39625f..391693b59 100644 --- a/applications/osb-portal/src/apiclient/workspaces/models/index.ts +++ b/applications/osb-portal/src/apiclient/workspaces/models/index.ts @@ -1,4 +1,5 @@ export * from './BiomodelsRepositoryResource'; +export * from './EbrainsRepositoryResource'; export * from './DandiRepositoryResource'; export * from './DownloadResource'; export * from './FigshareRepositoryResource'; From 38dd106cda6927da878e6873b7b18c86798a2ca3 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Thu, 1 May 2025 16:37:31 +0100 Subject: [PATCH 15/22] chore: "Ebrains" -> "EBRAINS" --- .../workspaces/models/RepositoryType.ts | 4 +- ...source.ts => EBRAINSRepositoryResource.ts} | 22 ++--- .../workspaces/models/RepositoryType.ts | 4 +- .../src/apiclient/workspaces/models/index.ts | 2 +- .../osb-portal/src/pages/RepositoryPage.tsx | 4 + applications/workspaces/api/openapi.yaml | 4 +- .../server/workspaces/models/__init__.py | 2 +- .../models/ebrains_repository_resource.py | 80 +++++++++---------- .../server/workspaces/openapi/openapi.yaml | 4 +- .../service/osbrepository/__init__.py | 4 +- .../osbrepository/adapters/__init__.py | 2 +- .../model/ebrains_repository_resource.py | 6 +- 12 files changed, 72 insertions(+), 66 deletions(-) rename applications/osb-portal/src/apiclient/workspaces/models/{EbrainsRepositoryResource.ts => EBRAINSRepositoryResource.ts} (76%) diff --git a/applications/backoffice/src/apiclient/workspaces/models/RepositoryType.ts b/applications/backoffice/src/apiclient/workspaces/models/RepositoryType.ts index 9e8e537dd..cbb19b0d3 100644 --- a/applications/backoffice/src/apiclient/workspaces/models/RepositoryType.ts +++ b/applications/backoffice/src/apiclient/workspaces/models/RepositoryType.ts @@ -23,7 +23,9 @@ export enum RepositoryType { Dandi = 'dandi', Figshare = 'figshare', - Github = 'github' + Github = 'github', + Biomodels = 'biomodels', + EBRAINS = "ebrains" } export function RepositoryTypeFromJSON(json: any): RepositoryType { diff --git a/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts b/applications/osb-portal/src/apiclient/workspaces/models/EBRAINSRepositoryResource.ts similarity index 76% rename from applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts rename to applications/osb-portal/src/apiclient/workspaces/models/EBRAINSRepositoryResource.ts index 8bad9089d..954892255 100644 --- a/applications/osb-portal/src/apiclient/workspaces/models/EbrainsRepositoryResource.ts +++ b/applications/osb-portal/src/apiclient/workspaces/models/EBRAINSRepositoryResource.ts @@ -23,46 +23,46 @@ import { /** * figshare repository resource * @export - * @interface EbrainsRepositoryResource + * @interface EBRAINSRepositoryResource */ -export interface EbrainsRepositoryResource { +export interface EBRAINSRepositoryResource { /** * file name * @type {string} - * @memberof EbrainsRepositoryResource + * @memberof EBRAINSRepositoryResource */ name?: string; /** * Download URL of the Resource * @type {string} - * @memberof EbrainsRepositoryResource + * @memberof EBRAINSRepositoryResource */ path?: string; /** * OSB Repository id * @type {number} - * @memberof EbrainsRepositoryResource + * @memberof EBRAINSRepositoryResource */ osbrepositoryId?: number; /** * File size in bytes of the RepositoryResource * @type {number} - * @memberof EbrainsRepositoryResource + * @memberof EBRAINSRepositoryResource */ size?: number; /** * Date/time the ReposityResource is last modified * @type {Date} - * @memberof EbrainsRepositoryResource + * @memberof EBRAINSRepositoryResource */ timestampModified?: Date; } -export function EbrainsRepositoryResourceFromJSON(json: any): EbrainsRepositoryResource { - return EbrainsRepositoryResourceFromJSONTyped(json, false); +export function EBRAINSRepositoryResourceFromJSON(json: any): EBRAINSRepositoryResource { + return EBRAINSRepositoryResourceFromJSONTyped(json, false); } -export function EbrainsRepositoryResourceFromJSONTyped(json: any, ignoreDiscriminator: boolean): EbrainsRepositoryResource { +export function EBRAINSRepositoryResourceFromJSONTyped(json: any, ignoreDiscriminator: boolean): EBRAINSRepositoryResource { if ((json === undefined) || (json === null)) { return json; } @@ -76,7 +76,7 @@ export function EbrainsRepositoryResourceFromJSONTyped(json: any, ignoreDiscrimi }; } -export function EbrainsRepositoryResourceToJSON(value?: EbrainsRepositoryResource | null): any { +export function EBRAINSRepositoryResourceToJSON(value?: EBRAINSRepositoryResource | null): any { if (value === undefined) { return undefined; } diff --git a/applications/osb-portal/src/apiclient/workspaces/models/RepositoryType.ts b/applications/osb-portal/src/apiclient/workspaces/models/RepositoryType.ts index 76f3602f5..1fa7320b6 100644 --- a/applications/osb-portal/src/apiclient/workspaces/models/RepositoryType.ts +++ b/applications/osb-portal/src/apiclient/workspaces/models/RepositoryType.ts @@ -18,7 +18,7 @@ * * figshare - FigShare repository * * github - Github repository * * biomodels - Biomodels repository - * * ebrains - Ebrains repository + * * ebrains - EBRAINS repository * @export * @enum {string} */ @@ -27,7 +27,7 @@ export enum RepositoryType { Figshare = 'figshare', Github = 'github', Biomodels = 'biomodels', - Ebrains = 'ebrains' + EBRAINS = 'ebrains' } export function RepositoryTypeFromJSON(json: any): RepositoryType { diff --git a/applications/osb-portal/src/apiclient/workspaces/models/index.ts b/applications/osb-portal/src/apiclient/workspaces/models/index.ts index 391693b59..7152a64a0 100644 --- a/applications/osb-portal/src/apiclient/workspaces/models/index.ts +++ b/applications/osb-portal/src/apiclient/workspaces/models/index.ts @@ -1,5 +1,5 @@ export * from './BiomodelsRepositoryResource'; -export * from './EbrainsRepositoryResource'; +export * from './EBRAINSRepositoryResource'; export * from './DandiRepositoryResource'; export * from './DownloadResource'; export * from './FigshareRepositoryResource'; diff --git a/applications/osb-portal/src/pages/RepositoryPage.tsx b/applications/osb-portal/src/pages/RepositoryPage.tsx index e540a16f0..6bddfe039 100644 --- a/applications/osb-portal/src/pages/RepositoryPage.tsx +++ b/applications/osb-portal/src/pages/RepositoryPage.tsx @@ -264,6 +264,10 @@ export const RepositoryPage = (props: any) => { "_blank" ); break; + // For figshare, there does not seem to be a version specific URL + case "ebrains": + window.open(`${repository.uri}`, "_blank"); + break; default: window.open(`#`, "_blank"); } diff --git a/applications/workspaces/api/openapi.yaml b/applications/workspaces/api/openapi.yaml index d2be29c9f..9a55b12a4 100644 --- a/applications/workspaces/api/openapi.yaml +++ b/applications/workspaces/api/openapi.yaml @@ -1299,8 +1299,8 @@ components: allOf: - $ref: '#/components/schemas/RepositoryResourceBase' - EbrainsRepositoryResource: - description: ebrains repository resource + EBRAINSRepositoryResource: + description: EBRAINS repository resource allOf: - $ref: '#/components/schemas/RepositoryResourceBase' diff --git a/applications/workspaces/server/workspaces/models/__init__.py b/applications/workspaces/server/workspaces/models/__init__.py index 8904300e7..2d595d8d4 100644 --- a/applications/workspaces/server/workspaces/models/__init__.py +++ b/applications/workspaces/server/workspaces/models/__init__.py @@ -46,4 +46,4 @@ from workspaces.models.workspace_resource_entity_all_of import WorkspaceResourceEntityAllOf from workspaces.models.repository_info import RepositoryInfo from workspaces.models.biomodels_repository_resource import BiomodelsRepositoryResource -from workspaces.models.ebrains_repository_resource import EbrainsRepositoryResource +from workspaces.models.ebrains_repository_resource import EBRAINSRepositoryResource diff --git a/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py b/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py index 299bc1570..5bf917183 100644 --- a/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py +++ b/applications/workspaces/server/workspaces/models/ebrains_repository_resource.py @@ -11,28 +11,28 @@ from workspaces.models.repository_resource_base import RepositoryResourceBase # noqa: E501 -class EbrainsRepositoryResource(Model): +class EBRAINSRepositoryResource(Model): """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). Do not edit the class manually. """ def __init__(self, name=None, path=None, osbrepository_id=None, size=None, timestamp_modified=None, ref=None, sha=None): # noqa: E501 - """EbrainsRepositoryResource - a model defined in OpenAPI + """EBRAINSRepositoryResource - a model defined in OpenAPI - :param name: The name of this EbrainsRepositoryResource. # noqa: E501 + :param name: The name of this EBRAINSRepositoryResource. # noqa: E501 :type name: str - :param path: The path of this EbrainsRepositoryResource. # noqa: E501 + :param path: The path of this EBRAINSRepositoryResource. # noqa: E501 :type path: str - :param osbrepository_id: The osbrepository_id of this EbrainsRepositoryResource. # noqa: E501 + :param osbrepository_id: The osbrepository_id of this EBRAINSRepositoryResource. # noqa: E501 :type osbrepository_id: int - :param size: The size of this EbrainsRepositoryResource. # noqa: E501 + :param size: The size of this EBRAINSRepositoryResource. # noqa: E501 :type size: int - :param timestamp_modified: The timestamp_modified of this EbrainsRepositoryResource. # noqa: E501 + :param timestamp_modified: The timestamp_modified of this EBRAINSRepositoryResource. # noqa: E501 :type timestamp_modified: datetime - :param ref: The ref of this EbrainsRepositoryResource. # noqa: E501 + :param ref: The ref of this EBRAINSRepositoryResource. # noqa: E501 :type ref: str - :param sha: The sha of this EbrainsRepositoryResource. # noqa: E501 + :param sha: The sha of this EBRAINSRepositoryResource. # noqa: E501 :type sha: str """ self.openapi_types = { @@ -64,34 +64,34 @@ def __init__(self, name=None, path=None, osbrepository_id=None, size=None, times self._sha = sha @classmethod - def from_dict(cls, dikt) -> 'EbrainsRepositoryResource': + def from_dict(cls, dikt) -> 'EBRAINSRepositoryResource': """Returns the dict as a model :param dikt: A dict. :type: dict - :return: The EbrainsRepositoryResource of this EbrainsRepositoryResource. # noqa: E501 - :rtype: EbrainsRepositoryResource + :return: The EBRAINSRepositoryResource of this EBRAINSRepositoryResource. # noqa: E501 + :rtype: EBRAINSRepositoryResource """ return util.deserialize_model(dikt, cls) @property def name(self): - """Gets the name of this EbrainsRepositoryResource. + """Gets the name of this EBRAINSRepositoryResource. file name # noqa: E501 - :return: The name of this EbrainsRepositoryResource. + :return: The name of this EBRAINSRepositoryResource. :rtype: str """ return self._name @name.setter def name(self, name): - """Sets the name of this EbrainsRepositoryResource. + """Sets the name of this EBRAINSRepositoryResource. file name # noqa: E501 - :param name: The name of this EbrainsRepositoryResource. + :param name: The name of this EBRAINSRepositoryResource. :type name: str """ @@ -99,22 +99,22 @@ def name(self, name): @property def path(self): - """Gets the path of this EbrainsRepositoryResource. + """Gets the path of this EBRAINSRepositoryResource. Download URL of the Resource # noqa: E501 - :return: The path of this EbrainsRepositoryResource. + :return: The path of this EBRAINSRepositoryResource. :rtype: str """ return self._path @path.setter def path(self, path): - """Sets the path of this EbrainsRepositoryResource. + """Sets the path of this EBRAINSRepositoryResource. Download URL of the Resource # noqa: E501 - :param path: The path of this EbrainsRepositoryResource. + :param path: The path of this EBRAINSRepositoryResource. :type path: str """ @@ -122,22 +122,22 @@ def path(self, path): @property def osbrepository_id(self): - """Gets the osbrepository_id of this EbrainsRepositoryResource. + """Gets the osbrepository_id of this EBRAINSRepositoryResource. OSB Repository id # noqa: E501 - :return: The osbrepository_id of this EbrainsRepositoryResource. + :return: The osbrepository_id of this EBRAINSRepositoryResource. :rtype: int """ return self._osbrepository_id @osbrepository_id.setter def osbrepository_id(self, osbrepository_id): - """Sets the osbrepository_id of this EbrainsRepositoryResource. + """Sets the osbrepository_id of this EBRAINSRepositoryResource. OSB Repository id # noqa: E501 - :param osbrepository_id: The osbrepository_id of this EbrainsRepositoryResource. + :param osbrepository_id: The osbrepository_id of this EBRAINSRepositoryResource. :type osbrepository_id: int """ @@ -145,22 +145,22 @@ def osbrepository_id(self, osbrepository_id): @property def size(self): - """Gets the size of this EbrainsRepositoryResource. + """Gets the size of this EBRAINSRepositoryResource. File size in bytes of the RepositoryResource # noqa: E501 - :return: The size of this EbrainsRepositoryResource. + :return: The size of this EBRAINSRepositoryResource. :rtype: int """ return self._size @size.setter def size(self, size): - """Sets the size of this EbrainsRepositoryResource. + """Sets the size of this EBRAINSRepositoryResource. File size in bytes of the RepositoryResource # noqa: E501 - :param size: The size of this EbrainsRepositoryResource. + :param size: The size of this EBRAINSRepositoryResource. :type size: int """ @@ -168,22 +168,22 @@ def size(self, size): @property def timestamp_modified(self): - """Gets the timestamp_modified of this EbrainsRepositoryResource. + """Gets the timestamp_modified of this EBRAINSRepositoryResource. Date/time the ReposityResource is last modified # noqa: E501 - :return: The timestamp_modified of this EbrainsRepositoryResource. + :return: The timestamp_modified of this EBRAINSRepositoryResource. :rtype: datetime """ return self._timestamp_modified @timestamp_modified.setter def timestamp_modified(self, timestamp_modified): - """Sets the timestamp_modified of this EbrainsRepositoryResource. + """Sets the timestamp_modified of this EBRAINSRepositoryResource. Date/time the ReposityResource is last modified # noqa: E501 - :param timestamp_modified: The timestamp_modified of this EbrainsRepositoryResource. + :param timestamp_modified: The timestamp_modified of this EBRAINSRepositoryResource. :type timestamp_modified: datetime """ @@ -191,22 +191,22 @@ def timestamp_modified(self, timestamp_modified): @property def ref(self): - """Gets the ref of this EbrainsRepositoryResource. + """Gets the ref of this EBRAINSRepositoryResource. The GIT ref # noqa: E501 - :return: The ref of this EbrainsRepositoryResource. + :return: The ref of this EBRAINSRepositoryResource. :rtype: str """ return self._ref @ref.setter def ref(self, ref): - """Sets the ref of this EbrainsRepositoryResource. + """Sets the ref of this EBRAINSRepositoryResource. The GIT ref # noqa: E501 - :param ref: The ref of this EbrainsRepositoryResource. + :param ref: The ref of this EBRAINSRepositoryResource. :type ref: str """ @@ -214,22 +214,22 @@ def ref(self, ref): @property def sha(self): - """Gets the sha of this EbrainsRepositoryResource. + """Gets the sha of this EBRAINSRepositoryResource. The GIT sha of the resource # noqa: E501 - :return: The sha of this EbrainsRepositoryResource. + :return: The sha of this EBRAINSRepositoryResource. :rtype: str """ return self._sha @sha.setter def sha(self, sha): - """Sets the sha of this EbrainsRepositoryResource. + """Sets the sha of this EBRAINSRepositoryResource. The GIT sha of the resource # noqa: E501 - :param sha: The sha of this EbrainsRepositoryResource. + :param sha: The sha of this EBRAINSRepositoryResource. :type sha: str """ diff --git a/applications/workspaces/server/workspaces/openapi/openapi.yaml b/applications/workspaces/server/workspaces/openapi/openapi.yaml index d2be29c9f..9a55b12a4 100644 --- a/applications/workspaces/server/workspaces/openapi/openapi.yaml +++ b/applications/workspaces/server/workspaces/openapi/openapi.yaml @@ -1299,8 +1299,8 @@ components: allOf: - $ref: '#/components/schemas/RepositoryResourceBase' - EbrainsRepositoryResource: - description: ebrains repository resource + EBRAINSRepositoryResource: + description: EBRAINS repository resource allOf: - $ref: '#/components/schemas/RepositoryResourceBase' diff --git a/applications/workspaces/server/workspaces/service/osbrepository/__init__.py b/applications/workspaces/server/workspaces/service/osbrepository/__init__.py index 8a559632c..db4c29089 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/__init__.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/__init__.py @@ -7,7 +7,7 @@ -from workspaces.service.osbrepository.adapters import DandiAdapter, FigShareAdapter, GitHubAdapter, BiomodelsAdapter, EbrainsAdapter +from workspaces.service.osbrepository.adapters import DandiAdapter, FigShareAdapter, GitHubAdapter, BiomodelsAdapter, EBRAINSAdapter def get_repository_adapter(osbrepository: OSBRepository=None, repository_type=None, uri=None, *args, **kwargs): @@ -23,7 +23,7 @@ def get_repository_adapter(osbrepository: OSBRepository=None, repository_type=No elif repository_type == "biomodels": return BiomodelsAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs) elif repository_type == "ebrains": - return EbrainsAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs) + return EBRAINSAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs) return None diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py index daffc3242..330efb5e9 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py @@ -2,4 +2,4 @@ from workspaces.service.osbrepository.adapters.figshareadapter import FigShareAdapter from workspaces.service.osbrepository.adapters.githubadapter import GitHubAdapter from workspaces.service.osbrepository.adapters.biomodelsadapter import BiomodelsAdapter -from workspaces.service.osbrepository.adapters.ebrainsadapter import EbrainsAdapter +from workspaces.service.osbrepository.adapters.ebrainsadapter import EBRAINSAdapter diff --git a/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py b/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py index 39273f7d5..93bbd398f 100644 --- a/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py +++ b/libraries/client/workspaces/workspaces_cli/model/ebrains_repository_resource.py @@ -34,7 +34,7 @@ def lazy_import(): globals()['RepositoryResourceBase'] = RepositoryResourceBase -class EbrainsRepositoryResource(ModelComposed): +class EBRAINSRepositoryResource(ModelComposed): """NOTE: This class is auto generated by OpenAPI Generator. Ref: https://openapi-generator.tech @@ -113,7 +113,7 @@ def discriminator(): @classmethod @convert_js_args_to_python_args def _from_openapi_data(cls, *args, **kwargs): # noqa: E501 - """EbrainsRepositoryResource - a model defined in OpenAPI + """EBRAINSRepositoryResource - a model defined in OpenAPI Keyword Args: _check_type (bool): if True, values for parameters in openapi_types @@ -217,7 +217,7 @@ def _from_openapi_data(cls, *args, **kwargs): # noqa: E501 @convert_js_args_to_python_args def __init__(self, *args, **kwargs): # noqa: E501 - """EbrainsRepositoryResource - a model defined in OpenAPI + """EBRAINSRepositoryResource - a model defined in OpenAPI Keyword Args: _check_type (bool): if True, values for parameters in openapi_types From 66971f6d8a7cea1842d1131708ae22537a5e6eaf Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Thu, 1 May 2025 18:24:50 +0100 Subject: [PATCH 16/22] feat(ebrains): complete repo addition --- .../osbrepository/adapters/ebrainsadapter.py | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index 266d2eb7d..c904a977a 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -5,25 +5,25 @@ from functools import cache, cached_property -from fairgraph import KGClient, KGProxy +from fairgraph import KGClient, KGProxy, KGQuery from fairgraph.errors import ResolutionFailure from fairgraph.openminds.core import FileRepository, Model, ModelVersion from cloudharness import log as logger from workspaces.models import RepositoryResourceNode, RepositoryInfo from workspaces.models.resource_origin import ResourceOrigin -from workspaces.models.ebrains_repository_resource import EbrainsRepositoryResource +from workspaces.models.ebrains_repository_resource import EBRAINSRepositoryResource from .utils import add_to_tree from .githubadapter import GitHubAdapter -class EbrainsException(Exception): +class EBRAINSException(Exception): pass -class EbrainsAdapter: +class EBRAINSAdapter: """ - Adapter for Ebrains + Adapter for EBRAINS https://search.kg.ebrains.eu/ """ @@ -31,25 +31,31 @@ class EbrainsAdapter: def __init__(self, osbrepository, uri=None): self.osbrepository = osbrepository self.uri = uri if uri else osbrepository.uri - self.api_url = "https://search.kg.ebrains.eu/" + self.api_url = "https://search.kg.ebrains.eu" # TODO: get permanent application auth token from EBRAINS - # self.kg_client = KGClient(token="", host="core.kg.ebrains.eu") + # self.kg_client = KGClient(token=token, host="core.kg.ebrains.eu") self.kg_client = KGClient(client_id="SOME ID", client_secret="SOME SECRET", host="core.kg.ebrains.eu") try: self.model_id = re.search( - f"{self.api_url}/instances/(\\w+)", + f"{self.api_url}/instances/([\\w-]+)", self.uri.strip("/")).group(1) except AttributeError: - raise EbrainsException(f"{uri} is not a valid Ebrains URL") + raise EBRAINSException(f"{uri} is not a valid EBRAINS URL") @cache def get_model(self, uri: Optional[str] = None) -> Model: """Get model object using FairGraph""" logger.debug(f"Getting: {self.model_id}") - model = Model.from_id(id=self.model_id, client=self.kg_client) - + # if it's a Model + try: + model: Model = Model.from_id(id=self.model_id, client=self.kg_client) + # if it's a ModelVersion + except TypeError: + model_version: ModelVersion = ModelVersion.from_id(id=self.model_id, client=self.kg_client) + model_query: KGQuery = model_version.is_version_of + model = model_query.resolve(self.kg_client) return model def get_base_uri(self): @@ -59,15 +65,15 @@ def get_base_uri(self): def get_info(self) -> RepositoryInfo: """Get repository metadata from model object""" model = self.get_model() - return RepositoryInfo(name=model.name, contexts=self.get_contexts(), tags=self._get_keywords(model), summary=model.description) + return RepositoryInfo(name=model.name, contexts=self.get_contexts(), tags=self.get_tags("foobar"), summary="A test model") - def _get_keywords(self, model: Model) -> list[str]: + def _get_keywords(self) -> list[str]: """Get keywords from model - :param model: model object :returns: list of keywords """ + model = self.get_model() keywords: list[str] = [] if model.study_targets: if isinstance(model.study_targets, KGProxy): @@ -331,7 +337,7 @@ def get_resources(self, context): files = ["TODO: handle other special cases"] tree = RepositoryResourceNode( - resource=EbrainsRepositoryResource( + resource=EBRAINSRepositoryResource( name="/", path="/", osbrepository_id=self.osbrepository.id, @@ -361,16 +367,14 @@ def get_description(self, context): return "" def get_tags(self, context): - # using the format name for the moment, since they don't do explict - # tags/keywords + # all versions have same tags for EBRAINS, so we pass any rubbish to the argument logger.debug(f"Getting tags: {context}") - model = self.get_model() - return self._get_keywords(model) + return self._get_keywords() def create_copy_task(self, workspace_id, origins: List[ResourceOrigin]): import workspaces.service.workflow as workflow - # no file tree in Ebrains from the looks of it + # no file tree in EBRAINS from the looks of it folder = self.osbrepository.name download_url = self._get_file_storage_url(self.osbrepository.default_context) From 3f6497459bee2560225be51e247d2fa9dee64812 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 6 May 2025 13:38:46 +0100 Subject: [PATCH 17/22] feat(ebrains): complete cscs single file use case --- .../osbrepository/adapters/ebrainsadapter.py | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index c904a977a..0ae857fd7 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -46,7 +46,11 @@ def __init__(self, osbrepository, uri=None): @cache def get_model(self, uri: Optional[str] = None) -> Model: - """Get model object using FairGraph""" + """Get model object using FairGraph + + This returns the main model, not the ModelVersion. If a ModelVersion URL is + passed, it finds the URL of the Model and returns that. + """ logger.debug(f"Getting: {self.model_id}") # if it's a Model try: @@ -65,7 +69,7 @@ def get_base_uri(self): def get_info(self) -> RepositoryInfo: """Get repository metadata from model object""" model = self.get_model() - return RepositoryInfo(name=model.name, contexts=self.get_contexts(), tags=self.get_tags("foobar"), summary="A test model") + return RepositoryInfo(name=model.name, contexts=self.get_contexts(), tags=self.get_tags("foobar"), summary=self.get_description()) def _get_keywords(self) -> list[str]: """Get keywords from model @@ -166,7 +170,7 @@ def _get_file_storage_url(self, context: str) -> Optional[str]: return None - def _get_ebrains_data_proxy_file_list(self, url: str) -> list[str]: + def _get_ebrains_data_proxy_file_list(self, url: str) -> dict[str, str]: """Get the list of files from an ebrains data proxy URL. The complete url will be of this form: @@ -272,22 +276,25 @@ def _get_cscs_file_list(self, url: str) -> dict[str, str]: logger.debug(f"Getting file list for {url}") if ".zip?" in url: + logger.debug(f"Cscs url format with zip: {url}") url_portions: list[str] = url.split(".zip")[0].split("/") file_list_url = "/".join(url_portions[:6]) file_list_string = "/".join(url_portions[6:]) # assume it's with prefix elif "?prefix=" in url: + logger.debug(f"Cscs url format with prefix: {url}") file_list_url = url.split("?prefix=")[0] file_list_string = url.split("?prefix=")[1] else: - logger.warning(f"Other cscs url format: {url}") + logger.debug(f"Other cscs url format: {url}") + # test if it's one of the file types + url_portions: list[str] = url.split("/") + file_string = "/".join(url_portions[6:]) for suf in special_suffixes: if url.endswith(suf): - file_list = {url: url} - break - - return file_list + file_list = {file_string: url} + return file_list r = requests.get(file_list_url) if r.status_code == 200: @@ -310,11 +317,13 @@ def get_resources(self, context): logger.debug(f"Getting resources: {context}") download_url = self._get_file_storage_url(context) - if "github" in download_url: + if "github" in download_url.lower(): + logger.debug("GITHUB resource") gh_adapter = GitHubAdapter(self.osbrepository, download_url) return gh_adapter.get_resources(context) elif "modeldb" in download_url.lower(): + logger.debug("Modeldb resource") model_id = "" # urls with model id after the last slash # https://modeldb.science/249408?tab=7 @@ -330,12 +339,16 @@ def get_resources(self, context): return gh_adapter.get_resources(context) elif "cscs.ch" in download_url: + logger.debug("CSCS resource") files = self._get_cscs_file_list(download_url) elif "data-proxy.ebrains.eu" in download_url: + logger.debug("Data-proxy resource") files = self._get_ebrains_data_proxy_file_list(download_url) else: files = ["TODO: handle other special cases"] + logger.info(f"Files are: {files}") + tree = RepositoryResourceNode( resource=EBRAINSRepositoryResource( name="/", @@ -349,14 +362,14 @@ def get_resources(self, context): for afile, url in files.items(): add_to_tree( tree=tree, - tree_path=afile, + tree_path=afile.split("/"), path=url, osbrepository_id=self.osbrepository.id, ) return tree - def get_description(self, context): + def get_description(self, context: str = "foobar"): logger.debug(f"Getting description: {context}") try: result = self.get_model() From 6094e47d60cd6da9f9dbbd394f8c02f97a4a43f3 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 6 May 2025 13:55:23 +0100 Subject: [PATCH 18/22] feat(ebrains): complete github file sources use case --- .../service/osbrepository/adapters/ebrainsadapter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index 0ae857fd7..e4e193f47 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -319,7 +319,10 @@ def get_resources(self, context): download_url = self._get_file_storage_url(context) if "github" in download_url.lower(): logger.debug("GITHUB resource") - gh_adapter = GitHubAdapter(self.osbrepository, download_url) + # these may have things like "tree" in them, we only want github.com/user/repository + github_url_parts = download_url.split("/") + github_url = "/".join(github_url_parts[:5]) + gh_adapter = GitHubAdapter(self.osbrepository, github_url) return gh_adapter.get_resources(context) elif "modeldb" in download_url.lower(): @@ -392,7 +395,10 @@ def create_copy_task(self, workspace_id, origins: List[ResourceOrigin]): download_url = self._get_file_storage_url(self.osbrepository.default_context) if "github" in download_url: - gh_adapter = GitHubAdapter(self.osbrepository, download_url) + # these may have things like "tree" in them, we only want github.com/user/repository + github_url_parts = download_url.split("/") + github_url = "/".join(github_url_parts[:5]) + gh_adapter = GitHubAdapter(self.osbrepository, uri=github_url) return gh_adapter.create_copy_task(workspace_id, origins) elif "modeldb" in download_url.lower(): From bf0ad06d7e3c04c9f87f03bc43ccd8004c649722 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 6 May 2025 15:09:52 +0100 Subject: [PATCH 19/22] feat(ebrains): handle plain cscs case --- .../service/osbrepository/adapters/ebrainsadapter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index e4e193f47..d4f1804ad 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -273,6 +273,8 @@ def _get_cscs_file_list(self, url: str) -> dict[str, str]: "zip", "ipynb", ] + # if prefixed by data.kg, get rid of it so that we have only the cscs URL + url.replace("https://data.kg.ebrains.eu/zip?container=", "") logger.debug(f"Getting file list for {url}") if ".zip?" in url: @@ -296,6 +298,10 @@ def _get_cscs_file_list(self, url: str) -> dict[str, str]: file_list = {file_string: url} return file_list + # default case + url_portions: list[str] = url.split("/") + file_list_url = "/".join(url_portions[:6]) + file_list_string = "/".join(url_portions[6:]) r = requests.get(file_list_url) if r.status_code == 200: for line in r.text.split(): From c9b0a771b634cb44816ef071ac9778358cfd2115 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 6 May 2025 15:10:04 +0100 Subject: [PATCH 20/22] feat(ebrains): handle modeldb to github case Versions on EBRAINS have nothing to do with versions on ModelDB/GitHub, so we need to get the branches from Github ourselves. --- .../service/osbrepository/adapters/ebrainsadapter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index d4f1804ad..6563cbc5e 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -345,7 +345,10 @@ def get_resources(self, context): modeldb_url = f"https://github.com/ModelDBRepository/{model_id}" gh_adapter = GitHubAdapter(self.osbrepository, modeldb_url) - return gh_adapter.get_resources(context) + # versions on EBRAINS do not match the versions on ModelDB/GitHub + # so we use the first context + contexts = gh_adapter.get_contexts() + return gh_adapter.get_resources(contexts[0]) elif "cscs.ch" in download_url: logger.debug("CSCS resource") From f7eed44bd82aafb102ac3c435e4f6927c52df6d3 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 6 May 2025 16:26:56 +0100 Subject: [PATCH 21/22] feat(ebrains): complete cscs handling --- .../osbrepository/adapters/ebrainsadapter.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index 6563cbc5e..e369c3c6a 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -274,34 +274,36 @@ def _get_cscs_file_list(self, url: str) -> dict[str, str]: "ipynb", ] # if prefixed by data.kg, get rid of it so that we have only the cscs URL - url.replace("https://data.kg.ebrains.eu/zip?container=", "") + new_url = url.replace("https://data.kg.ebrains.eu/zip?container=", "") - logger.debug(f"Getting file list for {url}") - if ".zip?" in url: - logger.debug(f"Cscs url format with zip: {url}") - url_portions: list[str] = url.split(".zip")[0].split("/") + logger.debug(f"Getting file list for {new_url}") + # default + url_portions: list[str] = new_url.split("/") + file_list_url = "/".join(url_portions[:6]) + file_list_string = "/".join(url_portions[6:]) + + # special cases + if ".zip?" in new_url: + logger.debug(f"Cscs url format with zip: {new_url}") + url_portions: list[str] = new_url.split(".zip")[0].split("/") file_list_url = "/".join(url_portions[:6]) file_list_string = "/".join(url_portions[6:]) - # assume it's with prefix - elif "?prefix=" in url: - logger.debug(f"Cscs url format with prefix: {url}") - file_list_url = url.split("?prefix=")[0] - file_list_string = url.split("?prefix=")[1] + elif "?prefix=" in new_url: + logger.debug(f"Cscs url format with prefix: {new_url}") + file_list_url = new_url.split("?prefix=")[0] + file_list_string = new_url.split("?prefix=")[1] else: - logger.debug(f"Other cscs url format: {url}") - - # test if it's one of the file types - url_portions: list[str] = url.split("/") - file_string = "/".join(url_portions[6:]) + # handle single files: + # it is possible that they provide zip URLs but all the files are also + # individually available. there's no way for us to know that the whole file list + # is also available, though, so we simply provide the zip URL too. + logger.debug(f"Other cscs url format: {new_url}") for suf in special_suffixes: - if url.endswith(suf): - file_list = {file_string: url} + if new_url.endswith(suf): + file_list = {file_list_string: new_url} return file_list - # default case - url_portions: list[str] = url.split("/") - file_list_url = "/".join(url_portions[:6]) - file_list_string = "/".join(url_portions[6:]) + # handle file lists r = requests.get(file_list_url) if r.status_code == 200: for line in r.text.split(): @@ -359,7 +361,7 @@ def get_resources(self, context): else: files = ["TODO: handle other special cases"] - logger.info(f"Files are: {files}") + logger.debug(f"Files are: {files}") tree = RepositoryResourceNode( resource=EBRAINSRepositoryResource( From a3abd8090b96ef580a296f7de930e252daa4d99e Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Tue, 6 May 2025 17:02:41 +0100 Subject: [PATCH 22/22] feat(ebrains): use cloud harness secret utils --- .../osbrepository/adapters/ebrainsadapter.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py index e369c3c6a..fa23e3eff 100644 --- a/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/ebrainsadapter.py @@ -9,6 +9,7 @@ from fairgraph.errors import ResolutionFailure from fairgraph.openminds.core import FileRepository, Model, ModelVersion from cloudharness import log as logger +from cloudharness.utils.secrets import get_secret from workspaces.models import RepositoryResourceNode, RepositoryInfo from workspaces.models.resource_origin import ResourceOrigin from workspaces.models.ebrains_repository_resource import EBRAINSRepositoryResource @@ -32,9 +33,24 @@ def __init__(self, osbrepository, uri=None): self.osbrepository = osbrepository self.uri = uri if uri else osbrepository.uri self.api_url = "https://search.kg.ebrains.eu" + # TODO: get permanent application auth token from EBRAINS - # self.kg_client = KGClient(token=token, host="core.kg.ebrains.eu") - self.kg_client = KGClient(client_id="SOME ID", client_secret="SOME SECRET", host="core.kg.ebrains.eu") + try: + kg_client = get_secret("ebrains-user") + except: + kg_client = None + try: + kg_secret = get_secret("ebrains-secret") + except: + kg_secret = None + if kg_user and kg_secret: + self.kg_client = KGClient(client_id=kg_client, client_secret=kg_secret, host="core.kg.ebrains.eu") + else: + token = "" + self.kg_client = KGClient(token=token, host="core.kg.ebrains.eu") + + if not self.kg_client: + raise EBRAINSException("Could not initialise EBRAINS KG client") try: self.model_id = re.search( @@ -60,6 +76,10 @@ def get_model(self, uri: Optional[str] = None) -> Model: model_version: ModelVersion = ModelVersion.from_id(id=self.model_id, client=self.kg_client) model_query: KGQuery = model_version.is_version_of model = model_query.resolve(self.kg_client) + + if not model: + raise EBRAINSException("Could not fetch EBRAINS model") + return model def get_base_uri(self):