feat: Filebrowser in storage-proxy

changes/38.feature

@@ -0,0 +1 @@
+Move file browser to storage proxy.

config/filebrowser/start.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+useradd -r -u $1 -g $2 work
+su work
+/bin/filebrowser config init -d /filebrowser_dir/db/filebrowser_$3.db -p $3
+/bin/filebrowser users add admin admin -d /filebrowser_dir/db/filebrowser_$3.db
+/bin/filebrowser config import /filebrowser_dir/config.json -d /filebrowser_dir/db/filebrowser_$3.db
+/bin/filebrowser -c /filebrowser_dir/settings.json -d /filebrowser_dir/db/filebrowser_$3.db -p $3
+
+exit 0

config/sample.toml

@@ -34,6 +34,50 @@ session-expire = "1d"
 # user = 1000
 # group = 1000
 
+[filebrowser]
+# A locally available container image to run as filebrowser containers
+image = "cr.backend.ai/stable/filebrowser:21.02-ubuntu20.04"
+
+# The IP address to expose the filebrowser ports
+service_ip = "127.0.0.1"
+
+# The service port number by which user will be able to connect to filerowser UI.
+service_port = 8080
+
+# Settings pass to configure filebrowser application in the docker container.
+# The specific file 'settings.json' should be places in given path, 
+# otherwise the default settings file will be genrated in the following pass
+# For more configuration file info see https://filebrowser.org/installation
+settings_path = "./filebrowser_dir/"
+
+# The maxium number of CPU cores available to each filebrowser container
+max_cpu = 1
+
+# The maximum size of memory available to each filebrowser container
+max_mem = "1g"
+
+# The maximum limit of the number of filebrowser containers
+max_containers = 32
+
+# The SQLite db path and name
+db_path = './filebrowser_dir/containers.db'
+
+# The group to be used by the filebrowser container
+user_id = 1001
+group_id = 100
+
+# The frequency to check network I/O status of containers.
+activity_check_freq = 1
+
+# The timeout to mark a container idle when there are no activities observed.
+activity_check_timeout = 30
+
+# The timeout to destroy a container after it is marked as idle.
+idle_timeout = 30
+
+# Define port range which can be used by filebrowser containers.
+# For example, port_range="4000-5000"
+port_range="4000-5000"
 
 [api.client]
 # Client-facing API

filebrowser_dir/start.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+/bin/filebrowser config init -d /filebrowser_dir/db/filebrowser_$1.db -p $1;
+/bin/filebrowser users add admin admin -d /filebrowser_dir/db/filebrowser_$1.db;
+/bin/filebrowser config import /filebrowser_dir/config.json -d /filebrowser_dir/db/filebrowser_$1.db;
+/bin/filebrowser -c /filebrowser_dir/settings.json -d /filebrowser_dir/db/filebrowser_$1.db -p $1;
+
+exit 0;

setup.cfg

@@ -33,6 +33,7 @@ python_requires = >=3.10
 setup_requires =
     setuptools>=50.3.2
 install_requires =
+    aiodocker~=0.21
     aiofiles>=0.7.0
     types-aiofiles>=0.1.9
     aiohttp~=3.8.0
@@ -45,6 +46,8 @@ install_requires =
     uvloop>=0.16.0
     zipstream~=1.1.4
     backend.ai-common~=22.3.0
+    sqlalchemy~=1.4.32
+    types-setuptools~=57.4.11
 zip_safe = false
 include_package_data = true
 

src/ai/backend/storage/api/manager.py

@@ -22,6 +22,7 @@
 from ..abc import AbstractVolume
 from ..context import Context
 from ..exception import InvalidSubpathError, VFolderNotFoundError
+from ..filebrowser import filebrowser
 from ..types import VFolderCreationOptions
 from ..utils import check_params, log_manager_api_entry
 
@@ -358,6 +359,7 @@ async def get_vfolder_usage(request: web.Request) -> web.Response:
             },
         ),
     ) as params:
+
         try:
             await log_manager_api_entry(log, "get_vfolder_usage", params)
             ctx: Context = request.app["ctx"]
@@ -626,6 +628,49 @@ async def delete_files(request: web.Request) -> web.Response:
         )
 
 
+async def create_or_update_filebrowser(request: web.Request) -> web.Response:
+    ctx: Context = request.app["ctx"]
+    params = await request.json()
+    host: str
+    port: int
+    container_id: str
+    host, port, container_id = await filebrowser.create_or_update(
+        ctx,
+        params["host"],
+        params["vfolders"],
+    )
+    return web.json_response(
+        {
+            "addr": f"http://{host}:{port}",  # TODO: SSL?
+            "container_id": f"{container_id}",
+            "status": "ok",
+        },
+    )
+
+
+async def destroy_filebrowser(request: web.Request) -> web.Response:
+    async with check_params(
+        request,
+        t.Dict(
+            {
+                t.Key("auth_token"): t.String,
+                t.Key("container_id"): t.String,
+            },
+        ),
+    ) as params:
+        await log_manager_api_entry(log, "destroy_filebrowser", params)
+        ctx: Context = request.app["ctx"]
+        try:
+            await filebrowser.destroy_container(ctx, params["container_id"])
+            return web.json_response(
+                {
+                    "status": "ok",
+                },
+            )
+        except Exception:
+            raise Exception
+
+
 async def init_manager_app(ctx: Context) -> web.Application:
     app = web.Application(
         middlewares=[
@@ -655,4 +700,10 @@ async def init_manager_app(ctx: Context) -> web.Application:
     app.router.add_route("POST", "/folder/file/download", create_download_session)
     app.router.add_route("POST", "/folder/file/upload", create_upload_session)
     app.router.add_route("POST", "/folder/file/delete", delete_files)
+    app.router.add_route(
+        "POST",
+        "/storage/filebrowser/create",
+        create_or_update_filebrowser,
+    )
+    app.router.add_route("DELETE", "/storage/filebrowser/destroy", destroy_filebrowser)
     return app

src/ai/backend/storage/config.py

@@ -1,4 +1,5 @@
 import os
+from contextlib import asynccontextmanager
 from pathlib import Path
 
 import trafaret as t
@@ -13,6 +14,14 @@
 _file_perm = (Path(__file__).parent / "server.py").stat()
 
 
+@asynccontextmanager
+async def closing_async(thing):
+    try:
+        yield thing
+    finally:
+        await thing.close()
+
+
 local_config_iv = (
     t.Dict(
         {
@@ -38,6 +47,34 @@
                     ),
                 },
             ),
+            t.Key("filebrowser"): t.Dict(
+                {
+                    t.Key("image"): t.String,
+                    t.Key("service_ip"): t.IP,
+                    t.Key("max_cpu", default=1): t.Int[1:_max_cpu_count],
+                    t.Key("max_mem", default="1g"): tx.BinarySize,
+                    t.Key("max_containers", default=32): t.Int[1:],
+                    t.Key("user_id", default=None): tx.UserID(
+                        default_uid=_file_perm.st_uid,
+                    ),
+                    t.Key("group_id", default=None): tx.GroupID(
+                        default_gid=_file_perm.st_gid,
+                    ),
+                    t.Key("settings_path", default=None): tx.Path(type="dir"),
+                    t.Key("service_port", default=None): t.Int,
+                    t.Key("mount_path", default=None): tx.Path(type="dir"),
+                    t.Key("max_containers", default=None): t.Int,
+                    t.Key("db_path", default=None): tx.Path(
+                        type="file",
+                        allow_nonexisting=True,
+                        allow_devnull=True,
+                    ),
+                    t.Key("activity_check_timeout", default=30): t.Int,
+                    t.Key("activity_check_freq", default=1): t.Int,
+                    t.Key("idle_timeout", default=300): t.Int,
+                    t.Key("port_range", default="4000-5000"): t.String,
+                },
+            ),
             t.Key("logging"): logging_config_iv,
             t.Key("api"): t.Dict(
                 {

src/ai/backend/storage/context.py

@@ -29,6 +29,7 @@ class Context:
     pid: int
     etcd: AsyncEtcd
     local_config: Mapping[str, Any]
+    container_id: Any
 
     def __init__(
         self,