More plugin API stuff

maubot/__main__.py

@@ -28,7 +28,7 @@ from .config import Config
 from .db import Base, init as init_db
 from .server import MaubotServer
 from .client import Client, init as init_client
-from .loader import ZippedPluginLoader, MaubotZipImportError, IDConflictError
+from .loader import ZippedPluginLoader
 from .instance import PluginInstance, init as init_plugin_instance_class
 from .management.api import init as init_management
 from .__meta__ import __version__
@@ -64,31 +64,9 @@ init_plugin_instance_class(db_session, config)
 management_api = init_management(config, loop)
 server = MaubotServer(config, management_api, loop)
 
-trash_path = config["plugin_directories.trash"]
+ZippedPluginLoader.trash_path = config["plugin_directories.trash"]
-
+ZippedPluginLoader.directories = config["plugin_directories.load"]
-
+ZippedPluginLoader.load_all()
-def trash(file_path: str, new_name: Optional[str] = None) -> None:
-    if trash_path == "delete":
-        os.remove(file_path)
-    else:
-        new_name = new_name or f"{int(time())}-{os.path.basename(file_path)}"
-        os.rename(file_path, os.path.abspath(os.path.join(trash_path, new_name)))
-
-
-ZippedPluginLoader.log.debug("Preloading plugins...")
-for directory in config["plugin_directories.load"]:
-    for file in os.listdir(directory):
-        if not file.endswith(".mbp"):
-            continue
-        path = os.path.abspath(os.path.join(directory, file))
-        try:
-            ZippedPluginLoader.get(path)
-        except MaubotZipImportError:
-            ZippedPluginLoader.log.exception(f"Failed to load plugin at {path}, trashing...")
-            trash(path)
-        except IDConflictError:
-            ZippedPluginLoader.log.warn(f"Duplicate plugin ID at {path}, trashing...")
-            trash(path)
 
 plugins = PluginInstance.all()
 

maubot/loader/abc.py

@@ -55,7 +55,7 @@ class PluginLoader(ABC):
         pass
 
     @abstractmethod
-    def read_file(self, path: str) -> bytes:
+    async def read_file(self, path: str) -> bytes:
         pass
 
     async def stop_instances(self) -> None:
@@ -65,17 +65,17 @@ class PluginLoader(ABC):
         await asyncio.gather([instance.start() for instance in self.references if instance.enabled])
 
     @abstractmethod
-    def load(self) -> Type[PluginClass]:
+    async def load(self) -> Type[PluginClass]:
         pass
 
     @abstractmethod
-    def reload(self) -> Type[PluginClass]:
+    async def reload(self) -> Type[PluginClass]:
         pass
 
     @abstractmethod
-    def unload(self) -> None:
+    async def unload(self) -> None:
         pass
 
     @abstractmethod
-    def delete(self) -> None:
+    async def delete(self) -> None:
         pass

maubot/loader/zip.py

@@ -13,8 +13,9 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, List, Type, Tuple
+from typing import Dict, List, Type, Tuple, Optional
 from zipfile import ZipFile, BadZipFile
+from time import time
 import configparser
 import logging
 import sys
@@ -31,7 +32,9 @@ class MaubotZipImportError(Exception):
 
 class ZippedPluginLoader(PluginLoader):
     path_cache: Dict[str, 'ZippedPluginLoader'] = {}
-    log = logging.getLogger("maubot.loader.zip")
+    log: logging.Logger = logging.getLogger("maubot.loader.zip")
+    trash_path: str = "delete"
+    directories: List[str] = []
 
     path: str
     id: str
@@ -84,7 +87,7 @@ class ZippedPluginLoader(PluginLoader):
                 f"id='{self.id}' "
                 f"loaded={self._loaded is not None}>")
 
-    def read_file(self, path: str) -> bytes:
+    async def read_file(self, path: str) -> bytes:
         return self._file.read(path)
 
     @staticmethod
@@ -159,7 +162,14 @@ class ZippedPluginLoader(PluginLoader):
             except ZipImportError as e:
                 raise MaubotZipImportError(f"Module {module} not found in file") from e
 
-    def load(self, reset_cache: bool = False) -> Type[PluginClass]:
+    async def load(self, reset_cache: bool = False) -> Type[PluginClass]:
+        try:
+            return self._load(reset_cache)
+        except MaubotZipImportError:
+            self.log.exception(f"Failed to load {self.id} v{self.version}")
+            raise
+
+    def _load(self, reset_cache: bool = False) -> Type[PluginClass]:
         if self._loaded is not None and not reset_cache:
             return self._loaded
         importer = self._get_importer(reset_cache=reset_cache)
@@ -176,19 +186,19 @@ class ZippedPluginLoader(PluginLoader):
         self.log.debug(f"Loaded and imported plugin {self.id} from {self.path}")
         return plugin
 
-    def reload(self) -> Type[PluginClass]:
+    async def reload(self) -> Type[PluginClass]:
-        self.unload()
+        await self.unload()
-        return self.load(reset_cache=True)
+        return await self.load(reset_cache=True)
 
-    def unload(self) -> None:
+    async def unload(self) -> None:
         for name, mod in list(sys.modules.items()):
             if getattr(mod, "__file__", "").startswith(self.path):
                 del sys.modules[name]
         self._loaded = None
         self.log.debug(f"Unloaded plugin {self.id} at {self.path}")
 
-    def delete(self) -> None:
+    async def delete(self) -> None:
-        self.unload()
+        await self.unload()
         try:
             del self.path_cache[self.path]
         except KeyError:
@@ -206,3 +216,28 @@ class ZippedPluginLoader(PluginLoader):
         self.path = None
         self.version = None
         self.modules = None
+
+    @classmethod
+    def trash(cls, file_path: str, new_name: Optional[str] = None, reason: str = "error") -> None:
+        if cls.trash_path == "delete":
+            os.remove(file_path)
+        else:
+            new_name = new_name or f"{int(time())}-{reason}-{os.path.basename(file_path)}"
+            os.rename(file_path, os.path.abspath(os.path.join(cls.trash_path, new_name)))
+
+    @classmethod
+    def load_all(cls):
+        cls.log.debug("Preloading plugins...")
+        for directory in cls.directories:
+            for file in os.listdir(directory):
+                if not file.endswith(".mbp"):
+                    continue
+                path = os.path.abspath(os.path.join(directory, file))
+                try:
+                    cls.get(path)
+                except MaubotZipImportError:
+                    cls.log.exception(f"Failed to load plugin at {path}, trashing...")
+                    cls.trash(path)
+                except IDConflictError:
+                    cls.log.error(f"Duplicate plugin ID at {path}, trashing...")
+                    cls.trash(path)

maubot/management/api/plugin.py

@@ -18,7 +18,8 @@ from io import BytesIO
 import os.path
 
 from ...loader import PluginLoader, ZippedPluginLoader, MaubotZipImportError
-from .responses import ErrPluginNotFound, ErrPluginInUse, RespDeleted
+from .responses import (ErrPluginNotFound, ErrPluginInUse, ErrInputPluginInvalid,
+                        ErrPluginReloadFailed, RespDeleted, RespOK)
 from . import routes, config
 
 
@@ -51,10 +52,19 @@ async def delete_plugin(request: web.Request) -> web.Response:
         return ErrPluginNotFound
     elif len(plugin.references) > 0:
         return ErrPluginInUse
-    plugin.delete()
+    await plugin.delete()
     return RespDeleted
 
 
+@routes.post("/plugin/{id}/reload")
+async def reload_plugin(request: web.Request) -> web.Response:
+    plugin_id = request.match_info.get("id", None)
+    plugin = PluginLoader.id_cache.get(plugin_id, None)
+    if not plugin:
+        return ErrPluginNotFound
+    return await reload(plugin)
+
+
 @routes.post("/plugins/upload")
 async def upload_plugin(request: web.Request) -> web.Response:
     content = await request.read()
@@ -62,22 +72,40 @@ async def upload_plugin(request: web.Request) -> web.Response:
     try:
         pid, version = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
-        return web.json_response({
+        return ErrInputPluginInvalid(e)
-            "error": str(e),
-            "errcode": "invalid_plugin",
-        }, status=web.HTTPBadRequest)
     plugin = PluginLoader.id_cache.get(pid, None)
     if not plugin:
-        path = os.path.join(config["plugin_directories.upload"], f"{pid}-{version}.mbp")
+        path = os.path.join(config["plugin_directories.upload"], f"{pid}-v{version}.mbp")
         with open(path, "wb") as p:
             p.write(content)
         try:
             ZippedPluginLoader.get(path)
         except MaubotZipImportError as e:
-            trash(path)
+            ZippedPluginLoader.trash(path)
-            return web.json_response({
+            # TODO log error?
-                "error": str(e),
+            return ErrInputPluginInvalid(e)
-                "errcode": "invalid_plugin",
+    elif isinstance(plugin, ZippedPluginLoader):
-            }, status=web.HTTPBadRequest)
+        dirname = os.path.dirname(plugin.path)
+        filename = os.path.basename(plugin.path)
+        if plugin.version in filename:
+            filename = filename.replace(plugin.version, version)
+        else:
+            filename = filename.rstrip(".mbp") + version + ".mbp"
+        path = os.path.join(dirname, filename)
+        with open(path, "wb") as p:
+            p.write(content)
+        ZippedPluginLoader.trash(plugin.path, reason="update")
+        plugin.path = path
+        return await reload(plugin)
     else:
-        pass
+        return web.json_response({})
+
+
+async def reload(plugin: PluginLoader) -> web.Response:
+    await plugin.stop_instances()
+    try:
+        await plugin.reload()
+    except MaubotZipImportError as e:
+        return ErrPluginReloadFailed(e)
+    await plugin.start_instances()
+    return RespOK

maubot/management/api/responses.py

@@ -33,6 +33,30 @@ ErrPluginNotFound = web.json_response({
 ErrPluginInUse = web.json_response({
     "error": "Plugin instances of this type still exist",
     "errcode": "plugin_in_use",
-})
+}, status=web.HTTPPreconditionFailed)
 
-RespDeleted = web.Response(status=204)
+
+def ErrInputPluginInvalid(error) -> web.Response:
+    return web.json_response({
+        "error": str(error),
+        "errcode": "plugin_invalid",
+    }, status=web.HTTPBadRequest)
+
+
+def ErrPluginReloadFailed(error) -> web.Response:
+    return web.json_response({
+        "error": str(error),
+        "errcode": "plugin_invalid",
+    }, status=web.HTTPInternalServerError)
+
+
+ErrNotImplemented = web.json_response({
+    "error": "Not implemented",
+    "errcode": "not_implemented",
+}, status=web.HTTPNotImplemented)
+
+RespOK = web.json_response({
+    "success": True,
+}, status=web.HTTPOk)
+
+RespDeleted = web.Response(status=web.HTTPNoContent)

maubot/management/api/spec.yaml

@@ -32,6 +32,7 @@ paths:
     post:
       operationId: upload_plugin
       summary: Upload a new plugin
+      description: Upload a new plugin. If the plugin already exists, enabled instances will be restarted.
       tags: [Plugin]
       responses:
         200:
@@ -81,10 +82,11 @@ paths:
         401:
           $ref: '#/components/responses/Unauthorized'
         404:
-          description: Plugin not found
+          $ref: '#/components/responses/PluginNotFound'
     delete:
       operationId: delete_plugin
       summary: Delete a plugin
+      description: Delete a plugin. All instances of the plugin must be deleted before deleting the plugin.
       tags: [Plugin]
       responses:
         204:
@@ -92,9 +94,28 @@ paths:
         401:
           $ref: '#/components/responses/Unauthorized'
         404:
-          description: Plugin not found
+          $ref: '#/components/responses/PluginNotFound'
         412:
           description: One or more plugin instances of this type exist
+  /plugin/{id}/reload:
+    parameters:
+    - name: id
+      in: path
+      description: The ID of the plugin to get
+      required: true
+      schema:
+        type: string
+    post:
+      operationId: reload_plugin
+      summary: Reload a plugin from disk
+      tags: [Plugin]
+      responses:
+        200:
+          description: Plugin reloaded
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/PluginNotFound'
 
   /instances:
     get:
@@ -134,7 +155,7 @@ paths:
         401:
           $ref: '#/components/responses/Unauthorized'
         404:
-          description: Plugin or instance not found
+          $ref: '#/components/responses/InstanceNotFound'
     delete:
       operationId: delete_instance
       summary: Delete a specific plugin instance
@@ -145,7 +166,7 @@ paths:
         401:
           $ref: '#/components/responses/Unauthorized'
         404:
-          description: Plugin or instance not found
+          $ref: '#/components/responses/InstanceNotFound'
     put:
       operationId: update_instance
       summary: Create a plugin instance or edit the details of an existing plugin instance
@@ -158,7 +179,7 @@ paths:
         401:
           $ref: '#/components/responses/Unauthorized'
         404:
-          description: Plugin or instance not found
+          $ref: '#/components/responses/InstanceNotFound'
 
   '/clients':
     get:
@@ -196,7 +217,7 @@ paths:
         401:
           $ref: '#/components/responses/Unauthorized'
         404:
-          description: Client not found
+          $ref: '#/components/responses/ClientNotFound'
     put:
       operationId: update_client
       summary: Create or update a Matrix client
@@ -217,7 +238,7 @@ paths:
         401:
           $ref: '#/components/responses/Unauthorized'
         404:
-          description: Client not found
+          $ref: '#/components/responses/ClientNotFound'
     delete:
       operationId: delete_client
       summary: Delete a Matrix client
@@ -228,7 +249,7 @@ paths:
         401:
           $ref: '#/components/responses/Unauthorized'
         404:
-          description: Client not found
+          $ref: '#/components/responses/ClientNotFound'
         412:
           description: One or more plugin instances with this as their primary client exist
 
@@ -236,6 +257,12 @@ components:
   responses:
     Unauthorized:
       description: Invalid or missing access token
+    PluginNotFound:
+      description: Plugin not found
+    ClientNotFound:
+      description: Client not found
+    InstanceNotFound:
+      description: Plugin instance not found
   securitySchemes:
     bearer:
       type: http