More plugin API stuff

This commit is contained in:
Tulir Asokan 2018-10-30 00:50:38 +02:00
parent f2449e2eba
commit d5353430a8
6 changed files with 156 additions and 64 deletions

View File

@ -28,7 +28,7 @@ from .config import Config
from .db import Base, init as init_db from .db import Base, init as init_db
from .server import MaubotServer from .server import MaubotServer
from .client import Client, init as init_client from .client import Client, init as init_client
from .loader import ZippedPluginLoader, MaubotZipImportError, IDConflictError from .loader import ZippedPluginLoader
from .instance import PluginInstance, init as init_plugin_instance_class from .instance import PluginInstance, init as init_plugin_instance_class
from .management.api import init as init_management from .management.api import init as init_management
from .__meta__ import __version__ from .__meta__ import __version__
@ -64,31 +64,9 @@ init_plugin_instance_class(db_session, config)
management_api = init_management(config, loop) management_api = init_management(config, loop)
server = MaubotServer(config, management_api, loop) server = MaubotServer(config, management_api, loop)
trash_path = config["plugin_directories.trash"] ZippedPluginLoader.trash_path = config["plugin_directories.trash"]
ZippedPluginLoader.directories = config["plugin_directories.load"]
ZippedPluginLoader.load_all()
def trash(file_path: str, new_name: Optional[str] = None) -> None:
if trash_path == "delete":
os.remove(file_path)
else:
new_name = new_name or f"{int(time())}-{os.path.basename(file_path)}"
os.rename(file_path, os.path.abspath(os.path.join(trash_path, new_name)))
ZippedPluginLoader.log.debug("Preloading plugins...")
for directory in config["plugin_directories.load"]:
for file in os.listdir(directory):
if not file.endswith(".mbp"):
continue
path = os.path.abspath(os.path.join(directory, file))
try:
ZippedPluginLoader.get(path)
except MaubotZipImportError:
ZippedPluginLoader.log.exception(f"Failed to load plugin at {path}, trashing...")
trash(path)
except IDConflictError:
ZippedPluginLoader.log.warn(f"Duplicate plugin ID at {path}, trashing...")
trash(path)
plugins = PluginInstance.all() plugins = PluginInstance.all()

View File

@ -55,7 +55,7 @@ class PluginLoader(ABC):
pass pass
@abstractmethod @abstractmethod
def read_file(self, path: str) -> bytes: async def read_file(self, path: str) -> bytes:
pass pass
async def stop_instances(self) -> None: async def stop_instances(self) -> None:
@ -65,17 +65,17 @@ class PluginLoader(ABC):
await asyncio.gather([instance.start() for instance in self.references if instance.enabled]) await asyncio.gather([instance.start() for instance in self.references if instance.enabled])
@abstractmethod @abstractmethod
def load(self) -> Type[PluginClass]: async def load(self) -> Type[PluginClass]:
pass pass
@abstractmethod @abstractmethod
def reload(self) -> Type[PluginClass]: async def reload(self) -> Type[PluginClass]:
pass pass
@abstractmethod @abstractmethod
def unload(self) -> None: async def unload(self) -> None:
pass pass
@abstractmethod @abstractmethod
def delete(self) -> None: async def delete(self) -> None:
pass pass

View File

@ -13,8 +13,9 @@
# #
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Dict, List, Type, Tuple from typing import Dict, List, Type, Tuple, Optional
from zipfile import ZipFile, BadZipFile from zipfile import ZipFile, BadZipFile
from time import time
import configparser import configparser
import logging import logging
import sys import sys
@ -31,7 +32,9 @@ class MaubotZipImportError(Exception):
class ZippedPluginLoader(PluginLoader): class ZippedPluginLoader(PluginLoader):
path_cache: Dict[str, 'ZippedPluginLoader'] = {} path_cache: Dict[str, 'ZippedPluginLoader'] = {}
log = logging.getLogger("maubot.loader.zip") log: logging.Logger = logging.getLogger("maubot.loader.zip")
trash_path: str = "delete"
directories: List[str] = []
path: str path: str
id: str id: str
@ -84,7 +87,7 @@ class ZippedPluginLoader(PluginLoader):
f"id='{self.id}' " f"id='{self.id}' "
f"loaded={self._loaded is not None}>") f"loaded={self._loaded is not None}>")
def read_file(self, path: str) -> bytes: async def read_file(self, path: str) -> bytes:
return self._file.read(path) return self._file.read(path)
@staticmethod @staticmethod
@ -159,7 +162,14 @@ class ZippedPluginLoader(PluginLoader):
except ZipImportError as e: except ZipImportError as e:
raise MaubotZipImportError(f"Module {module} not found in file") from e raise MaubotZipImportError(f"Module {module} not found in file") from e
def load(self, reset_cache: bool = False) -> Type[PluginClass]: async def load(self, reset_cache: bool = False) -> Type[PluginClass]:
try:
return self._load(reset_cache)
except MaubotZipImportError:
self.log.exception(f"Failed to load {self.id} v{self.version}")
raise
def _load(self, reset_cache: bool = False) -> Type[PluginClass]:
if self._loaded is not None and not reset_cache: if self._loaded is not None and not reset_cache:
return self._loaded return self._loaded
importer = self._get_importer(reset_cache=reset_cache) importer = self._get_importer(reset_cache=reset_cache)
@ -176,19 +186,19 @@ class ZippedPluginLoader(PluginLoader):
self.log.debug(f"Loaded and imported plugin {self.id} from {self.path}") self.log.debug(f"Loaded and imported plugin {self.id} from {self.path}")
return plugin return plugin
def reload(self) -> Type[PluginClass]: async def reload(self) -> Type[PluginClass]:
self.unload() await self.unload()
return self.load(reset_cache=True) return await self.load(reset_cache=True)
def unload(self) -> None: async def unload(self) -> None:
for name, mod in list(sys.modules.items()): for name, mod in list(sys.modules.items()):
if getattr(mod, "__file__", "").startswith(self.path): if getattr(mod, "__file__", "").startswith(self.path):
del sys.modules[name] del sys.modules[name]
self._loaded = None self._loaded = None
self.log.debug(f"Unloaded plugin {self.id} at {self.path}") self.log.debug(f"Unloaded plugin {self.id} at {self.path}")
def delete(self) -> None: async def delete(self) -> None:
self.unload() await self.unload()
try: try:
del self.path_cache[self.path] del self.path_cache[self.path]
except KeyError: except KeyError:
@ -206,3 +216,28 @@ class ZippedPluginLoader(PluginLoader):
self.path = None self.path = None
self.version = None self.version = None
self.modules = None self.modules = None
@classmethod
def trash(cls, file_path: str, new_name: Optional[str] = None, reason: str = "error") -> None:
if cls.trash_path == "delete":
os.remove(file_path)
else:
new_name = new_name or f"{int(time())}-{reason}-{os.path.basename(file_path)}"
os.rename(file_path, os.path.abspath(os.path.join(cls.trash_path, new_name)))
@classmethod
def load_all(cls):
cls.log.debug("Preloading plugins...")
for directory in cls.directories:
for file in os.listdir(directory):
if not file.endswith(".mbp"):
continue
path = os.path.abspath(os.path.join(directory, file))
try:
cls.get(path)
except MaubotZipImportError:
cls.log.exception(f"Failed to load plugin at {path}, trashing...")
cls.trash(path)
except IDConflictError:
cls.log.error(f"Duplicate plugin ID at {path}, trashing...")
cls.trash(path)

View File

@ -18,7 +18,8 @@ from io import BytesIO
import os.path import os.path
from ...loader import PluginLoader, ZippedPluginLoader, MaubotZipImportError from ...loader import PluginLoader, ZippedPluginLoader, MaubotZipImportError
from .responses import ErrPluginNotFound, ErrPluginInUse, RespDeleted from .responses import (ErrPluginNotFound, ErrPluginInUse, ErrInputPluginInvalid,
ErrPluginReloadFailed, RespDeleted, RespOK)
from . import routes, config from . import routes, config
@ -51,10 +52,19 @@ async def delete_plugin(request: web.Request) -> web.Response:
return ErrPluginNotFound return ErrPluginNotFound
elif len(plugin.references) > 0: elif len(plugin.references) > 0:
return ErrPluginInUse return ErrPluginInUse
plugin.delete() await plugin.delete()
return RespDeleted return RespDeleted
@routes.post("/plugin/{id}/reload")
async def reload_plugin(request: web.Request) -> web.Response:
plugin_id = request.match_info.get("id", None)
plugin = PluginLoader.id_cache.get(plugin_id, None)
if not plugin:
return ErrPluginNotFound
return await reload(plugin)
@routes.post("/plugins/upload") @routes.post("/plugins/upload")
async def upload_plugin(request: web.Request) -> web.Response: async def upload_plugin(request: web.Request) -> web.Response:
content = await request.read() content = await request.read()
@ -62,22 +72,40 @@ async def upload_plugin(request: web.Request) -> web.Response:
try: try:
pid, version = ZippedPluginLoader.verify_meta(file) pid, version = ZippedPluginLoader.verify_meta(file)
except MaubotZipImportError as e: except MaubotZipImportError as e:
return web.json_response({ return ErrInputPluginInvalid(e)
"error": str(e),
"errcode": "invalid_plugin",
}, status=web.HTTPBadRequest)
plugin = PluginLoader.id_cache.get(pid, None) plugin = PluginLoader.id_cache.get(pid, None)
if not plugin: if not plugin:
path = os.path.join(config["plugin_directories.upload"], f"{pid}-{version}.mbp") path = os.path.join(config["plugin_directories.upload"], f"{pid}-v{version}.mbp")
with open(path, "wb") as p: with open(path, "wb") as p:
p.write(content) p.write(content)
try: try:
ZippedPluginLoader.get(path) ZippedPluginLoader.get(path)
except MaubotZipImportError as e: except MaubotZipImportError as e:
trash(path) ZippedPluginLoader.trash(path)
return web.json_response({ # TODO log error?
"error": str(e), return ErrInputPluginInvalid(e)
"errcode": "invalid_plugin", elif isinstance(plugin, ZippedPluginLoader):
}, status=web.HTTPBadRequest) dirname = os.path.dirname(plugin.path)
filename = os.path.basename(plugin.path)
if plugin.version in filename:
filename = filename.replace(plugin.version, version)
else: else:
pass filename = filename.rstrip(".mbp") + version + ".mbp"
path = os.path.join(dirname, filename)
with open(path, "wb") as p:
p.write(content)
ZippedPluginLoader.trash(plugin.path, reason="update")
plugin.path = path
return await reload(plugin)
else:
return web.json_response({})
async def reload(plugin: PluginLoader) -> web.Response:
await plugin.stop_instances()
try:
await plugin.reload()
except MaubotZipImportError as e:
return ErrPluginReloadFailed(e)
await plugin.start_instances()
return RespOK

View File

@ -33,6 +33,30 @@ ErrPluginNotFound = web.json_response({
ErrPluginInUse = web.json_response({ ErrPluginInUse = web.json_response({
"error": "Plugin instances of this type still exist", "error": "Plugin instances of this type still exist",
"errcode": "plugin_in_use", "errcode": "plugin_in_use",
}) }, status=web.HTTPPreconditionFailed)
RespDeleted = web.Response(status=204)
def ErrInputPluginInvalid(error) -> web.Response:
return web.json_response({
"error": str(error),
"errcode": "plugin_invalid",
}, status=web.HTTPBadRequest)
def ErrPluginReloadFailed(error) -> web.Response:
return web.json_response({
"error": str(error),
"errcode": "plugin_invalid",
}, status=web.HTTPInternalServerError)
ErrNotImplemented = web.json_response({
"error": "Not implemented",
"errcode": "not_implemented",
}, status=web.HTTPNotImplemented)
RespOK = web.json_response({
"success": True,
}, status=web.HTTPOk)
RespDeleted = web.Response(status=web.HTTPNoContent)

View File

@ -32,6 +32,7 @@ paths:
post: post:
operationId: upload_plugin operationId: upload_plugin
summary: Upload a new plugin summary: Upload a new plugin
description: Upload a new plugin. If the plugin already exists, enabled instances will be restarted.
tags: [Plugin] tags: [Plugin]
responses: responses:
200: 200:
@ -81,10 +82,11 @@ paths:
401: 401:
$ref: '#/components/responses/Unauthorized' $ref: '#/components/responses/Unauthorized'
404: 404:
description: Plugin not found $ref: '#/components/responses/PluginNotFound'
delete: delete:
operationId: delete_plugin operationId: delete_plugin
summary: Delete a plugin summary: Delete a plugin
description: Delete a plugin. All instances of the plugin must be deleted before deleting the plugin.
tags: [Plugin] tags: [Plugin]
responses: responses:
204: 204:
@ -92,9 +94,28 @@ paths:
401: 401:
$ref: '#/components/responses/Unauthorized' $ref: '#/components/responses/Unauthorized'
404: 404:
description: Plugin not found $ref: '#/components/responses/PluginNotFound'
412: 412:
description: One or more plugin instances of this type exist description: One or more plugin instances of this type exist
/plugin/{id}/reload:
parameters:
- name: id
in: path
description: The ID of the plugin to get
required: true
schema:
type: string
post:
operationId: reload_plugin
summary: Reload a plugin from disk
tags: [Plugin]
responses:
200:
description: Plugin reloaded
401:
$ref: '#/components/responses/Unauthorized'
404:
$ref: '#/components/responses/PluginNotFound'
/instances: /instances:
get: get:
@ -134,7 +155,7 @@ paths:
401: 401:
$ref: '#/components/responses/Unauthorized' $ref: '#/components/responses/Unauthorized'
404: 404:
description: Plugin or instance not found $ref: '#/components/responses/InstanceNotFound'
delete: delete:
operationId: delete_instance operationId: delete_instance
summary: Delete a specific plugin instance summary: Delete a specific plugin instance
@ -145,7 +166,7 @@ paths:
401: 401:
$ref: '#/components/responses/Unauthorized' $ref: '#/components/responses/Unauthorized'
404: 404:
description: Plugin or instance not found $ref: '#/components/responses/InstanceNotFound'
put: put:
operationId: update_instance operationId: update_instance
summary: Create a plugin instance or edit the details of an existing plugin instance summary: Create a plugin instance or edit the details of an existing plugin instance
@ -158,7 +179,7 @@ paths:
401: 401:
$ref: '#/components/responses/Unauthorized' $ref: '#/components/responses/Unauthorized'
404: 404:
description: Plugin or instance not found $ref: '#/components/responses/InstanceNotFound'
'/clients': '/clients':
get: get:
@ -196,7 +217,7 @@ paths:
401: 401:
$ref: '#/components/responses/Unauthorized' $ref: '#/components/responses/Unauthorized'
404: 404:
description: Client not found $ref: '#/components/responses/ClientNotFound'
put: put:
operationId: update_client operationId: update_client
summary: Create or update a Matrix client summary: Create or update a Matrix client
@ -217,7 +238,7 @@ paths:
401: 401:
$ref: '#/components/responses/Unauthorized' $ref: '#/components/responses/Unauthorized'
404: 404:
description: Client not found $ref: '#/components/responses/ClientNotFound'
delete: delete:
operationId: delete_client operationId: delete_client
summary: Delete a Matrix client summary: Delete a Matrix client
@ -228,7 +249,7 @@ paths:
401: 401:
$ref: '#/components/responses/Unauthorized' $ref: '#/components/responses/Unauthorized'
404: 404:
description: Client not found $ref: '#/components/responses/ClientNotFound'
412: 412:
description: One or more plugin instances with this as their primary client exist description: One or more plugin instances with this as their primary client exist
@ -236,6 +257,12 @@ components:
responses: responses:
Unauthorized: Unauthorized:
description: Invalid or missing access token description: Invalid or missing access token
PluginNotFound:
description: Plugin not found
ClientNotFound:
description: Client not found
InstanceNotFound:
description: Plugin instance not found
securitySchemes: securitySchemes:
bearer: bearer:
type: http type: http