agatha/maubot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch to yaml for plugin metadata. Fixes #33

Browse Source
This commit is contained in:
Tulir Asokan 2018-12-09 15:49:57 +02:00
parent 55685dfd6e
commit 07fe46e7f9
6 changed files with 86 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
maubot/instance.py
View File

@ -135,7 +135,7 @@ class PluginInstance:
self.db_instance.enabled = False self.db_instance.enabled = False
return return
self.started = True self.started = True
self.log.info(f"Started instance of {self.loader.id} v{self.loader.version} " self.log.info(f"Started instance of {self.loader.meta.id} v{self.loader.meta.version} "
f"with user {self.client.id}") f"with user {self.client.id}")
async def stop(self) -> None: async def stop(self) -> None:
@ -199,12 +199,12 @@ class PluginInstance:
except KeyError: except KeyError:
return False return False
await self.stop() await self.stop()
self.db_instance.type = loader.id self.db_instance.type = loader.meta.id
self.loader.references.remove(self) self.loader.references.remove(self)
self.loader = loader self.loader = loader
self.loader.references.add(self) self.loader.references.add(self)
await self.start() await self.start()
self.log.debug(f"Type switched to {self.loader.id}") self.log.debug(f"Type switched to {self.loader.meta.id}")
return True return True
async def update_started(self, started: bool) -> None: async def update_started(self, started: bool) -> None:

39
maubot/loader/abc.py
View File

@ -13,10 +13,15 @@
# #
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import TypeVar, Type, Dict, Set, TYPE_CHECKING from typing import TypeVar, Type, Dict, Set, List, TYPE_CHECKING
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
import asyncio import asyncio
from attr import dataclass
from packaging.version import Version, InvalidVersion
from mautrix.client.api.types.util import (SerializableAttrs, SerializerError, serializer,
deserializer)
from ..plugin_base import Plugin from ..plugin_base import Plugin
if TYPE_CHECKING: if TYPE_CHECKING:
@ -29,12 +34,36 @@ class IDConflictError(Exception):
pass pass
@serializer(Version)
def serialize_version(version: Version) -> str:
return str(version)
@deserializer(Version)
def deserialize_version(version: str) -> Version:
try:
return Version(version)
except InvalidVersion as e:
raise SerializerError("Invalid version") from e
@dataclass
class PluginMeta(SerializableAttrs['PluginMeta']):
id: str
version: Version
license: str
modules: List[str]
main_class: str
extra_files: List[str] = []
dependencies: List[str] = []
soft_dependencies: List[str] = []
class PluginLoader(ABC): class PluginLoader(ABC):
id_cache: Dict[str, 'PluginLoader'] = {} id_cache: Dict[str, 'PluginLoader'] = {}
meta: PluginMeta
references: Set['PluginInstance'] references: Set['PluginInstance']
id: str
version: str
def __init__(self): def __init__(self):
self.references = set() self.references = set()
@ -45,8 +74,8 @@ class PluginLoader(ABC):
def to_dict(self) -> dict: def to_dict(self) -> dict:
return { return {
"id": self.id, "id": self.meta.id,
"version": self.version, "version": str(self.meta.version),
"instances": [instance.to_dict() for instance in self.references], "instances": [instance.to_dict() for instance in self.references],
} }

93
maubot/loader/zip.py
View File

@ -16,15 +16,20 @@
from typing import Dict, List, Type, Tuple, Optional from typing import Dict, List, Type, Tuple, Optional
from zipfile import ZipFile, BadZipFile from zipfile import ZipFile, BadZipFile
from time import time from time import time
import configparser
import logging import logging
import sys import sys
import os import os
from ruamel.yaml import YAML, YAMLError
from packaging.version import Version
from mautrix.client.api.types.util import SerializerError
from ..lib.zipimport import zipimporter, ZipImportError from ..lib.zipimport import zipimporter, ZipImportError
from ..plugin_base import Plugin from ..plugin_base import Plugin
from ..config import Config from ..config import Config
from .abc import PluginLoader, PluginClass, IDConflictError from .abc import PluginLoader, PluginClass, PluginMeta, IDConflictError
yaml = YAML()
class MaubotZipImportError(Exception): class MaubotZipImportError(Exception):
@ -50,9 +55,7 @@ class ZippedPluginLoader(PluginLoader):
directories: List[str] = [] directories: List[str] = []
path: str path: str
id: str meta: PluginMeta
version: str
modules: List[str]
main_class: str main_class: str
main_module: str main_module: str
_loaded: Type[PluginClass] _loaded: Type[PluginClass]
@ -62,20 +65,21 @@ class ZippedPluginLoader(PluginLoader):
def __init__(self, path: str) -> None: def __init__(self, path: str) -> None:
super().__init__() super().__init__()
self.path = path self.path = path
self.id = None self.meta = None
self._loaded = None self._loaded = None
self._importer = None self._importer = None
self._file = None self._file = None
self._load_meta() self._load_meta()
self._run_preload_checks(self._get_importer()) self._run_preload_checks(self._get_importer())
try: try:
existing = self.id_cache[self.id] existing = self.id_cache[self.meta.id]
raise IDConflictError(f"Plugin with id {self.id} already loaded from {existing.source}") raise IDConflictError(
f"Plugin with id {self.meta.id} already loaded from {existing.source}")
except KeyError: except KeyError:
pass pass
self.path_cache[self.path] = self self.path_cache[self.path] = self
self.id_cache[self.id] = self self.id_cache[self.meta.id] = self
self.log.debug(f"Preloaded plugin {self.id} from {self.path}") self.log.debug(f"Preloaded plugin {self.meta.id} from {self.path}")
def to_dict(self) -> dict: def to_dict(self) -> dict:
return { return {
@ -98,57 +102,48 @@ class ZippedPluginLoader(PluginLoader):
def __repr__(self) -> str: def __repr__(self) -> str:
return ("<ZippedPlugin " return ("<ZippedPlugin "
f"path='{self.path}' " f"path='{self.path}' "
f"id='{self.id}' " f"meta={self.meta} "
f"loaded={self._loaded is not None}>") f"loaded={self._loaded is not None}>")
async def read_file(self, path: str) -> bytes: async def read_file(self, path: str) -> bytes:
return self._file.read(path) return self._file.read(path)
@staticmethod @staticmethod
def _open_meta(source) -> Tuple[ZipFile, configparser.ConfigParser]: def _read_meta(source) -> Tuple[ZipFile, PluginMeta]:
try: try:
file = ZipFile(source) file = ZipFile(source)
data = file.read("maubot.ini") data = file.read("maubot.yaml")
except FileNotFoundError as e: except FileNotFoundError as e:
raise MaubotZipMetaError("Maubot plugin not found") from e raise MaubotZipMetaError("Maubot plugin not found") from e
except BadZipFile as e: except BadZipFile as e:
raise MaubotZipMetaError("File is not a maubot plugin") from e raise MaubotZipMetaError("File is not a maubot plugin") from e
except KeyError as e: except KeyError as e:
raise MaubotZipMetaError("File does not contain a maubot plugin definition") from e raise MaubotZipMetaError("File does not contain a maubot plugin definition") from e
config = configparser.ConfigParser()
try: try:
config.read_string(data.decode("utf-8")) meta_dict = yaml.load(data)
except (configparser.Error, KeyError, IndexError, ValueError) as e: except (YAMLError, KeyError, IndexError, ValueError) as e:
raise MaubotZipMetaError("Maubot plugin definition file is not valid YAML") from e
try:
meta = PluginMeta.deserialize(meta_dict)
except SerializerError as e:
raise MaubotZipMetaError("Maubot plugin definition in file is invalid") from e raise MaubotZipMetaError("Maubot plugin definition in file is invalid") from e
return file, config return file, meta
@classmethod @classmethod
def _read_meta(cls, config: configparser.ConfigParser) -> Tuple[str, str, List[str], str, str]: def verify_meta(cls, source) -> Tuple[str, Version]:
try: _, meta = cls._read_meta(source)
meta = config["maubot"] return meta.id, meta.version
meta_id = meta["ID"]
version = meta["Version"]
modules = [mod.strip() for mod in meta["Modules"].split(",")]
main_class = meta["MainClass"]
main_module = modules[-1]
if "/" in main_class:
main_module, main_class = main_class.split("/")[:2]
except (configparser.Error, KeyError, IndexError, ValueError) as e:
raise MaubotZipMetaError("Maubot plugin definition in file is invalid") from e
return meta_id, version, modules, main_class, main_module
@classmethod
def verify_meta(cls, source) -> Tuple[str, str]:
_, config = cls._open_meta(source)
meta = cls._read_meta(config)
return meta[0], meta[1]
def _load_meta(self) -> None: def _load_meta(self) -> None:
file, config = self._open_meta(self.path) file, meta = self._read_meta(self.path)
meta = self._read_meta(config) if self.meta and meta.id != self.meta.id:
if self.id and meta[0] != self.id:
raise MaubotZipMetaError("Maubot plugin ID changed during reload") raise MaubotZipMetaError("Maubot plugin ID changed during reload")
self.id, self.version, self.modules, self.main_class, self.main_module = meta self.meta = meta
if "/" in meta.main_class:
self.main_module, self.main_class = meta.main_class.split("/")[:2]
else:
self.main_module = meta.modules[0]
self.main_class = meta.main_class
self._file = file self._file = file
def _get_importer(self, reset_cache: bool = False) -> zipimporter: def _get_importer(self, reset_cache: bool = False) -> zipimporter:
@ -170,7 +165,7 @@ class ZippedPluginLoader(PluginLoader):
except ZipImportError as e: except ZipImportError as e:
raise MaubotZipPreLoadError( raise MaubotZipPreLoadError(
f"Main module {self.main_module} not found in file") from e f"Main module {self.main_module} not found in file") from e
for module in self.modules: for module in self.meta.modules:
try: try:
importer.find_module(module) importer.find_module(module)
except ZipImportError as e: except ZipImportError as e:
@ -180,7 +175,7 @@ class ZippedPluginLoader(PluginLoader):
try: try:
return self._load(reset_cache) return self._load(reset_cache)
except MaubotZipImportError: except MaubotZipImportError:
self.log.exception(f"Failed to load {self.id} v{self.version}") self.log.exception(f"Failed to load {self.meta.id} v{self.meta.version}")
raise raise
def _load(self, reset_cache: bool = False) -> Type[PluginClass]: def _load(self, reset_cache: bool = False) -> Type[PluginClass]:
@ -190,8 +185,8 @@ class ZippedPluginLoader(PluginLoader):
importer = self._get_importer(reset_cache=reset_cache) importer = self._get_importer(reset_cache=reset_cache)
self._run_preload_checks(importer) self._run_preload_checks(importer)
if reset_cache: if reset_cache:
self.log.debug(f"Re-preloaded plugin {self.id} from {self.path}") self.log.debug(f"Re-preloaded plugin {self.meta.id} from {self.meta.path}")
for module in self.modules: for module in self.meta.modules:
try: try:
importer.load_module(module) importer.load_module(module)
except ZipImportError: except ZipImportError:
@ -209,7 +204,7 @@ class ZippedPluginLoader(PluginLoader):
if not issubclass(plugin, Plugin): if not issubclass(plugin, Plugin):
raise MaubotZipLoadError("Main class of plugin does not extend maubot.Plugin") raise MaubotZipLoadError("Main class of plugin does not extend maubot.Plugin")
self._loaded = plugin self._loaded = plugin
self.log.debug(f"Loaded and imported plugin {self.id} from {self.path}") self.log.debug(f"Loaded and imported plugin {self.meta.id} from {self.path}")
return plugin return plugin
async def reload(self, new_path: Optional[str] = None) -> Type[PluginClass]: async def reload(self, new_path: Optional[str] = None) -> Type[PluginClass]:
@ -223,7 +218,7 @@ class ZippedPluginLoader(PluginLoader):
if getattr(mod, "__file__", "").startswith(self.path): if getattr(mod, "__file__", "").startswith(self.path):
del sys.modules[name] del sys.modules[name]
self._loaded = None self._loaded = None
self.log.debug(f"Unloaded plugin {self.id} at {self.path}") self.log.debug(f"Unloaded plugin {self.meta.id} at {self.path}")
async def delete(self) -> None: async def delete(self) -> None:
await self.unload() await self.unload()
@ -232,7 +227,7 @@ class ZippedPluginLoader(PluginLoader):
except KeyError: except KeyError:
pass pass
try: try:
del self.id_cache[self.id] del self.id_cache[self.meta.id]
except KeyError: except KeyError:
pass pass
if self._importer: if self._importer:
@ -240,10 +235,8 @@ class ZippedPluginLoader(PluginLoader):
self._importer = None self._importer = None
self._loaded = None self._loaded = None
self.trash(self.path, reason="delete") self.trash(self.path, reason="delete")
self.id = None self.meta = None
self.path = None self.path = None
self.version = None
self.modules = None
@classmethod @classmethod
def trash(cls, file_path: str, new_name: Optional[str] = None, reason: str = "error") -> None: def trash(cls, file_path: str, new_name: Optional[str] = None, reason: str = "error") -> None:

8
maubot/management/api/plugin.py
View File

@ -13,7 +13,6 @@
# #
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
from http import HTTPStatus
from io import BytesIO from io import BytesIO
from time import time from time import time
import traceback import traceback
@ -21,6 +20,7 @@ import os.path
import re import re
from aiohttp import web from aiohttp import web
from packaging.version import Version
from ...loader import PluginLoader, ZippedPluginLoader, MaubotZipImportError from ...loader import PluginLoader, ZippedPluginLoader, MaubotZipImportError
from .responses import resp from .responses import resp
@ -108,7 +108,7 @@ async def upload_plugin(request: web.Request) -> web.Response:
return resp.unsupported_plugin_loader return resp.unsupported_plugin_loader
async def upload_new_plugin(content: bytes, pid: str, version: str) -> web.Response: async def upload_new_plugin(content: bytes, pid: str, version: Version) -> web.Response:
path = os.path.join(get_config()["plugin_directories.upload"], f"{pid}-v{version}.mbp") path = os.path.join(get_config()["plugin_directories.upload"], f"{pid}-v{version}.mbp")
with open(path, "wb") as p: with open(path, "wb") as p:
p.write(content) p.write(content)
@ -120,8 +120,8 @@ async def upload_new_plugin(content: bytes, pid: str, version: str) -> web.Respo
return resp.created(plugin.to_dict()) return resp.created(plugin.to_dict())
async def upload_replacement_plugin(plugin: ZippedPluginLoader, content: bytes, new_version: str async def upload_replacement_plugin(plugin: ZippedPluginLoader, content: bytes,
) -> web.Response: new_version: Version) -> web.Response:
dirname = os.path.dirname(plugin.path) dirname = os.path.dirname(plugin.path)
old_filename = os.path.basename(plugin.path) old_filename = os.path.basename(plugin.path)
if plugin.version in old_filename: if plugin.version in old_filename:

1
requirements.txt
View File

@ -6,3 +6,4 @@ Markdown
ruamel.yaml ruamel.yaml
attrs attrs
bcrypt bcrypt
packaging

1
setup.py
View File

@ -29,6 +29,7 @@ setuptools.setup(
"ruamel.yaml>=0.15.35,<0.16", "ruamel.yaml>=0.15.35,<0.16",
"attrs>=18.1.0,<19", "attrs>=18.1.0,<19",
"bcrypt>=3.1.4,<4", "bcrypt>=3.1.4,<4",
"packaging>=10",
], ],
classifiers=[ classifiers=[