Add server with basic auth stuff

sticker/import.py

@@ -117,6 +117,8 @@ async def reupload_pack(client: TelegramClient, pack: StickerSetFull, output_dir
 pack_url_regex = re.compile(r"^(?:(?:https?://)?(?:t|telegram)\.(?:me|dog)/addstickers/)?"
                             r"([A-Za-z0-9-_]+)"
                             r"(?:\.json)?$")
+api_id = 298751
+api_hash = "cb676d6bae20553c9996996a8f52b4d7"
 
 parser = argparse.ArgumentParser()
 
@@ -132,7 +134,7 @@ parser.add_argument("pack", help="Sticker pack URLs to import", action="append",
 
 async def main(args: argparse.Namespace) -> None:
     await matrix.load_config(args.config)
-    client = TelegramClient(args.session, 298751, "cb676d6bae20553c9996996a8f52b4d7")
+    client = TelegramClient(args.session, api_id, api_hash)
     await client.start()
 
     if args.list:

sticker/server/__main__.py

@@ -0,0 +1,53 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.program import Program
+from mautrix.util.async_db import Database
+
+from .config import Config
+from .server import Server
+from .database import upgrade_table, Base
+from ..version import version
+
+
+class StickerServer(Program):
+    module = "sticker.server"
+    name = "maunium-stickerpicker server"
+    version = version
+    command = "python -m sticker.server"
+    description = "Server for maunium-stickerpicker"
+
+    config_class = Config
+
+    config: Config
+    server: Server
+    database: Database
+
+    async def start(self) -> None:
+        self.database = Database(url=self.config["database"], upgrade_table=upgrade_table)
+        Base.db = self.database
+        self.server = Server(self.config)
+
+        await self.database.start()
+        await self.server.start()
+
+        await super().start()
+
+    async def stop(self) -> None:
+        await super().stop()
+        await self.server.stop()
+
+
+StickerServer().run()

sticker/server/api/__init__.py

@@ -0,0 +1,38 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+
+from ..config import Config
+from .auth import (routes as auth_routes, init as auth_init,
+                   token_middleware, widget_secret_middleware)
+from .fed_connector import init as init_fed_connector
+from .packs import routes as packs_routes, init as packs_init
+from .setup import routes as setup_routes
+
+integrations_app = web.Application()
+integrations_app.add_routes(auth_routes)
+
+packs_app = web.Application(middlewares=[widget_secret_middleware])
+packs_app.add_routes(packs_routes)
+
+setup_app = web.Application(middlewares=[token_middleware])
+setup_app.add_routes(setup_routes)
+
+
+def init(config: Config) -> None:
+    init_fed_connector()
+    auth_init(config)
+    packs_init(config)

sticker/server/api/auth.py

@@ -0,0 +1,216 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Callable, Awaitable, Optional, TYPE_CHECKING
+import logging
+import json
+
+from mautrix.client import Client
+from mautrix.types import UserID
+from mautrix.util.logging import TraceLogger
+from aiohttp import web, hdrs, ClientError, ClientSession
+from yarl import URL
+
+from ..database import AccessToken, User
+from ..config import Config
+from .errors import Error
+from . import fed_connector
+
+if TYPE_CHECKING:
+    from typing import TypedDict
+
+
+    class OpenIDPayload(TypedDict):
+        access_token: str
+        token_type: str
+        matrix_server_name: str
+        expires_in: int
+
+
+    class OpenIDResponse(TypedDict):
+        sub: str
+
+Handler = Callable[[web.Request], Awaitable[web.Response]]
+
+log: TraceLogger = logging.getLogger("mau.api.auth")
+routes = web.RouteTableDef()
+config: Config
+
+
+def get_ip(request: web.Request) -> str:
+    if config["server.trust_forward_headers"]:
+        try:
+            return request.headers["X-Forwarded-For"]
+        except KeyError:
+            pass
+    return request.remote
+
+
+def get_auth_header(request: web.Request) -> str:
+    try:
+        auth = request.headers["Authorization"]
+        if not auth.startswith("Bearer "):
+            raise Error.invalid_auth_header
+        return auth[len("Bearer "):]
+    except KeyError:
+        raise Error.missing_auth_header
+
+
+async def get_user(request: web.Request) -> Tuple[User, AccessToken]:
+    auth = get_auth_header(request)
+    try:
+        token_id, token_val = auth.split(":")
+        token_id = int(token_id)
+    except ValueError:
+        raise Error.invalid_auth_token
+    token = await AccessToken.get(token_id)
+    if not token or not token.check(token_val):
+        raise Error.invalid_auth_token
+    elif token.expired:
+        raise Error.auth_token_expired
+    await token.update_ip(get_ip(request))
+    return await User.get(token.user_id), token
+
+
+@web.middleware
+async def token_middleware(request: web.Request, handler: Handler) -> web.Response:
+    if request.method == hdrs.METH_OPTIONS:
+        return await handler(request)
+    user, token = await get_user(request)
+    request["user"] = user
+    request["token"] = token
+    return await handler(request)
+
+
+async def get_widget_user(request: web.Request) -> User:
+    try:
+        user_id = UserID(request.headers["X-Matrix-User-ID"])
+    except KeyError:
+        raise Error.missing_user_id_header
+    user = await User.get(user_id)
+    if user is None:
+        raise Error.user_not_found
+    return user
+
+
+@web.middleware
+async def widget_secret_middleware(request: web.Request, handler: Handler) -> web.Response:
+    if request.method == hdrs.METH_OPTIONS:
+        return await handler(request)
+    user = await get_widget_user(request)
+    request["user"] = user
+    return await handler(request)
+
+
+account_cors_headers = {
+    "Access-Control-Allow-Origin": "*",
+    "Access-Control-Allow-Methods": "OPTIONS, GET, POST",
+    "Access-Control-Allow-Headers": "Authorization, Content-Type",
+}
+
+
+@routes.get("/account")
+async def get_auth(request: web.Request) -> web.Response:
+    user, token = await get_user(request)
+    return web.json_response({"user_id": token.user_id}, headers=account_cors_headers)
+
+
+async def check_openid_token(homeserver: str, token: str) -> Optional[UserID]:
+    server_info = await fed_connector.resolve_server_name(homeserver)
+    headers = {"Host": server_info.host_header}
+    userinfo_url = URL.build(scheme="https", host=server_info.host, port=server_info.port,
+                             path="/_matrix/federation/v1/openid/userinfo",
+                             query={"access_token": token})
+    try:
+        async with fed_connector.http.get(userinfo_url, headers=headers) as resp:
+            data: 'OpenIDResponse' = await resp.json()
+            return UserID(data["sub"])
+    except (ClientError, json.JSONDecodeError, KeyError, ValueError) as e:
+        log.debug(f"Failed to check OpenID token from {homeserver}", exc_info=True)
+        return None
+
+
+@routes.route(hdrs.METH_OPTIONS, "/account/register")
+@routes.route(hdrs.METH_OPTIONS, "/account/logout")
+@routes.route(hdrs.METH_OPTIONS, "/account")
+async def cors_token(_: web.Request) -> web.Response:
+    return web.Response(status=200, headers=account_cors_headers)
+
+
+async def resolve_client_well_known(server_name: str) -> str:
+    url = URL.build(scheme="https", host=server_name, port=443, path="/.well-known/matrix/client")
+    async with ClientSession() as sess, sess.get(url) as resp:
+        data = await resp.json()
+        return data["m.homeserver"]["base_url"]
+
+
+@routes.post("/account/register")
+async def exchange_token(request: web.Request) -> web.Response:
+    try:
+        data: 'OpenIDPayload' = await request.json()
+    except json.JSONDecodeError:
+        raise Error.request_not_json
+    try:
+        matrix_server_name = data["matrix_server_name"]
+        access_token = data["access_token"]
+    except KeyError:
+        raise Error.invalid_openid_payload
+    log.trace(f"Validating OpenID token from {matrix_server_name}")
+    user_id = await check_openid_token(matrix_server_name, access_token)
+    if user_id is None:
+        raise Error.invalid_openid_token
+    _, homeserver = Client.parse_user_id(user_id)
+    if homeserver != data["matrix_server_name"]:
+        raise Error.homeserver_mismatch
+
+    permissions = config.get_permissions(user_id)
+    if not permissions.access:
+        raise Error.no_access
+
+    try:
+        log.trace(f"Trying to resolve {matrix_server_name}'s client .well-known")
+        homeserver_url = await resolve_client_well_known(matrix_server_name)
+        log.trace(f"Got {homeserver_url} from {matrix_server_name}'s client .well-known")
+    except (ClientError, json.JSONDecodeError, KeyError, ValueError, TypeError):
+        log.trace(f"Failed to resolve {matrix_server_name}'s client .well-known", exc_info=True)
+        raise Error.client_well_known_error
+
+    user = await User.get(user_id)
+    if user is None:
+        log.debug(f"Creating user {user_id} with homeserver client URL {homeserver_url}")
+        user = User.new(user_id, homeserver_url=homeserver_url)
+        await user.insert()
+    elif user.homeserver_url != homeserver_url:
+        log.debug(f"Updating {user_id}'s homeserver client URL from {user.homeserver_url} "
+                  f"to {homeserver_url}")
+        await user.set_homeserver_url(homeserver_url)
+    token = await user.new_access_token(get_ip(request))
+    return web.json_response({
+        "user_id": user_id,
+        "token": token,
+        "permissions": permissions._asdict(),
+    }, headers=account_cors_headers)
+
+
+@routes.post("/account/logout")
+async def logout(request: web.Request) -> web.Response:
+    user, token = await get_user(request)
+    await token.delete()
+    return web.json_response({}, status=204, headers=account_cors_headers)
+
+
+def init(cfg: Config) -> None:
+    global config
+    config = cfg

sticker/server/api/errors.py

@@ -0,0 +1,110 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+import json
+
+from aiohttp import web
+
+
+class _ErrorMeta:
+    def __init__(self, *args, **kwargs) -> None:
+        pass
+
+    @staticmethod
+    def _make_error(errcode: str, error: str) -> Dict[str, str]:
+        return {
+            "body": json.dumps({
+                "error": error,
+                "errcode": errcode,
+            }).encode("utf-8"),
+            "content_type": "application/json",
+            "headers": {
+                "Access-Control-Allow-Origin": "*",
+                "Access-Control-Allow-Methods": "OPTIONS, GET, POST, PUT, DELETE, HEAD",
+                "Access-Control-Allow-Headers": "Authorization, Content-Type",
+            }
+        }
+
+    @property
+    def request_not_json(self) -> web.HTTPException:
+        return web.HTTPBadRequest(**self._make_error("M_NOT_JSON",
+                                                     "Request body is not valid JSON"))
+
+    @property
+    def missing_auth_header(self) -> web.HTTPException:
+        return web.HTTPForbidden(**self._make_error("M_MISSING_TOKEN",
+                                                    "Missing authorization header"))
+
+    @property
+    def missing_user_id_header(self) -> web.HTTPException:
+        return web.HTTPForbidden(**self._make_error("NET.MAUNIUM_MISSING_USER_ID",
+                                                    "Missing user ID header"))
+
+    @property
+    def user_not_found(self) -> web.HTTPException:
+        return web.HTTPNotFound(**self._make_error("NET.MAUNIUM_USER_NOT_FOUND",
+                                                   "User not found"))
+
+    @property
+    def invalid_auth_header(self) -> web.HTTPException:
+        return web.HTTPForbidden(**self._make_error("M_UNKNOWN_TOKEN",
+                                                    "Invalid authorization header"))
+
+    @property
+    def invalid_auth_token(self) -> web.HTTPException:
+        return web.HTTPForbidden(**self._make_error("M_UNKNOWN_TOKEN",
+                                                    "Invalid authorization token"))
+
+    @property
+    def auth_token_expired(self) -> web.HTTPException:
+        return web.HTTPForbidden(**self._make_error("NET.MAUNIUM_TOKEN_EXPIRED",
+                                                    "Authorization token has expired"))
+
+    @property
+    def invalid_openid_payload(self) -> web.HTTPException:
+        return web.HTTPBadRequest(**self._make_error("M_BAD_JSON", "Missing one or more "
+                                                                   "fields in OpenID payload"))
+
+    @property
+    def invalid_openid_token(self) -> web.HTTPException:
+        return web.HTTPForbidden(**self._make_error("M_UNKNOWN_TOKEN",
+                                                    "Invalid OpenID token"))
+
+    @property
+    def no_access(self) -> web.HTTPException:
+        return web.HTTPUnauthorized(**self._make_error(
+            "M_UNAUTHORIZED",
+            "You are not authorized to access this maunium-stickerpicker instance"))
+
+    @property
+    def homeserver_mismatch(self) -> web.HTTPException:
+        return web.HTTPUnauthorized(**self._make_error(
+            "M_UNAUTHORIZED", "Request matrix_server_name and OpenID sub homeserver don't match"))
+
+    @property
+    def pack_not_found(self) -> web.HTTPException:
+        return web.HTTPNotFound(**self._make_error("NET.MAUNIUM_PACK_NOT_FOUND",
+                                                   "Sticker pack not found"))
+
+    @property
+    def client_well_known_error(self) -> web.HTTPException:
+        return web.HTTPForbidden(**self._make_error("NET.MAUNIUM_CLIENT_WELL_KNOWN_ERROR",
+                                                    "Failed to resolve homeserver URL "
+                                                    "from client .well-known"))
+
+
+class Error(metaclass=_ErrorMeta):
+    pass

sticker/server/api/fed_connector.py

@@ -0,0 +1,110 @@
+from typing import Tuple, Any, NamedTuple, Dict, Optional
+from time import time
+import ipaddress
+import logging
+import asyncio
+import json
+
+from mautrix.util.logging import TraceLogger
+from aiohttp import ClientRequest, TCPConnector, ClientSession, ClientTimeout, ClientError
+from aiohttp.client_proto import ResponseHandler
+from yarl import URL
+import aiodns
+
+log: TraceLogger = logging.getLogger("mau.federation")
+
+
+class ResolvedServerName(NamedTuple):
+    host_header: str
+    host: str
+    port: int
+    expire: int
+
+
+class ServerNameSplit(NamedTuple):
+    host: str
+    port: Optional[int]
+    is_ip: bool
+
+
+dns_resolver: aiodns.DNSResolver
+http: ClientSession
+server_name_cache: Dict[str, ResolvedServerName] = {}
+
+
+class MatrixFederationTCPConnector(TCPConnector):
+    """An extension to aiohttp's TCPConnector that correctly sets the TLS SNI for Matrix federation
+    requests, where the TCP host may not match the SNI/Host header."""
+
+    async def _wrap_create_connection(self, *args: Any, server_hostname: str, req: ClientRequest,
+                                      **kwargs: Any) -> Tuple[asyncio.Transport, ResponseHandler]:
+        split = parse_server_name(req.headers["Host"])
+        return await super()._wrap_create_connection(*args, server_hostname=split.host,
+                                                     req=req, **kwargs)
+
+
+def parse_server_name(name: str) -> ServerNameSplit:
+    port_split = name.rsplit(":", 1)
+    if len(port_split) == 2 and port_split[1].isdecimal():
+        name, port = port_split
+    else:
+        port = None
+    try:
+        ipaddress.ip_address(name)
+        is_ip = True
+    except ValueError:
+        is_ip = False
+    res = ServerNameSplit(host=name, port=port, is_ip=is_ip)
+    log.trace(f"Parsed server name {name} into {res}")
+    return res
+
+
+async def resolve_server_name(server_name: str) -> ResolvedServerName:
+    try:
+        cached = server_name_cache[server_name]
+        if cached.expire > int(time()):
+            log.trace(f"Using cached server name resolution for {server_name}: {cached}")
+            return cached
+    except KeyError:
+        log.trace(f"No cached server name resolution for {server_name}")
+
+    host_header = server_name
+    hostname, port, is_ip = parse_server_name(host_header)
+    ttl = 86400
+    if port is None and not is_ip:
+        well_known_url = URL.build(scheme="https", host=host_header, port=443,
+                                   path="/.well-known/matrix/server")
+        try:
+            log.trace(f"Requesting {well_known_url} to resolve {server_name}'s .well-known")
+            async with http.get(well_known_url) as resp:
+                if resp.status == 200:
+                    well_known_data = await resp.json()
+                    host_header = well_known_data["m.server"]
+                    log.debug(f"Got {host_header} from {server_name}'s .well-known")
+                    hostname, port, is_ip = parse_server_name(host_header)
+                else:
+                    log.trace(f"Got non-200 status {resp.status} from {server_name}'s .well-known")
+        except (ClientError, json.JSONDecodeError, KeyError, ValueError) as e:
+            log.debug(f"Failed to fetch .well-known for {server_name}: {e}")
+    if port is None and not is_ip:
+        log.trace(f"Querying SRV at _matrix._tcp.{host_header}")
+        res = await dns_resolver.query(f"_matrix._tcp.{host_header}", "SRV")
+        if res:
+            hostname = res[0].host
+            port = res[0].port
+            ttl = max(res[0].ttl, 300)
+            log.debug(f"Got {hostname}:{port} from {host_header}'s Matrix SRV record")
+        else:
+            log.trace(f"No SRV records found at _matrix._tcp.{host_header}")
+    result = ResolvedServerName(host_header=host_header, host=hostname, port=port or 8448,
+                                expire=int(time()) + ttl)
+    server_name_cache[server_name] = result
+    log.debug(f"Resolved server name {server_name} -> {result}")
+    return result
+
+
+def init():
+    global http, dns_resolver
+    dns_resolver = aiodns.DNSResolver(loop=asyncio.get_running_loop())
+    http = ClientSession(timeout=ClientTimeout(total=10),
+                         connector=MatrixFederationTCPConnector())

sticker/server/api/packs.py

@@ -0,0 +1,52 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+
+from ..database import User
+from ..config import Config
+from .errors import Error
+
+routes = web.RouteTableDef()
+config: Config
+
+
+@routes.get("/index.json")
+async def get_packs(req: web.Request) -> web.Response:
+    user: User = req["user"]
+    packs = await user.get_packs()
+    return web.json_response({
+        "homeserver_url": user.homeserver_url,
+        "is_sticker_server": True,
+        "packs": [f"{pack.id}.json" for pack in packs],
+    })
+
+
+@routes.get("/{pack_id}.json")
+async def get_pack(req: web.Request) -> web.Response:
+    user: User = req["user"]
+    pack = await user.get_pack(req.match_info["pack_id"])
+    if pack is None:
+        raise Error.pack_not_found
+    stickers = await pack.get_stickers()
+    return web.json_response({
+        **pack.to_dict(),
+        "stickers": [sticker.to_dict() for sticker in stickers],
+    })
+
+
+def init(cfg: Config) -> None:
+    global config
+    config = cfg

sticker/server/api/setup.py

@@ -0,0 +1,19 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from aiohttp import web
+
+routes = web.RouteTableDef()

sticker/server/config.py

@@ -0,0 +1,55 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import NamedTuple
+
+from mautrix.util.config import BaseFileConfig, ConfigUpdateHelper
+from mautrix.types import UserID
+from mautrix.client import Client
+
+
+class Permissions(NamedTuple):
+    access: bool = False
+    create_packs: bool = False
+    telegram_import: bool = False
+
+
+class Config(BaseFileConfig):
+    def do_update(self, helper: ConfigUpdateHelper) -> None:
+        copy = helper.copy
+
+        copy("database")
+
+        copy("server.host")
+        copy("server.port")
+        copy("server.public_url")
+        copy("server.override_resource_path")
+        copy("server.trust_forward_headers")
+
+        copy("telegram_import.bot_token")
+        copy("telegram_import.homeserver.address")
+        copy("telegram_import.homeserver.access_token")
+
+        copy("permissions")
+
+        copy("logging")
+
+    def get_permissions(self, mxid: UserID) -> Permissions:
+        _, homeserver = Client.parse_user_id(mxid)
+        return Permissions(**{
+            **self["permissions"].get("*", {}),
+            **self["permissions"].get(homeserver, {}),
+            **self["permissions"].get(mxid, {}),
+        })

sticker/server/database/__init__.py

@@ -0,0 +1,6 @@
+from .base import Base
+from .upgrade import upgrade_table
+from .sticker import Sticker
+from .pack import Pack
+from .access_token import AccessToken
+from .user import User

sticker/server/database/access_token.py

@@ -0,0 +1,71 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, ClassVar
+from datetime import datetime, timedelta
+import hashlib
+
+from attr import dataclass
+import asyncpg
+
+from mautrix.types import UserID
+
+from .base import Base
+
+
+@dataclass(kw_only=True)
+class AccessToken(Base):
+    token_expiry: ClassVar[timedelta] = timedelta(days=1)
+
+    user_id: UserID
+    token_id: int
+    token_hash: bytes
+    last_seen_ip: str
+    last_seen_date: datetime
+
+    @classmethod
+    async def get(cls, token_id: int) -> Optional['AccessToken']:
+        q = "SELECT user_id, token_hash, last_seen_ip, last_seen_date FROM pack WHERE token_id=$1"
+        row: asyncpg.Record = await cls.db.fetchrow(q, token_id)
+        if row is None:
+            return None
+        return cls(**row, token_id=token_id)
+
+    async def update_ip(self, ip: str) -> None:
+        if self.last_seen_ip == ip and (self.last_seen_date.replace(second=0, microsecond=0)
+                                        == datetime.now().replace(second=0, microsecond=0)):
+            # Same IP and last seen on this minute, skip update
+            return
+        q = ("UPDATE access_token SET last_seen_ip=$3, last_seen_date=current_timestamp "
+             "WHERE token_id=$1 RETURNING last_seen_date")
+        self.last_seen_date = await self.db.fetchval(q, self.token_id, ip)
+        self.last_seen_ip = ip
+
+    def check(self, token: str) -> bool:
+        return self.token_hash == hashlib.sha256(token.encode("utf-8")).digest()
+
+    @property
+    def expired(self) -> bool:
+        return self.last_seen_date + self.token_expiry < datetime.now()
+
+    async def delete(self) -> None:
+        await self.db.execute("DELETE FROM access_token WHERE token_id=$1", self.token_id)
+
+    @classmethod
+    async def insert(cls, user_id: UserID, token: str, ip: str) -> int:
+        q = ("INSERT INTO access_token (user_id, token_hash, last_seen_ip, last_seen_date) "
+             "VALUES ($1, $2, $3, current_timestamp) RETURNING token_id")
+        hashed = hashlib.sha256(token.encode("utf-8")).digest()
+        return await cls.db.fetchval(q, user_id, hashed, ip)

sticker/server/database/base.py

@@ -0,0 +1,9 @@
+from typing import ClassVar, TYPE_CHECKING
+
+from mautrix.util.async_db import Database
+
+fake_db = Database("") if TYPE_CHECKING else None
+
+
+class Base:
+    db: ClassVar[Database] = fake_db

sticker/server/database/pack.py

@@ -0,0 +1,58 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Dict, Any
+
+from attr import dataclass
+
+from mautrix.types import UserID
+
+from .base import Base
+from .sticker import Sticker
+
+
+@dataclass(kw_only=True)
+class Pack(Base):
+    id: str
+    owner: UserID
+    title: str
+    meta: Dict[str, Any]
+
+    async def delete(self) -> None:
+        await self.db.execute("DELETE FROM pack WHERE id=$1", self.id)
+
+    async def insert(self) -> None:
+        await self.db.execute("INSERT INTO pack (id, owner, title, meta) VALUES ($1, $2, $3, $4)",
+                              self.id, self.owner, self.title, self.meta)
+
+    async def get_stickers(self) -> List[Sticker]:
+        res = await self.db.fetch('SELECT id, url, body, meta, "order" '
+                                  'FROM sticker WHERE pack_id=$1 ORDER BY "order"', self.id)
+        return [Sticker(**row, pack_id=self.id) for row in res]
+
+    async def set_stickers(self, stickers: List[Sticker]) -> None:
+        data = ((sticker.id, self.id, sticker.url, sticker.body, sticker.meta, order)
+                for order, sticker in enumerate(stickers))
+        columns = ["id", "pack_id", "url", "body", "meta", "order"]
+        async with self.db.acquire() as conn, conn.transaction():
+            await conn.execute("DELETE FROM sticker WHERE pack_id=$1", self.id)
+            await conn.copy_records_to_table("sticker", records=data, columns=columns)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            **self.meta,
+            "title": self.title,
+            "id": self.id,
+        }

sticker/server/database/sticker.py

@@ -0,0 +1,49 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Any
+
+from attr import dataclass
+import attr
+
+from mautrix.types import ContentURI
+
+from .base import Base
+
+
+@dataclass(kw_only=True)
+class Sticker(Base):
+    pack_id: str
+    order: int
+    id: str
+    url: ContentURI = attr.ib(order=False)
+    body: str = attr.ib(order=False)
+    meta: Dict[str, Any] = attr.ib(order=False)
+
+    async def delete(self) -> None:
+        await self.db.execute("DELETE FROM sticker WHERE id=$1", self.id)
+
+    async def insert(self) -> None:
+        await self.db.execute('INSERT INTO sticker (id, pack_id, url, body, meta, "order") '
+                              "VALUES ($1, $2, $3, $4, $5, $6)",
+                              self.id, self.pack_id, self.url, self.body, self.meta, self.order)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            **self.meta,
+            "body": self.body,
+            "url": self.url,
+            "id": self.id,
+        }

sticker/server/database/upgrade.py

@@ -0,0 +1,56 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from mautrix.util.async_db.upgrade import UpgradeTable
+
+upgrade_table = UpgradeTable()
+
+
+@upgrade_table.register(description="Initial revision")
+async def upgrade_v1(conn: Connection) -> None:
+    await conn.execute("""CREATE TABLE "user" (
+        id             TEXT PRIMARY KEY,
+        widget_secret  TEXT NOT NULL,
+        homeserver_url TEXT NOT NULL
+    )""")
+    await conn.execute("""CREATE TABLE access_token (
+        token_id       SERIAL PRIMARY KEY,
+        user_id        TEXT   NOT NULL REFERENCES "user"(id) ON DELETE CASCADE,
+        token_hash     BYTEA  NOT NULL,
+        last_seen_ip   TEXT,
+        last_seen_date TIMESTAMP
+    )""")
+    await conn.execute("""CREATE TABLE pack (
+        id    TEXT  PRIMARY KEY,
+        owner TEXT  REFERENCES "user"(id) ON DELETE SET NULL,
+        title TEXT  NOT NULL,
+        meta  JSONB NOT NULL
+    )""")
+    await conn.execute("""CREATE TABLE user_pack (
+        user_id TEXT REFERENCES "user"(id) ON DELETE CASCADE,
+        pack_id TEXT REFERENCES pack(id) ON DELETE CASCADE,
+        "order" INT  NOT NULL DEFAULT 0,
+        PRIMARY KEY (user_id, pack_id)
+    )""")
+    await conn.execute("""CREATE TABLE sticker (
+        id      TEXT  PRIMARY KEY,
+        pack_id TEXT  NOT NULL REFERENCES pack(id) ON DELETE CASCADE,
+        url     TEXT  NOT NULL,
+        body    TEXT  NOT NULL,
+        meta    JSONB NOT NULL,
+        "order" INT   NOT NULL DEFAULT 0
+    )""")

sticker/server/database/user.py

@@ -0,0 +1,95 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, ClassVar
+import random
+import string
+
+from attr import dataclass
+import asyncpg
+
+from mautrix.types import UserID
+
+from .base import Base
+from .pack import Pack
+from .access_token import AccessToken
+
+
+@dataclass(kw_only=True)
+class User(Base):
+    token_charset: ClassVar[str] = string.ascii_letters + string.digits
+
+    id: UserID
+    widget_secret: str
+    homeserver_url: str
+
+    @classmethod
+    def _random_token(cls) -> str:
+        return "".join(random.choices(cls.token_charset, k=64))
+
+    @classmethod
+    def new(cls, id: UserID, homeserver_url: str) -> 'User':
+        return User(id=id, widget_secret=cls._random_token(), homeserver_url=homeserver_url)
+
+    @classmethod
+    async def get(cls, id: UserID) -> Optional['User']:
+        q = 'SELECT id, widget_secret, homeserver_url FROM "user" WHERE id=$1'
+        row: asyncpg.Record = await cls.db.fetchrow(q, id)
+        if row is None:
+            return None
+        return cls(**row)
+
+    async def regenerate_widget_secret(self) -> None:
+        self.widget_secret = self._random_token()
+        await self.db.execute('UPDATE "user" SET widget_secret=$1 WHERE id=$2',
+                              self.widget_secret, self.id)
+
+    async def set_homeserver_url(self, url: str) -> None:
+        self.homeserver_url = url
+        await self.db.execute('UPDATE "user" SET homeserver_url=$1 WHERE id=$2', url, self.id)
+
+    async def new_access_token(self, ip: str) -> str:
+        token = self._random_token()
+        token_id = await AccessToken.insert(self.id, token, ip)
+        return f"{token_id}:{token}"
+
+    async def delete(self) -> None:
+        await self.db.execute('DELETE FROM "user" WHERE id=$1', self.id)
+
+    async def insert(self) -> None:
+        q = 'INSERT INTO "user" (id, widget_secret, homeserver_url) VALUES ($1, $2, $3)'
+        await self.db.execute(q, self.id, self.widget_secret, self.homeserver_url)
+
+    async def get_packs(self) -> List[Pack]:
+        res = await self.db.fetch("SELECT id, owner, title, meta FROM user_pack "
+                                  "LEFT JOIN pack ON pack.id=user_pack.pack_id "
+                                  'WHERE user_id=$1 ORDER BY "order"', self.id)
+        return [Pack(**row) for row in res]
+
+    async def get_pack(self, pack_id: str) -> Optional[Pack]:
+        row = await self.db.fetchrow("SELECT id, owner, title, meta FROM user_pack "
+                                     "LEFT JOIN pack ON pack.id=user_pack.pack_id "
+                                     "WHERE user_id=$1 AND pack_id=$2", self.id, pack_id)
+        if row is None:
+            return None
+        return Pack(**row)
+
+    async def set_packs(self, packs: List[Pack]) -> None:
+        data = ((self.id, pack.id, order)
+                for order, pack in enumerate(packs))
+        columns = ["user_id", "pack_id", "order"]
+        async with self.db.acquire() as conn, conn.transaction():
+            await conn.execute("DELETE FROM user_pack WHERE user_id=$1", self.id)
+            await conn.copy_records_to_table("user_pack", records=data, columns=columns)

sticker/server/example-config.yaml

@@ -0,0 +1,74 @@
+# Postgres database URL for storing sticker packs and other things.
+database: postgres://username:password@hostname/dbname
+
+# Settings for the actual HTTP server
+server:
+    # The IP and port to listen to.
+    hostname: 0.0.0.0
+    port: 29329
+    # Public base URL where the server is visible.
+    public_url: https://example.com
+    # Override path from where to load UI resources.
+    # Set to false to using pkg_resources to find the path.
+    override_resource_path: false
+    # Whether or not to trust X-Forwarded-For headers for determining the request IP.
+    trust_forward_headers: false
+
+# Telegram configuration for downloading sticker packs. In the future, this will be client-side and
+# none of this configuration will be necessary.
+telegram_import:
+    # Create your own bot at https://t.me/BotFather
+    bot_token: null
+
+    # Matrix homeserver access details. This is only used for uploading Telegram-imported stickers.
+    homeserver:
+        address: https://example.com
+        access_token: null
+
+# Permissions for who is allowed to use the sticker picker.
+#
+# Values are objects that should contain boolean values each permission:
+#             access - Access the sticker picker and use existing packs.
+#       create_packs - Create packs by uploading images.
+#    telegram_import - Create packs by importing from Telegram. Images are stored on
+#
+# Permission keys may be user IDs, server names or "*". If a server name or user ID permission
+# doesn't specify some keys, they'll be inherited from the higher level.
+permissions:
+    "*":
+        access: true
+        create_packs: true
+        telegram_import: false
+    "example.com":
+        telegram_import: true
+
+# Python logging configuration.
+#
+# See Configuration dictionary schema in the Python documentation for more info:
+# https://docs.python.org/3.9/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        colored:
+            (): mautrix.util.logging.ColorFormatter
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+        normal:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: normal
+            filename: ./sticker.server.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: colored
+    loggers:
+        mau:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

sticker/server/frontend

@@ -0,0 +1 @@
+../../web/

sticker/server/server.py

@@ -0,0 +1,50 @@
+# maunium-stickerpicker - A fast and simple Matrix sticker picker widget.
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from pkg_resources import resource_filename
+from aiohttp import web
+
+from .api import packs_app, setup_app, integrations_app, init as init_api
+from .static import StaticResource
+from .config import Config
+
+
+class Server:
+    config: Config
+    runner: web.AppRunner
+    app: web.Application
+    site: web.TCPSite
+
+    def __init__(self, config: Config) -> None:
+        init_api(config)
+        self.config = config
+        self.app = web.Application()
+        self.app.add_subapp("/_matrix/integrations/v1", integrations_app)
+        self.app.add_subapp("/setup/api", setup_app)
+        self.app.add_subapp("/packs", packs_app)
+
+        resource_path = (config["server.override_resource_path"]
+                         or resource_filename("sticker.server", "frontend"))
+        self.app.router.register_resource(StaticResource("/", resource_path, name="frontend"))
+        self.runner = web.AppRunner(self.app)
+
+    async def start(self) -> None:
+        await self.runner.setup()
+        self.site = web.TCPSite(self.runner, self.config["server.host"],
+                                self.config["server.port"])
+        await self.site.start()
+
+    async def stop(self) -> None:
+        await self.runner.cleanup()

sticker/server/static.py

@@ -0,0 +1,106 @@
+# Simplified version of aiohttp's StaticResource with support for index.html
+# https://github.com/aio-libs/aiohttp/blob/v3.6.2/aiohttp/web_urldispatcher.py#L496-L678
+# Licensed under Apache 2.0
+from typing import Callable, Awaitable, Tuple, Optional, Union, Dict, Set, Iterator, Any
+from pathlib import Path, PurePath
+
+from aiohttp.web import (Request, StreamResponse, FileResponse, ResourceRoute, AbstractResource,
+                         AbstractRoute, UrlMappingMatchInfo, HTTPNotFound, HTTPForbidden)
+from aiohttp.abc import AbstractMatchInfo
+from yarl import URL
+
+Handler = Callable[[Request], Awaitable[StreamResponse]]
+
+
+class StaticResource(AbstractResource):
+    def __init__(self, prefix: str, directory: Union[str, PurePath], *, name: Optional[str] = None,
+                 error_path: Optional[str] = "index.html", chunk_size: int = 256 * 1024) -> None:
+        super().__init__(name=name)
+        try:
+            directory = Path(directory).resolve()
+            if not directory.is_dir():
+                raise ValueError("Not a directory")
+        except (FileNotFoundError, ValueError) as error:
+            raise ValueError(f"No directory exists at '{directory}'") from error
+        self._directory = directory
+        self._chunk_size = chunk_size
+        self._prefix = prefix
+        self._error_file = (directory / error_path) if error_path else None
+
+        self._routes = {
+            "GET": ResourceRoute("GET", self._handle, self),
+            "HEAD": ResourceRoute("HEAD", self._handle, self),
+        }
+
+    @property
+    def canonical(self) -> str:
+        return self._prefix
+
+    def add_prefix(self, prefix: str) -> None:
+        assert prefix.startswith("/")
+        assert not prefix.endswith("/")
+        assert len(prefix) > 1
+        self._prefix = prefix + self._prefix
+
+    def raw_match(self, prefix: str) -> bool:
+        return False
+
+    def url_for(self, *, filename: Union[str, Path]) -> URL:
+        if isinstance(filename, Path):
+            filename = str(filename)
+        while filename.startswith("/"):
+            filename = filename[1:]
+        return URL.build(path=f"{self._prefix}/{filename}")
+
+    def get_info(self) -> Dict[str, Any]:
+        return {
+            "directory": self._directory,
+            "prefix": self._prefix,
+        }
+
+    def set_options_route(self, handler: Handler) -> None:
+        if "OPTIONS" in self._routes:
+            raise RuntimeError("OPTIONS route was set already")
+        self._routes["OPTIONS"] = ResourceRoute("OPTIONS", handler, self)
+
+    async def resolve(self, request: Request) -> Tuple[Optional[AbstractMatchInfo], Set[str]]:
+        path = request.rel_url.raw_path
+        method = request.method
+        allowed_methods = set(self._routes)
+        if not path.startswith(self._prefix):
+            return None, set()
+
+        if method not in allowed_methods:
+            return None, allowed_methods
+
+        return UrlMappingMatchInfo({
+            "filename": URL.build(path=path[len(self._prefix):], encoded=True).path
+        }, self._routes[method]), allowed_methods
+
+    def __len__(self) -> int:
+        return len(self._routes)
+
+    def __iter__(self) -> Iterator[AbstractRoute]:
+        return iter(self._routes.values())
+
+    async def _handle(self, request: Request) -> StreamResponse:
+        try:
+            filename = Path(request.match_info["filename"])
+            if not filename.anchor:
+                filepath = (self._directory / filename).resolve()
+                if filepath.is_file():
+                    return FileResponse(filepath, chunk_size=self._chunk_size)
+                index_path = (self._directory / filename / "index.html").resolve()
+                if index_path.is_file():
+                    return FileResponse(index_path, chunk_size=self._chunk_size)
+        except (ValueError, FileNotFoundError) as error:
+            raise HTTPNotFound() from error
+        except HTTPForbidden:
+            raise
+        except Exception as error:
+            request.app.logger.exception("Error while trying to serve static file")
+            raise HTTPNotFound() from error
+
+    def __repr__(self) -> str:
+        name = f"'{self.name}'" if self.name is not None else ""
+        return f"<StaticResource {name} {self._prefix} -> {self._directory!r}>"