From 60d4605ac13d752236d7d04b3e6291ac8a1358fa Mon Sep 17 00:00:00 2001 From: Midgard Date: Sat, 23 Nov 2024 18:15:06 +0100 Subject: [PATCH] Get emoji statically --- .gitignore | 3 +- emoji.py | 2 +- emoji_proxy/emoji_proxy.py | 95 ------------------- .../mattermost_credentials.json.sample | 3 - gen.sh | 3 +- get_custom_emoji.py | 69 ++++++++++++++ 6 files changed, 73 insertions(+), 102 deletions(-) delete mode 100755 emoji_proxy/emoji_proxy.py delete mode 100644 emoji_proxy/mattermost_credentials.json.sample create mode 100755 get_custom_emoji.py diff --git a/.gitignore b/.gitignore index 14d2b11..849ddff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ -mattermost_channels.html -*_credentials.json +dist/ diff --git a/emoji.py b/emoji.py index 5d0d976..066b300 100755 --- a/emoji.py +++ b/emoji.py @@ -22,7 +22,7 @@ def translate_emoji(match_obj): ) for emoji in custom_emoji: if emoji["name"] == emoji_name: - return f":{emoji_name}:" + return f":{emoji_name}:" return f":{emoji_name}:" # From https://mattermost.example.org//emoji/add: diff --git a/emoji_proxy/emoji_proxy.py b/emoji_proxy/emoji_proxy.py deleted file mode 100755 index 1326499..0000000 --- a/emoji_proxy/emoji_proxy.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -import json -import urllib.request -import urllib.error -from wsgiref.simple_server import make_server, demo_app - -software_name = "emoji_proxy" - -os.chdir(os.path.dirname(__file__)) - -with open("./mattermost_credentials.json") as fh: - credentials = json.load(fh) - - -HTTP_OK = "200 OK" -HTTP_NOT_FOUND = "404 Not Found" -HTTP_INTERNAL_SERVER_ERROR = "500 Internal Server Error" - - -class HTTPError(Exception): - def __init__(self, status, message): - self.status = status - self.message = message - - def __repr__(self): - return f"" - - -default_headers = [ - ("Server", software_name), -] - - -def app(environ, start_response): - path = environ["PATH_INFO"] - - m = re.fullmatch(r'/emoji_proxy/(.+)/([a-z0-9]{26})', path) - if not m: - raise HTTPError(HTTP_NOT_FOUND, "Path not recognized, expecting /emoji_proxy//\n may include slashes to specify a path, though this is not typical.") - - server = m.group(1) - emoji_id = m.group(2) - - if server not in credentials: - raise HTTPError(HTTP_NOT_FOUND, f"Server {server} not supported") - access_token = credentials[server]["access_token"] - - req = urllib.request.Request(f"https://{server}/api/v4/emoji/{emoji_id}/image") - req.add_header("Referer", "https://{server}/") - req.add_header("User-Agent", software_name) - req.add_header("Authorization", "Bearer " + access_token) - try: - with urllib.request.urlopen(req) as upstream_response: - start_response(HTTP_OK, default_headers + [ - ("Cache-Control", "max-age=86400, public"), - ] + [ - (header_name, upstream_response.headers[header_name]) - for header_name in upstream_response.headers - if header_name not in {"Server", "Connection", "Accept-Ranges", "Transfer-Encoding"} and not header_name.startswith("X-") - ]) - - chunk = upstream_response.read1() - while chunk != b"": - yield chunk - chunk = upstream_response.read1() - - except urllib.error.HTTPError as e: - if e.status == 404: - raise HTTPError(HTTP_NOT_FOUND, f"Emoji not found") - raise HTTPError(f"{e.status} {e.reason}", [b"The Mattermost server reported an error:\n\n", e.read()]) - except urllib.error.URLError as e: - raise e - - -def app_wrap(environ, start_response): - try: - yield from app(environ, start_response) - except HTTPError as e: - start_response(e.status, default_headers + [ - ("Content-Type", "text/plain; charset=utf-8"), - ]) - if isinstance(e.message, str): - yield from [e.message.encode("utf-8")] - else: - yield from e.message - - -with make_server("", 8000, app_wrap) as httpd: - print("Serving HTTP on port 8000...") - - # Respond to requests until process is killed - httpd.serve_forever() diff --git a/emoji_proxy/mattermost_credentials.json.sample b/emoji_proxy/mattermost_credentials.json.sample deleted file mode 100644 index 5dfbd89..0000000 --- a/emoji_proxy/mattermost_credentials.json.sample +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mattermost.example.org": {"access_token": "1ab2cd3ef4gh5ij6kl7mn8op9q"} -} diff --git a/gen.sh b/gen.sh index 78971cc..0492ebc 100755 --- a/gen.sh +++ b/gen.sh @@ -7,7 +7,8 @@ team_name="zeus" dir="$(dirname "$0")" -out=mattermost_channels.html +out="$dir/dist/channels/index.html" +mkdir -p "$(dirname "$out")" custom_emoji_file="$(mktemp --tmpdir custom_emoji.XXXXXXXXXX.json)" mmcli listcustomemoji | jq -s > "$custom_emoji_file" diff --git a/get_custom_emoji.py b/get_custom_emoji.py new file mode 100755 index 0000000..4cf8279 --- /dev/null +++ b/get_custom_emoji.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import sys +import os +import re +import json +import mimetypes +import urllib.request + + +destination_dir = "dist/emoji/images" + +os.chdir(os.path.dirname(__file__)) +os.makedirs(destination_dir, exist_ok=True) + +user_agent = "emoji crawler by the MI5 Delegation for Guarding of Access to Resource Development" + +MM_SERVER = os.environ["MM_SERVER"] +MM_ACCESSTOKEN = os.environ["MM_ACCESSTOKEN"] + + +def mm_api_get(path): + req = urllib.request.Request(f"https://{MM_SERVER}/api{path}") + req.add_header("Referer", f"https://{MM_SERVER}/") + req.add_header("User-Agent", user_agent) + req.add_header("Authorization", "Bearer " + MM_ACCESSTOKEN) + return urllib.request.urlopen(req) + +def mm_api_get_json(*args, **kwargs): + with mm_api_get(*args, **kwargs) as response: + return json.load(response) + + +def get_list_of_custom_emoji(): + page = 0 + per_page = 200 # Maximum allowed in API + response = [] + while page == 0 or response: + response = mm_api_get_json(f"/v4/emoji?page={page}&per_page={per_page}&sort=name") + for emoji in response: + yield(emoji) + page += 1 + + +def get_emoji_image(name, emoji_id): + print(f"Downloading {name}", file=sys.stderr, flush=True) + with mm_api_get(f"/v4/emoji/{emoji_id}/image") as upstream_response: + headers = {k.lower(): v for (k, v) in upstream_response.headers.items()} + mime = headers.get("content-type") + if not mime.startswith("image/"): + raise Exception(f"Got response with MIME type {mime}, not an image") + extension = mimetypes.guess_extension(mime) + + filepath = f"{destination_dir}/{name}{extension}" + with open(filepath, "wb") as fh: + chunk = upstream_response.read1() + while chunk != b"": + fh.write(chunk) + chunk = upstream_response.read1() + print(f"Saved to {filepath}", file=sys.stderr, flush=True) + + +def main(): + for emoji in get_list_of_custom_emoji(): + get_emoji_image(emoji["name"], emoji["id"]) + + +if __name__ == "__main__": + main()