diff --git a/.gitignore b/.gitignore index 5e861b6..706c774 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ data/ +data_backup/ drive-temp/ db.json .env @@ -6,4 +7,4 @@ db.json venv/ __pycache__/ .idea -users.toml +config.toml \ No newline at end of file diff --git a/.tool-versions b/.tool-versions index 47179ff..53bf831 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -python 3.10.9 +python 3.12.1 \ No newline at end of file diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 0000000..fdce466 --- /dev/null +++ b/config.example.toml @@ -0,0 +1,35 @@ +[mattermost] + +server_url = "https://mattermost.zeus.gent/zeus" +scrape_channel_ids = [ + "hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN + "uda7ax9poprduq8ob56e1fqk4e" # bestuur +] +selected_user = "my_username" + +[mattermost.users.my_username] + +name = "my_username" +password = "..." + + +[codimd] + +server_url = "https://codimd.zeus.gent" +email = "" +password = "" + + +[gitlab] + +host = "https://git.zeus.gent" +# access_token_name = "..." +# access_token = "..." + +local_repo_folder = "drive" + + +[gitea] + +# Generate at gitea.example.com/user/settings/applications +access_token = "..." \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 69b6604..b7fba76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,10 @@ tabulate +humanize colored + mattermostdriver hug + gitpython -python-gitlab \ No newline at end of file +python-gitlab +giteapy \ No newline at end of file diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..1a1cdde --- /dev/null +++ b/src/config.py @@ -0,0 +1,8 @@ +import tomllib +from pprint import pprint + +with open("config.toml", mode="rb") as config_toml: + config = tomllib.load(config_toml) + + +# pprint(config) \ No newline at end of file diff --git a/src/dir_utils.py b/src/dir_utils.py index e6ab2f5..a0ef4f0 100644 --- a/src/dir_utils.py +++ b/src/dir_utils.py @@ -12,24 +12,27 @@ def find_metadata(filename): metadata = {} - start_str = ":::spoiler Gitlab sync\n" + start_strings = [ ":::spoiler Gitlab sync\n" , ":::spoiler git drive sync\n" ] end_str = "\n:::" - start_i = data.find(start_str) - if start_i >= 0: - start_i += len(start_str) - end_i = data.find(end_str, start_i + 1) - file_data = data[start_i:end_i] - for line in file_data.split("\n"): - key_index = 2 - value_index = line.find(": ") - key = line[key_index:value_index] - value = line[value_index + 2 :] - metadata[key] = value - print("Valid report") - print(metadata) - else: - print("Not a valid report") - return None + for start_str in start_strings: + start_i = data.find(start_str) + if start_i >= 0: + start_i += len(start_str) + end_i = data.find(end_str, start_i + 1) + file_data = data[start_i:end_i] + for line in file_data.split("\n"): + key_index = 2 + value_index = line.find(": ") + key = line[key_index:value_index] + value = line[value_index + 2 :] + metadata[key] = value + print("Valid report") + print(metadata) - return metadata + return metadata + + print("Not a valid report") + return None + + diff --git a/src/mattermost_client.py b/src/mattermost_client.py index fda10c9..7b624f1 100644 --- a/src/mattermost_client.py +++ b/src/mattermost_client.py @@ -7,8 +7,8 @@ from enum import Enum from time import sleep from typing import Dict, List -import toml -from colored import style +from config import config +from colored import Style from mattermostdriver import Driver from tabulate import tabulate @@ -56,20 +56,17 @@ users: {str: [User]} = {} def loadusers(): - with open("users.toml") as f: - usersstring = f.read() - usersdict = toml.loads(usersstring) - usr = None - for name, data in usersdict.items(): - if "token" in data: - usr = TokenUser(token=data["token"]) - elif "name" in data and "password" in data: - usr = NormalUser(login_id=data["name"], password=data["password"]) - else: - print("Invalid user '{}' in toml file".format(name)) - exit(1) - users[name] = usr + usr = None + for name, data in config["mattermost"]["users"].items(): + if "token" in data: + usr = TokenUser(token=data["token"]) + elif "name" in data and "password" in data: + usr = NormalUser(login_id=data["name"], password=data["password"]) + else: + print("Invalid user '{}' in toml file".format(name)) + exit(1) + users[name] = usr loadusers() @@ -103,11 +100,11 @@ class MMApi(Driver): @staticmethod def print_response(resp, title="Response"): print("--------") - print(style.BOLD + title + style.RESET) + print(Style.BOLD + title + Style.RESET) pp.pprint(resp) def log(self, text: str, log_level: LogLevel = LogLevel.INFO): - print(f"{style.BOLD}[{log_level.value}]{style.RESET} {text}") + print(f"{Style.BOLD}[{log_level.value}]{Style.RESET} {text}") def get_channel_id(self, channel_name): resp = self.channels.get_channel_by_name(self.team_id, channel_name) diff --git a/src/mattermost_communication.py b/src/mattermost_communication.py index 0ff8d99..b6062b1 100644 --- a/src/mattermost_communication.py +++ b/src/mattermost_communication.py @@ -3,6 +3,8 @@ import mattermostdriver.exceptions import mattermost_client from mattermost_client import ChannelApi, MMApi +from config import config + def send_message(file_id, file_info, message): channel_id = file_info["originating_mm_post_channel_id"] @@ -27,14 +29,14 @@ def send_message(file_id, file_info, message): ) except mattermostdriver.exceptions.InvalidOrMissingParameters as e: # This will occur when we try to react to a file in a channel that is not the same as the originating channel. - unique_post_url = f"https://mattermost.zeus.gent/zeus/pl/{post_id}" + unique_post_url = f"{config['mattermost']['server_url']}/pl/{post_id}" channel.create_post( f"{unique_post_url}\n\n{message}", ) def report_newly_found_file(file_id, file_info): - message = f"I found a new CodiMD file in this post! Making work of putting it on gitlab :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}" + message = f"I found a new CodiMD file in this post! Making work of putting it on git :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}" send_message(file_id, file_info, message) @@ -47,7 +49,7 @@ You can easily add the correct info and I will do the rest of the work for you! Just add the following lines to your file, the location in your file is not important but at the top would be my recommendation. ``` -:::spoiler Gitlab sync +:::spoiler git drive sync - sync-to: ::: ```""" diff --git a/src/mattermost_objects.py b/src/mattermost_objects.py index 8492817..588d1ff 100644 --- a/src/mattermost_objects.py +++ b/src/mattermost_objects.py @@ -64,6 +64,7 @@ class MMPostProps(NamedTuple): message: str = None attachments: str = None from_bot: str = False + disable_group_highlight: str = None class MMPost(NamedTuple): @@ -77,16 +78,19 @@ class MMPost(NamedTuple): message: str metadata: Dict original_id: str - parent_id: str pending_post_id: str root_id: str type: str update_at: int user_id: str + parent_id: str = None message_source: str = None has_reactions: bool = None file_ids: List[str] = None props: MMPostProps = None + reply_count: int = None + last_reply_at: str = None + participants: any = None def from_human(self): return self.props is None or ( @@ -115,8 +119,10 @@ class MMChannelPosts(NamedTuple): next_post_id: str order: List[str] posts: Dict[str, MMPost] - disable_group_highlight: any - reply_count: any + has_next: any + first_inaccessible_post_time: any + reply_count: any = None + disable_group_highlight: any = None @staticmethod def load(data): diff --git a/src/sync_gitea.py b/src/sync_gitea.py new file mode 100644 index 0000000..6f5524d --- /dev/null +++ b/src/sync_gitea.py @@ -0,0 +1,34 @@ +from __future__ import print_function +import time +import giteapy +from giteapy.rest import ApiException +from pprint import pprint + +from config import config + +def init_sync(): + pass + + +if __name__ == "__main__": + configuration = giteapy.Configuration() + configuration.host = "https://git.zeus.gent/api/v1" + configuration.api_key['token'] = config["gitea"]["access_token"] + configuration.debug = True + + # create an instance of the API class + api_instance = giteapy.RepositoryApi(giteapy.ApiClient(configuration)) + # username = 'username_example' # str | username of the user that will own the created organization + # organization = giteapy.CreateOrgOption() # CreateOrgOption | + + pull_request = giteapy.CreatePullRequestOption( + body="Test 123" + ) + + try: + # Create an organization + # api_response = api_instance.admin_create_org(username, organization) + api_response = api_instance.repo_create_pull_request("ZeusWPI", "drive", async_req=False, body=pull_request) + pprint(api_response) + except ApiException as e: + print("Exception when calling RepositoryApi->repo_create_pull_request: %s\n" % e) \ No newline at end of file diff --git a/src/sync.py b/src/sync_gitlab.py similarity index 97% rename from src/sync.py rename to src/sync_gitlab.py index 9ff38b7..734cfdb 100755 --- a/src/sync.py +++ b/src/sync_gitlab.py @@ -7,9 +7,12 @@ import pathlib import git import gitlab +from config import config + TOKEN_NAME = os.environ["GITLAB_ACCESS_TOKEN_NAME"] TOKEN = os.environ["GITLAB_ACCESS_TOKEN"] -REPO_FOLDER = "drive" + +REPO_FOLDER = config["gitlab"]["local_repo_folder"] def get_repo(): diff --git a/src/web.py b/src/web.py index bd4e83d..e9bc9f1 100644 --- a/src/web.py +++ b/src/web.py @@ -11,17 +11,15 @@ import db import dir_utils import mattermost_client import mattermost_communication -import sync +import sync_gitea as sync import gitlab from utils import id_to_url, url_to_id +from config import config def find_codimd_files_on_mattermost(): mattermost = mattermost_client.MMApi() - channels = [ - "hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN - "uda7ax9poprduq8ob56e1fqk4e", # bestuur - ] + channels = config["mattermost"]["scrape_channel_ids"] last_fetch_time = db.get_latest_sync_time() current_fetch_time = int(time.time() * 1000) @@ -71,7 +69,7 @@ def download_files(): db.set_local_file_path(file_id, local_file_path) -def validate_downloaded_files(): +def validate_downloaded_files(post_mattermost_hint=True): path = "data" dir_list = os.listdir(path) @@ -84,7 +82,7 @@ def validate_downloaded_files(): mattermost_communication.report_newly_found_file(file_id, new_file_info) else: changed, new_file_info = db.mark_file_invalid(file_id) - if changed: + if changed and post_mattermost_hint: mattermost_communication.report_newly_found_but_invalid_file( file_id, new_file_info ) @@ -115,11 +113,11 @@ def sync_mattermost(): print() print("================================================") print("== Finding valid files in the downloaded ones ==") - validate_downloaded_files() + validate_downloaded_files(post_mattermost_hint=True) print() print("================================================") print("== Syncing files to gitlab ==") - sync_files_to_gitlab() + # sync_files_to_gitlab() print() - return db._load_db() + # return db._load_db()