mattermost fixes, better config file, start on gitea integration

.gitignore

@@ -1,4 +1,5 @@
 data/
+data_backup/
 drive-temp/
 db.json
 .env
@@ -6,4 +7,4 @@ db.json
 venv/
 __pycache__/
 .idea
-users.toml
+config.toml

.tool-versions

@@ -1 +1 @@
-python 3.10.9
+python 3.12.1

config.example.toml

@@ -0,0 +1,35 @@
+[mattermost]
+
+server_url = "https://mattermost.zeus.gent/zeus"
+scrape_channel_ids = [
+    "hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN
+    "uda7ax9poprduq8ob56e1fqk4e"  # bestuur
+]
+selected_user = "my_username"
+
+[mattermost.users.my_username]
+
+name = "my_username"
+password = "..."
+
+
+[codimd]
+
+server_url = "https://codimd.zeus.gent"
+email = ""
+password = ""
+
+
+[gitlab]
+
+host = "https://git.zeus.gent"
+# access_token_name = "..."
+# access_token = "..."
+
+local_repo_folder = "drive"
+
+
+[gitea]
+
+# Generate at gitea.example.com/user/settings/applications
+access_token = "..."

requirements.txt

@@ -1,6 +1,10 @@
 tabulate
+humanize
 colored
+
 mattermostdriver
 hug
+
 gitpython
-python-gitlab
+python-gitlab
+giteapy

src/config.py

@@ -0,0 +1,8 @@
+import tomllib
+from pprint import pprint
+
+with open("config.toml", mode="rb") as config_toml:
+    config = tomllib.load(config_toml)
+
+
+# pprint(config)

src/dir_utils.py

@@ -12,24 +12,27 @@ def find_metadata(filename):
 
         metadata = {}
 
-        start_str = ":::spoiler Gitlab sync\n"
+        start_strings = [ ":::spoiler Gitlab sync\n" , ":::spoiler git drive sync\n" ]
         end_str = "\n:::"
 
-        start_i = data.find(start_str)
+        for start_str in start_strings:
-        if start_i >= 0:
+            start_i = data.find(start_str)
-            start_i += len(start_str)
+            if start_i >= 0:
-            end_i = data.find(end_str, start_i + 1)
+                start_i += len(start_str)
-            file_data = data[start_i:end_i]
+                end_i = data.find(end_str, start_i + 1)
-            for line in file_data.split("\n"):
+                file_data = data[start_i:end_i]
-                key_index = 2
+                for line in file_data.split("\n"):
-                value_index = line.find(": ")
+                    key_index = 2
-                key = line[key_index:value_index]
+                    value_index = line.find(": ")
-                value = line[value_index + 2 :]
+                    key = line[key_index:value_index]
-                metadata[key] = value
+                    value = line[value_index + 2 :]
-            print("Valid report")
+                    metadata[key] = value
-            print(metadata)
+                print("Valid report")
-        else:
+                print(metadata)
-            print("Not a valid report")
-            return None
 
-        return metadata
+                return metadata
+            
+        print("Not a valid report")
+        return None
+
+            

src/mattermost_client.py

@@ -7,8 +7,8 @@ from enum import Enum
 from time import sleep
 from typing import Dict, List
 
-import toml
+from config import config
-from colored import style
+from colored import Style
 from mattermostdriver import Driver
 from tabulate import tabulate
 
@@ -56,20 +56,17 @@ users: {str: [User]} = {}
 
 
 def loadusers():
-    with open("users.toml") as f:
-        usersstring = f.read()
-        usersdict = toml.loads(usersstring)
 
-        usr = None
+    usr = None
-        for name, data in usersdict.items():
+    for name, data in config["mattermost"]["users"].items():
-            if "token" in data:
+        if "token" in data:
-                usr = TokenUser(token=data["token"])
+            usr = TokenUser(token=data["token"])
-            elif "name" in data and "password" in data:
+        elif "name" in data and "password" in data:
-                usr = NormalUser(login_id=data["name"], password=data["password"])
+            usr = NormalUser(login_id=data["name"], password=data["password"])
-            else:
+        else:
-                print("Invalid user '{}' in toml file".format(name))
+            print("Invalid user '{}' in toml file".format(name))
-                exit(1)
+            exit(1)
-            users[name] = usr
+        users[name] = usr
 
 
 loadusers()
@@ -103,11 +100,11 @@ class MMApi(Driver):
     @staticmethod
     def print_response(resp, title="Response"):
         print("--------")
-        print(style.BOLD + title + style.RESET)
+        print(Style.BOLD + title + Style.RESET)
         pp.pprint(resp)
 
     def log(self, text: str, log_level: LogLevel = LogLevel.INFO):
-        print(f"{style.BOLD}[{log_level.value}]{style.RESET} {text}")
+        print(f"{Style.BOLD}[{log_level.value}]{Style.RESET} {text}")
 
     def get_channel_id(self, channel_name):
         resp = self.channels.get_channel_by_name(self.team_id, channel_name)

src/mattermost_communication.py

@@ -3,6 +3,8 @@ import mattermostdriver.exceptions
 import mattermost_client
 from mattermost_client import ChannelApi, MMApi
 
+from config import config
+
 
 def send_message(file_id, file_info, message):
     channel_id = file_info["originating_mm_post_channel_id"]
@@ -27,14 +29,14 @@ def send_message(file_id, file_info, message):
         )
     except mattermostdriver.exceptions.InvalidOrMissingParameters as e:
         # This will occur when we try to react to a file in a channel that is not the same as the originating channel.
-        unique_post_url = f"https://mattermost.zeus.gent/zeus/pl/{post_id}"
+        unique_post_url = f"{config['mattermost']['server_url']}/pl/{post_id}"
         channel.create_post(
             f"{unique_post_url}\n\n{message}",
         )
 
 
 def report_newly_found_file(file_id, file_info):
-    message = f"I found a new CodiMD file in this post! Making work of putting it on gitlab :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}"
+    message = f"I found a new CodiMD file in this post! Making work of putting it on git :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}"
     send_message(file_id, file_info, message)
 
 
@@ -47,7 +49,7 @@ You can easily add the correct info and I will do the rest of the work for you!
 Just add the following lines to your file, the location in your file is not important but at the top would be my recommendation.
 
 ```
-:::spoiler Gitlab sync
+:::spoiler git drive sync
 - sync-to: <a valid path on the DRIVE, for ex.: verslagen/21-22/2022-05-13.md>
 :::
 ```"""

src/mattermost_objects.py

@@ -64,6 +64,7 @@ class MMPostProps(NamedTuple):
     message: str = None
     attachments: str = None
     from_bot: str = False
+    disable_group_highlight: str = None
 
 
 class MMPost(NamedTuple):
@@ -77,16 +78,19 @@ class MMPost(NamedTuple):
     message: str
     metadata: Dict
     original_id: str
-    parent_id: str
     pending_post_id: str
     root_id: str
     type: str
     update_at: int
     user_id: str
+    parent_id: str = None
     message_source: str = None
     has_reactions: bool = None
     file_ids: List[str] = None
     props: MMPostProps = None
+    reply_count: int = None
+    last_reply_at: str = None
+    participants: any = None
 
     def from_human(self):
         return self.props is None or (
@@ -115,8 +119,10 @@ class MMChannelPosts(NamedTuple):
     next_post_id: str
     order: List[str]
     posts: Dict[str, MMPost]
-    disable_group_highlight: any
+    has_next: any
-    reply_count: any
+    first_inaccessible_post_time: any
+    reply_count: any = None
+    disable_group_highlight: any = None
 
     @staticmethod
     def load(data):

src/sync_gitea.py

@@ -0,0 +1,34 @@
+from __future__ import print_function
+import time
+import giteapy
+from giteapy.rest import ApiException
+from pprint import pprint
+
+from config import config
+
+def init_sync():
+    pass
+
+
+if __name__ == "__main__":
+    configuration = giteapy.Configuration()
+    configuration.host = "https://git.zeus.gent/api/v1"
+    configuration.api_key['token'] = config["gitea"]["access_token"]
+    configuration.debug = True
+
+    # create an instance of the API class
+    api_instance = giteapy.RepositoryApi(giteapy.ApiClient(configuration))
+    # username = 'username_example' # str | username of the user that will own the created organization
+    # organization = giteapy.CreateOrgOption() # CreateOrgOption |
+
+    pull_request = giteapy.CreatePullRequestOption(
+        body="Test 123"
+    )
+
+    try:
+        # Create an organization
+        # api_response = api_instance.admin_create_org(username, organization)
+        api_response = api_instance.repo_create_pull_request("ZeusWPI", "drive", async_req=False, body=pull_request)
+        pprint(api_response)
+    except ApiException as e:
+        print("Exception when calling RepositoryApi->repo_create_pull_request: %s\n" % e)

src/sync_gitlab.py

@@ -7,9 +7,12 @@ import pathlib
 import git
 import gitlab
 
+from config import config
+
 TOKEN_NAME = os.environ["GITLAB_ACCESS_TOKEN_NAME"]
 TOKEN = os.environ["GITLAB_ACCESS_TOKEN"]
-REPO_FOLDER = "drive"
+
+REPO_FOLDER = config["gitlab"]["local_repo_folder"]
 
 
 def get_repo():

src/web.py

@@ -11,17 +11,15 @@ import db
 import dir_utils
 import mattermost_client
 import mattermost_communication
-import sync
+import sync_gitea as sync
 import gitlab
 from utils import id_to_url, url_to_id
+from config import config
 
 
 def find_codimd_files_on_mattermost():
     mattermost = mattermost_client.MMApi()
-    channels = [
+    channels = config["mattermost"]["scrape_channel_ids"]
-        "hrx6pgfswjbttcj8nim3jrwe7w",  # bestuur-INTERN
-        "uda7ax9poprduq8ob56e1fqk4e",  # bestuur
-    ]
 
     last_fetch_time = db.get_latest_sync_time()
     current_fetch_time = int(time.time() * 1000)
@@ -71,7 +69,7 @@ def download_files():
             db.set_local_file_path(file_id, local_file_path)
 
 
-def validate_downloaded_files():
+def validate_downloaded_files(post_mattermost_hint=True):
     path = "data"
     dir_list = os.listdir(path)
 
@@ -84,7 +82,7 @@ def validate_downloaded_files():
                 mattermost_communication.report_newly_found_file(file_id, new_file_info)
         else:
             changed, new_file_info = db.mark_file_invalid(file_id)
-            if changed:
+            if changed and post_mattermost_hint:
                 mattermost_communication.report_newly_found_but_invalid_file(
                     file_id, new_file_info
                 )
@@ -115,11 +113,11 @@ def sync_mattermost():
     print()
     print("================================================")
     print("== Finding valid files in the downloaded ones ==")
-    validate_downloaded_files()
+    validate_downloaded_files(post_mattermost_hint=True)
     print()
     print("================================================")
     print("== Syncing files to gitlab ==")
-    sync_files_to_gitlab()
+    # sync_files_to_gitlab()
     print()
 
-    return db._load_db()
+    # return db._load_db()