mattermost fixes, better config file, start on gitea integration

This commit is contained in:
mcbloch 2024-03-27 00:00:54 +01:00
parent 7cef7bce11
commit 9da2481df9
12 changed files with 146 additions and 55 deletions

3
.gitignore vendored
View file

@ -1,4 +1,5 @@
data/ data/
data_backup/
drive-temp/ drive-temp/
db.json db.json
.env .env
@ -6,4 +7,4 @@ db.json
venv/ venv/
__pycache__/ __pycache__/
.idea .idea
users.toml config.toml

View file

@ -1 +1 @@
python 3.10.9 python 3.12.1

35
config.example.toml Normal file
View file

@ -0,0 +1,35 @@
[mattermost]
server_url = "https://mattermost.zeus.gent/zeus"
scrape_channel_ids = [
"hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN
"uda7ax9poprduq8ob56e1fqk4e" # bestuur
]
selected_user = "my_username"
[mattermost.users.my_username]
name = "my_username"
password = "..."
[codimd]
server_url = "https://codimd.zeus.gent"
email = ""
password = ""
[gitlab]
host = "https://git.zeus.gent"
# access_token_name = "..."
# access_token = "..."
local_repo_folder = "drive"
[gitea]
# Generate at gitea.example.com/user/settings/applications
access_token = "..."

View file

@ -1,6 +1,10 @@
tabulate tabulate
humanize
colored colored
mattermostdriver mattermostdriver
hug hug
gitpython gitpython
python-gitlab python-gitlab
giteapy

8
src/config.py Normal file
View file

@ -0,0 +1,8 @@
import tomllib
from pprint import pprint
with open("config.toml", mode="rb") as config_toml:
config = tomllib.load(config_toml)
# pprint(config)

View file

@ -12,9 +12,10 @@ def find_metadata(filename):
metadata = {} metadata = {}
start_str = ":::spoiler Gitlab sync\n" start_strings = [ ":::spoiler Gitlab sync\n" , ":::spoiler git drive sync\n" ]
end_str = "\n:::" end_str = "\n:::"
for start_str in start_strings:
start_i = data.find(start_str) start_i = data.find(start_str)
if start_i >= 0: if start_i >= 0:
start_i += len(start_str) start_i += len(start_str)
@ -28,8 +29,10 @@ def find_metadata(filename):
metadata[key] = value metadata[key] = value
print("Valid report") print("Valid report")
print(metadata) print(metadata)
else:
return metadata
print("Not a valid report") print("Not a valid report")
return None return None
return metadata

View file

@ -7,8 +7,8 @@ from enum import Enum
from time import sleep from time import sleep
from typing import Dict, List from typing import Dict, List
import toml from config import config
from colored import style from colored import Style
from mattermostdriver import Driver from mattermostdriver import Driver
from tabulate import tabulate from tabulate import tabulate
@ -56,12 +56,9 @@ users: {str: [User]} = {}
def loadusers(): def loadusers():
with open("users.toml") as f:
usersstring = f.read()
usersdict = toml.loads(usersstring)
usr = None usr = None
for name, data in usersdict.items(): for name, data in config["mattermost"]["users"].items():
if "token" in data: if "token" in data:
usr = TokenUser(token=data["token"]) usr = TokenUser(token=data["token"])
elif "name" in data and "password" in data: elif "name" in data and "password" in data:
@ -103,11 +100,11 @@ class MMApi(Driver):
@staticmethod @staticmethod
def print_response(resp, title="Response"): def print_response(resp, title="Response"):
print("--------") print("--------")
print(style.BOLD + title + style.RESET) print(Style.BOLD + title + Style.RESET)
pp.pprint(resp) pp.pprint(resp)
def log(self, text: str, log_level: LogLevel = LogLevel.INFO): def log(self, text: str, log_level: LogLevel = LogLevel.INFO):
print(f"{style.BOLD}[{log_level.value}]{style.RESET} {text}") print(f"{Style.BOLD}[{log_level.value}]{Style.RESET} {text}")
def get_channel_id(self, channel_name): def get_channel_id(self, channel_name):
resp = self.channels.get_channel_by_name(self.team_id, channel_name) resp = self.channels.get_channel_by_name(self.team_id, channel_name)

View file

@ -3,6 +3,8 @@ import mattermostdriver.exceptions
import mattermost_client import mattermost_client
from mattermost_client import ChannelApi, MMApi from mattermost_client import ChannelApi, MMApi
from config import config
def send_message(file_id, file_info, message): def send_message(file_id, file_info, message):
channel_id = file_info["originating_mm_post_channel_id"] channel_id = file_info["originating_mm_post_channel_id"]
@ -27,14 +29,14 @@ def send_message(file_id, file_info, message):
) )
except mattermostdriver.exceptions.InvalidOrMissingParameters as e: except mattermostdriver.exceptions.InvalidOrMissingParameters as e:
# This will occur when we try to react to a file in a channel that is not the same as the originating channel. # This will occur when we try to react to a file in a channel that is not the same as the originating channel.
unique_post_url = f"https://mattermost.zeus.gent/zeus/pl/{post_id}" unique_post_url = f"{config['mattermost']['server_url']}/pl/{post_id}"
channel.create_post( channel.create_post(
f"{unique_post_url}\n\n{message}", f"{unique_post_url}\n\n{message}",
) )
def report_newly_found_file(file_id, file_info): def report_newly_found_file(file_id, file_info):
message = f"I found a new CodiMD file in this post! Making work of putting it on gitlab :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}" message = f"I found a new CodiMD file in this post! Making work of putting it on git :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}"
send_message(file_id, file_info, message) send_message(file_id, file_info, message)
@ -47,7 +49,7 @@ You can easily add the correct info and I will do the rest of the work for you!
Just add the following lines to your file, the location in your file is not important but at the top would be my recommendation. Just add the following lines to your file, the location in your file is not important but at the top would be my recommendation.
``` ```
:::spoiler Gitlab sync :::spoiler git drive sync
- sync-to: <a valid path on the DRIVE, for ex.: verslagen/21-22/2022-05-13.md> - sync-to: <a valid path on the DRIVE, for ex.: verslagen/21-22/2022-05-13.md>
::: :::
```""" ```"""

View file

@ -64,6 +64,7 @@ class MMPostProps(NamedTuple):
message: str = None message: str = None
attachments: str = None attachments: str = None
from_bot: str = False from_bot: str = False
disable_group_highlight: str = None
class MMPost(NamedTuple): class MMPost(NamedTuple):
@ -77,16 +78,19 @@ class MMPost(NamedTuple):
message: str message: str
metadata: Dict metadata: Dict
original_id: str original_id: str
parent_id: str
pending_post_id: str pending_post_id: str
root_id: str root_id: str
type: str type: str
update_at: int update_at: int
user_id: str user_id: str
parent_id: str = None
message_source: str = None message_source: str = None
has_reactions: bool = None has_reactions: bool = None
file_ids: List[str] = None file_ids: List[str] = None
props: MMPostProps = None props: MMPostProps = None
reply_count: int = None
last_reply_at: str = None
participants: any = None
def from_human(self): def from_human(self):
return self.props is None or ( return self.props is None or (
@ -115,8 +119,10 @@ class MMChannelPosts(NamedTuple):
next_post_id: str next_post_id: str
order: List[str] order: List[str]
posts: Dict[str, MMPost] posts: Dict[str, MMPost]
disable_group_highlight: any has_next: any
reply_count: any first_inaccessible_post_time: any
reply_count: any = None
disable_group_highlight: any = None
@staticmethod @staticmethod
def load(data): def load(data):

34
src/sync_gitea.py Normal file
View file

@ -0,0 +1,34 @@
from __future__ import print_function
import time
import giteapy
from giteapy.rest import ApiException
from pprint import pprint
from config import config
def init_sync():
pass
if __name__ == "__main__":
configuration = giteapy.Configuration()
configuration.host = "https://git.zeus.gent/api/v1"
configuration.api_key['token'] = config["gitea"]["access_token"]
configuration.debug = True
# create an instance of the API class
api_instance = giteapy.RepositoryApi(giteapy.ApiClient(configuration))
# username = 'username_example' # str | username of the user that will own the created organization
# organization = giteapy.CreateOrgOption() # CreateOrgOption |
pull_request = giteapy.CreatePullRequestOption(
body="Test 123"
)
try:
# Create an organization
# api_response = api_instance.admin_create_org(username, organization)
api_response = api_instance.repo_create_pull_request("ZeusWPI", "drive", async_req=False, body=pull_request)
pprint(api_response)
except ApiException as e:
print("Exception when calling RepositoryApi->repo_create_pull_request: %s\n" % e)

View file

@ -7,9 +7,12 @@ import pathlib
import git import git
import gitlab import gitlab
from config import config
TOKEN_NAME = os.environ["GITLAB_ACCESS_TOKEN_NAME"] TOKEN_NAME = os.environ["GITLAB_ACCESS_TOKEN_NAME"]
TOKEN = os.environ["GITLAB_ACCESS_TOKEN"] TOKEN = os.environ["GITLAB_ACCESS_TOKEN"]
REPO_FOLDER = "drive"
REPO_FOLDER = config["gitlab"]["local_repo_folder"]
def get_repo(): def get_repo():

View file

@ -11,17 +11,15 @@ import db
import dir_utils import dir_utils
import mattermost_client import mattermost_client
import mattermost_communication import mattermost_communication
import sync import sync_gitea as sync
import gitlab import gitlab
from utils import id_to_url, url_to_id from utils import id_to_url, url_to_id
from config import config
def find_codimd_files_on_mattermost(): def find_codimd_files_on_mattermost():
mattermost = mattermost_client.MMApi() mattermost = mattermost_client.MMApi()
channels = [ channels = config["mattermost"]["scrape_channel_ids"]
"hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN
"uda7ax9poprduq8ob56e1fqk4e", # bestuur
]
last_fetch_time = db.get_latest_sync_time() last_fetch_time = db.get_latest_sync_time()
current_fetch_time = int(time.time() * 1000) current_fetch_time = int(time.time() * 1000)
@ -71,7 +69,7 @@ def download_files():
db.set_local_file_path(file_id, local_file_path) db.set_local_file_path(file_id, local_file_path)
def validate_downloaded_files(): def validate_downloaded_files(post_mattermost_hint=True):
path = "data" path = "data"
dir_list = os.listdir(path) dir_list = os.listdir(path)
@ -84,7 +82,7 @@ def validate_downloaded_files():
mattermost_communication.report_newly_found_file(file_id, new_file_info) mattermost_communication.report_newly_found_file(file_id, new_file_info)
else: else:
changed, new_file_info = db.mark_file_invalid(file_id) changed, new_file_info = db.mark_file_invalid(file_id)
if changed: if changed and post_mattermost_hint:
mattermost_communication.report_newly_found_but_invalid_file( mattermost_communication.report_newly_found_but_invalid_file(
file_id, new_file_info file_id, new_file_info
) )
@ -115,11 +113,11 @@ def sync_mattermost():
print() print()
print("================================================") print("================================================")
print("== Finding valid files in the downloaded ones ==") print("== Finding valid files in the downloaded ones ==")
validate_downloaded_files() validate_downloaded_files(post_mattermost_hint=True)
print() print()
print("================================================") print("================================================")
print("== Syncing files to gitlab ==") print("== Syncing files to gitlab ==")
sync_files_to_gitlab() # sync_files_to_gitlab()
print() print()
return db._load_db() # return db._load_db()