mattermost fixes, better config file, start on gitea integration

This commit is contained in:
mcbloch 2024-03-27 00:00:54 +01:00
parent 7cef7bce11
commit 9da2481df9
12 changed files with 146 additions and 55 deletions

3
.gitignore vendored
View file

@ -1,4 +1,5 @@
data/
data_backup/
drive-temp/
db.json
.env
@ -6,4 +7,4 @@ db.json
venv/
__pycache__/
.idea
users.toml
config.toml

View file

@ -1 +1 @@
python 3.10.9
python 3.12.1

35
config.example.toml Normal file
View file

@ -0,0 +1,35 @@
[mattermost]
server_url = "https://mattermost.zeus.gent/zeus"
scrape_channel_ids = [
"hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN
"uda7ax9poprduq8ob56e1fqk4e" # bestuur
]
selected_user = "my_username"
[mattermost.users.my_username]
name = "my_username"
password = "..."
[codimd]
server_url = "https://codimd.zeus.gent"
email = ""
password = ""
[gitlab]
host = "https://git.zeus.gent"
# access_token_name = "..."
# access_token = "..."
local_repo_folder = "drive"
[gitea]
# Generate at gitea.example.com/user/settings/applications
access_token = "..."

View file

@ -1,6 +1,10 @@
tabulate
humanize
colored
mattermostdriver
hug
gitpython
python-gitlab
python-gitlab
giteapy

8
src/config.py Normal file
View file

@ -0,0 +1,8 @@
import tomllib
from pprint import pprint
with open("config.toml", mode="rb") as config_toml:
config = tomllib.load(config_toml)
# pprint(config)

View file

@ -12,24 +12,27 @@ def find_metadata(filename):
metadata = {}
start_str = ":::spoiler Gitlab sync\n"
start_strings = [ ":::spoiler Gitlab sync\n" , ":::spoiler git drive sync\n" ]
end_str = "\n:::"
start_i = data.find(start_str)
if start_i >= 0:
start_i += len(start_str)
end_i = data.find(end_str, start_i + 1)
file_data = data[start_i:end_i]
for line in file_data.split("\n"):
key_index = 2
value_index = line.find(": ")
key = line[key_index:value_index]
value = line[value_index + 2 :]
metadata[key] = value
print("Valid report")
print(metadata)
else:
print("Not a valid report")
return None
for start_str in start_strings:
start_i = data.find(start_str)
if start_i >= 0:
start_i += len(start_str)
end_i = data.find(end_str, start_i + 1)
file_data = data[start_i:end_i]
for line in file_data.split("\n"):
key_index = 2
value_index = line.find(": ")
key = line[key_index:value_index]
value = line[value_index + 2 :]
metadata[key] = value
print("Valid report")
print(metadata)
return metadata
return metadata
print("Not a valid report")
return None

View file

@ -7,8 +7,8 @@ from enum import Enum
from time import sleep
from typing import Dict, List
import toml
from colored import style
from config import config
from colored import Style
from mattermostdriver import Driver
from tabulate import tabulate
@ -56,20 +56,17 @@ users: {str: [User]} = {}
def loadusers():
with open("users.toml") as f:
usersstring = f.read()
usersdict = toml.loads(usersstring)
usr = None
for name, data in usersdict.items():
if "token" in data:
usr = TokenUser(token=data["token"])
elif "name" in data and "password" in data:
usr = NormalUser(login_id=data["name"], password=data["password"])
else:
print("Invalid user '{}' in toml file".format(name))
exit(1)
users[name] = usr
usr = None
for name, data in config["mattermost"]["users"].items():
if "token" in data:
usr = TokenUser(token=data["token"])
elif "name" in data and "password" in data:
usr = NormalUser(login_id=data["name"], password=data["password"])
else:
print("Invalid user '{}' in toml file".format(name))
exit(1)
users[name] = usr
loadusers()
@ -103,11 +100,11 @@ class MMApi(Driver):
@staticmethod
def print_response(resp, title="Response"):
print("--------")
print(style.BOLD + title + style.RESET)
print(Style.BOLD + title + Style.RESET)
pp.pprint(resp)
def log(self, text: str, log_level: LogLevel = LogLevel.INFO):
print(f"{style.BOLD}[{log_level.value}]{style.RESET} {text}")
print(f"{Style.BOLD}[{log_level.value}]{Style.RESET} {text}")
def get_channel_id(self, channel_name):
resp = self.channels.get_channel_by_name(self.team_id, channel_name)

View file

@ -3,6 +3,8 @@ import mattermostdriver.exceptions
import mattermost_client
from mattermost_client import ChannelApi, MMApi
from config import config
def send_message(file_id, file_info, message):
channel_id = file_info["originating_mm_post_channel_id"]
@ -27,14 +29,14 @@ def send_message(file_id, file_info, message):
)
except mattermostdriver.exceptions.InvalidOrMissingParameters as e:
# This will occur when we try to react to a file in a channel that is not the same as the originating channel.
unique_post_url = f"https://mattermost.zeus.gent/zeus/pl/{post_id}"
unique_post_url = f"{config['mattermost']['server_url']}/pl/{post_id}"
channel.create_post(
f"{unique_post_url}\n\n{message}",
)
def report_newly_found_file(file_id, file_info):
message = f"I found a new CodiMD file in this post! Making work of putting it on gitlab :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}"
message = f"I found a new CodiMD file in this post! Making work of putting it on git :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}"
send_message(file_id, file_info, message)
@ -47,7 +49,7 @@ You can easily add the correct info and I will do the rest of the work for you!
Just add the following lines to your file, the location in your file is not important but at the top would be my recommendation.
```
:::spoiler Gitlab sync
:::spoiler git drive sync
- sync-to: <a valid path on the DRIVE, for ex.: verslagen/21-22/2022-05-13.md>
:::
```"""

View file

@ -64,6 +64,7 @@ class MMPostProps(NamedTuple):
message: str = None
attachments: str = None
from_bot: str = False
disable_group_highlight: str = None
class MMPost(NamedTuple):
@ -77,16 +78,19 @@ class MMPost(NamedTuple):
message: str
metadata: Dict
original_id: str
parent_id: str
pending_post_id: str
root_id: str
type: str
update_at: int
user_id: str
parent_id: str = None
message_source: str = None
has_reactions: bool = None
file_ids: List[str] = None
props: MMPostProps = None
reply_count: int = None
last_reply_at: str = None
participants: any = None
def from_human(self):
return self.props is None or (
@ -115,8 +119,10 @@ class MMChannelPosts(NamedTuple):
next_post_id: str
order: List[str]
posts: Dict[str, MMPost]
disable_group_highlight: any
reply_count: any
has_next: any
first_inaccessible_post_time: any
reply_count: any = None
disable_group_highlight: any = None
@staticmethod
def load(data):

34
src/sync_gitea.py Normal file
View file

@ -0,0 +1,34 @@
from __future__ import print_function
import time
import giteapy
from giteapy.rest import ApiException
from pprint import pprint
from config import config
def init_sync():
pass
if __name__ == "__main__":
configuration = giteapy.Configuration()
configuration.host = "https://git.zeus.gent/api/v1"
configuration.api_key['token'] = config["gitea"]["access_token"]
configuration.debug = True
# create an instance of the API class
api_instance = giteapy.RepositoryApi(giteapy.ApiClient(configuration))
# username = 'username_example' # str | username of the user that will own the created organization
# organization = giteapy.CreateOrgOption() # CreateOrgOption |
pull_request = giteapy.CreatePullRequestOption(
body="Test 123"
)
try:
# Create an organization
# api_response = api_instance.admin_create_org(username, organization)
api_response = api_instance.repo_create_pull_request("ZeusWPI", "drive", async_req=False, body=pull_request)
pprint(api_response)
except ApiException as e:
print("Exception when calling RepositoryApi->repo_create_pull_request: %s\n" % e)

View file

@ -7,9 +7,12 @@ import pathlib
import git
import gitlab
from config import config
TOKEN_NAME = os.environ["GITLAB_ACCESS_TOKEN_NAME"]
TOKEN = os.environ["GITLAB_ACCESS_TOKEN"]
REPO_FOLDER = "drive"
REPO_FOLDER = config["gitlab"]["local_repo_folder"]
def get_repo():

View file

@ -11,17 +11,15 @@ import db
import dir_utils
import mattermost_client
import mattermost_communication
import sync
import sync_gitea as sync
import gitlab
from utils import id_to_url, url_to_id
from config import config
def find_codimd_files_on_mattermost():
mattermost = mattermost_client.MMApi()
channels = [
"hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN
"uda7ax9poprduq8ob56e1fqk4e", # bestuur
]
channels = config["mattermost"]["scrape_channel_ids"]
last_fetch_time = db.get_latest_sync_time()
current_fetch_time = int(time.time() * 1000)
@ -71,7 +69,7 @@ def download_files():
db.set_local_file_path(file_id, local_file_path)
def validate_downloaded_files():
def validate_downloaded_files(post_mattermost_hint=True):
path = "data"
dir_list = os.listdir(path)
@ -84,7 +82,7 @@ def validate_downloaded_files():
mattermost_communication.report_newly_found_file(file_id, new_file_info)
else:
changed, new_file_info = db.mark_file_invalid(file_id)
if changed:
if changed and post_mattermost_hint:
mattermost_communication.report_newly_found_but_invalid_file(
file_id, new_file_info
)
@ -115,11 +113,11 @@ def sync_mattermost():
print()
print("================================================")
print("== Finding valid files in the downloaded ones ==")
validate_downloaded_files()
validate_downloaded_files(post_mattermost_hint=True)
print()
print("================================================")
print("== Syncing files to gitlab ==")
sync_files_to_gitlab()
# sync_files_to_gitlab()
print()
return db._load_db()
# return db._load_db()