mattermost fixes, better config file, start on gitea integration
This commit is contained in:
parent
7cef7bce11
commit
9da2481df9
12 changed files with 146 additions and 55 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -1,4 +1,5 @@
|
|||
data/
|
||||
data_backup/
|
||||
drive-temp/
|
||||
db.json
|
||||
.env
|
||||
|
@ -6,4 +7,4 @@ db.json
|
|||
venv/
|
||||
__pycache__/
|
||||
.idea
|
||||
users.toml
|
||||
config.toml
|
|
@ -1 +1 @@
|
|||
python 3.10.9
|
||||
python 3.12.1
|
35
config.example.toml
Normal file
35
config.example.toml
Normal file
|
@ -0,0 +1,35 @@
|
|||
[mattermost]
|
||||
|
||||
server_url = "https://mattermost.zeus.gent/zeus"
|
||||
scrape_channel_ids = [
|
||||
"hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN
|
||||
"uda7ax9poprduq8ob56e1fqk4e" # bestuur
|
||||
]
|
||||
selected_user = "my_username"
|
||||
|
||||
[mattermost.users.my_username]
|
||||
|
||||
name = "my_username"
|
||||
password = "..."
|
||||
|
||||
|
||||
[codimd]
|
||||
|
||||
server_url = "https://codimd.zeus.gent"
|
||||
email = ""
|
||||
password = ""
|
||||
|
||||
|
||||
[gitlab]
|
||||
|
||||
host = "https://git.zeus.gent"
|
||||
# access_token_name = "..."
|
||||
# access_token = "..."
|
||||
|
||||
local_repo_folder = "drive"
|
||||
|
||||
|
||||
[gitea]
|
||||
|
||||
# Generate at gitea.example.com/user/settings/applications
|
||||
access_token = "..."
|
|
@ -1,6 +1,10 @@
|
|||
tabulate
|
||||
humanize
|
||||
colored
|
||||
|
||||
mattermostdriver
|
||||
hug
|
||||
|
||||
gitpython
|
||||
python-gitlab
|
||||
python-gitlab
|
||||
giteapy
|
8
src/config.py
Normal file
8
src/config.py
Normal file
|
@ -0,0 +1,8 @@
|
|||
import tomllib
|
||||
from pprint import pprint
|
||||
|
||||
with open("config.toml", mode="rb") as config_toml:
|
||||
config = tomllib.load(config_toml)
|
||||
|
||||
|
||||
# pprint(config)
|
|
@ -12,24 +12,27 @@ def find_metadata(filename):
|
|||
|
||||
metadata = {}
|
||||
|
||||
start_str = ":::spoiler Gitlab sync\n"
|
||||
start_strings = [ ":::spoiler Gitlab sync\n" , ":::spoiler git drive sync\n" ]
|
||||
end_str = "\n:::"
|
||||
|
||||
start_i = data.find(start_str)
|
||||
if start_i >= 0:
|
||||
start_i += len(start_str)
|
||||
end_i = data.find(end_str, start_i + 1)
|
||||
file_data = data[start_i:end_i]
|
||||
for line in file_data.split("\n"):
|
||||
key_index = 2
|
||||
value_index = line.find(": ")
|
||||
key = line[key_index:value_index]
|
||||
value = line[value_index + 2 :]
|
||||
metadata[key] = value
|
||||
print("Valid report")
|
||||
print(metadata)
|
||||
else:
|
||||
print("Not a valid report")
|
||||
return None
|
||||
for start_str in start_strings:
|
||||
start_i = data.find(start_str)
|
||||
if start_i >= 0:
|
||||
start_i += len(start_str)
|
||||
end_i = data.find(end_str, start_i + 1)
|
||||
file_data = data[start_i:end_i]
|
||||
for line in file_data.split("\n"):
|
||||
key_index = 2
|
||||
value_index = line.find(": ")
|
||||
key = line[key_index:value_index]
|
||||
value = line[value_index + 2 :]
|
||||
metadata[key] = value
|
||||
print("Valid report")
|
||||
print(metadata)
|
||||
|
||||
return metadata
|
||||
return metadata
|
||||
|
||||
print("Not a valid report")
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@ from enum import Enum
|
|||
from time import sleep
|
||||
from typing import Dict, List
|
||||
|
||||
import toml
|
||||
from colored import style
|
||||
from config import config
|
||||
from colored import Style
|
||||
from mattermostdriver import Driver
|
||||
from tabulate import tabulate
|
||||
|
||||
|
@ -56,20 +56,17 @@ users: {str: [User]} = {}
|
|||
|
||||
|
||||
def loadusers():
|
||||
with open("users.toml") as f:
|
||||
usersstring = f.read()
|
||||
usersdict = toml.loads(usersstring)
|
||||
|
||||
usr = None
|
||||
for name, data in usersdict.items():
|
||||
if "token" in data:
|
||||
usr = TokenUser(token=data["token"])
|
||||
elif "name" in data and "password" in data:
|
||||
usr = NormalUser(login_id=data["name"], password=data["password"])
|
||||
else:
|
||||
print("Invalid user '{}' in toml file".format(name))
|
||||
exit(1)
|
||||
users[name] = usr
|
||||
usr = None
|
||||
for name, data in config["mattermost"]["users"].items():
|
||||
if "token" in data:
|
||||
usr = TokenUser(token=data["token"])
|
||||
elif "name" in data and "password" in data:
|
||||
usr = NormalUser(login_id=data["name"], password=data["password"])
|
||||
else:
|
||||
print("Invalid user '{}' in toml file".format(name))
|
||||
exit(1)
|
||||
users[name] = usr
|
||||
|
||||
|
||||
loadusers()
|
||||
|
@ -103,11 +100,11 @@ class MMApi(Driver):
|
|||
@staticmethod
|
||||
def print_response(resp, title="Response"):
|
||||
print("--------")
|
||||
print(style.BOLD + title + style.RESET)
|
||||
print(Style.BOLD + title + Style.RESET)
|
||||
pp.pprint(resp)
|
||||
|
||||
def log(self, text: str, log_level: LogLevel = LogLevel.INFO):
|
||||
print(f"{style.BOLD}[{log_level.value}]{style.RESET} {text}")
|
||||
print(f"{Style.BOLD}[{log_level.value}]{Style.RESET} {text}")
|
||||
|
||||
def get_channel_id(self, channel_name):
|
||||
resp = self.channels.get_channel_by_name(self.team_id, channel_name)
|
||||
|
|
|
@ -3,6 +3,8 @@ import mattermostdriver.exceptions
|
|||
import mattermost_client
|
||||
from mattermost_client import ChannelApi, MMApi
|
||||
|
||||
from config import config
|
||||
|
||||
|
||||
def send_message(file_id, file_info, message):
|
||||
channel_id = file_info["originating_mm_post_channel_id"]
|
||||
|
@ -27,14 +29,14 @@ def send_message(file_id, file_info, message):
|
|||
)
|
||||
except mattermostdriver.exceptions.InvalidOrMissingParameters as e:
|
||||
# This will occur when we try to react to a file in a channel that is not the same as the originating channel.
|
||||
unique_post_url = f"https://mattermost.zeus.gent/zeus/pl/{post_id}"
|
||||
unique_post_url = f"{config['mattermost']['server_url']}/pl/{post_id}"
|
||||
channel.create_post(
|
||||
f"{unique_post_url}\n\n{message}",
|
||||
)
|
||||
|
||||
|
||||
def report_newly_found_file(file_id, file_info):
|
||||
message = f"I found a new CodiMD file in this post! Making work of putting it on gitlab :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}"
|
||||
message = f"I found a new CodiMD file in this post! Making work of putting it on git :)\n - Requested location in the [drive](https://git.zeus.gent/bestuur/drive): {file_info['metadata']['sync-to']}"
|
||||
send_message(file_id, file_info, message)
|
||||
|
||||
|
||||
|
@ -47,7 +49,7 @@ You can easily add the correct info and I will do the rest of the work for you!
|
|||
Just add the following lines to your file, the location in your file is not important but at the top would be my recommendation.
|
||||
|
||||
```
|
||||
:::spoiler Gitlab sync
|
||||
:::spoiler git drive sync
|
||||
- sync-to: <a valid path on the DRIVE, for ex.: verslagen/21-22/2022-05-13.md>
|
||||
:::
|
||||
```"""
|
||||
|
|
|
@ -64,6 +64,7 @@ class MMPostProps(NamedTuple):
|
|||
message: str = None
|
||||
attachments: str = None
|
||||
from_bot: str = False
|
||||
disable_group_highlight: str = None
|
||||
|
||||
|
||||
class MMPost(NamedTuple):
|
||||
|
@ -77,16 +78,19 @@ class MMPost(NamedTuple):
|
|||
message: str
|
||||
metadata: Dict
|
||||
original_id: str
|
||||
parent_id: str
|
||||
pending_post_id: str
|
||||
root_id: str
|
||||
type: str
|
||||
update_at: int
|
||||
user_id: str
|
||||
parent_id: str = None
|
||||
message_source: str = None
|
||||
has_reactions: bool = None
|
||||
file_ids: List[str] = None
|
||||
props: MMPostProps = None
|
||||
reply_count: int = None
|
||||
last_reply_at: str = None
|
||||
participants: any = None
|
||||
|
||||
def from_human(self):
|
||||
return self.props is None or (
|
||||
|
@ -115,8 +119,10 @@ class MMChannelPosts(NamedTuple):
|
|||
next_post_id: str
|
||||
order: List[str]
|
||||
posts: Dict[str, MMPost]
|
||||
disable_group_highlight: any
|
||||
reply_count: any
|
||||
has_next: any
|
||||
first_inaccessible_post_time: any
|
||||
reply_count: any = None
|
||||
disable_group_highlight: any = None
|
||||
|
||||
@staticmethod
|
||||
def load(data):
|
||||
|
|
34
src/sync_gitea.py
Normal file
34
src/sync_gitea.py
Normal file
|
@ -0,0 +1,34 @@
|
|||
from __future__ import print_function
|
||||
import time
|
||||
import giteapy
|
||||
from giteapy.rest import ApiException
|
||||
from pprint import pprint
|
||||
|
||||
from config import config
|
||||
|
||||
def init_sync():
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
configuration = giteapy.Configuration()
|
||||
configuration.host = "https://git.zeus.gent/api/v1"
|
||||
configuration.api_key['token'] = config["gitea"]["access_token"]
|
||||
configuration.debug = True
|
||||
|
||||
# create an instance of the API class
|
||||
api_instance = giteapy.RepositoryApi(giteapy.ApiClient(configuration))
|
||||
# username = 'username_example' # str | username of the user that will own the created organization
|
||||
# organization = giteapy.CreateOrgOption() # CreateOrgOption |
|
||||
|
||||
pull_request = giteapy.CreatePullRequestOption(
|
||||
body="Test 123"
|
||||
)
|
||||
|
||||
try:
|
||||
# Create an organization
|
||||
# api_response = api_instance.admin_create_org(username, organization)
|
||||
api_response = api_instance.repo_create_pull_request("ZeusWPI", "drive", async_req=False, body=pull_request)
|
||||
pprint(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling RepositoryApi->repo_create_pull_request: %s\n" % e)
|
|
@ -7,9 +7,12 @@ import pathlib
|
|||
import git
|
||||
import gitlab
|
||||
|
||||
from config import config
|
||||
|
||||
TOKEN_NAME = os.environ["GITLAB_ACCESS_TOKEN_NAME"]
|
||||
TOKEN = os.environ["GITLAB_ACCESS_TOKEN"]
|
||||
REPO_FOLDER = "drive"
|
||||
|
||||
REPO_FOLDER = config["gitlab"]["local_repo_folder"]
|
||||
|
||||
|
||||
def get_repo():
|
18
src/web.py
18
src/web.py
|
@ -11,17 +11,15 @@ import db
|
|||
import dir_utils
|
||||
import mattermost_client
|
||||
import mattermost_communication
|
||||
import sync
|
||||
import sync_gitea as sync
|
||||
import gitlab
|
||||
from utils import id_to_url, url_to_id
|
||||
from config import config
|
||||
|
||||
|
||||
def find_codimd_files_on_mattermost():
|
||||
mattermost = mattermost_client.MMApi()
|
||||
channels = [
|
||||
"hrx6pgfswjbttcj8nim3jrwe7w", # bestuur-INTERN
|
||||
"uda7ax9poprduq8ob56e1fqk4e", # bestuur
|
||||
]
|
||||
channels = config["mattermost"]["scrape_channel_ids"]
|
||||
|
||||
last_fetch_time = db.get_latest_sync_time()
|
||||
current_fetch_time = int(time.time() * 1000)
|
||||
|
@ -71,7 +69,7 @@ def download_files():
|
|||
db.set_local_file_path(file_id, local_file_path)
|
||||
|
||||
|
||||
def validate_downloaded_files():
|
||||
def validate_downloaded_files(post_mattermost_hint=True):
|
||||
path = "data"
|
||||
dir_list = os.listdir(path)
|
||||
|
||||
|
@ -84,7 +82,7 @@ def validate_downloaded_files():
|
|||
mattermost_communication.report_newly_found_file(file_id, new_file_info)
|
||||
else:
|
||||
changed, new_file_info = db.mark_file_invalid(file_id)
|
||||
if changed:
|
||||
if changed and post_mattermost_hint:
|
||||
mattermost_communication.report_newly_found_but_invalid_file(
|
||||
file_id, new_file_info
|
||||
)
|
||||
|
@ -115,11 +113,11 @@ def sync_mattermost():
|
|||
print()
|
||||
print("================================================")
|
||||
print("== Finding valid files in the downloaded ones ==")
|
||||
validate_downloaded_files()
|
||||
validate_downloaded_files(post_mattermost_hint=True)
|
||||
print()
|
||||
print("================================================")
|
||||
print("== Syncing files to gitlab ==")
|
||||
sync_files_to_gitlab()
|
||||
# sync_files_to_gitlab()
|
||||
print()
|
||||
|
||||
return db._load_db()
|
||||
# return db._load_db()
|
||||
|
|
Loading…
Reference in a new issue