From af5cd69f7b60c07c4830f2eca9b8b1544c7c4972 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Tue, 31 May 2022 21:08:56 +0200 Subject: [PATCH 01/51] set up gprc server --- planetwars-server/Cargo.toml | 5 ++++ planetwars-server/build.rs | 9 +++++++ planetwars-server/src/modules/bot_api.rs | 30 ++++++++++++++++++++++++ planetwars-server/src/modules/mod.rs | 1 + proto/bot_api.proto | 15 ++++++++++++ 5 files changed, 60 insertions(+) create mode 100644 planetwars-server/build.rs create mode 100644 planetwars-server/src/modules/bot_api.rs create mode 100644 proto/bot_api.proto diff --git a/planetwars-server/Cargo.toml b/planetwars-server/Cargo.toml index 4c6ddfc..6b96b04 100644 --- a/planetwars-server/Cargo.toml +++ b/planetwars-server/Cargo.toml @@ -26,9 +26,14 @@ toml = "0.5" planetwars-matchrunner = { path = "../planetwars-matchrunner" } config = { version = "0.12", features = ["toml"] } thiserror = "1.0.31" +prost = "0.10" +tonic = "0.7.2" # TODO: remove me shlex = "1.1" +[build-dependencies] +tonic-build = "0.7.2" + [dev-dependencies] parking_lot = "0.11" diff --git a/planetwars-server/build.rs b/planetwars-server/build.rs new file mode 100644 index 0000000..97bf355 --- /dev/null +++ b/planetwars-server/build.rs @@ -0,0 +1,9 @@ +extern crate tonic_build; + +fn main() -> Result<(), Box> { + tonic_build::configure() + .build_server(true) + .build_client(false) + .compile(&["../proto/bot_api.proto"], &["../proto"])?; + Ok(()) +} diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs new file mode 100644 index 0000000..1941136 --- /dev/null +++ b/planetwars-server/src/modules/bot_api.rs @@ -0,0 +1,30 @@ +pub mod pb { + tonic::include_proto!("grpc.planetwars.bot_api"); +} + +use std::net::SocketAddr; + +use tonic; +use tonic::transport::Server; +use tonic::{Request, Response, Status}; + +pub struct BotApiServer {} + +#[tonic::async_trait] +impl pb::test_service_server::TestService for BotApiServer { + async fn greet(&self, req: Request) -> Result, Status> { + Ok(Response::new(pb::HelloResponse { + response: format!("hallo {}", req.get_ref().hello_message), + })) + } +} + +pub async fn run_bot_api() { + let server = BotApiServer {}; + let addr = SocketAddr::from(([127, 0, 0, 1], 50051)); + Server::builder() + .add_service(pb::test_service_server::TestServiceServer::new(server)) + .serve(addr) + .await + .unwrap() +} diff --git a/planetwars-server/src/modules/mod.rs b/planetwars-server/src/modules/mod.rs index bea28e0..43c2507 100644 --- a/planetwars-server/src/modules/mod.rs +++ b/planetwars-server/src/modules/mod.rs @@ -1,5 +1,6 @@ // This module implements general domain logic, not directly // tied to the database or API layers. +pub mod bot_api; pub mod bots; pub mod matches; pub mod ranking; diff --git a/proto/bot_api.proto b/proto/bot_api.proto new file mode 100644 index 0000000..ad0ee2f --- /dev/null +++ b/proto/bot_api.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package grpc.planetwars.bot_api; + +message Hello { + string hello_message = 1; +} + +message HelloResponse { + string response = 1; +} + +service TestService { + rpc greet(Hello) returns (HelloResponse); +} From 0f80b196149a0fb75d84b61c8bbbeb9a71267129 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Wed, 1 Jun 2022 20:19:13 +0200 Subject: [PATCH 02/51] set up stub grpc client --- Cargo.toml | 2 +- planetwars-client/Cargo.toml | 14 ++++++++++++++ planetwars-client/build.rs | 9 +++++++++ planetwars-client/src/main.rs | 21 +++++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 planetwars-client/Cargo.toml create mode 100644 planetwars-client/build.rs create mode 100644 planetwars-client/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index cebf247..0fe450a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,6 @@ members = [ "planetwars-rules", "planetwars-matchrunner", - "planetwars-cli", "planetwars-server", + "planetwars-client", ] diff --git a/planetwars-client/Cargo.toml b/planetwars-client/Cargo.toml new file mode 100644 index 0000000..10de887 --- /dev/null +++ b/planetwars-client/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "planetwars-client" +version = "0.0.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = { version = "1.15", features = ["full"] } +prost = "0.10" +tonic = "0.7.2" + +[build-dependencies] +tonic-build = "0.7.2" diff --git a/planetwars-client/build.rs b/planetwars-client/build.rs new file mode 100644 index 0000000..acabd08 --- /dev/null +++ b/planetwars-client/build.rs @@ -0,0 +1,9 @@ +extern crate tonic_build; + +fn main() -> Result<(), Box> { + tonic_build::configure() + .build_server(false) + .build_client(true) + .compile(&["../proto/bot_api.proto"], &["../proto"])?; + Ok(()) +} diff --git a/planetwars-client/src/main.rs b/planetwars-client/src/main.rs new file mode 100644 index 0000000..9d9bdab --- /dev/null +++ b/planetwars-client/src/main.rs @@ -0,0 +1,21 @@ +pub mod pb { + tonic::include_proto!("grpc.planetwars.bot_api"); +} + +use pb::test_service_client::TestServiceClient; +use pb::{Hello, HelloResponse}; +use tonic::Response; + +#[tokio::main] +async fn main() { + let mut client = TestServiceClient::connect("http://localhost:50051") + .await + .unwrap(); + let response: Response = client + .greet(Hello { + hello_message: "robbe".to_string(), + }) + .await + .unwrap(); + println!("{}", response.get_ref().response); +} From c3d32e051cfeb1deffffbdfe533d17736f72aeda Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Fri, 3 Jun 2022 21:24:18 +0200 Subject: [PATCH 03/51] basic bot api proto definition --- proto/bot_api.proto | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/proto/bot_api.proto b/proto/bot_api.proto index ad0ee2f..0892270 100644 --- a/proto/bot_api.proto +++ b/proto/bot_api.proto @@ -10,6 +10,17 @@ message HelloResponse { string response = 1; } -service TestService { - rpc greet(Hello) returns (HelloResponse); +message PlayerRequest { + int32 request_id = 1; + bytes content = 2; +} + +message PlayerRequestResponse { + int32 request_id = 1; + bytes content = 2; +} + +service BotApiService { + // server sends requests to the player, player responds + rpc ConnectBot(stream PlayerRequestResponse) returns (stream PlayerRequest); } From 90ecb13a1772dfdab20a006b421102c0aa584f60 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sun, 5 Jun 2022 21:22:38 +0200 Subject: [PATCH 04/51] baby steps towards a working bot api --- planetwars-client/Cargo.toml | 1 + planetwars-client/src/main.rs | 32 +++-- planetwars-server/Cargo.toml | 1 + planetwars-server/src/modules/bot_api.rs | 150 +++++++++++++++++++++-- 4 files changed, 165 insertions(+), 19 deletions(-) diff --git a/planetwars-client/Cargo.toml b/planetwars-client/Cargo.toml index 10de887..52c3c64 100644 --- a/planetwars-client/Cargo.toml +++ b/planetwars-client/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] tokio = { version = "1.15", features = ["full"] } +tokio-stream = "0.1.9" prost = "0.10" tonic = "0.7.2" diff --git a/planetwars-client/src/main.rs b/planetwars-client/src/main.rs index 9d9bdab..d995ebc 100644 --- a/planetwars-client/src/main.rs +++ b/planetwars-client/src/main.rs @@ -2,20 +2,34 @@ pub mod pb { tonic::include_proto!("grpc.planetwars.bot_api"); } -use pb::test_service_client::TestServiceClient; -use pb::{Hello, HelloResponse}; -use tonic::Response; +use pb::bot_api_service_client::BotApiServiceClient; +use tokio_stream::wrappers::UnboundedReceiverStream; + +use tokio::sync::mpsc; #[tokio::main] async fn main() { - let mut client = TestServiceClient::connect("http://localhost:50051") + let mut client = BotApiServiceClient::connect("http://localhost:50051") .await .unwrap(); - let response: Response = client - .greet(Hello { - hello_message: "robbe".to_string(), + + let (tx, rx) = mpsc::unbounded_channel(); + let mut stream = client + .connect_bot(UnboundedReceiverStream::new(rx)) + .await + .unwrap() + .into_inner(); + while let Some(message) = stream.message().await.unwrap() { + let state = String::from_utf8(message.content).unwrap(); + println!("{}", state); + let response = r#"{ moves: [] }"#; + tx.send(pb::PlayerRequestResponse { + request_id: message.request_id, + content: response.as_bytes().to_vec(), }) - .await .unwrap(); - println!("{}", response.get_ref().response); + } + std::mem::drop(tx); + // for clean exit + std::mem::drop(client); } diff --git a/planetwars-server/Cargo.toml b/planetwars-server/Cargo.toml index 6b96b04..0ceabbc 100644 --- a/planetwars-server/Cargo.toml +++ b/planetwars-server/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] tokio = { version = "1.15", features = ["full"] } +tokio-stream = "0.1.9" hyper = "0.14" axum = { version = "0.4", features = ["json", "headers", "multipart"] } diesel = { version = "1.4.4", features = ["postgres", "chrono"] } diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 1941136..0f1ff82 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -3,27 +3,157 @@ pub mod pb { } use std::net::SocketAddr; +use std::ops::DerefMut; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use runner::match_context::{EventBus, PlayerHandle, RequestMessage}; +use runner::match_log::MatchLogger; +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::wrappers::UnboundedReceiverStream; use tonic; use tonic::transport::Server; -use tonic::{Request, Response, Status}; +use tonic::{Request, Response, Status, Streaming}; -pub struct BotApiServer {} +use planetwars_matchrunner as runner; + +use crate::db; +use crate::{ConnectionPool, MAPS_DIR, MATCHES_DIR}; + +use super::matches::code_bundle_to_botspec; + +pub struct BotApiServer { + sync_thing: ServerSyncThing, +} #[tonic::async_trait] -impl pb::test_service_server::TestService for BotApiServer { - async fn greet(&self, req: Request) -> Result, Status> { - Ok(Response::new(pb::HelloResponse { - response: format!("hallo {}", req.get_ref().hello_message), - })) +impl pb::bot_api_service_server::BotApiService for BotApiServer { + type ConnectBotStream = UnboundedReceiverStream>; + + async fn connect_bot( + &self, + req: Request>, + ) -> Result, Status> { + println!("bot connected"); + let stream = req.into_inner(); + let sync_data = self.sync_thing.streams.lock().unwrap().take().unwrap(); + sync_data.tx.send(stream).unwrap(); + Ok(Response::new(UnboundedReceiverStream::new( + sync_data.server_messages, + ))) } } -pub async fn run_bot_api() { - let server = BotApiServer {}; +#[derive(Clone)] +struct ServerSyncThing { + streams: Arc>>, +} + +struct SyncThingData { + tx: oneshot::Sender>, + server_messages: mpsc::UnboundedReceiver>, +} + +impl ServerSyncThing { + fn new() -> Self { + ServerSyncThing { + streams: Arc::new(Mutex::new(None)), + } + } +} + +struct RemoteBotSpec { + sync_thing: ServerSyncThing, +} + +#[tonic::async_trait] +impl runner::BotSpec for RemoteBotSpec { + async fn run_bot( + &self, + player_id: u32, + event_bus: Arc>, + _match_logger: MatchLogger, + ) -> Box { + let (tx, rx) = oneshot::channel(); + let (server_msg_snd, server_msg_recv) = mpsc::unbounded_channel(); + *self.sync_thing.streams.lock().unwrap().deref_mut() = Some(SyncThingData { + tx, + server_messages: server_msg_recv, + }); + + let client_messages = rx.await.unwrap(); + tokio::spawn(handle_bot_messages(player_id, event_bus, client_messages)); + + Box::new(RemoteBotHandle { + sender: server_msg_snd, + }) + } +} + +async fn handle_bot_messages( + player_id: u32, + event_bus: Arc>, + mut messages: Streaming, +) { + while let Some(message) = messages.message().await.unwrap() { + let request_id = (player_id, message.request_id as u32); + event_bus + .lock() + .unwrap() + .resolve_request(request_id, Ok(message.content)); + } +} + +struct RemoteBotHandle { + sender: mpsc::UnboundedSender>, +} + +impl PlayerHandle for RemoteBotHandle { + fn send_request(&mut self, r: RequestMessage) { + self.sender + .send(Ok(pb::PlayerRequest { + request_id: r.request_id as i32, + content: r.content, + })) + .unwrap(); + } +} + +async fn run_match(sync_thing: ServerSyncThing, pool: ConnectionPool) { + let conn = pool.get().await.unwrap(); + + let opponent = db::bots::find_bot_by_name("simplebot", &conn).unwrap(); + let opponent_code_bundle = db::bots::active_code_bundle(opponent.id, &conn).unwrap(); + + let log_file_name = "remote_match.log"; + + let remote_bot_spec = RemoteBotSpec { sync_thing }; + + let match_config = runner::MatchConfig { + map_path: PathBuf::from(MAPS_DIR).join("hex.json"), + map_name: "hex".to_string(), + log_path: PathBuf::from(MATCHES_DIR).join(&log_file_name), + players: vec![ + runner::MatchPlayer { + bot_spec: Box::new(remote_bot_spec), + }, + runner::MatchPlayer { + bot_spec: code_bundle_to_botspec(&opponent_code_bundle), + }, + ], + }; + + runner::run_match(match_config).await; +} + +pub async fn run_bot_api(pool: ConnectionPool) { + let sync_thing = ServerSyncThing::new(); + tokio::spawn(run_match(sync_thing.clone(), pool)); + let server = BotApiServer { sync_thing }; + let addr = SocketAddr::from(([127, 0, 0, 1], 50051)); Server::builder() - .add_service(pb::test_service_server::TestServiceServer::new(server)) + .add_service(pb::bot_api_service_server::BotApiServiceServer::new(server)) .serve(addr) .await .unwrap() From d0faec7d1f4deb132554db7f946df4b9d4e9711b Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 6 Jun 2022 13:08:43 +0200 Subject: [PATCH 05/51] implement PlayerRouter --- planetwars-server/src/modules/bot_api.rs | 71 +++++++++++++++--------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 0f1ff82..2face62 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -2,8 +2,8 @@ pub mod pb { tonic::include_proto!("grpc.planetwars.bot_api"); } +use std::collections::HashMap; use std::net::SocketAddr; -use std::ops::DerefMut; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -23,7 +23,35 @@ use crate::{ConnectionPool, MAPS_DIR, MATCHES_DIR}; use super::matches::code_bundle_to_botspec; pub struct BotApiServer { - sync_thing: ServerSyncThing, + router: PlayerRouter, +} + +/// Routes players to their handler +#[derive(Clone)] +struct PlayerRouter { + routing_table: Arc>>, +} + +impl PlayerRouter { + pub fn new() -> Self { + PlayerRouter { + routing_table: Arc::new(Mutex::new(HashMap::new())), + } + } +} + +// TODO: implement a way to expire entries +impl PlayerRouter { + fn put(&self, player_id: String, entry: SyncThingData) { + let mut routing_table = self.routing_table.lock().unwrap(); + routing_table.insert(player_id, entry); + } + + fn get(&self, player_id: &str) -> Option { + // TODO: this design does not allow for reconnects. Is this desired? + let mut routing_table = self.routing_table.lock().unwrap(); + routing_table.remove(player_id) + } } #[tonic::async_trait] @@ -36,7 +64,8 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { ) -> Result, Status> { println!("bot connected"); let stream = req.into_inner(); - let sync_data = self.sync_thing.streams.lock().unwrap().take().unwrap(); + // TODO: return error when player does not exist + let sync_data = self.router.get("test_player").unwrap(); sync_data.tx.send(stream).unwrap(); Ok(Response::new(UnboundedReceiverStream::new( sync_data.server_messages, @@ -44,26 +73,13 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { } } -#[derive(Clone)] -struct ServerSyncThing { - streams: Arc>>, -} - struct SyncThingData { tx: oneshot::Sender>, server_messages: mpsc::UnboundedReceiver>, } -impl ServerSyncThing { - fn new() -> Self { - ServerSyncThing { - streams: Arc::new(Mutex::new(None)), - } - } -} - struct RemoteBotSpec { - sync_thing: ServerSyncThing, + router: PlayerRouter, } #[tonic::async_trait] @@ -76,10 +92,13 @@ impl runner::BotSpec for RemoteBotSpec { ) -> Box { let (tx, rx) = oneshot::channel(); let (server_msg_snd, server_msg_recv) = mpsc::unbounded_channel(); - *self.sync_thing.streams.lock().unwrap().deref_mut() = Some(SyncThingData { - tx, - server_messages: server_msg_recv, - }); + self.router.put( + "test_player".to_string(), + SyncThingData { + tx, + server_messages: server_msg_recv, + }, + ); let client_messages = rx.await.unwrap(); tokio::spawn(handle_bot_messages(player_id, event_bus, client_messages)); @@ -119,7 +138,7 @@ impl PlayerHandle for RemoteBotHandle { } } -async fn run_match(sync_thing: ServerSyncThing, pool: ConnectionPool) { +async fn run_match(router: PlayerRouter, pool: ConnectionPool) { let conn = pool.get().await.unwrap(); let opponent = db::bots::find_bot_by_name("simplebot", &conn).unwrap(); @@ -127,7 +146,7 @@ async fn run_match(sync_thing: ServerSyncThing, pool: ConnectionPool) { let log_file_name = "remote_match.log"; - let remote_bot_spec = RemoteBotSpec { sync_thing }; + let remote_bot_spec = RemoteBotSpec { router }; let match_config = runner::MatchConfig { map_path: PathBuf::from(MAPS_DIR).join("hex.json"), @@ -147,9 +166,9 @@ async fn run_match(sync_thing: ServerSyncThing, pool: ConnectionPool) { } pub async fn run_bot_api(pool: ConnectionPool) { - let sync_thing = ServerSyncThing::new(); - tokio::spawn(run_match(sync_thing.clone(), pool)); - let server = BotApiServer { sync_thing }; + let router = PlayerRouter::new(); + tokio::spawn(run_match(router.clone(), pool)); + let server = BotApiServer { router }; let addr = SocketAddr::from(([127, 0, 0, 1], 50051)); Server::builder() From 2f915af91982073644be94bb2c68e095ffd35596 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 6 Jun 2022 14:25:56 +0200 Subject: [PATCH 06/51] send player_id through request metadata --- planetwars-client/src/main.rs | 10 +++++++++- planetwars-server/src/modules/bot_api.rs | 19 ++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/planetwars-client/src/main.rs b/planetwars-client/src/main.rs index d995ebc..0fbcdb2 100644 --- a/planetwars-client/src/main.rs +++ b/planetwars-client/src/main.rs @@ -6,13 +6,21 @@ use pb::bot_api_service_client::BotApiServiceClient; use tokio_stream::wrappers::UnboundedReceiverStream; use tokio::sync::mpsc; +use tonic::{metadata::MetadataValue, transport::Channel, Request}; #[tokio::main] async fn main() { - let mut client = BotApiServiceClient::connect("http://localhost:50051") + let channel = Channel::from_static("http://localhost:50051") + .connect() .await .unwrap(); + let mut client = BotApiServiceClient::with_interceptor(channel, |mut req: Request<()>| { + let player_id: MetadataValue<_> = "test_player".parse().unwrap(); + req.metadata_mut().insert("player_id", player_id); + Ok(req) + }); + let (tx, rx) = mpsc::unbounded_channel(); let mut stream = client .connect_bot(UnboundedReceiverStream::new(rx)) diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 2face62..f6e4d5c 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -62,10 +62,23 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { &self, req: Request>, ) -> Result, Status> { - println!("bot connected"); + // TODO: clean up errors + let player_id = req + .metadata() + .get("player_id") + .ok_or_else(|| Status::unauthenticated("no player_id provided"))?; + + let player_id_str = player_id + .to_str() + .map_err(|_| Status::invalid_argument("unreadable string"))?; + + let sync_data = self + .router + .get(player_id_str) + .ok_or_else(|| Status::not_found("player_id not found"))?; + let stream = req.into_inner(); - // TODO: return error when player does not exist - let sync_data = self.router.get("test_player").unwrap(); + sync_data.tx.send(stream).unwrap(); Ok(Response::new(UnboundedReceiverStream::new( sync_data.server_messages, From 69421d7b25090724eaa9399f83f83ca36deab882 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 6 Jun 2022 20:23:01 +0200 Subject: [PATCH 07/51] bot api: handle timeouts and disconnects --- planetwars-server/src/modules/bot_api.rs | 63 +++++++++++++++++++++--- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index f6e4d5c..f5aae20 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -6,8 +6,9 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::path::PathBuf; use std::sync::{Arc, Mutex}; +use std::time::Duration; -use runner::match_context::{EventBus, PlayerHandle, RequestMessage}; +use runner::match_context::{EventBus, PlayerHandle, RequestError, RequestMessage}; use runner::match_log::MatchLogger; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::UnboundedReceiverStream; @@ -114,10 +115,16 @@ impl runner::BotSpec for RemoteBotSpec { ); let client_messages = rx.await.unwrap(); - tokio::spawn(handle_bot_messages(player_id, event_bus, client_messages)); + tokio::spawn(handle_bot_messages( + player_id, + event_bus.clone(), + client_messages, + )); Box::new(RemoteBotHandle { sender: server_msg_snd, + player_id, + event_bus, }) } } @@ -138,19 +145,59 @@ async fn handle_bot_messages( struct RemoteBotHandle { sender: mpsc::UnboundedSender>, + player_id: u32, + event_bus: Arc>, } impl PlayerHandle for RemoteBotHandle { fn send_request(&mut self, r: RequestMessage) { - self.sender - .send(Ok(pb::PlayerRequest { - request_id: r.request_id as i32, - content: r.content, - })) - .unwrap(); + let res = self.sender.send(Ok(pb::PlayerRequest { + request_id: r.request_id as i32, + content: r.content, + })); + match res { + Ok(()) => { + // schedule a timeout. See comments at method implementation + tokio::spawn(schedule_timeout( + (self.player_id, r.request_id), + r.timeout, + self.event_bus.clone(), + )); + } + Err(_send_error) => { + // cannot contact the remote bot anymore; + // directly mark all requests as timed out. + // TODO: create a dedicated error type for this. + // should it be logged? + self.event_bus + .lock() + .unwrap() + .resolve_request((self.player_id, r.request_id), Err(RequestError::Timeout)); + } + } } } +// TODO: this will spawn a task for every request, which might not be ideal. +// Some alternatives: +// - create a single task that manages all time-outs. +// - intersperse timeouts with incoming client messages +// - push timeouts upwards, into the matchrunner logic (before we hit the playerhandle). +// This was initially not done to allow timer start to be delayed until the message actually arrived +// with the player. Is this still needed, or is there a different way to do this? +// +async fn schedule_timeout( + request_id: (u32, u32), + duration: Duration, + event_bus: Arc>, +) { + tokio::time::sleep(duration).await; + event_bus + .lock() + .unwrap() + .resolve_request(request_id, Err(RequestError::Timeout)); +} + async fn run_match(router: PlayerRouter, pool: ConnectionPool) { let conn = pool.get().await.unwrap(); From ff061f2a7a0e3a62792ffcef8f2cd3ec6ddc5710 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Tue, 7 Jun 2022 19:12:49 +0200 Subject: [PATCH 08/51] timeout when player never connects --- planetwars-server/src/modules/bot_api.rs | 32 +++++++++++++++++------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index f5aae20..2fffc79 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -48,7 +48,7 @@ impl PlayerRouter { routing_table.insert(player_id, entry); } - fn get(&self, player_id: &str) -> Option { + fn take(&self, player_id: &str) -> Option { // TODO: this design does not allow for reconnects. Is this desired? let mut routing_table = self.routing_table.lock().unwrap(); routing_table.remove(player_id) @@ -75,7 +75,7 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { let sync_data = self .router - .get(player_id_str) + .take(player_id_str) .ok_or_else(|| Status::not_found("player_id not found"))?; let stream = req.into_inner(); @@ -106,21 +106,35 @@ impl runner::BotSpec for RemoteBotSpec { ) -> Box { let (tx, rx) = oneshot::channel(); let (server_msg_snd, server_msg_recv) = mpsc::unbounded_channel(); + let player_key = "test_player".to_string(); self.router.put( - "test_player".to_string(), + player_key.clone(), SyncThingData { tx, server_messages: server_msg_recv, }, ); - let client_messages = rx.await.unwrap(); - tokio::spawn(handle_bot_messages( - player_id, - event_bus.clone(), - client_messages, - )); + let fut = tokio::time::timeout(Duration::from_secs(10), rx); + match fut.await { + Ok(Ok(client_messages)) => { + // let client_messages = rx.await.unwrap(); + tokio::spawn(handle_bot_messages( + player_id, + event_bus.clone(), + client_messages, + )); + } + _ => { + // ensure router cleanup + self.router.take(&player_key); + } + }; + // If the player did not connect, the receiving half of `sender` + // will be dropped here, resulting in a time-out for every turn. + // This is fine for now, but + // TODO: provide a formal mechanism for player startup failure Box::new(RemoteBotHandle { sender: server_msg_snd, player_id, From 028d4a99e4033f9289239600c0dd6ec499a99c04 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Tue, 7 Jun 2022 20:16:42 +0200 Subject: [PATCH 09/51] run bot process in client --- planetwars-client/Cargo.toml | 3 +++ planetwars-client/simplebot.toml | 2 ++ planetwars-client/src/main.rs | 29 +++++++++++++++++++++++------ 3 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 planetwars-client/simplebot.toml diff --git a/planetwars-client/Cargo.toml b/planetwars-client/Cargo.toml index 52c3c64..9c68391 100644 --- a/planetwars-client/Cargo.toml +++ b/planetwars-client/Cargo.toml @@ -10,6 +10,9 @@ tokio = { version = "1.15", features = ["full"] } tokio-stream = "0.1.9" prost = "0.10" tonic = "0.7.2" +serde = { version = "1.0", features = ["derive"] } +toml = "0.5" +planetwars-matchrunner = { path = "../planetwars-matchrunner" } [build-dependencies] tonic-build = "0.7.2" diff --git a/planetwars-client/simplebot.toml b/planetwars-client/simplebot.toml new file mode 100644 index 0000000..dfee25c --- /dev/null +++ b/planetwars-client/simplebot.toml @@ -0,0 +1,2 @@ +name = "simplebot" +command = ["python", "../simplebot/simplebot.py"] diff --git a/planetwars-client/src/main.rs b/planetwars-client/src/main.rs index 0fbcdb2..8840a89 100644 --- a/planetwars-client/src/main.rs +++ b/planetwars-client/src/main.rs @@ -3,13 +3,25 @@ pub mod pb { } use pb::bot_api_service_client::BotApiServiceClient; -use tokio_stream::wrappers::UnboundedReceiverStream; - +use planetwars_matchrunner::bot_runner::Bot; +use serde::Deserialize; +use std::path::PathBuf; use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; use tonic::{metadata::MetadataValue, transport::Channel, Request}; +#[derive(Deserialize)] +struct BotConfig { + #[allow(dead_code)] + name: String, + command: Vec, +} + #[tokio::main] async fn main() { + let content = std::fs::read_to_string("simplebot.toml").unwrap(); + let bot_config: BotConfig = toml::from_str(&content).unwrap(); + let channel = Channel::from_static("http://localhost:50051") .connect() .await @@ -21,6 +33,12 @@ async fn main() { Ok(req) }); + let mut bot_process = Bot { + working_dir: PathBuf::from("."), + argv: bot_config.command, + } + .spawn_process(); + let (tx, rx) = mpsc::unbounded_channel(); let mut stream = client .connect_bot(UnboundedReceiverStream::new(rx)) @@ -28,12 +46,11 @@ async fn main() { .unwrap() .into_inner(); while let Some(message) = stream.message().await.unwrap() { - let state = String::from_utf8(message.content).unwrap(); - println!("{}", state); - let response = r#"{ moves: [] }"#; + let state = std::str::from_utf8(&message.content).unwrap(); + let moves = bot_process.communicate(&message.content).await.unwrap(); tx.send(pb::PlayerRequestResponse { request_id: message.request_id, - content: response.as_bytes().to_vec(), + content: moves.as_bytes().to_vec(), }) .unwrap(); } From 1b2472fbfc876c3f8b6cf5dd6164308123fed133 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Wed, 8 Jun 2022 22:37:38 +0200 Subject: [PATCH 10/51] implement grpc match creation PoC --- planetwars-server/src/modules/bot_api.rs | 86 +++++++++++++++--------- proto/bot_api.proto | 10 +++ 2 files changed, 64 insertions(+), 32 deletions(-) diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 2fffc79..8aa5d29 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -19,11 +19,13 @@ use tonic::{Request, Response, Status, Streaming}; use planetwars_matchrunner as runner; use crate::db; +use crate::util::gen_alphanumeric; use crate::{ConnectionPool, MAPS_DIR, MATCHES_DIR}; use super::matches::code_bundle_to_botspec; pub struct BotApiServer { + conn_pool: ConnectionPool, router: PlayerRouter, } @@ -85,6 +87,50 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { sync_data.server_messages, ))) } + + async fn create_match( + &self, + req: Request, + ) -> Result, Status> { + // TODO: unify with matchrunner module + let conn = self.conn_pool.get().await.unwrap(); + + let match_request = req.get_ref(); + + let opponent = db::bots::find_bot_by_name(&match_request.opponent_name, &conn) + .map_err(|_| Status::not_found("opponent not found"))?; + let opponent_code_bundle = db::bots::active_code_bundle(opponent.id, &conn) + .map_err(|_| Status::not_found("opponent has no code"))?; + + let log_file_name = "remote_match.log"; + let player_key = gen_alphanumeric(32); + + let remote_bot_spec = RemoteBotSpec { + player_key: player_key.clone(), + router: self.router.clone(), + }; + + let match_config = runner::MatchConfig { + map_path: PathBuf::from(MAPS_DIR).join("hex.json"), + map_name: "hex".to_string(), + log_path: PathBuf::from(MATCHES_DIR).join(&log_file_name), + players: vec![ + runner::MatchPlayer { + bot_spec: Box::new(remote_bot_spec), + }, + runner::MatchPlayer { + bot_spec: code_bundle_to_botspec(&opponent_code_bundle), + }, + ], + }; + + tokio::spawn(runner::run_match(match_config)); + Ok(Response::new(pb::CreatedMatch { + // TODO + match_id: 0, + player_key, + })) + } } struct SyncThingData { @@ -93,6 +139,7 @@ struct SyncThingData { } struct RemoteBotSpec { + player_key: String, router: PlayerRouter, } @@ -106,9 +153,8 @@ impl runner::BotSpec for RemoteBotSpec { ) -> Box { let (tx, rx) = oneshot::channel(); let (server_msg_snd, server_msg_recv) = mpsc::unbounded_channel(); - let player_key = "test_player".to_string(); self.router.put( - player_key.clone(), + self.player_key.clone(), SyncThingData { tx, server_messages: server_msg_recv, @@ -127,7 +173,7 @@ impl runner::BotSpec for RemoteBotSpec { } _ => { // ensure router cleanup - self.router.take(&player_key); + self.router.take(&self.player_key); } }; @@ -183,6 +229,7 @@ impl PlayerHandle for RemoteBotHandle { // directly mark all requests as timed out. // TODO: create a dedicated error type for this. // should it be logged? + println!("send error: {:?}", _send_error); self.event_bus .lock() .unwrap() @@ -212,37 +259,12 @@ async fn schedule_timeout( .resolve_request(request_id, Err(RequestError::Timeout)); } -async fn run_match(router: PlayerRouter, pool: ConnectionPool) { - let conn = pool.get().await.unwrap(); - - let opponent = db::bots::find_bot_by_name("simplebot", &conn).unwrap(); - let opponent_code_bundle = db::bots::active_code_bundle(opponent.id, &conn).unwrap(); - - let log_file_name = "remote_match.log"; - - let remote_bot_spec = RemoteBotSpec { router }; - - let match_config = runner::MatchConfig { - map_path: PathBuf::from(MAPS_DIR).join("hex.json"), - map_name: "hex".to_string(), - log_path: PathBuf::from(MATCHES_DIR).join(&log_file_name), - players: vec![ - runner::MatchPlayer { - bot_spec: Box::new(remote_bot_spec), - }, - runner::MatchPlayer { - bot_spec: code_bundle_to_botspec(&opponent_code_bundle), - }, - ], - }; - - runner::run_match(match_config).await; -} - pub async fn run_bot_api(pool: ConnectionPool) { let router = PlayerRouter::new(); - tokio::spawn(run_match(router.clone(), pool)); - let server = BotApiServer { router }; + let server = BotApiServer { + router, + conn_pool: pool.clone(), + }; let addr = SocketAddr::from(([127, 0, 0, 1], 50051)); Server::builder() diff --git a/proto/bot_api.proto b/proto/bot_api.proto index 0892270..08839f0 100644 --- a/proto/bot_api.proto +++ b/proto/bot_api.proto @@ -20,7 +20,17 @@ message PlayerRequestResponse { bytes content = 2; } +message MatchRequest { + string opponent_name = 1; +} + +message CreatedMatch { + int32 match_id = 1; + string player_key = 2; +} + service BotApiService { + rpc CreateMatch(MatchRequest) returns (CreatedMatch); // server sends requests to the player, player responds rpc ConnectBot(stream PlayerRequestResponse) returns (stream PlayerRequest); } From e3cf0df4509fe5cb0ad114040f52f61436a6663f Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Wed, 8 Jun 2022 23:24:32 +0200 Subject: [PATCH 11/51] update client to request matches --- planetwars-client/src/main.rs | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/planetwars-client/src/main.rs b/planetwars-client/src/main.rs index 8840a89..9df100d 100644 --- a/planetwars-client/src/main.rs +++ b/planetwars-client/src/main.rs @@ -5,10 +5,10 @@ pub mod pb { use pb::bot_api_service_client::BotApiServiceClient; use planetwars_matchrunner::bot_runner::Bot; use serde::Deserialize; -use std::path::PathBuf; +use std::{path::PathBuf, time::Duration}; use tokio::sync::mpsc; use tokio_stream::wrappers::UnboundedReceiverStream; -use tonic::{metadata::MetadataValue, transport::Channel, Request}; +use tonic::{metadata::MetadataValue, transport::Channel, Request, Status}; #[derive(Deserialize)] struct BotConfig { @@ -27,8 +27,24 @@ async fn main() { .await .unwrap(); + let created_match = create_match(channel.clone()).await.unwrap(); + run_player(bot_config, created_match.player_key, channel).await; + tokio::time::sleep(Duration::from_secs(1)).await; +} + +async fn create_match(channel: Channel) -> Result { + let mut client = BotApiServiceClient::new(channel); + let res = client + .create_match(Request::new(pb::MatchRequest { + opponent_name: "simplebot".to_string(), + })) + .await; + res.map(|response| response.into_inner()) +} + +async fn run_player(bot_config: BotConfig, player_key: String, channel: Channel) { let mut client = BotApiServiceClient::with_interceptor(channel, |mut req: Request<()>| { - let player_id: MetadataValue<_> = "test_player".parse().unwrap(); + let player_id: MetadataValue<_> = player_key.parse().unwrap(); req.metadata_mut().insert("player_id", player_id); Ok(req) }); @@ -46,7 +62,6 @@ async fn main() { .unwrap() .into_inner(); while let Some(message) = stream.message().await.unwrap() { - let state = std::str::from_utf8(&message.content).unwrap(); let moves = bot_process.communicate(&message.content).await.unwrap(); tx.send(pb::PlayerRequestResponse { request_id: message.request_id, @@ -54,7 +69,4 @@ async fn main() { }) .unwrap(); } - std::mem::drop(tx); - // for clean exit - std::mem::drop(client); } From d1977b95c82f608bc558432cdfba8026aaf0648d Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Thu, 9 Jun 2022 20:57:45 +0200 Subject: [PATCH 12/51] consistently use player_key and player_id --- planetwars-client/src/main.rs | 4 ++-- planetwars-server/src/modules/bot_api.rs | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/planetwars-client/src/main.rs b/planetwars-client/src/main.rs index 9df100d..3ece5b3 100644 --- a/planetwars-client/src/main.rs +++ b/planetwars-client/src/main.rs @@ -44,8 +44,8 @@ async fn create_match(channel: Channel) -> Result { async fn run_player(bot_config: BotConfig, player_key: String, channel: Channel) { let mut client = BotApiServiceClient::with_interceptor(channel, |mut req: Request<()>| { - let player_id: MetadataValue<_> = player_key.parse().unwrap(); - req.metadata_mut().insert("player_id", player_id); + let player_key: MetadataValue<_> = player_key.parse().unwrap(); + req.metadata_mut().insert("player_key", player_key); Ok(req) }); diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 8aa5d29..4eb13c1 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -45,15 +45,15 @@ impl PlayerRouter { // TODO: implement a way to expire entries impl PlayerRouter { - fn put(&self, player_id: String, entry: SyncThingData) { + fn put(&self, player_key: String, entry: SyncThingData) { let mut routing_table = self.routing_table.lock().unwrap(); - routing_table.insert(player_id, entry); + routing_table.insert(player_key, entry); } - fn take(&self, player_id: &str) -> Option { + fn take(&self, player_key: &str) -> Option { // TODO: this design does not allow for reconnects. Is this desired? let mut routing_table = self.routing_table.lock().unwrap(); - routing_table.remove(player_id) + routing_table.remove(player_key) } } @@ -66,19 +66,19 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { req: Request>, ) -> Result, Status> { // TODO: clean up errors - let player_id = req + let player_key = req .metadata() - .get("player_id") - .ok_or_else(|| Status::unauthenticated("no player_id provided"))?; + .get("player_key") + .ok_or_else(|| Status::unauthenticated("no player_key provided"))?; - let player_id_str = player_id + let player_key_str = player_key .to_str() .map_err(|_| Status::invalid_argument("unreadable string"))?; let sync_data = self .router - .take(player_id_str) - .ok_or_else(|| Status::not_found("player_id not found"))?; + .take(player_key_str) + .ok_or_else(|| Status::not_found("player_key not found"))?; let stream = req.into_inner(); From 5ee66c9c9b4156692c739a861c9cdbaf0c65aec8 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Fri, 10 Jun 2022 21:09:33 +0200 Subject: [PATCH 13/51] allow match_player code_bundle_id to be null --- .../down.sql | 1 + .../up.sql | 1 + planetwars-server/src/db/matches.rs | 23 +++++++++++++------ planetwars-server/src/routes/bots.rs | 2 +- planetwars-server/src/routes/demo.rs | 4 ++-- planetwars-server/src/routes/matches.rs | 4 ++-- planetwars-server/src/schema.rs | 2 +- 7 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 planetwars-server/migrations/2022-06-10-180418_nullable_match_player_code_bundle/down.sql create mode 100644 planetwars-server/migrations/2022-06-10-180418_nullable_match_player_code_bundle/up.sql diff --git a/planetwars-server/migrations/2022-06-10-180418_nullable_match_player_code_bundle/down.sql b/planetwars-server/migrations/2022-06-10-180418_nullable_match_player_code_bundle/down.sql new file mode 100644 index 0000000..bb0b613 --- /dev/null +++ b/planetwars-server/migrations/2022-06-10-180418_nullable_match_player_code_bundle/down.sql @@ -0,0 +1 @@ +ALTER TABLE match_players ALTER COLUMN code_bundle_id SET NOT NULL; diff --git a/planetwars-server/migrations/2022-06-10-180418_nullable_match_player_code_bundle/up.sql b/planetwars-server/migrations/2022-06-10-180418_nullable_match_player_code_bundle/up.sql new file mode 100644 index 0000000..86ab65d --- /dev/null +++ b/planetwars-server/migrations/2022-06-10-180418_nullable_match_player_code_bundle/up.sql @@ -0,0 +1 @@ +ALTER TABLE match_players ALTER COLUMN code_bundle_id DROP NOT NULL; diff --git a/planetwars-server/src/db/matches.rs b/planetwars-server/src/db/matches.rs index ee25e85..6ec1389 100644 --- a/planetwars-server/src/db/matches.rs +++ b/planetwars-server/src/db/matches.rs @@ -44,7 +44,7 @@ pub struct MatchBase { pub struct MatchPlayer { pub match_id: i32, pub player_id: i32, - pub code_bundle_id: i32, + pub code_bundle_id: Option, } pub struct MatchPlayerData { @@ -92,7 +92,10 @@ pub fn list_matches(conn: &PgConnection) -> QueryResult> { let matches = matches::table.get_results::(conn)?; let match_players = MatchPlayer::belonging_to(&matches) - .inner_join(code_bundles::table) + .left_join( + code_bundles::table + .on(match_players::code_bundle_id.eq(code_bundles::id.nullable())), + ) .left_join(bots::table.on(code_bundles::bot_id.eq(bots::id.nullable()))) .load::(conn)? .grouped_by(&matches); @@ -120,7 +123,7 @@ pub struct FullMatchData { // #[primary_key(base.match_id, base::player_id)] pub struct FullMatchPlayerData { pub base: MatchPlayer, - pub code_bundle: CodeBundle, + pub code_bundle: Option, pub bot: Option, } @@ -142,7 +145,10 @@ pub fn find_match(id: i32, conn: &PgConnection) -> QueryResult { let match_base = matches::table.find(id).get_result::(conn)?; let match_players = MatchPlayer::belonging_to(&match_base) - .inner_join(code_bundles::table) + .left_join( + code_bundles::table + .on(match_players::code_bundle_id.eq(code_bundles::id.nullable())), + ) .left_join(bots::table.on(code_bundles::bot_id.eq(bots::id.nullable()))) .load::(conn)?; @@ -160,14 +166,17 @@ pub fn find_match_base(id: i32, conn: &PgConnection) -> QueryResult { } pub enum MatchResult { - Finished { winner: Option } + Finished { winner: Option }, } pub fn save_match_result(id: i32, result: MatchResult, conn: &PgConnection) -> QueryResult<()> { let MatchResult::Finished { winner } = result; diesel::update(matches::table.find(id)) - .set((matches::winner.eq(winner), matches::state.eq(MatchState::Finished))) + .set(( + matches::winner.eq(winner), + matches::state.eq(MatchState::Finished), + )) .execute(conn)?; Ok(()) -} \ No newline at end of file +} diff --git a/planetwars-server/src/routes/bots.rs b/planetwars-server/src/routes/bots.rs index 3bbaa1a..df0c4d0 100644 --- a/planetwars-server/src/routes/bots.rs +++ b/planetwars-server/src/routes/bots.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use thiserror; use crate::db::bots::{self, CodeBundle}; -use crate::db::ratings::{RankedBot, self}; +use crate::db::ratings::{self, RankedBot}; use crate::db::users::User; use crate::modules::bots::save_code_bundle; use crate::{DatabaseConnection, BOTS_DIR}; diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 7f7ba71..3318dfd 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -58,12 +58,12 @@ pub async fn submit_bot( match_players: vec![ FullMatchPlayerData { base: match_data.match_players[0].clone(), - code_bundle: player_code_bundle, + code_bundle: Some(player_code_bundle), bot: None, }, FullMatchPlayerData { base: match_data.match_players[1].clone(), - code_bundle: opponent_code_bundle, + code_bundle: Some(opponent_code_bundle), bot: Some(opponent), }, ], diff --git a/planetwars-server/src/routes/matches.rs b/planetwars-server/src/routes/matches.rs index b61008d..7169ebe 100644 --- a/planetwars-server/src/routes/matches.rs +++ b/planetwars-server/src/routes/matches.rs @@ -107,7 +107,7 @@ pub struct ApiMatch { #[derive(Serialize, Deserialize)] pub struct ApiMatchPlayer { - code_bundle_id: i32, + code_bundle_id: Option, bot_id: Option, bot_name: Option, } @@ -127,7 +127,7 @@ pub fn match_data_to_api(data: matches::FullMatchData) -> ApiMatch { .match_players .iter() .map(|_p| ApiMatchPlayer { - code_bundle_id: _p.code_bundle.id, + code_bundle_id: _p.code_bundle.as_ref().map(|cb| cb.id), bot_id: _p.bot.as_ref().map(|b| b.id), bot_name: _p.bot.as_ref().map(|b| b.name.clone()), }) diff --git a/planetwars-server/src/schema.rs b/planetwars-server/src/schema.rs index be3e858..92acc8e 100644 --- a/planetwars-server/src/schema.rs +++ b/planetwars-server/src/schema.rs @@ -31,7 +31,7 @@ table! { match_players (match_id, player_id) { match_id -> Int4, player_id -> Int4, - code_bundle_id -> Int4, + code_bundle_id -> Nullable, } } From a3766980735851e9aa4b56a80e91c0b77cf63adb Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Fri, 10 Jun 2022 21:49:32 +0200 Subject: [PATCH 14/51] update RunMatch helper to allow for remote bots --- planetwars-server/src/db/matches.rs | 4 +- planetwars-server/src/modules/matches.rs | 50 +++++++++++++++++------- planetwars-server/src/modules/ranking.rs | 9 +++-- planetwars-server/src/routes/demo.rs | 7 +++- planetwars-server/src/routes/matches.rs | 2 +- 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/planetwars-server/src/db/matches.rs b/planetwars-server/src/db/matches.rs index 6ec1389..54fd113 100644 --- a/planetwars-server/src/db/matches.rs +++ b/planetwars-server/src/db/matches.rs @@ -25,7 +25,7 @@ pub struct NewMatchPlayer { /// player id within the match pub player_id: i32, /// id of the bot behind this player - pub code_bundle_id: i32, + pub code_bundle_id: Option, } #[derive(Queryable, Identifiable)] @@ -48,7 +48,7 @@ pub struct MatchPlayer { } pub struct MatchPlayerData { - pub code_bundle_id: i32, + pub code_bundle_id: Option, } pub fn create_match( diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index a254bac..6d9261d 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -16,32 +16,54 @@ use crate::{ const PYTHON_IMAGE: &str = "python:3.10-slim-buster"; -pub struct RunMatch<'a> { +pub struct RunMatch { log_file_name: String, - player_code_bundles: Vec<&'a db::bots::CodeBundle>, + players: Vec, match_id: Option, } -impl<'a> RunMatch<'a> { - pub fn from_players(player_code_bundles: Vec<&'a db::bots::CodeBundle>) -> Self { +pub struct MatchPlayer { + bot_spec: Box, + // meta that will be passed on to database + code_bundle_id: Option, +} + +impl MatchPlayer { + pub fn from_code_bundle(code_bundle: &db::bots::CodeBundle) -> Self { + MatchPlayer { + bot_spec: code_bundle_to_botspec(code_bundle), + code_bundle_id: Some(code_bundle.id), + } + } + + pub fn from_bot_spec(bot_spec: Box) -> Self { + MatchPlayer { + bot_spec, + code_bundle_id: None, + } + } +} + +impl RunMatch { + pub fn from_players(players: Vec) -> Self { let log_file_name = format!("{}.log", gen_alphanumeric(16)); RunMatch { log_file_name, - player_code_bundles, + players, match_id: None, } } - pub fn runner_config(&self) -> runner::MatchConfig { + pub fn into_runner_config(self) -> runner::MatchConfig { runner::MatchConfig { map_path: PathBuf::from(MAPS_DIR).join("hex.json"), map_name: "hex".to_string(), log_path: PathBuf::from(MATCHES_DIR).join(&self.log_file_name), players: self - .player_code_bundles - .iter() - .map(|b| runner::MatchPlayer { - bot_spec: code_bundle_to_botspec(b), + .players + .into_iter() + .map(|player| runner::MatchPlayer { + bot_spec: player.bot_spec, }) .collect(), } @@ -56,10 +78,10 @@ impl<'a> RunMatch<'a> { log_path: &self.log_file_name, }; let new_match_players = self - .player_code_bundles + .players .iter() - .map(|b| db::matches::MatchPlayerData { - code_bundle_id: b.id, + .map(|p| db::matches::MatchPlayerData { + code_bundle_id: p.code_bundle_id, }) .collect::>(); @@ -70,7 +92,7 @@ impl<'a> RunMatch<'a> { pub fn spawn(self, pool: ConnectionPool) -> JoinHandle { let match_id = self.match_id.expect("match must be saved before running"); - let runner_config = self.runner_config(); + let runner_config = self.into_runner_config(); tokio::spawn(run_match_task(pool, runner_config, match_id)) } } diff --git a/planetwars-server/src/modules/ranking.rs b/planetwars-server/src/modules/ranking.rs index f76fbae..d83debb 100644 --- a/planetwars-server/src/modules/ranking.rs +++ b/planetwars-server/src/modules/ranking.rs @@ -1,7 +1,7 @@ use crate::{db::bots::Bot, DbPool}; use crate::db; -use crate::modules::matches::RunMatch; +use crate::modules::matches::{MatchPlayer, RunMatch}; use rand::seq::SliceRandom; use std::time::Duration; use tokio; @@ -43,9 +43,12 @@ async fn play_ranking_match(selected_bots: Vec, db_pool: DbPool) { code_bundles.push(code_bundle); } - let code_bundle_refs = code_bundles.iter().map(|b| b).collect::>(); + let players = code_bundles + .iter() + .map(MatchPlayer::from_code_bundle) + .collect::>(); - let mut run_match = RunMatch::from_players(code_bundle_refs); + let mut run_match = RunMatch::from_players(players); run_match .store_in_database(&db_conn) .expect("could not store match in db"); diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 3318dfd..33dc02d 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -1,7 +1,7 @@ use crate::db; use crate::db::matches::{FullMatchData, FullMatchPlayerData}; use crate::modules::bots::save_code_bundle; -use crate::modules::matches::RunMatch; +use crate::modules::matches::{MatchPlayer, RunMatch}; use crate::ConnectionPool; use axum::extract::Extension; use axum::Json; @@ -46,7 +46,10 @@ pub async fn submit_bot( // TODO: can we recover from this? .expect("could not save bot code"); - let mut run_match = RunMatch::from_players(vec![&player_code_bundle, &opponent_code_bundle]); + let mut run_match = RunMatch::from_players(vec![ + MatchPlayer::from_code_bundle(&player_code_bundle), + MatchPlayer::from_code_bundle(&opponent_code_bundle), + ]); let match_data = run_match .store_in_database(&conn) .expect("failed to save match"); diff --git a/planetwars-server/src/routes/matches.rs b/planetwars-server/src/routes/matches.rs index 7169ebe..874c775 100644 --- a/planetwars-server/src/routes/matches.rs +++ b/planetwars-server/src/routes/matches.rs @@ -61,7 +61,7 @@ pub async fn play_match( }); bot_ids.push(matches::MatchPlayerData { - code_bundle_id: code_bundle.id, + code_bundle_id: Some(code_bundle.id), }); } From 7a3b801f58752a78b65e3e7e7b998b6479f980f7 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sat, 11 Jun 2022 17:50:44 +0200 Subject: [PATCH 15/51] use RunMatch in bot_api service --- planetwars-server/src/modules/bot_api.rs | 43 +++++++++++------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 4eb13c1..0ecbf71 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -4,7 +4,6 @@ pub mod pb { use std::collections::HashMap; use std::net::SocketAddr; -use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -20,9 +19,9 @@ use planetwars_matchrunner as runner; use crate::db; use crate::util::gen_alphanumeric; -use crate::{ConnectionPool, MAPS_DIR, MATCHES_DIR}; +use crate::ConnectionPool; -use super::matches::code_bundle_to_botspec; +use super::matches::{MatchPlayer, RunMatch}; pub struct BotApiServer { conn_pool: ConnectionPool, @@ -43,6 +42,12 @@ impl PlayerRouter { } } +impl Default for PlayerRouter { + fn default() -> Self { + Self::new() + } +} + // TODO: implement a way to expire entries impl PlayerRouter { fn put(&self, player_key: String, entry: SyncThingData) { @@ -102,37 +107,29 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { let opponent_code_bundle = db::bots::active_code_bundle(opponent.id, &conn) .map_err(|_| Status::not_found("opponent has no code"))?; - let log_file_name = "remote_match.log"; let player_key = gen_alphanumeric(32); - let remote_bot_spec = RemoteBotSpec { + let remote_bot_spec = Box::new(RemoteBotSpec { player_key: player_key.clone(), router: self.router.clone(), - }; + }); + let mut run_match = RunMatch::from_players(vec![ + MatchPlayer::from_bot_spec(remote_bot_spec), + MatchPlayer::from_code_bundle(&opponent_code_bundle), + ]); + let created_match = run_match + .store_in_database(&conn) + .expect("failed to save match"); + run_match.spawn(self.conn_pool.clone()); - let match_config = runner::MatchConfig { - map_path: PathBuf::from(MAPS_DIR).join("hex.json"), - map_name: "hex".to_string(), - log_path: PathBuf::from(MATCHES_DIR).join(&log_file_name), - players: vec![ - runner::MatchPlayer { - bot_spec: Box::new(remote_bot_spec), - }, - runner::MatchPlayer { - bot_spec: code_bundle_to_botspec(&opponent_code_bundle), - }, - ], - }; - - tokio::spawn(runner::run_match(match_config)); Ok(Response::new(pb::CreatedMatch { - // TODO - match_id: 0, + match_id: created_match.base.id, player_key, })) } } +// TODO: please rename me struct SyncThingData { tx: oneshot::Sender>, server_messages: mpsc::UnboundedReceiver>, From dde0bc820e47a372c9b1042249637c708a323188 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sun, 12 Jun 2022 21:03:41 +0200 Subject: [PATCH 16/51] accept docker push --- planetwars-server/Cargo.toml | 2 + planetwars-server/src/lib.rs | 12 ++ planetwars-server/src/modules/mod.rs | 1 + planetwars-server/src/modules/registry.rs | 215 ++++++++++++++++++++++ 4 files changed, 230 insertions(+) create mode 100644 planetwars-server/src/modules/registry.rs diff --git a/planetwars-server/Cargo.toml b/planetwars-server/Cargo.toml index 4c6ddfc..3e1f05e 100644 --- a/planetwars-server/Cargo.toml +++ b/planetwars-server/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +futures = "0.3" tokio = { version = "1.15", features = ["full"] } hyper = "0.14" axum = { version = "0.4", features = ["json", "headers", "multipart"] } @@ -26,6 +27,7 @@ toml = "0.5" planetwars-matchrunner = { path = "../planetwars-matchrunner" } config = { version = "0.12", features = ["toml"] } thiserror = "1.0.31" +sha2 = "0.10" # TODO: remove me shlex = "1.1" diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index 28d7a76..e50003c 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -16,6 +16,7 @@ use bb8_diesel::{self, DieselConnectionManager}; use config::ConfigError; use diesel::{Connection, PgConnection}; use modules::ranking::run_ranker; +use modules::registry::registry_service; use serde::Deserialize; use axum::{ @@ -104,11 +105,22 @@ pub fn get_config() -> Result { .try_deserialize() } +async fn run_registry(_db_pool: DbPool) { + // TODO: put in config + let addr = SocketAddr::from(([127, 0, 0, 1], 9001)); + + axum::Server::bind(&addr) + .serve(registry_service().into_make_service()) + .await + .unwrap(); +} + pub async fn run_app() { let configuration = get_config().unwrap(); let db_pool = prepare_db(&configuration.database_url).await; tokio::spawn(run_ranker(db_pool.clone())); + tokio::spawn(run_registry(db_pool.clone())); let api_service = Router::new() .nest("/api", api()) diff --git a/planetwars-server/src/modules/mod.rs b/planetwars-server/src/modules/mod.rs index bea28e0..d66f568 100644 --- a/planetwars-server/src/modules/mod.rs +++ b/planetwars-server/src/modules/mod.rs @@ -3,3 +3,4 @@ pub mod bots; pub mod matches; pub mod ranking; +pub mod registry; diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs new file mode 100644 index 0000000..d63621a --- /dev/null +++ b/planetwars-server/src/modules/registry.rs @@ -0,0 +1,215 @@ +use axum::body::Body; +use axum::extract::{BodyStream, Path, Query}; +use axum::handler::Handler; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, head, post, put}; +use axum::Router; +use hyper::StatusCode; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::path::PathBuf; +use tokio::io::AsyncWriteExt; + +use crate::util::gen_alphanumeric; + +const REGISTRY_PATH: &'static str = "./data/registry"; +pub fn registry_service() -> Router { + Router::new() + .nest("/v2", registry_api_v2()) + .fallback(fallback.into_service()) +} + +fn registry_api_v2() -> Router { + Router::new() + .route("/", get(root_handler)) + .route("/:name/blobs/:digest", head(blob_check).get(blob_check)) + .route("/:name/blobs/uploads/", post(blob_upload)) + .route( + "/:name/blobs/uploads/:uuid", + put(put_handler).patch(handle_upload), + ) + .route("/:name/manifests/:reference", put(put_manifest)) +} + +async fn fallback(request: axum::http::Request) -> impl IntoResponse { + // for debugging + println!("no route for {} {}", request.method(), request.uri()); + StatusCode::NOT_FOUND +} + +// root should return 200 OK to confirm api compliance +async fn root_handler() -> Response { + Response::builder() + .status(StatusCode::OK) + .header("Docker-Distribution-API-Version", "registry/2.0") + .body(Body::empty()) + .unwrap() +} + +#[derive(Serialize)] +pub struct RegistryErrors { + errors: Vec, +} + +#[derive(Serialize)] +pub struct RegistryError { + code: String, + message: String, + detail: serde_json::Value, +} + +async fn blob_check( + Path((_repository_name, raw_digest)): Path<(String, String)>, +) -> impl IntoResponse { + let digest = raw_digest.strip_prefix("sha256:").unwrap(); + let blob_path = PathBuf::from(REGISTRY_PATH).join(&digest); + if blob_path.exists() { + StatusCode::OK + } else { + StatusCode::NOT_FOUND + } +} + +async fn blob_upload(Path(repository_name): Path) -> impl IntoResponse { + // let value = json!({ + // "errors": [ + // { + // "code": "UNSUPPORTED", + // "message": "not implemented yet lol", + // } + // ] + // }); + + let uuid = gen_alphanumeric(16); + tokio::fs::File::create(PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid)) + .await + .unwrap(); + + Response::builder() + .status(StatusCode::ACCEPTED) + .header( + "Location", + format!("/v2/{}/blobs/uploads/{}", repository_name, uuid), + ) + .header("Docker-Upload-UUID", uuid) + .header("Range", "bytes=0-0") + .body(Body::empty()) + .unwrap() +} + +use futures::StreamExt; + +async fn handle_upload( + Path((repository_name, uuid)): Path<(String, String)>, + mut stream: BodyStream, +) -> impl IntoResponse { + // let content_length = headers.get("Content-Length").unwrap(); + // let content_range = headers.get("Content-Range").unwrap(); + // let content_type = headers.get("Content-Type").unwrap(); + // assert!(content_type == "application/octet-stream"); + let mut len = 0; + let upload_path = PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid); + let mut file = tokio::fs::OpenOptions::new() + .read(false) + .write(true) + .append(true) + .create(false) + .open(upload_path) + .await + .unwrap(); + while let Some(Ok(chunk)) = stream.next().await { + let n_bytes = file.write(&chunk).await.unwrap(); + len += n_bytes; + } + + Response::builder() + .status(StatusCode::ACCEPTED) + .header( + "Location", + format!("/v2/{}/blobs/uploads/{}", repository_name, uuid), + ) + .header("Docker-Upload-UUID", uuid) + .header("Range", format!("0-{}", len)) + .body(Body::empty()) + .unwrap() +} + +use serde::Deserialize; +#[derive(Deserialize)] +struct UploadParams { + digest: String, +} + +async fn put_handler( + Path((repository_name, uuid)): Path<(String, String)>, + Query(params): Query, + mut stream: BodyStream, +) -> impl IntoResponse { + let mut _len = 0; + let upload_path = PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid); + let mut file = tokio::fs::OpenOptions::new() + .read(false) + .write(true) + .append(true) + .create(false) + .open(&upload_path) + .await + .unwrap(); + + while let Some(Ok(chunk)) = stream.next().await { + let n_bytes = file.write(&chunk).await.unwrap(); + _len += n_bytes; + } + let digest = params.digest.strip_prefix("sha256:").unwrap(); + // TODO: check the digest + let target_path = PathBuf::from(REGISTRY_PATH).join(&digest); + tokio::fs::rename(&upload_path, &target_path).await.unwrap(); + println!("DIGEST {}", digest); + Response::builder() + .status(StatusCode::CREATED) + .header( + "Location", + format!("/v2/{}/blobs/{}", repository_name, digest), + ) + .header("Docker-Upload-UUID", uuid) + // .header("Range", format!("0-{}", len)) + .header("Docker-Content-Digest", digest) + .body(Body::empty()) + .unwrap() +} + +async fn put_manifest( + Path((repository_name, reference)): Path<(String, String)>, + mut stream: BodyStream, +) -> impl IntoResponse { + let repository_dir = PathBuf::from(REGISTRY_PATH).join(&repository_name); + + tokio::fs::create_dir_all(&repository_dir).await.unwrap(); + + let mut hasher = Sha256::new(); + { + let manifest_path = repository_dir.join(&reference).with_extension("json"); + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&manifest_path) + .await + .unwrap(); + while let Some(Ok(chunk)) = stream.next().await { + hasher.update(&chunk); + file.write(&chunk).await.unwrap(); + } + } + let digest = hasher.finalize(); + + Response::builder() + .status(StatusCode::CREATED) + .header( + "Location", + format!("/v2/{}/manifests/{}", repository_name, reference), + ) + .header("Docker-Content-Digest", format!("sha256:{:x}", digest)) + .body(Body::empty()) + .unwrap() +} From b90b3d3635f57bb84450d90544df536bf58e8588 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Fri, 17 Jun 2022 19:01:40 +0200 Subject: [PATCH 17/51] store blobs in sha256 directory --- planetwars-server/src/modules/registry.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index d63621a..d10532a 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -62,7 +62,7 @@ async fn blob_check( Path((_repository_name, raw_digest)): Path<(String, String)>, ) -> impl IntoResponse { let digest = raw_digest.strip_prefix("sha256:").unwrap(); - let blob_path = PathBuf::from(REGISTRY_PATH).join(&digest); + let blob_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); if blob_path.exists() { StatusCode::OK } else { @@ -162,9 +162,9 @@ async fn put_handler( } let digest = params.digest.strip_prefix("sha256:").unwrap(); // TODO: check the digest - let target_path = PathBuf::from(REGISTRY_PATH).join(&digest); + let target_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); tokio::fs::rename(&upload_path, &target_path).await.unwrap(); - println!("DIGEST {}", digest); + Response::builder() .status(StatusCode::CREATED) .header( @@ -182,7 +182,9 @@ async fn put_manifest( Path((repository_name, reference)): Path<(String, String)>, mut stream: BodyStream, ) -> impl IntoResponse { - let repository_dir = PathBuf::from(REGISTRY_PATH).join(&repository_name); + let repository_dir = PathBuf::from(REGISTRY_PATH) + .join("manifests") + .join(&repository_name); tokio::fs::create_dir_all(&repository_dir).await.unwrap(); From 2cde7ec673b38b51db0dce8d5e8496ba2d92aa12 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sat, 18 Jun 2022 12:42:03 +0200 Subject: [PATCH 18/51] support docker pull --- planetwars-server/src/modules/registry.rs | 52 ++++++++++++++++++++--- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index d10532a..6095527 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -1,4 +1,4 @@ -use axum::body::Body; +use axum::body::{Body, StreamBody}; use axum::extract::{BodyStream, Path, Query}; use axum::handler::Handler; use axum::response::{IntoResponse, Response}; @@ -9,6 +9,7 @@ use serde::Serialize; use sha2::{Digest, Sha256}; use std::path::PathBuf; use tokio::io::AsyncWriteExt; +use tokio_util::io::ReaderStream; use crate::util::gen_alphanumeric; @@ -22,13 +23,16 @@ pub fn registry_service() -> Router { fn registry_api_v2() -> Router { Router::new() .route("/", get(root_handler)) - .route("/:name/blobs/:digest", head(blob_check).get(blob_check)) + .route("/:name/blobs/:digest", head(blob_check).get(get_blob)) .route("/:name/blobs/uploads/", post(blob_upload)) .route( "/:name/blobs/uploads/:uuid", put(put_handler).patch(handle_upload), ) - .route("/:name/manifests/:reference", put(put_manifest)) + .route( + "/:name/manifests/:reference", + get(get_manifest).put(put_manifest), + ) } async fn fallback(request: axum::http::Request) -> impl IntoResponse { @@ -70,6 +74,20 @@ async fn blob_check( } } +async fn get_blob( + Path((_repository_name, raw_digest)): Path<(String, String)>, +) -> impl IntoResponse { + let digest = raw_digest.strip_prefix("sha256:").unwrap(); + let blob_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); + if !blob_path.exists() { + return Err(StatusCode::NOT_FOUND); + } + let file = tokio::fs::File::open(&blob_path).await.unwrap(); + let reader_stream = ReaderStream::new(file); + let stream_body = StreamBody::new(reader_stream); + Ok(stream_body) +} + async fn blob_upload(Path(repository_name): Path) -> impl IntoResponse { // let value = json!({ // "errors": [ @@ -178,6 +196,26 @@ async fn put_handler( .unwrap() } +async fn get_manifest( + Path((repository_name, reference)): Path<(String, String)>, +) -> impl IntoResponse { + let manifest_path = PathBuf::from(REGISTRY_PATH) + .join("manifests") + .join(&repository_name) + .join(&reference) + .with_extension("json"); + let data = tokio::fs::read(&manifest_path).await.unwrap(); + + let manifest: serde_json::Map = + serde_json::from_slice(&data).unwrap(); + let media_type = manifest.get("mediaType").unwrap().as_str().unwrap(); + Response::builder() + .status(StatusCode::OK) + .header("Content-Type", media_type) + .body(axum::body::Full::from(data)) + .unwrap() +} + async fn put_manifest( Path((repository_name, reference)): Path<(String, String)>, mut stream: BodyStream, @@ -189,8 +227,8 @@ async fn put_manifest( tokio::fs::create_dir_all(&repository_dir).await.unwrap(); let mut hasher = Sha256::new(); + let manifest_path = repository_dir.join(&reference).with_extension("json"); { - let manifest_path = repository_dir.join(&reference).with_extension("json"); let mut file = tokio::fs::OpenOptions::new() .write(true) .create(true) @@ -204,6 +242,10 @@ async fn put_manifest( } } let digest = hasher.finalize(); + // TODO: store content-adressable manifests separately + let content_digest = format!("sha256:{:x}", digest); + let digest_path = repository_dir.join(&content_digest).with_extension("json"); + tokio::fs::copy(manifest_path, digest_path).await.unwrap(); Response::builder() .status(StatusCode::CREATED) @@ -211,7 +253,7 @@ async fn put_manifest( "Location", format!("/v2/{}/manifests/{}", repository_name, reference), ) - .header("Docker-Content-Digest", format!("sha256:{:x}", digest)) + .header("Docker-Content-Digest", content_digest) .body(Body::empty()) .unwrap() } From 478094abcf6f79ddb4e13e5763f5827208363ae7 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sun, 19 Jun 2022 22:33:44 +0200 Subject: [PATCH 19/51] basic docker login PoC --- planetwars-server/Cargo.toml | 1 + planetwars-server/src/modules/registry.rs | 57 +++++++++++++++++------ 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/planetwars-server/Cargo.toml b/planetwars-server/Cargo.toml index 3e1f05e..d85a011 100644 --- a/planetwars-server/Cargo.toml +++ b/planetwars-server/Cargo.toml @@ -28,6 +28,7 @@ planetwars-matchrunner = { path = "../planetwars-matchrunner" } config = { version = "0.12", features = ["toml"] } thiserror = "1.0.31" sha2 = "0.10" +tokio-util = { version="0.7.3", features=["io"] } # TODO: remove me shlex = "1.1" diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index 6095527..9d71dd7 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -1,9 +1,11 @@ -use axum::body::{Body, StreamBody}; -use axum::extract::{BodyStream, Path, Query}; +use axum::body::{Body, Bytes, StreamBody}; +use axum::extract::{BodyStream, FromRequest, Path, Query, RequestParts, TypedHeader}; use axum::handler::Handler; +use axum::headers::authorization::Basic; +use axum::headers::Authorization; use axum::response::{IntoResponse, Response}; use axum::routing::{get, head, post, put}; -use axum::Router; +use axum::{async_trait, Router}; use hyper::StatusCode; use serde::Serialize; use sha2::{Digest, Sha256}; @@ -16,7 +18,8 @@ use crate::util::gen_alphanumeric; const REGISTRY_PATH: &'static str = "./data/registry"; pub fn registry_service() -> Router { Router::new() - .nest("/v2", registry_api_v2()) + // The docker API requires this trailing slash + .nest("/v2/", registry_api_v2()) .fallback(fallback.into_service()) } @@ -41,8 +44,41 @@ async fn fallback(request: axum::http::Request) -> impl IntoResponse { StatusCode::NOT_FOUND } -// root should return 200 OK to confirm api compliance -async fn root_handler() -> Response { +type AuthorizationHeader = TypedHeader>; + +struct RegistryAuth; + +#[async_trait] +impl FromRequest for RegistryAuth +where + B: Send, +{ + type Rejection = Response>; + + async fn from_request(req: &mut RequestParts) -> Result { + let TypedHeader(Authorization(_basic)) = + AuthorizationHeader::from_request(req).await.map_err(|_| { + let err = RegistryErrors { + errors: vec![RegistryError { + code: "UNAUTHORIZED".to_string(), + message: "please log in".to_string(), + detail: serde_json::Value::Null, + }], + }; + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("Docker-Distribution-API-Version", "registry/2.0") + .header("WWW-Authenticate", "Basic") + .body(axum::body::Full::from(serde_json::to_vec(&err).unwrap())) + .unwrap() + })?; + + Ok(RegistryAuth) + } +} + +async fn root_handler(_auth: RegistryAuth) -> impl IntoResponse { + // root should return 200 OK to confirm api compliance Response::builder() .status(StatusCode::OK) .header("Docker-Distribution-API-Version", "registry/2.0") @@ -89,15 +125,6 @@ async fn get_blob( } async fn blob_upload(Path(repository_name): Path) -> impl IntoResponse { - // let value = json!({ - // "errors": [ - // { - // "code": "UNSUPPORTED", - // "message": "not implemented yet lol", - // } - // ] - // }); - let uuid = gen_alphanumeric(16); tokio::fs::File::create(PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid)) .await From a2a8a41689ad07eb2236ee438e9d01266946008d Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 20 Jun 2022 20:27:51 +0200 Subject: [PATCH 20/51] rename route handler methods --- planetwars-server/src/modules/registry.rs | 27 +++++++++++++---------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index 9d71dd7..61652d9 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -25,17 +25,20 @@ pub fn registry_service() -> Router { fn registry_api_v2() -> Router { Router::new() - .route("/", get(root_handler)) - .route("/:name/blobs/:digest", head(blob_check).get(get_blob)) - .route("/:name/blobs/uploads/", post(blob_upload)) - .route( - "/:name/blobs/uploads/:uuid", - put(put_handler).patch(handle_upload), - ) + .route("/", get(get_root)) .route( "/:name/manifests/:reference", get(get_manifest).put(put_manifest), ) + .route( + "/:name/blobs/:digest", + head(check_blob_exists).get(get_blob), + ) + .route("/:name/blobs/uploads/", post(create_upload)) + .route( + "/:name/blobs/uploads/:uuid", + put(put_upload).patch(patch_upload), + ) } async fn fallback(request: axum::http::Request) -> impl IntoResponse { @@ -77,7 +80,7 @@ where } } -async fn root_handler(_auth: RegistryAuth) -> impl IntoResponse { +async fn get_root(_auth: RegistryAuth) -> impl IntoResponse { // root should return 200 OK to confirm api compliance Response::builder() .status(StatusCode::OK) @@ -98,7 +101,7 @@ pub struct RegistryError { detail: serde_json::Value, } -async fn blob_check( +async fn check_blob_exists( Path((_repository_name, raw_digest)): Path<(String, String)>, ) -> impl IntoResponse { let digest = raw_digest.strip_prefix("sha256:").unwrap(); @@ -124,7 +127,7 @@ async fn get_blob( Ok(stream_body) } -async fn blob_upload(Path(repository_name): Path) -> impl IntoResponse { +async fn create_upload(Path(repository_name): Path) -> impl IntoResponse { let uuid = gen_alphanumeric(16); tokio::fs::File::create(PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid)) .await @@ -144,7 +147,7 @@ async fn blob_upload(Path(repository_name): Path) -> impl IntoResponse { use futures::StreamExt; -async fn handle_upload( +async fn patch_upload( Path((repository_name, uuid)): Path<(String, String)>, mut stream: BodyStream, ) -> impl IntoResponse { @@ -185,7 +188,7 @@ struct UploadParams { digest: String, } -async fn put_handler( +async fn put_upload( Path((repository_name, uuid)): Path<(String, String)>, Query(params): Query, mut stream: BodyStream, From 951cb293111db9ea0947cd65872da744bce92d31 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 20 Jun 2022 22:01:26 +0200 Subject: [PATCH 21/51] upgrade to axum 0.5 --- planetwars-server/Cargo.toml | 2 +- planetwars-server/src/db/matches.rs | 9 ++++++--- planetwars-server/src/lib.rs | 12 ++++++++---- planetwars-server/src/routes/bots.rs | 2 +- planetwars-server/src/routes/users.rs | 6 +++--- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/planetwars-server/Cargo.toml b/planetwars-server/Cargo.toml index d85a011..99161bb 100644 --- a/planetwars-server/Cargo.toml +++ b/planetwars-server/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" futures = "0.3" tokio = { version = "1.15", features = ["full"] } hyper = "0.14" -axum = { version = "0.4", features = ["json", "headers", "multipart"] } +axum = { version = "0.5", features = ["json", "headers", "multipart"] } diesel = { version = "1.4.4", features = ["postgres", "chrono"] } diesel-derive-enum = { version = "1.1", features = ["postgres"] } bb8 = "0.7" diff --git a/planetwars-server/src/db/matches.rs b/planetwars-server/src/db/matches.rs index ee25e85..dfff3cf 100644 --- a/planetwars-server/src/db/matches.rs +++ b/planetwars-server/src/db/matches.rs @@ -160,14 +160,17 @@ pub fn find_match_base(id: i32, conn: &PgConnection) -> QueryResult { } pub enum MatchResult { - Finished { winner: Option } + Finished { winner: Option }, } pub fn save_match_result(id: i32, result: MatchResult, conn: &PgConnection) -> QueryResult<()> { let MatchResult::Finished { winner } = result; diesel::update(matches::table.find(id)) - .set((matches::winner.eq(winner), matches::state.eq(MatchState::Finished))) + .set(( + matches::winner.eq(winner), + matches::state.eq(MatchState::Finished), + )) .execute(conn)?; Ok(()) -} \ No newline at end of file +} diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index e50003c..9c9a03c 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -24,7 +24,7 @@ use axum::{ extract::{Extension, FromRequest, RequestParts}, http::StatusCode, routing::{get, post}, - AddExtensionLayer, Router, + Router, }; // TODO: make these configurable @@ -105,12 +105,16 @@ pub fn get_config() -> Result { .try_deserialize() } -async fn run_registry(_db_pool: DbPool) { +async fn run_registry(db_pool: DbPool) { // TODO: put in config let addr = SocketAddr::from(([127, 0, 0, 1], 9001)); axum::Server::bind(&addr) - .serve(registry_service().into_make_service()) + .serve( + registry_service() + .layer(Extension(db_pool)) + .into_make_service(), + ) .await .unwrap(); } @@ -124,7 +128,7 @@ pub async fn run_app() { let api_service = Router::new() .nest("/api", api()) - .layer(AddExtensionLayer::new(db_pool)) + .layer(Extension(db_pool)) .into_make_service(); // TODO: put in config diff --git a/planetwars-server/src/routes/bots.rs b/planetwars-server/src/routes/bots.rs index 3bbaa1a..df0c4d0 100644 --- a/planetwars-server/src/routes/bots.rs +++ b/planetwars-server/src/routes/bots.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use thiserror; use crate::db::bots::{self, CodeBundle}; -use crate::db::ratings::{RankedBot, self}; +use crate::db::ratings::{self, RankedBot}; use crate::db::users::User; use crate::modules::bots::save_code_bundle; use crate::{DatabaseConnection, BOTS_DIR}; diff --git a/planetwars-server/src/routes/users.rs b/planetwars-server/src/routes/users.rs index 54ddd09..1989904 100644 --- a/planetwars-server/src/routes/users.rs +++ b/planetwars-server/src/routes/users.rs @@ -5,7 +5,7 @@ use axum::extract::{FromRequest, RequestParts, TypedHeader}; use axum::headers::authorization::Bearer; use axum::headers::Authorization; use axum::http::StatusCode; -use axum::response::{Headers, IntoResponse, Response}; +use axum::response::{IntoResponse, Response}; use axum::{async_trait, Json}; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -163,9 +163,9 @@ pub async fn login(conn: DatabaseConnection, params: Json) -> Respo Some(user) => { let session = sessions::create_session(&user, &conn); let user_data: UserData = user.into(); - let headers = Headers(vec![("Token", &session.token)]); + let headers = [("Token", &session.token)]; - (headers, Json(user_data)).into_response() + (StatusCode::OK, headers, Json(user_data)).into_response() } } } From 059cd4fa0e1da6e3d9b2edaae62d2e58e2f37924 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 20 Jun 2022 22:14:15 +0200 Subject: [PATCH 22/51] implement basic auth checking --- planetwars-server/src/modules/registry.rs | 84 +++++++++++++++++------ 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index 61652d9..a866dce 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -1,4 +1,4 @@ -use axum::body::{Body, Bytes, StreamBody}; +use axum::body::{Body, StreamBody}; use axum::extract::{BodyStream, FromRequest, Path, Query, RequestParts, TypedHeader}; use axum::handler::Handler; use axum::headers::authorization::Basic; @@ -14,8 +14,12 @@ use tokio::io::AsyncWriteExt; use tokio_util::io::ReaderStream; use crate::util::gen_alphanumeric; +use crate::DatabaseConnection; + +use crate::db::users::{authenticate_user, Credentials, User}; + +const REGISTRY_PATH: &str = "./data/registry"; -const REGISTRY_PATH: &'static str = "./data/registry"; pub fn registry_service() -> Router { Router::new() // The docker API requires this trailing slash @@ -49,34 +53,61 @@ async fn fallback(request: axum::http::Request) -> impl IntoResponse { type AuthorizationHeader = TypedHeader>; -struct RegistryAuth; +enum RegistryAuth { + User(User), +} + +enum RegistryAuthError { + NoAuthHeader, + InvalidCredentials, +} + +impl IntoResponse for RegistryAuthError { + fn into_response(self) -> Response { + // TODO: create enum for registry errors + let err = RegistryErrors { + errors: vec![RegistryError { + code: "UNAUTHORIZED".to_string(), + message: "please log in".to_string(), + detail: serde_json::Value::Null, + }], + }; + + ( + StatusCode::UNAUTHORIZED, + [ + ("Docker-Distribution-API-Version", "registry/2.0"), + ("WWW-Authenticate", "Basic"), + ], + serde_json::to_string(&err).unwrap(), + ) + .into_response() + } +} #[async_trait] impl FromRequest for RegistryAuth where B: Send, { - type Rejection = Response>; + type Rejection = RegistryAuthError; async fn from_request(req: &mut RequestParts) -> Result { - let TypedHeader(Authorization(_basic)) = - AuthorizationHeader::from_request(req).await.map_err(|_| { - let err = RegistryErrors { - errors: vec![RegistryError { - code: "UNAUTHORIZED".to_string(), - message: "please log in".to_string(), - detail: serde_json::Value::Null, - }], - }; - Response::builder() - .status(StatusCode::UNAUTHORIZED) - .header("Docker-Distribution-API-Version", "registry/2.0") - .header("WWW-Authenticate", "Basic") - .body(axum::body::Full::from(serde_json::to_vec(&err).unwrap())) - .unwrap() - })?; + let db_conn = DatabaseConnection::from_request(req).await.unwrap(); - Ok(RegistryAuth) + let TypedHeader(Authorization(basic)) = AuthorizationHeader::from_request(req) + .await + .map_err(|_| RegistryAuthError::NoAuthHeader)?; + + // TODO: Into would be nice + let credentials = Credentials { + username: basic.username(), + password: basic.password(), + }; + let user = authenticate_user(&credentials, &db_conn) + .ok_or(RegistryAuthError::InvalidCredentials)?; + + Ok(RegistryAuth::User(user)) } } @@ -102,6 +133,7 @@ pub struct RegistryError { } async fn check_blob_exists( + _auth: RegistryAuth, Path((_repository_name, raw_digest)): Path<(String, String)>, ) -> impl IntoResponse { let digest = raw_digest.strip_prefix("sha256:").unwrap(); @@ -114,6 +146,7 @@ async fn check_blob_exists( } async fn get_blob( + _auth: RegistryAuth, Path((_repository_name, raw_digest)): Path<(String, String)>, ) -> impl IntoResponse { let digest = raw_digest.strip_prefix("sha256:").unwrap(); @@ -127,7 +160,10 @@ async fn get_blob( Ok(stream_body) } -async fn create_upload(Path(repository_name): Path) -> impl IntoResponse { +async fn create_upload( + _auth: RegistryAuth, + Path(repository_name): Path, +) -> impl IntoResponse { let uuid = gen_alphanumeric(16); tokio::fs::File::create(PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid)) .await @@ -148,6 +184,7 @@ async fn create_upload(Path(repository_name): Path) -> impl IntoResponse use futures::StreamExt; async fn patch_upload( + _auth: RegistryAuth, Path((repository_name, uuid)): Path<(String, String)>, mut stream: BodyStream, ) -> impl IntoResponse { @@ -189,6 +226,7 @@ struct UploadParams { } async fn put_upload( + _auth: RegistryAuth, Path((repository_name, uuid)): Path<(String, String)>, Query(params): Query, mut stream: BodyStream, @@ -227,6 +265,7 @@ async fn put_upload( } async fn get_manifest( + _auth: RegistryAuth, Path((repository_name, reference)): Path<(String, String)>, ) -> impl IntoResponse { let manifest_path = PathBuf::from(REGISTRY_PATH) @@ -247,6 +286,7 @@ async fn get_manifest( } async fn put_manifest( + _auth: RegistryAuth, Path((repository_name, reference)): Path<(String, String)>, mut stream: BodyStream, ) -> impl IntoResponse { From 381ce040fda929f65c681d4134a03e3143659243 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Tue, 21 Jun 2022 22:45:59 +0200 Subject: [PATCH 23/51] add auth to all registry routes --- planetwars-server/src/modules/registry.rs | 106 +++++++++++++++------- 1 file changed, 75 insertions(+), 31 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index a866dce..8bc3a7d 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -6,6 +6,7 @@ use axum::headers::Authorization; use axum::response::{IntoResponse, Response}; use axum::routing::{get, head, post, put}; use axum::{async_trait, Router}; +use futures::StreamExt; use hyper::StatusCode; use serde::Serialize; use sha2::{Digest, Sha256}; @@ -14,7 +15,7 @@ use tokio::io::AsyncWriteExt; use tokio_util::io::ReaderStream; use crate::util::gen_alphanumeric; -use crate::DatabaseConnection; +use crate::{db, DatabaseConnection}; use crate::db::users::{authenticate_user, Credentials, User}; @@ -133,22 +134,28 @@ pub struct RegistryError { } async fn check_blob_exists( - _auth: RegistryAuth, - Path((_repository_name, raw_digest)): Path<(String, String)>, -) -> impl IntoResponse { + db_conn: DatabaseConnection, + auth: RegistryAuth, + Path((repository_name, raw_digest)): Path<(String, String)>, +) -> Result { + check_access(&repository_name, &auth, &db_conn)?; + let digest = raw_digest.strip_prefix("sha256:").unwrap(); let blob_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); if blob_path.exists() { - StatusCode::OK + Ok(StatusCode::OK) } else { - StatusCode::NOT_FOUND + Err(StatusCode::NOT_FOUND) } } async fn get_blob( - _auth: RegistryAuth, - Path((_repository_name, raw_digest)): Path<(String, String)>, -) -> impl IntoResponse { + db_conn: DatabaseConnection, + auth: RegistryAuth, + Path((repository_name, raw_digest)): Path<(String, String)>, +) -> Result { + check_access(&repository_name, &auth, &db_conn)?; + let digest = raw_digest.strip_prefix("sha256:").unwrap(); let blob_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); if !blob_path.exists() { @@ -161,15 +168,18 @@ async fn get_blob( } async fn create_upload( - _auth: RegistryAuth, + db_conn: DatabaseConnection, + auth: RegistryAuth, Path(repository_name): Path, -) -> impl IntoResponse { +) -> Result { + check_access(&repository_name, &auth, &db_conn)?; + let uuid = gen_alphanumeric(16); tokio::fs::File::create(PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid)) .await .unwrap(); - Response::builder() + Ok(Response::builder() .status(StatusCode::ACCEPTED) .header( "Location", @@ -178,16 +188,17 @@ async fn create_upload( .header("Docker-Upload-UUID", uuid) .header("Range", "bytes=0-0") .body(Body::empty()) - .unwrap() + .unwrap()) } -use futures::StreamExt; - async fn patch_upload( - _auth: RegistryAuth, + db_conn: DatabaseConnection, + auth: RegistryAuth, Path((repository_name, uuid)): Path<(String, String)>, mut stream: BodyStream, -) -> impl IntoResponse { +) -> Result { + check_access(&repository_name, &auth, &db_conn)?; + // let content_length = headers.get("Content-Length").unwrap(); // let content_range = headers.get("Content-Range").unwrap(); // let content_type = headers.get("Content-Type").unwrap(); @@ -207,7 +218,7 @@ async fn patch_upload( len += n_bytes; } - Response::builder() + Ok(Response::builder() .status(StatusCode::ACCEPTED) .header( "Location", @@ -216,7 +227,7 @@ async fn patch_upload( .header("Docker-Upload-UUID", uuid) .header("Range", format!("0-{}", len)) .body(Body::empty()) - .unwrap() + .unwrap()) } use serde::Deserialize; @@ -226,11 +237,14 @@ struct UploadParams { } async fn put_upload( - _auth: RegistryAuth, + db_conn: DatabaseConnection, + auth: RegistryAuth, Path((repository_name, uuid)): Path<(String, String)>, Query(params): Query, mut stream: BodyStream, -) -> impl IntoResponse { +) -> Result { + check_access(&repository_name, &auth, &db_conn)?; + let mut _len = 0; let upload_path = PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid); let mut file = tokio::fs::OpenOptions::new() @@ -251,7 +265,7 @@ async fn put_upload( let target_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); tokio::fs::rename(&upload_path, &target_path).await.unwrap(); - Response::builder() + Ok(Response::builder() .status(StatusCode::CREATED) .header( "Location", @@ -261,13 +275,16 @@ async fn put_upload( // .header("Range", format!("0-{}", len)) .header("Docker-Content-Digest", digest) .body(Body::empty()) - .unwrap() + .unwrap()) } async fn get_manifest( - _auth: RegistryAuth, + db_conn: DatabaseConnection, + auth: RegistryAuth, Path((repository_name, reference)): Path<(String, String)>, -) -> impl IntoResponse { +) -> Result { + check_access(&repository_name, &auth, &db_conn)?; + let manifest_path = PathBuf::from(REGISTRY_PATH) .join("manifests") .join(&repository_name) @@ -278,18 +295,21 @@ async fn get_manifest( let manifest: serde_json::Map = serde_json::from_slice(&data).unwrap(); let media_type = manifest.get("mediaType").unwrap().as_str().unwrap(); - Response::builder() + Ok(Response::builder() .status(StatusCode::OK) .header("Content-Type", media_type) .body(axum::body::Full::from(data)) - .unwrap() + .unwrap()) } async fn put_manifest( - _auth: RegistryAuth, + db_conn: DatabaseConnection, + auth: RegistryAuth, Path((repository_name, reference)): Path<(String, String)>, mut stream: BodyStream, -) -> impl IntoResponse { +) -> Result { + check_access(&repository_name, &auth, &db_conn)?; + let repository_dir = PathBuf::from(REGISTRY_PATH) .join("manifests") .join(&repository_name); @@ -317,7 +337,7 @@ async fn put_manifest( let digest_path = repository_dir.join(&content_digest).with_extension("json"); tokio::fs::copy(manifest_path, digest_path).await.unwrap(); - Response::builder() + Ok(Response::builder() .status(StatusCode::CREATED) .header( "Location", @@ -325,5 +345,29 @@ async fn put_manifest( ) .header("Docker-Content-Digest", content_digest) .body(Body::empty()) - .unwrap() + .unwrap()) +} + +fn check_access( + repository_name: &str, + auth: &RegistryAuth, + db_conn: &DatabaseConnection, +) -> Result<(), StatusCode> { + use diesel::OptionalExtension; + + let res = db::bots::find_bot_by_name(repository_name, db_conn) + .optional() + .expect("could not run query"); + + match res { + None => Ok(()), // name has not been claimed yet (TODO: verify its validity) + Some(existing_bot) => { + let RegistryAuth::User(user) = auth; + if existing_bot.owner_id == Some(user.id) { + Ok(()) + } else { + Err(StatusCode::FORBIDDEN) + } + } + } } From f6fca3818a5f5e32afd02280c04fdbe77972075f Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Fri, 24 Jun 2022 19:32:22 +0200 Subject: [PATCH 24/51] don't allow accessing non-existing repositories --- planetwars-server/src/modules/registry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index 8bc3a7d..c0e12d0 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -360,7 +360,7 @@ fn check_access( .expect("could not run query"); match res { - None => Ok(()), // name has not been claimed yet (TODO: verify its validity) + None => Err(StatusCode::FORBIDDEN), Some(existing_bot) => { let RegistryAuth::User(user) = auth; if existing_bot.owner_id == Some(user.id) { From d7e4a1fd5cb1ab7438d281de6dfe26013623dc6b Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 27 Jun 2022 21:20:05 +0200 Subject: [PATCH 25/51] implement admin login --- planetwars-server/src/modules/registry.rs | 41 ++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index c0e12d0..346f5d9 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -52,10 +52,15 @@ async fn fallback(request: axum::http::Request) -> impl IntoResponse { StatusCode::NOT_FOUND } +const ADMIN_USERNAME: &str = "admin"; +// TODO: put this in some configuration +const ADMIN_PASSWORD: &str = "supersecretpassword"; + type AuthorizationHeader = TypedHeader>; enum RegistryAuth { User(User), + Admin, } enum RegistryAuthError { @@ -94,8 +99,6 @@ where type Rejection = RegistryAuthError; async fn from_request(req: &mut RequestParts) -> Result { - let db_conn = DatabaseConnection::from_request(req).await.unwrap(); - let TypedHeader(Authorization(basic)) = AuthorizationHeader::from_request(req) .await .map_err(|_| RegistryAuthError::NoAuthHeader)?; @@ -105,10 +108,20 @@ where username: basic.username(), password: basic.password(), }; - let user = authenticate_user(&credentials, &db_conn) - .ok_or(RegistryAuthError::InvalidCredentials)?; - Ok(RegistryAuth::User(user)) + if credentials.username == ADMIN_USERNAME { + if credentials.password == ADMIN_PASSWORD { + Ok(RegistryAuth::Admin) + } else { + Err(RegistryAuthError::InvalidCredentials) + } + } else { + let db_conn = DatabaseConnection::from_request(req).await.unwrap(); + let user = authenticate_user(&credentials, &db_conn) + .ok_or(RegistryAuthError::InvalidCredentials)?; + + Ok(RegistryAuth::User(user)) + } } } @@ -348,6 +361,8 @@ async fn put_manifest( .unwrap()) } +/// Ensure that the accessed repository exists +/// and the user is allowed to access ti fn check_access( repository_name: &str, auth: &RegistryAuth, @@ -355,15 +370,17 @@ fn check_access( ) -> Result<(), StatusCode> { use diesel::OptionalExtension; - let res = db::bots::find_bot_by_name(repository_name, db_conn) + // TODO: it would be nice to provide the found repository + // to the route handlers + let bot = db::bots::find_bot_by_name(repository_name, db_conn) .optional() - .expect("could not run query"); + .expect("could not run query") + .ok_or(StatusCode::NOT_FOUND)?; - match res { - None => Err(StatusCode::FORBIDDEN), - Some(existing_bot) => { - let RegistryAuth::User(user) = auth; - if existing_bot.owner_id == Some(user.id) { + match &auth { + RegistryAuth::Admin => Ok(()), + RegistryAuth::User(user) => { + if bot.owner_id == Some(user.id) { Ok(()) } else { Err(StatusCode::FORBIDDEN) From 4d1c0a3289a295ea27eea51ec0a91c4229a92edc Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Thu, 30 Jun 2022 20:28:37 +0200 Subject: [PATCH 26/51] make sure that all pushed data is actually written --- planetwars-server/src/modules/registry.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index 346f5d9..7adb764 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -227,8 +227,8 @@ async fn patch_upload( .await .unwrap(); while let Some(Ok(chunk)) = stream.next().await { - let n_bytes = file.write(&chunk).await.unwrap(); - len += n_bytes; + file.write_all(&chunk).await.unwrap(); + len += chunk.len(); } Ok(Response::builder() @@ -270,9 +270,10 @@ async fn put_upload( .unwrap(); while let Some(Ok(chunk)) = stream.next().await { - let n_bytes = file.write(&chunk).await.unwrap(); - _len += n_bytes; + file.write_all(&chunk).await.unwrap(); + _len += chunk.len(); } + let digest = params.digest.strip_prefix("sha256:").unwrap(); // TODO: check the digest let target_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); From 419029738dd914bd0c8edd9c8d4365cac2d53ad7 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Thu, 30 Jun 2022 20:49:10 +0200 Subject: [PATCH 27/51] verify blob digest on upload --- planetwars-server/src/modules/registry.rs | 26 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index 7adb764..6e29878 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -125,6 +125,15 @@ where } } +// Since async file io just calls spawn_blocking internally, it does not really make sense +// to make this an async function +fn file_sha256_digest(path: &std::path::Path) -> std::io::Result { + let mut file = std::fs::File::open(path)?; + let mut hasher = Sha256::new(); + let _n = std::io::copy(&mut file, &mut hasher)?; + Ok(format!("{:x}", hasher.finalize())) +} + async fn get_root(_auth: RegistryAuth) -> impl IntoResponse { // root should return 200 OK to confirm api compliance Response::builder() @@ -273,9 +282,15 @@ async fn put_upload( file.write_all(&chunk).await.unwrap(); _len += chunk.len(); } + file.flush().await.unwrap(); + + let expected_digest = params.digest.strip_prefix("sha256:").unwrap(); + let digest = file_sha256_digest(&upload_path).unwrap(); + if digest != expected_digest { + // TODO: return a docker error body + return Err(StatusCode::BAD_REQUEST); + } - let digest = params.digest.strip_prefix("sha256:").unwrap(); - // TODO: check the digest let target_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); tokio::fs::rename(&upload_path, &target_path).await.unwrap(); @@ -286,8 +301,9 @@ async fn put_upload( format!("/v2/{}/blobs/{}", repository_name, digest), ) .header("Docker-Upload-UUID", uuid) - // .header("Range", format!("0-{}", len)) - .header("Docker-Content-Digest", digest) + // TODO: set content-range + // .header("Content-Range", format!("0-{}", len)) + .header("Docker-Content-Digest", params.digest) .body(Body::empty()) .unwrap()) } @@ -342,7 +358,7 @@ async fn put_manifest( .unwrap(); while let Some(Ok(chunk)) = stream.next().await { hasher.update(&chunk); - file.write(&chunk).await.unwrap(); + file.write_all(&chunk).await.unwrap(); } } let digest = hasher.finalize(); From 7b88bb0502f67e913b6e8bca394428fd2df45cc2 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Fri, 1 Jul 2022 20:45:26 +0200 Subject: [PATCH 28/51] use file metadata for returning data ranges and lengths --- planetwars-server/src/modules/registry.rs | 30 ++++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index 6e29878..d73e7e9 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -134,6 +134,13 @@ fn file_sha256_digest(path: &std::path::Path) -> std::io::Result { Ok(format!("{:x}", hasher.finalize())) } +/// Get the index of the last byte in a file +async fn last_byte_pos(file: &tokio::fs::File) -> std::io::Result { + let n_bytes = file.metadata().await?.len(); + let pos = if n_bytes == 0 { 0 } else { n_bytes - 1 }; + Ok(pos) +} + async fn get_root(_auth: RegistryAuth) -> impl IntoResponse { // root should return 200 OK to confirm api compliance Response::builder() @@ -165,7 +172,8 @@ async fn check_blob_exists( let digest = raw_digest.strip_prefix("sha256:").unwrap(); let blob_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); if blob_path.exists() { - Ok(StatusCode::OK) + let metadata = std::fs::metadata(&blob_path).unwrap(); + Ok((StatusCode::OK, [("Content-Length", metadata.len())])) } else { Err(StatusCode::NOT_FOUND) } @@ -221,11 +229,7 @@ async fn patch_upload( ) -> Result { check_access(&repository_name, &auth, &db_conn)?; - // let content_length = headers.get("Content-Length").unwrap(); - // let content_range = headers.get("Content-Range").unwrap(); - // let content_type = headers.get("Content-Type").unwrap(); - // assert!(content_type == "application/octet-stream"); - let mut len = 0; + // TODO: support content range header in request let upload_path = PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid); let mut file = tokio::fs::OpenOptions::new() .read(false) @@ -237,9 +241,10 @@ async fn patch_upload( .unwrap(); while let Some(Ok(chunk)) = stream.next().await { file.write_all(&chunk).await.unwrap(); - len += chunk.len(); } + let last_byte = last_byte_pos(&file).await.unwrap(); + Ok(Response::builder() .status(StatusCode::ACCEPTED) .header( @@ -247,7 +252,8 @@ async fn patch_upload( format!("/v2/{}/blobs/uploads/{}", repository_name, uuid), ) .header("Docker-Upload-UUID", uuid) - .header("Range", format!("0-{}", len)) + // range indicating current progress of the upload + .header("Range", format!("0-{}", last_byte)) .body(Body::empty()) .unwrap()) } @@ -267,7 +273,6 @@ async fn put_upload( ) -> Result { check_access(&repository_name, &auth, &db_conn)?; - let mut _len = 0; let upload_path = PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid); let mut file = tokio::fs::OpenOptions::new() .read(false) @@ -278,11 +283,12 @@ async fn put_upload( .await .unwrap(); + let range_begin = last_byte_pos(&file).await.unwrap(); while let Some(Ok(chunk)) = stream.next().await { file.write_all(&chunk).await.unwrap(); - _len += chunk.len(); } file.flush().await.unwrap(); + let range_end = last_byte_pos(&file).await.unwrap(); let expected_digest = params.digest.strip_prefix("sha256:").unwrap(); let digest = file_sha256_digest(&upload_path).unwrap(); @@ -301,8 +307,8 @@ async fn put_upload( format!("/v2/{}/blobs/{}", repository_name, digest), ) .header("Docker-Upload-UUID", uuid) - // TODO: set content-range - // .header("Content-Range", format!("0-{}", len)) + // content range for bytes that were in the body of this request + .header("Content-Range", format!("{}-{}", range_begin, range_end)) .header("Docker-Content-Digest", params.digest) .body(Body::empty()) .unwrap()) From bbed87755419f97b0ee8967617af0c6573c168af Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 4 Jul 2022 20:11:29 +0200 Subject: [PATCH 29/51] cleanup and comments --- planetwars-server/src/modules/registry.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index d73e7e9..c8ec4fa 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -1,6 +1,7 @@ +// TODO: this module is functional, but it needs a good refactor for proper error handling. + use axum::body::{Body, StreamBody}; use axum::extract::{BodyStream, FromRequest, Path, Query, RequestParts, TypedHeader}; -use axum::handler::Handler; use axum::headers::authorization::Basic; use axum::headers::Authorization; use axum::response::{IntoResponse, Response}; @@ -19,13 +20,13 @@ use crate::{db, DatabaseConnection}; use crate::db::users::{authenticate_user, Credentials, User}; +// TODO: put this in a config file const REGISTRY_PATH: &str = "./data/registry"; pub fn registry_service() -> Router { Router::new() // The docker API requires this trailing slash .nest("/v2/", registry_api_v2()) - .fallback(fallback.into_service()) } fn registry_api_v2() -> Router { @@ -46,12 +47,6 @@ fn registry_api_v2() -> Router { ) } -async fn fallback(request: axum::http::Request) -> impl IntoResponse { - // for debugging - println!("no route for {} {}", request.method(), request.uri()); - StatusCode::NOT_FOUND -} - const ADMIN_USERNAME: &str = "admin"; // TODO: put this in some configuration const ADMIN_PASSWORD: &str = "supersecretpassword"; From ea05674b4473d9399f5aa6dca982ae73aad0ebcf Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 4 Jul 2022 22:33:35 +0200 Subject: [PATCH 30/51] remove obsolete create match route --- planetwars-server/src/lib.rs | 2 +- planetwars-server/src/routes/matches.rs | 103 +----------------------- 2 files changed, 4 insertions(+), 101 deletions(-) diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index 9c9a03c..8798945 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -85,7 +85,7 @@ pub fn api() -> Router { ) .route( "/matches", - get(routes::matches::list_matches).post(routes::matches::play_match), + get(routes::matches::list_matches), ) .route("/matches/:match_id", get(routes::matches::get_match_data)) .route( diff --git a/planetwars-server/src/routes/matches.rs b/planetwars-server/src/routes/matches.rs index 874c775..5f95ce9 100644 --- a/planetwars-server/src/routes/matches.rs +++ b/planetwars-server/src/routes/matches.rs @@ -1,102 +1,13 @@ use std::path::PathBuf; - -use axum::{ - extract::{Extension, Path}, - Json, -}; +use axum::{extract::Path, Json}; use hyper::StatusCode; -use planetwars_matchrunner::{docker_runner::DockerBotSpec, run_match, MatchConfig, MatchPlayer}; -use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; use crate::{ - db::{ - bots, - matches::{self, MatchState}, - users::User, - }, - ConnectionPool, DatabaseConnection, BOTS_DIR, MAPS_DIR, MATCHES_DIR, + db::matches::{self, MatchState}, + DatabaseConnection, MATCHES_DIR, }; -#[derive(Serialize, Deserialize, Debug)] -pub struct MatchParams { - // Just bot ids for now - players: Vec, -} - -pub async fn play_match( - _user: User, - Extension(pool): Extension, - Json(params): Json, -) -> Result<(), StatusCode> { - let conn = pool.get().await.expect("could not get database connection"); - let map_path = PathBuf::from(MAPS_DIR).join("hex.json"); - - let slug: String = rand::thread_rng() - .sample_iter(&Alphanumeric) - .take(16) - .map(char::from) - .collect(); - let log_file_name = format!("{}.log", slug); - - let mut players = Vec::new(); - let mut bot_ids = Vec::new(); - for bot_name in params.players { - let bot = bots::find_bot(bot_name, &conn).map_err(|_| StatusCode::BAD_REQUEST)?; - let code_bundle = - bots::active_code_bundle(bot.id, &conn).map_err(|_| StatusCode::BAD_REQUEST)?; - - let bundle_path = PathBuf::from(BOTS_DIR).join(&code_bundle.path); - let bot_config: BotConfig = std::fs::read_to_string(bundle_path.join("botconfig.toml")) - .and_then(|config_str| toml::from_str(&config_str).map_err(|e| e.into())) - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - - players.push(MatchPlayer { - bot_spec: Box::new(DockerBotSpec { - code_path: PathBuf::from(BOTS_DIR).join(code_bundle.path), - image: "python:3.10-slim-buster".to_string(), - argv: shlex::split(&bot_config.run_command) - .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?, - }), - }); - - bot_ids.push(matches::MatchPlayerData { - code_bundle_id: Some(code_bundle.id), - }); - } - - let match_config = MatchConfig { - map_name: "hex".to_string(), - map_path, - log_path: PathBuf::from(MATCHES_DIR).join(&log_file_name), - players, - }; - - tokio::spawn(run_match_task( - match_config, - log_file_name, - bot_ids, - pool.clone(), - )); - Ok(()) -} - -async fn run_match_task( - config: MatchConfig, - log_file_name: String, - match_players: Vec, - pool: ConnectionPool, -) { - let match_data = matches::NewMatch { - state: MatchState::Finished, - log_path: &log_file_name, - }; - - run_match(config).await; - let conn = pool.get().await.expect("could not get database connection"); - matches::create_match(&match_data, &match_players, &conn).expect("could not create match"); -} - #[derive(Serialize, Deserialize)] pub struct ApiMatch { id: i32, @@ -135,14 +46,6 @@ pub fn match_data_to_api(data: matches::FullMatchData) -> ApiMatch { } } -// TODO: this is duplicated from planetwars-cli -// clean this up and move to matchrunner crate -#[derive(Serialize, Deserialize)] -pub struct BotConfig { - pub name: String, - pub run_command: String, - pub build_command: Option, -} pub async fn get_match_data( Path(match_id): Path, From 8a47b948eb88f1e0ff649880a70f9416306bac92 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Tue, 5 Jul 2022 20:33:39 +0200 Subject: [PATCH 31/51] migrate code_bundles to bot_versions --- .../2022-07-04-200149_code_bundle_to_bot_version/down.sql | 4 ++++ .../2022-07-04-200149_code_bundle_to_bot_version/up.sql | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/down.sql create mode 100644 planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/up.sql diff --git a/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/down.sql b/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/down.sql new file mode 100644 index 0000000..bc2d7d6 --- /dev/null +++ b/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/down.sql @@ -0,0 +1,4 @@ +ALTER TABLE bot_versions DROP COLUMN container_digest; +ALTER TABLE bot_versions RENAME COLUMN code_bundle_path to path; +ALTER TABLE bot_versions ALTER COLUMN path SET NOT NULL; +ALTER TABLE bot_versions RENAME TO code_bundles; diff --git a/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/up.sql b/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/up.sql new file mode 100644 index 0000000..0b28df7 --- /dev/null +++ b/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/up.sql @@ -0,0 +1,4 @@ +ALTER TABLE code_bundles RENAME TO bot_versions; +ALTER TABLE bot_versions RENAME COLUMN path to code_bundle_path; +ALTER TABLE bot_versions ALTER COLUMN code_bundle_path DROP NOT NULL; +ALTER TABLE bot_versions ADD COLUMN container_digest TEXT; From b3df5c6f8cc59e099a2f1db3df8089af4abca02e Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Tue, 5 Jul 2022 20:34:20 +0200 Subject: [PATCH 32/51] migrate code_bundles to bot_versions --- planetwars-server/src/db/bots.rs | 21 ++++++++-------- planetwars-server/src/db/matches.rs | 14 +++++------ planetwars-server/src/lib.rs | 5 +--- planetwars-server/src/modules/bots.rs | 2 +- planetwars-server/src/modules/matches.rs | 5 ++-- planetwars-server/src/modules/ranking.rs | 2 +- planetwars-server/src/routes/bots.rs | 2 +- planetwars-server/src/routes/matches.rs | 3 +-- planetwars-server/src/schema.rs | 31 ++++++++++++------------ 9 files changed, 42 insertions(+), 43 deletions(-) diff --git a/planetwars-server/src/db/bots.rs b/planetwars-server/src/db/bots.rs index 108c692..964deaa 100644 --- a/planetwars-server/src/db/bots.rs +++ b/planetwars-server/src/db/bots.rs @@ -1,7 +1,7 @@ use diesel::prelude::*; use serde::{Deserialize, Serialize}; -use crate::schema::{bots, code_bundles}; +use crate::schema::{bot_versions, bots}; use chrono; #[derive(Insertable)] @@ -44,38 +44,39 @@ pub fn find_all_bots(conn: &PgConnection) -> QueryResult> { } #[derive(Insertable)] -#[table_name = "code_bundles"] +#[table_name = "bot_versions"] pub struct NewCodeBundle<'a> { pub bot_id: Option, - pub path: &'a str, + pub code_bundle_path: &'a str, } #[derive(Queryable, Serialize, Deserialize, Debug)] pub struct CodeBundle { pub id: i32, pub bot_id: Option, - pub path: String, + pub code_bundle_path: Option, pub created_at: chrono::NaiveDateTime, + pub container_digest: Option, } pub fn create_code_bundle( new_code_bundle: &NewCodeBundle, conn: &PgConnection, ) -> QueryResult { - diesel::insert_into(code_bundles::table) + diesel::insert_into(bot_versions::table) .values(new_code_bundle) .get_result(conn) } pub fn find_bot_code_bundles(bot_id: i32, conn: &PgConnection) -> QueryResult> { - code_bundles::table - .filter(code_bundles::bot_id.eq(bot_id)) + bot_versions::table + .filter(bot_versions::bot_id.eq(bot_id)) .get_results(conn) } pub fn active_code_bundle(bot_id: i32, conn: &PgConnection) -> QueryResult { - code_bundles::table - .filter(code_bundles::bot_id.eq(bot_id)) - .order(code_bundles::created_at.desc()) + bot_versions::table + .filter(bot_versions::bot_id.eq(bot_id)) + .order(bot_versions::created_at.desc()) .first(conn) } diff --git a/planetwars-server/src/db/matches.rs b/planetwars-server/src/db/matches.rs index 54fd113..d9d893c 100644 --- a/planetwars-server/src/db/matches.rs +++ b/planetwars-server/src/db/matches.rs @@ -6,7 +6,7 @@ use diesel::{ }; use diesel::{Connection, GroupedBy, PgConnection, QueryResult}; -use crate::schema::{bots, code_bundles, match_players, matches}; +use crate::schema::{bot_versions, bots, match_players, matches}; use super::bots::{Bot, CodeBundle}; @@ -93,10 +93,10 @@ pub fn list_matches(conn: &PgConnection) -> QueryResult> { let match_players = MatchPlayer::belonging_to(&matches) .left_join( - code_bundles::table - .on(match_players::code_bundle_id.eq(code_bundles::id.nullable())), + bot_versions::table + .on(match_players::code_bundle_id.eq(bot_versions::id.nullable())), ) - .left_join(bots::table.on(code_bundles::bot_id.eq(bots::id.nullable()))) + .left_join(bots::table.on(bot_versions::bot_id.eq(bots::id.nullable()))) .load::(conn)? .grouped_by(&matches); @@ -146,10 +146,10 @@ pub fn find_match(id: i32, conn: &PgConnection) -> QueryResult { let match_players = MatchPlayer::belonging_to(&match_base) .left_join( - code_bundles::table - .on(match_players::code_bundle_id.eq(code_bundles::id.nullable())), + bot_versions::table + .on(match_players::code_bundle_id.eq(bot_versions::id.nullable())), ) - .left_join(bots::table.on(code_bundles::bot_id.eq(bots::id.nullable()))) + .left_join(bots::table.on(bot_versions::bot_id.eq(bots::id.nullable()))) .load::(conn)?; let res = FullMatchData { diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index 8798945..7076604 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -83,10 +83,7 @@ pub fn api() -> Router { "/bots/:bot_id/upload", post(routes::bots::upload_code_multipart), ) - .route( - "/matches", - get(routes::matches::list_matches), - ) + .route("/matches", get(routes::matches::list_matches)) .route("/matches/:match_id", get(routes::matches::get_match_data)) .route( "/matches/:match_id/log", diff --git a/planetwars-server/src/modules/bots.rs b/planetwars-server/src/modules/bots.rs index 843e48d..ddc1589 100644 --- a/planetwars-server/src/modules/bots.rs +++ b/planetwars-server/src/modules/bots.rs @@ -17,7 +17,7 @@ pub fn save_code_bundle( let new_code_bundle = db::bots::NewCodeBundle { bot_id, - path: &bundle_name, + code_bundle_path: &bundle_name, }; db::bots::create_code_bundle(&new_code_bundle, conn) } diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index 6d9261d..7d6a1dc 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -98,7 +98,8 @@ impl RunMatch { } pub fn code_bundle_to_botspec(code_bundle: &db::bots::CodeBundle) -> Box { - let bundle_path = PathBuf::from(BOTS_DIR).join(&code_bundle.path); + // TODO: get rid of this unwrap + let bundle_path = PathBuf::from(BOTS_DIR).join(code_bundle.code_bundle_path.as_ref().unwrap()); Box::new(DockerBotSpec { code_path: bundle_path, @@ -126,5 +127,5 @@ async fn run_match_task( db::matches::save_match_result(match_id, result, &conn).expect("could not save match result"); - return outcome; + outcome } diff --git a/planetwars-server/src/modules/ranking.rs b/planetwars-server/src/modules/ranking.rs index 72156ee..b1ad0da 100644 --- a/planetwars-server/src/modules/ranking.rs +++ b/planetwars-server/src/modules/ranking.rs @@ -1,8 +1,8 @@ use crate::{db::bots::Bot, DbPool}; use crate::db; -use diesel::{PgConnection, QueryResult}; use crate::modules::matches::{MatchPlayer, RunMatch}; +use diesel::{PgConnection, QueryResult}; use rand::seq::SliceRandom; use std::collections::HashMap; use std::mem; diff --git a/planetwars-server/src/routes/bots.rs b/planetwars-server/src/routes/bots.rs index df0c4d0..6d5d7df 100644 --- a/planetwars-server/src/routes/bots.rs +++ b/planetwars-server/src/routes/bots.rs @@ -215,7 +215,7 @@ pub async fn upload_code_multipart( let bundle = bots::NewCodeBundle { bot_id: Some(bot.id), - path: &folder_name, + code_bundle_path: &folder_name, }; let code_bundle = bots::create_code_bundle(&bundle, &conn).expect("Failed to create code bundle"); diff --git a/planetwars-server/src/routes/matches.rs b/planetwars-server/src/routes/matches.rs index 5f95ce9..0c1bee4 100644 --- a/planetwars-server/src/routes/matches.rs +++ b/planetwars-server/src/routes/matches.rs @@ -1,7 +1,7 @@ -use std::path::PathBuf; use axum::{extract::Path, Json}; use hyper::StatusCode; use serde::{Deserialize, Serialize}; +use std::path::PathBuf; use crate::{ db::matches::{self, MatchState}, @@ -46,7 +46,6 @@ pub fn match_data_to_api(data: matches::FullMatchData) -> ApiMatch { } } - pub async fn get_match_data( Path(match_id): Path, conn: DatabaseConnection, diff --git a/planetwars-server/src/schema.rs b/planetwars-server/src/schema.rs index 92acc8e..d632a32 100644 --- a/planetwars-server/src/schema.rs +++ b/planetwars-server/src/schema.rs @@ -1,6 +1,19 @@ // This file is autogenerated by diesel #![allow(unused_imports)] +table! { + use diesel::sql_types::*; + use crate::db_types::*; + + bot_versions (id) { + id -> Int4, + bot_id -> Nullable, + code_bundle_path -> Nullable, + created_at -> Timestamp, + container_digest -> Nullable, + } +} + table! { use diesel::sql_types::*; use crate::db_types::*; @@ -12,18 +25,6 @@ table! { } } -table! { - use diesel::sql_types::*; - use crate::db_types::*; - - code_bundles (id) { - id -> Int4, - bot_id -> Nullable, - path -> Text, - created_at -> Timestamp, - } -} - table! { use diesel::sql_types::*; use crate::db_types::*; @@ -81,16 +82,16 @@ table! { } } +joinable!(bot_versions -> bots (bot_id)); joinable!(bots -> users (owner_id)); -joinable!(code_bundles -> bots (bot_id)); -joinable!(match_players -> code_bundles (code_bundle_id)); +joinable!(match_players -> bot_versions (code_bundle_id)); joinable!(match_players -> matches (match_id)); joinable!(ratings -> bots (bot_id)); joinable!(sessions -> users (user_id)); allow_tables_to_appear_in_same_query!( + bot_versions, bots, - code_bundles, match_players, matches, ratings, From d7b7585dd70f9d41184cf88c2ecbd88341898c38 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Wed, 6 Jul 2022 22:41:27 +0200 Subject: [PATCH 33/51] rename code_bundle to bot_version --- .../down.sql | 4 +++- .../up.sql | 2 ++ planetwars-server/src/db/bots.rs | 8 ++++---- planetwars-server/src/db/matches.rs | 12 ++++++------ planetwars-server/src/modules/bot_api.rs | 2 +- planetwars-server/src/modules/bots.rs | 2 +- planetwars-server/src/modules/matches.rs | 4 ++-- planetwars-server/src/modules/ranking.rs | 2 +- planetwars-server/src/routes/bots.rs | 8 ++++---- planetwars-server/src/routes/demo.rs | 14 +++++++------- planetwars-server/src/routes/matches.rs | 4 ++-- planetwars-server/src/schema.rs | 4 ++-- 12 files changed, 35 insertions(+), 31 deletions(-) diff --git a/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/down.sql b/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/down.sql index bc2d7d6..89058fe 100644 --- a/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/down.sql +++ b/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/down.sql @@ -1,4 +1,6 @@ +ALTER TABLE match_players RENAME COLUMN bot_version_id TO code_bundle_id; + ALTER TABLE bot_versions DROP COLUMN container_digest; -ALTER TABLE bot_versions RENAME COLUMN code_bundle_path to path; +ALTER TABLE bot_versions RENAME COLUMN code_bundle_path TO path; ALTER TABLE bot_versions ALTER COLUMN path SET NOT NULL; ALTER TABLE bot_versions RENAME TO code_bundles; diff --git a/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/up.sql b/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/up.sql index 0b28df7..91afc0c 100644 --- a/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/up.sql +++ b/planetwars-server/migrations/2022-07-04-200149_code_bundle_to_bot_version/up.sql @@ -2,3 +2,5 @@ ALTER TABLE code_bundles RENAME TO bot_versions; ALTER TABLE bot_versions RENAME COLUMN path to code_bundle_path; ALTER TABLE bot_versions ALTER COLUMN code_bundle_path DROP NOT NULL; ALTER TABLE bot_versions ADD COLUMN container_digest TEXT; + +ALTER TABLE match_players RENAME COLUMN code_bundle_id TO bot_version_id; diff --git a/planetwars-server/src/db/bots.rs b/planetwars-server/src/db/bots.rs index 964deaa..53c11b1 100644 --- a/planetwars-server/src/db/bots.rs +++ b/planetwars-server/src/db/bots.rs @@ -51,7 +51,7 @@ pub struct NewCodeBundle<'a> { } #[derive(Queryable, Serialize, Deserialize, Debug)] -pub struct CodeBundle { +pub struct BotVersion { pub id: i32, pub bot_id: Option, pub code_bundle_path: Option, @@ -62,19 +62,19 @@ pub struct CodeBundle { pub fn create_code_bundle( new_code_bundle: &NewCodeBundle, conn: &PgConnection, -) -> QueryResult { +) -> QueryResult { diesel::insert_into(bot_versions::table) .values(new_code_bundle) .get_result(conn) } -pub fn find_bot_code_bundles(bot_id: i32, conn: &PgConnection) -> QueryResult> { +pub fn find_bot_versions(bot_id: i32, conn: &PgConnection) -> QueryResult> { bot_versions::table .filter(bot_versions::bot_id.eq(bot_id)) .get_results(conn) } -pub fn active_code_bundle(bot_id: i32, conn: &PgConnection) -> QueryResult { +pub fn active_bot_version(bot_id: i32, conn: &PgConnection) -> QueryResult { bot_versions::table .filter(bot_versions::bot_id.eq(bot_id)) .order(bot_versions::created_at.desc()) diff --git a/planetwars-server/src/db/matches.rs b/planetwars-server/src/db/matches.rs index d9d893c..6590a37 100644 --- a/planetwars-server/src/db/matches.rs +++ b/planetwars-server/src/db/matches.rs @@ -8,7 +8,7 @@ use diesel::{Connection, GroupedBy, PgConnection, QueryResult}; use crate::schema::{bot_versions, bots, match_players, matches}; -use super::bots::{Bot, CodeBundle}; +use super::bots::{Bot, BotVersion}; #[derive(Insertable)] #[table_name = "matches"] @@ -25,7 +25,7 @@ pub struct NewMatchPlayer { /// player id within the match pub player_id: i32, /// id of the bot behind this player - pub code_bundle_id: Option, + pub bot_version_id: Option, } #[derive(Queryable, Identifiable)] @@ -67,7 +67,7 @@ pub fn create_match( .map(|(num, player_data)| NewMatchPlayer { match_id: match_base.id, player_id: num as i32, - code_bundle_id: player_data.code_bundle_id, + bot_version_id: player_data.code_bundle_id, }) .collect::>(); @@ -94,7 +94,7 @@ pub fn list_matches(conn: &PgConnection) -> QueryResult> { let match_players = MatchPlayer::belonging_to(&matches) .left_join( bot_versions::table - .on(match_players::code_bundle_id.eq(bot_versions::id.nullable())), + .on(match_players::bot_version_id.eq(bot_versions::id.nullable())), ) .left_join(bots::table.on(bot_versions::bot_id.eq(bots::id.nullable()))) .load::(conn)? @@ -123,7 +123,7 @@ pub struct FullMatchData { // #[primary_key(base.match_id, base::player_id)] pub struct FullMatchPlayerData { pub base: MatchPlayer, - pub code_bundle: Option, + pub bot_version: Option, pub bot: Option, } @@ -147,7 +147,7 @@ pub fn find_match(id: i32, conn: &PgConnection) -> QueryResult { let match_players = MatchPlayer::belonging_to(&match_base) .left_join( bot_versions::table - .on(match_players::code_bundle_id.eq(bot_versions::id.nullable())), + .on(match_players::bot_version_id.eq(bot_versions::id.nullable())), ) .left_join(bots::table.on(bot_versions::bot_id.eq(bots::id.nullable()))) .load::(conn)?; diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 0ecbf71..6324010 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -104,7 +104,7 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { let opponent = db::bots::find_bot_by_name(&match_request.opponent_name, &conn) .map_err(|_| Status::not_found("opponent not found"))?; - let opponent_code_bundle = db::bots::active_code_bundle(opponent.id, &conn) + let opponent_code_bundle = db::bots::active_bot_version(opponent.id, &conn) .map_err(|_| Status::not_found("opponent has no code"))?; let player_key = gen_alphanumeric(32); diff --git a/planetwars-server/src/modules/bots.rs b/planetwars-server/src/modules/bots.rs index ddc1589..cd26ee0 100644 --- a/planetwars-server/src/modules/bots.rs +++ b/planetwars-server/src/modules/bots.rs @@ -8,7 +8,7 @@ pub fn save_code_bundle( bot_code: &str, bot_id: Option, conn: &PgConnection, -) -> QueryResult { +) -> QueryResult { let bundle_name = gen_alphanumeric(16); let code_bundle_dir = PathBuf::from(BOTS_DIR).join(&bundle_name); diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index 7d6a1dc..4a5a980 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -29,7 +29,7 @@ pub struct MatchPlayer { } impl MatchPlayer { - pub fn from_code_bundle(code_bundle: &db::bots::CodeBundle) -> Self { + pub fn from_code_bundle(code_bundle: &db::bots::BotVersion) -> Self { MatchPlayer { bot_spec: code_bundle_to_botspec(code_bundle), code_bundle_id: Some(code_bundle.id), @@ -97,7 +97,7 @@ impl RunMatch { } } -pub fn code_bundle_to_botspec(code_bundle: &db::bots::CodeBundle) -> Box { +pub fn code_bundle_to_botspec(code_bundle: &db::bots::BotVersion) -> Box { // TODO: get rid of this unwrap let bundle_path = PathBuf::from(BOTS_DIR).join(code_bundle.code_bundle_path.as_ref().unwrap()); diff --git a/planetwars-server/src/modules/ranking.rs b/planetwars-server/src/modules/ranking.rs index b1ad0da..751c35e 100644 --- a/planetwars-server/src/modules/ranking.rs +++ b/planetwars-server/src/modules/ranking.rs @@ -39,7 +39,7 @@ async fn play_ranking_match(selected_bots: Vec, db_pool: DbPool) { let db_conn = db_pool.get().await.expect("could not get db pool"); let mut code_bundles = Vec::new(); for bot in &selected_bots { - let code_bundle = db::bots::active_code_bundle(bot.id, &db_conn) + let code_bundle = db::bots::active_bot_version(bot.id, &db_conn) .expect("could not get active code bundle"); code_bundles.push(code_bundle); } diff --git a/planetwars-server/src/routes/bots.rs b/planetwars-server/src/routes/bots.rs index 6d5d7df..1ffedef 100644 --- a/planetwars-server/src/routes/bots.rs +++ b/planetwars-server/src/routes/bots.rs @@ -11,7 +11,7 @@ use std::io::Cursor; use std::path::PathBuf; use thiserror; -use crate::db::bots::{self, CodeBundle}; +use crate::db::bots::{self, BotVersion}; use crate::db::ratings::{self, RankedBot}; use crate::db::users::User; use crate::modules::bots::save_code_bundle; @@ -148,8 +148,8 @@ pub async fn get_bot( Path(bot_id): Path, ) -> Result, StatusCode> { let bot = bots::find_bot(bot_id, &conn).map_err(|_| StatusCode::NOT_FOUND)?; - let bundles = bots::find_bot_code_bundles(bot.id, &conn) - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let bundles = + bots::find_bot_versions(bot.id, &conn).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(Json(json!({ "bot": bot, "bundles": bundles, @@ -183,7 +183,7 @@ pub async fn upload_code_multipart( user: User, Path(bot_id): Path, mut multipart: Multipart, -) -> Result, StatusCode> { +) -> Result, StatusCode> { let bots_dir = PathBuf::from(BOTS_DIR); let bot = bots::find_bot(bot_id, &conn).map_err(|_| StatusCode::NOT_FOUND)?; diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 33dc02d..1747bfe 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -39,16 +39,16 @@ pub async fn submit_bot( let opponent = db::bots::find_bot_by_name(&opponent_name, &conn).map_err(|_| StatusCode::BAD_REQUEST)?; - let opponent_code_bundle = - db::bots::active_code_bundle(opponent.id, &conn).map_err(|_| StatusCode::BAD_REQUEST)?; + let opponent_bot_version = + db::bots::active_bot_version(opponent.id, &conn).map_err(|_| StatusCode::BAD_REQUEST)?; - let player_code_bundle = save_code_bundle(¶ms.code, None, &conn) + let player_bot_version = save_code_bundle(¶ms.code, None, &conn) // TODO: can we recover from this? .expect("could not save bot code"); let mut run_match = RunMatch::from_players(vec![ - MatchPlayer::from_code_bundle(&player_code_bundle), - MatchPlayer::from_code_bundle(&opponent_code_bundle), + MatchPlayer::from_code_bundle(&player_bot_version), + MatchPlayer::from_code_bundle(&opponent_bot_version), ]); let match_data = run_match .store_in_database(&conn) @@ -61,12 +61,12 @@ pub async fn submit_bot( match_players: vec![ FullMatchPlayerData { base: match_data.match_players[0].clone(), - code_bundle: Some(player_code_bundle), + bot_version: Some(player_bot_version), bot: None, }, FullMatchPlayerData { base: match_data.match_players[1].clone(), - code_bundle: Some(opponent_code_bundle), + bot_version: Some(opponent_bot_version), bot: Some(opponent), }, ], diff --git a/planetwars-server/src/routes/matches.rs b/planetwars-server/src/routes/matches.rs index 0c1bee4..f33a5f1 100644 --- a/planetwars-server/src/routes/matches.rs +++ b/planetwars-server/src/routes/matches.rs @@ -18,7 +18,7 @@ pub struct ApiMatch { #[derive(Serialize, Deserialize)] pub struct ApiMatchPlayer { - code_bundle_id: Option, + bot_version_id: Option, bot_id: Option, bot_name: Option, } @@ -38,7 +38,7 @@ pub fn match_data_to_api(data: matches::FullMatchData) -> ApiMatch { .match_players .iter() .map(|_p| ApiMatchPlayer { - code_bundle_id: _p.code_bundle.as_ref().map(|cb| cb.id), + bot_version_id: _p.bot_version.as_ref().map(|cb| cb.id), bot_id: _p.bot.as_ref().map(|b| b.id), bot_name: _p.bot.as_ref().map(|b| b.name.clone()), }) diff --git a/planetwars-server/src/schema.rs b/planetwars-server/src/schema.rs index d632a32..0606ac4 100644 --- a/planetwars-server/src/schema.rs +++ b/planetwars-server/src/schema.rs @@ -32,7 +32,7 @@ table! { match_players (match_id, player_id) { match_id -> Int4, player_id -> Int4, - code_bundle_id -> Nullable, + bot_version_id -> Nullable, } } @@ -84,7 +84,7 @@ table! { joinable!(bot_versions -> bots (bot_id)); joinable!(bots -> users (owner_id)); -joinable!(match_players -> bot_versions (code_bundle_id)); +joinable!(match_players -> bot_versions (bot_version_id)); joinable!(match_players -> matches (match_id)); joinable!(ratings -> bots (bot_id)); joinable!(sessions -> users (user_id)); From 6ec792e3bd633a0b3971e401d29b2f8671f38b14 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Thu, 7 Jul 2022 18:57:46 +0200 Subject: [PATCH 34/51] NewBotVersion --- planetwars-server/src/db/bots.rs | 11 ++++++----- planetwars-server/src/modules/bots.rs | 7 ++++--- planetwars-server/src/routes/bots.rs | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/planetwars-server/src/db/bots.rs b/planetwars-server/src/db/bots.rs index 53c11b1..1654f43 100644 --- a/planetwars-server/src/db/bots.rs +++ b/planetwars-server/src/db/bots.rs @@ -45,9 +45,10 @@ pub fn find_all_bots(conn: &PgConnection) -> QueryResult> { #[derive(Insertable)] #[table_name = "bot_versions"] -pub struct NewCodeBundle<'a> { +pub struct NewBotVersion<'a> { pub bot_id: Option, - pub code_bundle_path: &'a str, + pub code_bundle_path: Option<&'a str>, + pub container_digest: Option<&'a str>, } #[derive(Queryable, Serialize, Deserialize, Debug)] @@ -59,12 +60,12 @@ pub struct BotVersion { pub container_digest: Option, } -pub fn create_code_bundle( - new_code_bundle: &NewCodeBundle, +pub fn create_bot_version( + new_bot_version: &NewBotVersion, conn: &PgConnection, ) -> QueryResult { diesel::insert_into(bot_versions::table) - .values(new_code_bundle) + .values(new_bot_version) .get_result(conn) } diff --git a/planetwars-server/src/modules/bots.rs b/planetwars-server/src/modules/bots.rs index cd26ee0..629ecf6 100644 --- a/planetwars-server/src/modules/bots.rs +++ b/planetwars-server/src/modules/bots.rs @@ -15,9 +15,10 @@ pub fn save_code_bundle( std::fs::create_dir(&code_bundle_dir).unwrap(); std::fs::write(code_bundle_dir.join("bot.py"), bot_code).unwrap(); - let new_code_bundle = db::bots::NewCodeBundle { + let new_code_bundle = db::bots::NewBotVersion { bot_id, - code_bundle_path: &bundle_name, + code_bundle_path: Some(&bundle_name), + container_digest: None, }; - db::bots::create_code_bundle(&new_code_bundle, conn) + db::bots::create_bot_version(&new_code_bundle, conn) } diff --git a/planetwars-server/src/routes/bots.rs b/planetwars-server/src/routes/bots.rs index 1ffedef..54c0d36 100644 --- a/planetwars-server/src/routes/bots.rs +++ b/planetwars-server/src/routes/bots.rs @@ -213,12 +213,13 @@ pub async fn upload_code_multipart( .extract(bots_dir.join(&folder_name)) .map_err(|_| StatusCode::BAD_REQUEST)?; - let bundle = bots::NewCodeBundle { + let bot_version = bots::NewBotVersion { bot_id: Some(bot.id), - code_bundle_path: &folder_name, + code_bundle_path: Some(&folder_name), + container_digest: None, }; let code_bundle = - bots::create_code_bundle(&bundle, &conn).expect("Failed to create code bundle"); + bots::create_bot_version(&bot_version, &conn).expect("Failed to create code bundle"); Ok(Json(code_bundle)) } From 0f14dee499f48b11fc329164c30cd475400a9f4d Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Thu, 7 Jul 2022 19:13:55 +0200 Subject: [PATCH 35/51] refactor: rename save_code_bundle to save_code_string --- planetwars-server/src/lib.rs | 2 +- planetwars-server/src/modules/bots.rs | 3 ++- planetwars-server/src/routes/bots.rs | 4 ++-- planetwars-server/src/routes/demo.rs | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index 7076604..fdaf800 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -51,7 +51,7 @@ pub async fn seed_simplebot(pool: &ConnectionPool) { let simplebot_code = std::fs::read_to_string(SIMPLEBOT_PATH).expect("could not read simplebot code"); - modules::bots::save_code_bundle(&simplebot_code, Some(simplebot.id), &conn)?; + modules::bots::save_code_string(&simplebot_code, Some(simplebot.id), &conn)?; println!("initialized simplebot"); diff --git a/planetwars-server/src/modules/bots.rs b/planetwars-server/src/modules/bots.rs index 629ecf6..b82ad41 100644 --- a/planetwars-server/src/modules/bots.rs +++ b/planetwars-server/src/modules/bots.rs @@ -4,7 +4,8 @@ use diesel::{PgConnection, QueryResult}; use crate::{db, util::gen_alphanumeric, BOTS_DIR}; -pub fn save_code_bundle( +/// Save a string containing bot code as a code bundle. +pub fn save_code_string( bot_code: &str, bot_id: Option, conn: &PgConnection, diff --git a/planetwars-server/src/routes/bots.rs b/planetwars-server/src/routes/bots.rs index 54c0d36..edb68ae 100644 --- a/planetwars-server/src/routes/bots.rs +++ b/planetwars-server/src/routes/bots.rs @@ -14,7 +14,7 @@ use thiserror; use crate::db::bots::{self, BotVersion}; use crate::db::ratings::{self, RankedBot}; use crate::db::users::User; -use crate::modules::bots::save_code_bundle; +use crate::modules::bots::save_code_string; use crate::{DatabaseConnection, BOTS_DIR}; use bots::Bot; @@ -120,7 +120,7 @@ pub async fn save_bot( } }; let _code_bundle = - save_code_bundle(¶ms.code, Some(bot.id), &conn).expect("failed to save code bundle"); + save_code_string(¶ms.code, Some(bot.id), &conn).expect("failed to save code bundle"); Ok(Json(bot)) } diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 1747bfe..4f83de0 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -1,6 +1,6 @@ use crate::db; use crate::db::matches::{FullMatchData, FullMatchPlayerData}; -use crate::modules::bots::save_code_bundle; +use crate::modules::bots::save_code_string; use crate::modules::matches::{MatchPlayer, RunMatch}; use crate::ConnectionPool; use axum::extract::Extension; @@ -42,7 +42,7 @@ pub async fn submit_bot( let opponent_bot_version = db::bots::active_bot_version(opponent.id, &conn).map_err(|_| StatusCode::BAD_REQUEST)?; - let player_bot_version = save_code_bundle(¶ms.code, None, &conn) + let player_bot_version = save_code_string(¶ms.code, None, &conn) // TODO: can we recover from this? .expect("could not save bot code"); From 7eb02a2efc8f0bb8ec411c5af0f648aeda939226 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Fri, 8 Jul 2022 20:40:20 +0200 Subject: [PATCH 36/51] create a new bot verison on docker push --- planetwars-server/src/modules/registry.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index c8ec4fa..7198a61 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -15,6 +15,7 @@ use std::path::PathBuf; use tokio::io::AsyncWriteExt; use tokio_util::io::ReaderStream; +use crate::db::bots::NewBotVersion; use crate::util::gen_alphanumeric; use crate::{db, DatabaseConnection}; @@ -339,7 +340,7 @@ async fn put_manifest( Path((repository_name, reference)): Path<(String, String)>, mut stream: BodyStream, ) -> Result { - check_access(&repository_name, &auth, &db_conn)?; + let bot = check_access(&repository_name, &auth, &db_conn)?; let repository_dir = PathBuf::from(REGISTRY_PATH) .join("manifests") @@ -368,6 +369,15 @@ async fn put_manifest( let digest_path = repository_dir.join(&content_digest).with_extension("json"); tokio::fs::copy(manifest_path, digest_path).await.unwrap(); + // Register the new image as a bot version + // TODO: how should tags be handled? + let new_version = NewBotVersion { + bot_id: Some(bot.id), + code_bundle_path: None, + container_digest: Some(&content_digest), + }; + db::bots::create_bot_version(&new_version, &db_conn).expect("could not save bot version"); + Ok(Response::builder() .status(StatusCode::CREATED) .header( @@ -380,12 +390,13 @@ async fn put_manifest( } /// Ensure that the accessed repository exists -/// and the user is allowed to access ti +/// and the user is allowed to access it. +/// Returns the associated bot. fn check_access( repository_name: &str, auth: &RegistryAuth, db_conn: &DatabaseConnection, -) -> Result<(), StatusCode> { +) -> Result { use diesel::OptionalExtension; // TODO: it would be nice to provide the found repository @@ -396,10 +407,10 @@ fn check_access( .ok_or(StatusCode::NOT_FOUND)?; match &auth { - RegistryAuth::Admin => Ok(()), + RegistryAuth::Admin => Ok(bot), RegistryAuth::User(user) => { if bot.owner_id == Some(user.id) { - Ok(()) + Ok(bot) } else { Err(StatusCode::FORBIDDEN) } From ec1d50f655c05d9dec0c4b01fd1039e9c5525f34 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sat, 9 Jul 2022 20:01:05 +0200 Subject: [PATCH 37/51] refactor: pass on both Bot and BotVersion to MatchPlayer --- planetwars-server/src/modules/bot_api.rs | 2 +- planetwars-server/src/modules/matches.rs | 41 +++++++++++++++++++----- planetwars-server/src/modules/ranking.rs | 14 +++----- planetwars-server/src/routes/demo.rs | 12 +++---- 4 files changed, 45 insertions(+), 24 deletions(-) diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 6324010..732aa21 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -115,7 +115,7 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { }); let mut run_match = RunMatch::from_players(vec![ MatchPlayer::from_bot_spec(remote_bot_spec), - MatchPlayer::from_code_bundle(&opponent_code_bundle), + MatchPlayer::from_bot_version(&opponent, &opponent_code_bundle), ]); let created_match = run_match .store_in_database(&conn) diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index 4a5a980..a8c7ca9 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -24,15 +24,28 @@ pub struct RunMatch { pub struct MatchPlayer { bot_spec: Box, - // meta that will be passed on to database + // metadata that will be passed on to database code_bundle_id: Option, } impl MatchPlayer { - pub fn from_code_bundle(code_bundle: &db::bots::BotVersion) -> Self { + pub fn from_bot_version(bot: &db::bots::Bot, version: &db::bots::BotVersion) -> Self { MatchPlayer { - bot_spec: code_bundle_to_botspec(code_bundle), - code_bundle_id: Some(code_bundle.id), + bot_spec: bot_version_to_botspec(bot, version), + code_bundle_id: Some(version.id), + } + } + + /// Construct a MatchPlayer from a BotVersion that certainly contains a code bundle path. + /// Will panic when this is not the case. + pub fn from_code_bundle_version(version: &db::bots::BotVersion) -> Self { + let code_bundle_path = version + .code_bundle_path + .as_ref() + .expect("no code_bundle_path found"); + MatchPlayer { + bot_spec: python_docker_bot_spec(code_bundle_path), + code_bundle_id: Some(version.id), } } @@ -97,12 +110,24 @@ impl RunMatch { } } -pub fn code_bundle_to_botspec(code_bundle: &db::bots::BotVersion) -> Box { - // TODO: get rid of this unwrap - let bundle_path = PathBuf::from(BOTS_DIR).join(code_bundle.code_bundle_path.as_ref().unwrap()); +pub fn bot_version_to_botspec( + _bot: &db::bots::Bot, + bot_version: &db::bots::BotVersion, +) -> Box { + if let Some(code_bundle_path) = &bot_version.code_bundle_path { + python_docker_bot_spec(code_bundle_path) + } else if let Some(_container_digest) = &bot_version.container_digest { + unimplemented!() + } else { + panic!("bad bot version") + } +} + +fn python_docker_bot_spec(code_bundle_path: &str) -> Box { + let code_bundle_abs_path = PathBuf::from(BOTS_DIR).join(code_bundle_path); Box::new(DockerBotSpec { - code_path: bundle_path, + code_path: code_bundle_abs_path, image: PYTHON_IMAGE.to_string(), argv: vec!["python".to_string(), "bot.py".to_string()], }) diff --git a/planetwars-server/src/modules/ranking.rs b/planetwars-server/src/modules/ranking.rs index 751c35e..3182ce2 100644 --- a/planetwars-server/src/modules/ranking.rs +++ b/planetwars-server/src/modules/ranking.rs @@ -37,18 +37,14 @@ pub async fn run_ranker(db_pool: DbPool) { async fn play_ranking_match(selected_bots: Vec, db_pool: DbPool) { let db_conn = db_pool.get().await.expect("could not get db pool"); - let mut code_bundles = Vec::new(); + let mut players = Vec::new(); for bot in &selected_bots { - let code_bundle = db::bots::active_bot_version(bot.id, &db_conn) - .expect("could not get active code bundle"); - code_bundles.push(code_bundle); + let version = db::bots::active_bot_version(bot.id, &db_conn) + .expect("could not get active bot version"); + let player = MatchPlayer::from_bot_version(bot, &version); + players.push(player); } - let players = code_bundles - .iter() - .map(MatchPlayer::from_code_bundle) - .collect::>(); - let mut run_match = RunMatch::from_players(players); run_match .store_in_database(&db_conn) diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 4f83de0..1a6ae9a 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -37,18 +37,18 @@ pub async fn submit_bot( .opponent_name .unwrap_or_else(|| DEFAULT_OPPONENT_NAME.to_string()); - let opponent = + let opponent_bot = db::bots::find_bot_by_name(&opponent_name, &conn).map_err(|_| StatusCode::BAD_REQUEST)?; - let opponent_bot_version = - db::bots::active_bot_version(opponent.id, &conn).map_err(|_| StatusCode::BAD_REQUEST)?; + let opponent_bot_version = db::bots::active_bot_version(opponent_bot.id, &conn) + .map_err(|_| StatusCode::BAD_REQUEST)?; let player_bot_version = save_code_string(¶ms.code, None, &conn) // TODO: can we recover from this? .expect("could not save bot code"); let mut run_match = RunMatch::from_players(vec![ - MatchPlayer::from_code_bundle(&player_bot_version), - MatchPlayer::from_code_bundle(&opponent_bot_version), + MatchPlayer::from_code_bundle_version(&player_bot_version), + MatchPlayer::from_bot_version(&opponent_bot, &opponent_bot_version), ]); let match_data = run_match .store_in_database(&conn) @@ -67,7 +67,7 @@ pub async fn submit_bot( FullMatchPlayerData { base: match_data.match_players[1].clone(), bot_version: Some(opponent_bot_version), - bot: Some(opponent), + bot: Some(opponent_bot), }, ], }; From 0b9a9f0eaafb68acb7896ade26b9ae4508096d5c Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 11 Jul 2022 20:43:10 +0200 Subject: [PATCH 38/51] tying it together: execute docker bots --- planetwars-matchrunner/src/docker_runner.rs | 14 ++++++------- planetwars-server/src/modules/matches.rs | 23 +++++++++++++++------ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/planetwars-matchrunner/src/docker_runner.rs b/planetwars-matchrunner/src/docker_runner.rs index 63a7a67..2d93273 100644 --- a/planetwars-matchrunner/src/docker_runner.rs +++ b/planetwars-matchrunner/src/docker_runner.rs @@ -1,5 +1,4 @@ use std::io; -use std::path::PathBuf; use std::pin::Pin; use std::sync::{Arc, Mutex}; @@ -19,8 +18,9 @@ use crate::BotSpec; #[derive(Clone, Debug)] pub struct DockerBotSpec { pub image: String, - pub code_path: PathBuf, - pub argv: Vec, + pub binds: Option>, + pub argv: Option>, + pub working_dir: Option, } #[async_trait] @@ -42,14 +42,12 @@ async fn spawn_docker_process( params: &DockerBotSpec, ) -> Result { let docker = Docker::connect_with_socket_defaults()?; - let bot_code_dir = std::fs::canonicalize(¶ms.code_path).unwrap(); - let code_dir_str = bot_code_dir.as_os_str().to_str().unwrap(); let memory_limit = 512 * 1024 * 1024; // 512MB let config = container::Config { image: Some(params.image.clone()), host_config: Some(bollard::models::HostConfig { - binds: Some(vec![format!("{}:{}", code_dir_str, "/workdir")]), + binds: params.binds.clone(), network_mode: Some("none".to_string()), memory: Some(memory_limit), memory_swap: Some(memory_limit), @@ -59,8 +57,8 @@ async fn spawn_docker_process( // cpu_quota: Some(10_000), ..Default::default() }), - working_dir: Some("/workdir".to_string()), - cmd: Some(params.argv.clone()), + working_dir: params.working_dir.clone(), + cmd: params.argv.clone(), attach_stdin: Some(true), attach_stdout: Some(true), attach_stderr: Some(true), diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index a8c7ca9..03be5db 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -111,25 +111,36 @@ impl RunMatch { } pub fn bot_version_to_botspec( - _bot: &db::bots::Bot, + bot: &db::bots::Bot, bot_version: &db::bots::BotVersion, ) -> Box { if let Some(code_bundle_path) = &bot_version.code_bundle_path { python_docker_bot_spec(code_bundle_path) - } else if let Some(_container_digest) = &bot_version.container_digest { - unimplemented!() + } else if let Some(container_digest) = &bot_version.container_digest { + // TODO: put this in config + let registry_url = "localhost:9001"; + Box::new(DockerBotSpec { + image: format!("{}/{}@{}", registry_url, bot.name, container_digest), + binds: None, + argv: None, + working_dir: None, + }) } else { panic!("bad bot version") } } fn python_docker_bot_spec(code_bundle_path: &str) -> Box { - let code_bundle_abs_path = PathBuf::from(BOTS_DIR).join(code_bundle_path); + let code_bundle_rel_path = PathBuf::from(BOTS_DIR).join(code_bundle_path); + let code_bundle_abs_path = std::fs::canonicalize(&code_bundle_rel_path).unwrap(); + let code_bundle_path_str = code_bundle_abs_path.as_os_str().to_str().unwrap(); + // TODO: it would be good to simplify this configuration Box::new(DockerBotSpec { - code_path: code_bundle_abs_path, image: PYTHON_IMAGE.to_string(), - argv: vec!["python".to_string(), "bot.py".to_string()], + binds: Some(vec![format!("{}:{}", code_bundle_path_str, "/workdir")]), + argv: Some(vec!["python".to_string(), "bot.py".to_string()]), + working_dir: Some("/workdir".to_string()), }) } From e69bd14f1d64b0d8b2438a40a069d3647c1edd73 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Tue, 12 Jul 2022 20:54:00 +0200 Subject: [PATCH 39/51] refactor: delay BotSpec construction in RunMatch --- planetwars-server/src/db/bots.rs | 2 +- planetwars-server/src/modules/bot_api.rs | 15 ++++-- planetwars-server/src/modules/matches.rs | 60 +++++++++--------------- planetwars-server/src/modules/ranking.rs | 5 +- planetwars-server/src/routes/demo.rs | 10 +++- 5 files changed, 45 insertions(+), 47 deletions(-) diff --git a/planetwars-server/src/db/bots.rs b/planetwars-server/src/db/bots.rs index 1654f43..a112a9a 100644 --- a/planetwars-server/src/db/bots.rs +++ b/planetwars-server/src/db/bots.rs @@ -51,7 +51,7 @@ pub struct NewBotVersion<'a> { pub container_digest: Option<&'a str>, } -#[derive(Queryable, Serialize, Deserialize, Debug)] +#[derive(Queryable, Serialize, Deserialize, Clone, Debug)] pub struct BotVersion { pub id: i32, pub bot_id: Option, diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 732aa21..962b33d 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -102,10 +102,10 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { let match_request = req.get_ref(); - let opponent = db::bots::find_bot_by_name(&match_request.opponent_name, &conn) + let opponent_bot = db::bots::find_bot_by_name(&match_request.opponent_name, &conn) .map_err(|_| Status::not_found("opponent not found"))?; - let opponent_code_bundle = db::bots::active_bot_version(opponent.id, &conn) - .map_err(|_| Status::not_found("opponent has no code"))?; + let opponent_bot_version = db::bots::active_bot_version(opponent_bot.id, &conn) + .map_err(|_| Status::not_found("no opponent version found"))?; let player_key = gen_alphanumeric(32); @@ -114,8 +114,13 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { router: self.router.clone(), }); let mut run_match = RunMatch::from_players(vec![ - MatchPlayer::from_bot_spec(remote_bot_spec), - MatchPlayer::from_bot_version(&opponent, &opponent_code_bundle), + MatchPlayer::BotSpec { + spec: remote_bot_spec, + }, + MatchPlayer::BotVersion { + bot: Some(opponent_bot), + version: opponent_bot_version, + }, ]); let created_match = run_match .store_in_database(&conn) diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index 03be5db..0496db7 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -22,39 +22,14 @@ pub struct RunMatch { match_id: Option, } -pub struct MatchPlayer { - bot_spec: Box, - // metadata that will be passed on to database - code_bundle_id: Option, -} - -impl MatchPlayer { - pub fn from_bot_version(bot: &db::bots::Bot, version: &db::bots::BotVersion) -> Self { - MatchPlayer { - bot_spec: bot_version_to_botspec(bot, version), - code_bundle_id: Some(version.id), - } - } - - /// Construct a MatchPlayer from a BotVersion that certainly contains a code bundle path. - /// Will panic when this is not the case. - pub fn from_code_bundle_version(version: &db::bots::BotVersion) -> Self { - let code_bundle_path = version - .code_bundle_path - .as_ref() - .expect("no code_bundle_path found"); - MatchPlayer { - bot_spec: python_docker_bot_spec(code_bundle_path), - code_bundle_id: Some(version.id), - } - } - - pub fn from_bot_spec(bot_spec: Box) -> Self { - MatchPlayer { - bot_spec, - code_bundle_id: None, - } - } +pub enum MatchPlayer { + BotVersion { + bot: Option, + version: db::bots::BotVersion, + }, + BotSpec { + spec: Box, + }, } impl RunMatch { @@ -76,7 +51,12 @@ impl RunMatch { .players .into_iter() .map(|player| runner::MatchPlayer { - bot_spec: player.bot_spec, + bot_spec: match player { + MatchPlayer::BotVersion { bot, version } => { + bot_version_to_botspec(bot.as_ref(), &version) + } + MatchPlayer::BotSpec { spec } => spec, + }, }) .collect(), } @@ -94,11 +74,14 @@ impl RunMatch { .players .iter() .map(|p| db::matches::MatchPlayerData { - code_bundle_id: p.code_bundle_id, + code_bundle_id: match p { + MatchPlayer::BotVersion { version, .. } => Some(version.id), + MatchPlayer::BotSpec { .. } => None, + }, }) .collect::>(); - let match_data = db::matches::create_match(&new_match_data, &new_match_players, &db_conn)?; + let match_data = db::matches::create_match(&new_match_data, &new_match_players, db_conn)?; self.match_id = Some(match_data.base.id); Ok(match_data) } @@ -111,12 +94,12 @@ impl RunMatch { } pub fn bot_version_to_botspec( - bot: &db::bots::Bot, + bot: Option<&db::bots::Bot>, bot_version: &db::bots::BotVersion, ) -> Box { if let Some(code_bundle_path) = &bot_version.code_bundle_path { python_docker_bot_spec(code_bundle_path) - } else if let Some(container_digest) = &bot_version.container_digest { + } else if let (Some(container_digest), Some(bot)) = (&bot_version.container_digest, bot) { // TODO: put this in config let registry_url = "localhost:9001"; Box::new(DockerBotSpec { @@ -126,6 +109,7 @@ pub fn bot_version_to_botspec( working_dir: None, }) } else { + // TODO: ideally this would not be possible panic!("bad bot version") } } diff --git a/planetwars-server/src/modules/ranking.rs b/planetwars-server/src/modules/ranking.rs index 3182ce2..7147b98 100644 --- a/planetwars-server/src/modules/ranking.rs +++ b/planetwars-server/src/modules/ranking.rs @@ -41,7 +41,10 @@ async fn play_ranking_match(selected_bots: Vec, db_pool: DbPool) { for bot in &selected_bots { let version = db::bots::active_bot_version(bot.id, &db_conn) .expect("could not get active bot version"); - let player = MatchPlayer::from_bot_version(bot, &version); + let player = MatchPlayer::BotVersion { + bot: Some(bot.clone()), + version, + }; players.push(player); } diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 1a6ae9a..f9929f7 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -47,8 +47,14 @@ pub async fn submit_bot( .expect("could not save bot code"); let mut run_match = RunMatch::from_players(vec![ - MatchPlayer::from_code_bundle_version(&player_bot_version), - MatchPlayer::from_bot_version(&opponent_bot, &opponent_bot_version), + MatchPlayer::BotVersion { + bot: None, + version: player_bot_version.clone(), + }, + MatchPlayer::BotVersion { + bot: Some(opponent_bot.clone()), + version: opponent_bot_version.clone(), + }, ]); let match_data = run_match .store_in_database(&conn) From 668409e76d8cc7797fe627b2e2c3d0223b3db684 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Wed, 13 Jul 2022 19:36:07 +0200 Subject: [PATCH 40/51] refactor: unify match save and spawn --- planetwars-server/src/modules/bot_api.rs | 8 +++--- planetwars-server/src/modules/matches.rs | 32 +++++++++++++----------- planetwars-server/src/modules/ranking.rs | 12 ++++----- planetwars-server/src/routes/demo.rs | 10 ++++---- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 962b33d..0ee9357 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -122,10 +122,10 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { version: opponent_bot_version, }, ]); - let created_match = run_match - .store_in_database(&conn) - .expect("failed to save match"); - run_match.spawn(self.conn_pool.clone()); + let (created_match, _) = run_match + .run(self.conn_pool.clone()) + .await + .expect("failed to create match"); Ok(Response::new(pb::CreatedMatch { match_id: created_match.base.id, diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index 0496db7..6caa8c2 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -19,7 +19,6 @@ const PYTHON_IMAGE: &str = "python:3.10-slim-buster"; pub struct RunMatch { log_file_name: String, players: Vec, - match_id: Option, } pub enum MatchPlayer { @@ -38,7 +37,6 @@ impl RunMatch { RunMatch { log_file_name, players, - match_id: None, } } @@ -62,10 +60,24 @@ impl RunMatch { } } - pub fn store_in_database(&mut self, db_conn: &PgConnection) -> QueryResult { - // don't store the same match twice - assert!(self.match_id.is_none()); + pub async fn run( + self, + conn_pool: ConnectionPool, + ) -> QueryResult<(MatchData, JoinHandle)> { + let match_data = { + // TODO: it would be nice to get an already-open connection here when possible. + // Maybe we need an additional abstraction, bundling a connection and connection pool? + let db_conn = conn_pool.get().await.expect("could not get a connection"); + self.store_in_database(&db_conn)? + }; + let runner_config = self.into_runner_config(); + let handle = tokio::spawn(run_match_task(conn_pool, runner_config, match_data.base.id)); + + Ok((match_data, handle)) + } + + fn store_in_database(&self, db_conn: &PgConnection) -> QueryResult { let new_match_data = db::matches::NewMatch { state: db::matches::MatchState::Playing, log_path: &self.log_file_name, @@ -81,15 +93,7 @@ impl RunMatch { }) .collect::>(); - let match_data = db::matches::create_match(&new_match_data, &new_match_players, db_conn)?; - self.match_id = Some(match_data.base.id); - Ok(match_data) - } - - pub fn spawn(self, pool: ConnectionPool) -> JoinHandle { - let match_id = self.match_id.expect("match must be saved before running"); - let runner_config = self.into_runner_config(); - tokio::spawn(run_match_task(pool, runner_config, match_id)) + db::matches::create_match(&new_match_data, &new_match_players, db_conn) } } diff --git a/planetwars-server/src/modules/ranking.rs b/planetwars-server/src/modules/ranking.rs index 7147b98..1c35394 100644 --- a/planetwars-server/src/modules/ranking.rs +++ b/planetwars-server/src/modules/ranking.rs @@ -48,14 +48,12 @@ async fn play_ranking_match(selected_bots: Vec, db_pool: DbPool) { players.push(player); } - let mut run_match = RunMatch::from_players(players); - run_match - .store_in_database(&db_conn) - .expect("could not store match in db"); - run_match - .spawn(db_pool.clone()) + let (_, handle) = RunMatch::from_players(players) + .run(db_pool.clone()) .await - .expect("running match failed"); + .expect("failed to run match"); + // wait for match to complete, so that only one ranking match can be running + let _outcome = handle.await; } fn recalculate_ratings(db_conn: &PgConnection) -> QueryResult<()> { diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index f9929f7..5ff02c7 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -46,7 +46,7 @@ pub async fn submit_bot( // TODO: can we recover from this? .expect("could not save bot code"); - let mut run_match = RunMatch::from_players(vec![ + let run_match = RunMatch::from_players(vec![ MatchPlayer::BotVersion { bot: None, version: player_bot_version.clone(), @@ -56,10 +56,10 @@ pub async fn submit_bot( version: opponent_bot_version.clone(), }, ]); - let match_data = run_match - .store_in_database(&conn) - .expect("failed to save match"); - run_match.spawn(pool.clone()); + let (match_data, _) = run_match + .run(pool.clone()) + .await + .expect("failed to run match"); // TODO: avoid clones let full_match_data = FullMatchData { From 00459f9e3d818f0fb84160862f02898d64f98110 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Thu, 14 Jul 2022 20:53:08 +0200 Subject: [PATCH 41/51] create a configuration to hold docker registry url --- planetwars-server/src/lib.rs | 11 ++++++-- planetwars-server/src/modules/bot_api.rs | 29 ++++++++++++--------- planetwars-server/src/modules/matches.rs | 33 ++++++++++++++++-------- planetwars-server/src/modules/ranking.rs | 15 ++++++++--- planetwars-server/src/routes/demo.rs | 28 ++++++++++++-------- 5 files changed, 76 insertions(+), 40 deletions(-) diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index fdaf800..eb69c82 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -10,13 +10,14 @@ pub mod util; use std::net::SocketAddr; use std::ops::Deref; +use std::sync::Arc; use bb8::{Pool, PooledConnection}; use bb8_diesel::{self, DieselConnectionManager}; use config::ConfigError; use diesel::{Connection, PgConnection}; -use modules::ranking::run_ranker; use modules::registry::registry_service; +use modules::{matches::MatchRunnerConfig, ranking::run_ranker}; use serde::Deserialize; use axum::{ @@ -120,12 +121,18 @@ pub async fn run_app() { let configuration = get_config().unwrap(); let db_pool = prepare_db(&configuration.database_url).await; - tokio::spawn(run_ranker(db_pool.clone())); + let runner_config = Arc::new(MatchRunnerConfig { + python_runner_image: "python:3.10-slim-buster".to_string(), + container_registry_url: "localhost:9001".to_string(), + }); + + tokio::spawn(run_ranker(runner_config.clone(), db_pool.clone())); tokio::spawn(run_registry(db_pool.clone())); let api_service = Router::new() .nest("/api", api()) .layer(Extension(db_pool)) + .layer(Extension(runner_config)) .into_make_service(); // TODO: put in config diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 0ee9357..4e7d737 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -21,10 +21,11 @@ use crate::db; use crate::util::gen_alphanumeric; use crate::ConnectionPool; -use super::matches::{MatchPlayer, RunMatch}; +use super::matches::{MatchPlayer, MatchRunnerConfig, RunMatch}; pub struct BotApiServer { conn_pool: ConnectionPool, + runner_config: Arc, router: PlayerRouter, } @@ -113,15 +114,18 @@ impl pb::bot_api_service_server::BotApiService for BotApiServer { player_key: player_key.clone(), router: self.router.clone(), }); - let mut run_match = RunMatch::from_players(vec![ - MatchPlayer::BotSpec { - spec: remote_bot_spec, - }, - MatchPlayer::BotVersion { - bot: Some(opponent_bot), - version: opponent_bot_version, - }, - ]); + let run_match = RunMatch::from_players( + self.runner_config.clone(), + vec![ + MatchPlayer::BotSpec { + spec: remote_bot_spec, + }, + MatchPlayer::BotVersion { + bot: Some(opponent_bot), + version: opponent_bot_version, + }, + ], + ); let (created_match, _) = run_match .run(self.conn_pool.clone()) .await @@ -261,11 +265,12 @@ async fn schedule_timeout( .resolve_request(request_id, Err(RequestError::Timeout)); } -pub async fn run_bot_api(pool: ConnectionPool) { +pub async fn run_bot_api(runner_config: Arc, pool: ConnectionPool) { let router = PlayerRouter::new(); let server = BotApiServer { router, - conn_pool: pool.clone(), + conn_pool: pool, + runner_config, }; let addr = SocketAddr::from(([127, 0, 0, 1], 50051)); diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index 6caa8c2..07dc68b 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; use diesel::{PgConnection, QueryResult}; use planetwars_matchrunner::{self as runner, docker_runner::DockerBotSpec, BotSpec, MatchConfig}; @@ -14,11 +14,16 @@ use crate::{ ConnectionPool, BOTS_DIR, MAPS_DIR, MATCHES_DIR, }; -const PYTHON_IMAGE: &str = "python:3.10-slim-buster"; +// TODO: add all paths +pub struct MatchRunnerConfig { + pub python_runner_image: String, + pub container_registry_url: String, +} pub struct RunMatch { log_file_name: String, players: Vec, + runner_config: Arc, } pub enum MatchPlayer { @@ -32,15 +37,16 @@ pub enum MatchPlayer { } impl RunMatch { - pub fn from_players(players: Vec) -> Self { + pub fn from_players(runner_config: Arc, players: Vec) -> Self { let log_file_name = format!("{}.log", gen_alphanumeric(16)); RunMatch { + runner_config, log_file_name, players, } } - pub fn into_runner_config(self) -> runner::MatchConfig { + fn into_runner_config(self) -> runner::MatchConfig { runner::MatchConfig { map_path: PathBuf::from(MAPS_DIR).join("hex.json"), map_name: "hex".to_string(), @@ -51,7 +57,7 @@ impl RunMatch { .map(|player| runner::MatchPlayer { bot_spec: match player { MatchPlayer::BotVersion { bot, version } => { - bot_version_to_botspec(bot.as_ref(), &version) + bot_version_to_botspec(&self.runner_config, bot.as_ref(), &version) } MatchPlayer::BotSpec { spec } => spec, }, @@ -98,16 +104,18 @@ impl RunMatch { } pub fn bot_version_to_botspec( + runner_config: &Arc, bot: Option<&db::bots::Bot>, bot_version: &db::bots::BotVersion, ) -> Box { if let Some(code_bundle_path) = &bot_version.code_bundle_path { - python_docker_bot_spec(code_bundle_path) + python_docker_bot_spec(runner_config, code_bundle_path) } else if let (Some(container_digest), Some(bot)) = (&bot_version.container_digest, bot) { - // TODO: put this in config - let registry_url = "localhost:9001"; Box::new(DockerBotSpec { - image: format!("{}/{}@{}", registry_url, bot.name, container_digest), + image: format!( + "{}/{}@{}", + runner_config.container_registry_url, bot.name, container_digest + ), binds: None, argv: None, working_dir: None, @@ -118,14 +126,17 @@ pub fn bot_version_to_botspec( } } -fn python_docker_bot_spec(code_bundle_path: &str) -> Box { +fn python_docker_bot_spec( + runner_config: &Arc, + code_bundle_path: &str, +) -> Box { let code_bundle_rel_path = PathBuf::from(BOTS_DIR).join(code_bundle_path); let code_bundle_abs_path = std::fs::canonicalize(&code_bundle_rel_path).unwrap(); let code_bundle_path_str = code_bundle_abs_path.as_os_str().to_str().unwrap(); // TODO: it would be good to simplify this configuration Box::new(DockerBotSpec { - image: PYTHON_IMAGE.to_string(), + image: runner_config.python_runner_image.clone(), binds: Some(vec![format!("{}:{}", code_bundle_path_str, "/workdir")]), argv: Some(vec!["python".to_string(), "bot.py".to_string()]), working_dir: Some("/workdir".to_string()), diff --git a/planetwars-server/src/modules/ranking.rs b/planetwars-server/src/modules/ranking.rs index 1c35394..e483d1c 100644 --- a/planetwars-server/src/modules/ranking.rs +++ b/planetwars-server/src/modules/ranking.rs @@ -6,12 +6,15 @@ use diesel::{PgConnection, QueryResult}; use rand::seq::SliceRandom; use std::collections::HashMap; use std::mem; +use std::sync::Arc; use std::time::{Duration, Instant}; use tokio; +use super::matches::MatchRunnerConfig; + const RANKER_INTERVAL: u64 = 60; -pub async fn run_ranker(db_pool: DbPool) { +pub async fn run_ranker(runner_config: Arc, db_pool: DbPool) { // TODO: make this configurable // play at most one match every n seconds let mut interval = tokio::time::interval(Duration::from_secs(RANKER_INTERVAL)); @@ -30,12 +33,16 @@ pub async fn run_ranker(db_pool: DbPool) { let mut rng = &mut rand::thread_rng(); bots.choose_multiple(&mut rng, 2).cloned().collect() }; - play_ranking_match(selected_bots, db_pool.clone()).await; + play_ranking_match(runner_config.clone(), selected_bots, db_pool.clone()).await; recalculate_ratings(&db_conn).expect("could not recalculate ratings"); } } -async fn play_ranking_match(selected_bots: Vec, db_pool: DbPool) { +async fn play_ranking_match( + runner_config: Arc, + selected_bots: Vec, + db_pool: DbPool, +) { let db_conn = db_pool.get().await.expect("could not get db pool"); let mut players = Vec::new(); for bot in &selected_bots { @@ -48,7 +55,7 @@ async fn play_ranking_match(selected_bots: Vec, db_pool: DbPool) { players.push(player); } - let (_, handle) = RunMatch::from_players(players) + let (_, handle) = RunMatch::from_players(runner_config, players) .run(db_pool.clone()) .await .expect("failed to run match"); diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 5ff02c7..6f2d5e6 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -1,7 +1,9 @@ +use std::sync::Arc; + use crate::db; use crate::db::matches::{FullMatchData, FullMatchPlayerData}; use crate::modules::bots::save_code_string; -use crate::modules::matches::{MatchPlayer, RunMatch}; +use crate::modules::matches::{MatchPlayer, MatchRunnerConfig, RunMatch}; use crate::ConnectionPool; use axum::extract::Extension; use axum::Json; @@ -30,6 +32,7 @@ pub struct SubmitBotResponse { pub async fn submit_bot( Json(params): Json, Extension(pool): Extension, + Extension(runner_config): Extension>, ) -> Result, StatusCode> { let conn = pool.get().await.expect("could not get database connection"); @@ -46,16 +49,19 @@ pub async fn submit_bot( // TODO: can we recover from this? .expect("could not save bot code"); - let run_match = RunMatch::from_players(vec![ - MatchPlayer::BotVersion { - bot: None, - version: player_bot_version.clone(), - }, - MatchPlayer::BotVersion { - bot: Some(opponent_bot.clone()), - version: opponent_bot_version.clone(), - }, - ]); + let run_match = RunMatch::from_players( + runner_config, + vec![ + MatchPlayer::BotVersion { + bot: None, + version: player_bot_version.clone(), + }, + MatchPlayer::BotVersion { + bot: Some(opponent_bot.clone()), + version: opponent_bot_version.clone(), + }, + ], + ); let (match_data, _) = run_match .run(pool.clone()) .await From ec5c91d37b46cb3cec4878176469c66d2304dadd Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Thu, 14 Jul 2022 21:50:42 +0200 Subject: [PATCH 42/51] change runnerconfig to globalconfig --- planetwars-server/src/lib.rs | 9 +++++++-- planetwars-server/src/modules/bot_api.rs | 7 ++++--- planetwars-server/src/modules/matches.rs | 16 +++++----------- planetwars-server/src/modules/ranking.rs | 16 +++++----------- planetwars-server/src/routes/demo.rs | 7 ++++--- 5 files changed, 25 insertions(+), 30 deletions(-) diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index eb69c82..d9f5e8e 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -16,8 +16,8 @@ use bb8::{Pool, PooledConnection}; use bb8_diesel::{self, DieselConnectionManager}; use config::ConfigError; use diesel::{Connection, PgConnection}; +use modules::ranking::run_ranker; use modules::registry::registry_service; -use modules::{matches::MatchRunnerConfig, ranking::run_ranker}; use serde::Deserialize; use axum::{ @@ -36,6 +36,11 @@ const SIMPLEBOT_PATH: &str = "../simplebot/simplebot.py"; type ConnectionPool = bb8::Pool>; +pub struct GlobalConfig { + pub python_runner_image: String, + pub container_registry_url: String, +} + pub async fn seed_simplebot(pool: &ConnectionPool) { let conn = pool.get().await.expect("could not get database connection"); // This transaction is expected to fail when simplebot already exists. @@ -121,7 +126,7 @@ pub async fn run_app() { let configuration = get_config().unwrap(); let db_pool = prepare_db(&configuration.database_url).await; - let runner_config = Arc::new(MatchRunnerConfig { + let runner_config = Arc::new(GlobalConfig { python_runner_image: "python:3.10-slim-buster".to_string(), container_registry_url: "localhost:9001".to_string(), }); diff --git a/planetwars-server/src/modules/bot_api.rs b/planetwars-server/src/modules/bot_api.rs index 4e7d737..33f5d87 100644 --- a/planetwars-server/src/modules/bot_api.rs +++ b/planetwars-server/src/modules/bot_api.rs @@ -20,12 +20,13 @@ use planetwars_matchrunner as runner; use crate::db; use crate::util::gen_alphanumeric; use crate::ConnectionPool; +use crate::GlobalConfig; -use super::matches::{MatchPlayer, MatchRunnerConfig, RunMatch}; +use super::matches::{MatchPlayer, RunMatch}; pub struct BotApiServer { conn_pool: ConnectionPool, - runner_config: Arc, + runner_config: Arc, router: PlayerRouter, } @@ -265,7 +266,7 @@ async fn schedule_timeout( .resolve_request(request_id, Err(RequestError::Timeout)); } -pub async fn run_bot_api(runner_config: Arc, pool: ConnectionPool) { +pub async fn run_bot_api(runner_config: Arc, pool: ConnectionPool) { let router = PlayerRouter::new(); let server = BotApiServer { router, diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index 07dc68b..dd5e523 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -11,19 +11,13 @@ use crate::{ matches::{MatchData, MatchResult}, }, util::gen_alphanumeric, - ConnectionPool, BOTS_DIR, MAPS_DIR, MATCHES_DIR, + ConnectionPool, GlobalConfig, BOTS_DIR, MAPS_DIR, MATCHES_DIR, }; -// TODO: add all paths -pub struct MatchRunnerConfig { - pub python_runner_image: String, - pub container_registry_url: String, -} - pub struct RunMatch { log_file_name: String, players: Vec, - runner_config: Arc, + runner_config: Arc, } pub enum MatchPlayer { @@ -37,7 +31,7 @@ pub enum MatchPlayer { } impl RunMatch { - pub fn from_players(runner_config: Arc, players: Vec) -> Self { + pub fn from_players(runner_config: Arc, players: Vec) -> Self { let log_file_name = format!("{}.log", gen_alphanumeric(16)); RunMatch { runner_config, @@ -104,7 +98,7 @@ impl RunMatch { } pub fn bot_version_to_botspec( - runner_config: &Arc, + runner_config: &Arc, bot: Option<&db::bots::Bot>, bot_version: &db::bots::BotVersion, ) -> Box { @@ -127,7 +121,7 @@ pub fn bot_version_to_botspec( } fn python_docker_bot_spec( - runner_config: &Arc, + runner_config: &Arc, code_bundle_path: &str, ) -> Box { let code_bundle_rel_path = PathBuf::from(BOTS_DIR).join(code_bundle_path); diff --git a/planetwars-server/src/modules/ranking.rs b/planetwars-server/src/modules/ranking.rs index e483d1c..a9f6419 100644 --- a/planetwars-server/src/modules/ranking.rs +++ b/planetwars-server/src/modules/ranking.rs @@ -1,4 +1,4 @@ -use crate::{db::bots::Bot, DbPool}; +use crate::{db::bots::Bot, DbPool, GlobalConfig}; use crate::db; use crate::modules::matches::{MatchPlayer, RunMatch}; @@ -10,11 +10,9 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tokio; -use super::matches::MatchRunnerConfig; - const RANKER_INTERVAL: u64 = 60; -pub async fn run_ranker(runner_config: Arc, db_pool: DbPool) { +pub async fn run_ranker(config: Arc, db_pool: DbPool) { // TODO: make this configurable // play at most one match every n seconds let mut interval = tokio::time::interval(Duration::from_secs(RANKER_INTERVAL)); @@ -33,16 +31,12 @@ pub async fn run_ranker(runner_config: Arc, db_pool: DbPool) let mut rng = &mut rand::thread_rng(); bots.choose_multiple(&mut rng, 2).cloned().collect() }; - play_ranking_match(runner_config.clone(), selected_bots, db_pool.clone()).await; + play_ranking_match(config.clone(), selected_bots, db_pool.clone()).await; recalculate_ratings(&db_conn).expect("could not recalculate ratings"); } } -async fn play_ranking_match( - runner_config: Arc, - selected_bots: Vec, - db_pool: DbPool, -) { +async fn play_ranking_match(config: Arc, selected_bots: Vec, db_pool: DbPool) { let db_conn = db_pool.get().await.expect("could not get db pool"); let mut players = Vec::new(); for bot in &selected_bots { @@ -55,7 +49,7 @@ async fn play_ranking_match( players.push(player); } - let (_, handle) = RunMatch::from_players(runner_config, players) + let (_, handle) = RunMatch::from_players(config, players) .run(db_pool.clone()) .await .expect("failed to run match"); diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 6f2d5e6..77f9e8d 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -3,8 +3,9 @@ use std::sync::Arc; use crate::db; use crate::db::matches::{FullMatchData, FullMatchPlayerData}; use crate::modules::bots::save_code_string; -use crate::modules::matches::{MatchPlayer, MatchRunnerConfig, RunMatch}; +use crate::modules::matches::{MatchPlayer, RunMatch}; use crate::ConnectionPool; +use crate::GlobalConfig; use axum::extract::Extension; use axum::Json; use hyper::StatusCode; @@ -32,7 +33,7 @@ pub struct SubmitBotResponse { pub async fn submit_bot( Json(params): Json, Extension(pool): Extension, - Extension(runner_config): Extension>, + Extension(config): Extension>, ) -> Result, StatusCode> { let conn = pool.get().await.expect("could not get database connection"); @@ -50,7 +51,7 @@ pub async fn submit_bot( .expect("could not save bot code"); let run_match = RunMatch::from_players( - runner_config, + config, vec![ MatchPlayer::BotVersion { bot: None, From d13d131130ab53fb8ee7d49d2b40718622a4ab11 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sat, 16 Jul 2022 21:22:03 +0200 Subject: [PATCH 43/51] move storage paths to GlobalConfig --- planetwars-server/src/lib.rs | 28 ++++++++++++++---------- planetwars-server/src/modules/bots.rs | 5 +++-- planetwars-server/src/modules/matches.rs | 25 ++++++++++----------- planetwars-server/src/routes/bots.rs | 13 ++++++----- planetwars-server/src/routes/demo.rs | 2 +- planetwars-server/src/routes/matches.rs | 9 ++++---- 6 files changed, 45 insertions(+), 37 deletions(-) diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index d9f5e8e..7e14add 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -29,9 +29,6 @@ use axum::{ }; // TODO: make these configurable -const BOTS_DIR: &str = "./data/bots"; -const MATCHES_DIR: &str = "./data/matches"; -const MAPS_DIR: &str = "./data/maps"; const SIMPLEBOT_PATH: &str = "../simplebot/simplebot.py"; type ConnectionPool = bb8::Pool>; @@ -39,9 +36,13 @@ type ConnectionPool = bb8::Pool>; pub struct GlobalConfig { pub python_runner_image: String, pub container_registry_url: String, + + pub bots_directory: String, + pub match_logs_directory: String, + pub maps_directory: String, } -pub async fn seed_simplebot(pool: &ConnectionPool) { +pub async fn seed_simplebot(config: &GlobalConfig, pool: &ConnectionPool) { let conn = pool.get().await.expect("could not get database connection"); // This transaction is expected to fail when simplebot already exists. let _res = conn.transaction::<(), diesel::result::Error, _>(|| { @@ -57,7 +58,7 @@ pub async fn seed_simplebot(pool: &ConnectionPool) { let simplebot_code = std::fs::read_to_string(SIMPLEBOT_PATH).expect("could not read simplebot code"); - modules::bots::save_code_string(&simplebot_code, Some(simplebot.id), &conn)?; + modules::bots::save_code_string(&simplebot_code, Some(simplebot.id), &conn, config)?; println!("initialized simplebot"); @@ -67,10 +68,10 @@ pub async fn seed_simplebot(pool: &ConnectionPool) { pub type DbPool = Pool>; -pub async fn prepare_db(database_url: &str) -> DbPool { +pub async fn prepare_db(database_url: &str, config: &GlobalConfig) -> DbPool { let manager = DieselConnectionManager::::new(database_url); let pool = bb8::Pool::builder().build(manager).await.unwrap(); - seed_simplebot(&pool).await; + seed_simplebot(&config, &pool).await; pool } @@ -124,20 +125,25 @@ async fn run_registry(db_pool: DbPool) { pub async fn run_app() { let configuration = get_config().unwrap(); - let db_pool = prepare_db(&configuration.database_url).await; - let runner_config = Arc::new(GlobalConfig { + let global_config = Arc::new(GlobalConfig { python_runner_image: "python:3.10-slim-buster".to_string(), container_registry_url: "localhost:9001".to_string(), + + bots_directory: "./data/bots".to_string(), + match_logs_directory: "./data/matches".to_string(), + maps_directory: "./data/maps".to_string(), }); - tokio::spawn(run_ranker(runner_config.clone(), db_pool.clone())); + let db_pool = prepare_db(&configuration.database_url, &global_config).await; + + tokio::spawn(run_ranker(global_config.clone(), db_pool.clone())); tokio::spawn(run_registry(db_pool.clone())); let api_service = Router::new() .nest("/api", api()) .layer(Extension(db_pool)) - .layer(Extension(runner_config)) + .layer(Extension(global_config)) .into_make_service(); // TODO: put in config diff --git a/planetwars-server/src/modules/bots.rs b/planetwars-server/src/modules/bots.rs index b82ad41..5513539 100644 --- a/planetwars-server/src/modules/bots.rs +++ b/planetwars-server/src/modules/bots.rs @@ -2,17 +2,18 @@ use std::path::PathBuf; use diesel::{PgConnection, QueryResult}; -use crate::{db, util::gen_alphanumeric, BOTS_DIR}; +use crate::{db, util::gen_alphanumeric, GlobalConfig}; /// Save a string containing bot code as a code bundle. pub fn save_code_string( bot_code: &str, bot_id: Option, conn: &PgConnection, + config: &GlobalConfig, ) -> QueryResult { let bundle_name = gen_alphanumeric(16); - let code_bundle_dir = PathBuf::from(BOTS_DIR).join(&bundle_name); + let code_bundle_dir = PathBuf::from(&config.bots_directory).join(&bundle_name); std::fs::create_dir(&code_bundle_dir).unwrap(); std::fs::write(code_bundle_dir.join("bot.py"), bot_code).unwrap(); diff --git a/planetwars-server/src/modules/matches.rs b/planetwars-server/src/modules/matches.rs index dd5e523..a1fe63d 100644 --- a/planetwars-server/src/modules/matches.rs +++ b/planetwars-server/src/modules/matches.rs @@ -11,13 +11,13 @@ use crate::{ matches::{MatchData, MatchResult}, }, util::gen_alphanumeric, - ConnectionPool, GlobalConfig, BOTS_DIR, MAPS_DIR, MATCHES_DIR, + ConnectionPool, GlobalConfig, }; pub struct RunMatch { log_file_name: String, players: Vec, - runner_config: Arc, + config: Arc, } pub enum MatchPlayer { @@ -31,10 +31,10 @@ pub enum MatchPlayer { } impl RunMatch { - pub fn from_players(runner_config: Arc, players: Vec) -> Self { + pub fn from_players(config: Arc, players: Vec) -> Self { let log_file_name = format!("{}.log", gen_alphanumeric(16)); RunMatch { - runner_config, + config, log_file_name, players, } @@ -42,16 +42,16 @@ impl RunMatch { fn into_runner_config(self) -> runner::MatchConfig { runner::MatchConfig { - map_path: PathBuf::from(MAPS_DIR).join("hex.json"), + map_path: PathBuf::from(&self.config.maps_directory).join("hex.json"), map_name: "hex".to_string(), - log_path: PathBuf::from(MATCHES_DIR).join(&self.log_file_name), + log_path: PathBuf::from(&self.config.match_logs_directory).join(&self.log_file_name), players: self .players .into_iter() .map(|player| runner::MatchPlayer { bot_spec: match player { MatchPlayer::BotVersion { bot, version } => { - bot_version_to_botspec(&self.runner_config, bot.as_ref(), &version) + bot_version_to_botspec(&self.config, bot.as_ref(), &version) } MatchPlayer::BotSpec { spec } => spec, }, @@ -98,7 +98,7 @@ impl RunMatch { } pub fn bot_version_to_botspec( - runner_config: &Arc, + runner_config: &GlobalConfig, bot: Option<&db::bots::Bot>, bot_version: &db::bots::BotVersion, ) -> Box { @@ -120,17 +120,14 @@ pub fn bot_version_to_botspec( } } -fn python_docker_bot_spec( - runner_config: &Arc, - code_bundle_path: &str, -) -> Box { - let code_bundle_rel_path = PathBuf::from(BOTS_DIR).join(code_bundle_path); +fn python_docker_bot_spec(config: &GlobalConfig, code_bundle_path: &str) -> Box { + let code_bundle_rel_path = PathBuf::from(&config.bots_directory).join(code_bundle_path); let code_bundle_abs_path = std::fs::canonicalize(&code_bundle_rel_path).unwrap(); let code_bundle_path_str = code_bundle_abs_path.as_os_str().to_str().unwrap(); // TODO: it would be good to simplify this configuration Box::new(DockerBotSpec { - image: runner_config.python_runner_image.clone(), + image: config.python_runner_image.clone(), binds: Some(vec![format!("{}:{}", code_bundle_path_str, "/workdir")]), argv: Some(vec!["python".to_string(), "bot.py".to_string()]), working_dir: Some("/workdir".to_string()), diff --git a/planetwars-server/src/routes/bots.rs b/planetwars-server/src/routes/bots.rs index edb68ae..9ddb109 100644 --- a/planetwars-server/src/routes/bots.rs +++ b/planetwars-server/src/routes/bots.rs @@ -1,7 +1,7 @@ use axum::extract::{Multipart, Path}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; -use axum::{body, Json}; +use axum::{body, Extension, Json}; use diesel::OptionalExtension; use rand::distributions::Alphanumeric; use rand::Rng; @@ -9,13 +9,14 @@ use serde::{Deserialize, Serialize}; use serde_json::{self, json, value::Value as JsonValue}; use std::io::Cursor; use std::path::PathBuf; +use std::sync::Arc; use thiserror; use crate::db::bots::{self, BotVersion}; use crate::db::ratings::{self, RankedBot}; use crate::db::users::User; use crate::modules::bots::save_code_string; -use crate::{DatabaseConnection, BOTS_DIR}; +use crate::{DatabaseConnection, GlobalConfig}; use bots::Bot; #[derive(Serialize, Deserialize, Debug)] @@ -96,6 +97,7 @@ pub async fn save_bot( Json(params): Json, user: User, conn: DatabaseConnection, + Extension(config): Extension>, ) -> Result, SaveBotError> { let res = bots::find_bot_by_name(¶ms.bot_name, &conn) .optional() @@ -119,8 +121,8 @@ pub async fn save_bot( bots::create_bot(&new_bot, &conn).expect("could not create bot") } }; - let _code_bundle = - save_code_string(¶ms.code, Some(bot.id), &conn).expect("failed to save code bundle"); + let _code_bundle = save_code_string(¶ms.code, Some(bot.id), &conn, &config) + .expect("failed to save code bundle"); Ok(Json(bot)) } @@ -183,8 +185,9 @@ pub async fn upload_code_multipart( user: User, Path(bot_id): Path, mut multipart: Multipart, + Extension(config): Extension>, ) -> Result, StatusCode> { - let bots_dir = PathBuf::from(BOTS_DIR); + let bots_dir = PathBuf::from(&config.bots_directory); let bot = bots::find_bot(bot_id, &conn).map_err(|_| StatusCode::NOT_FOUND)?; diff --git a/planetwars-server/src/routes/demo.rs b/planetwars-server/src/routes/demo.rs index 77f9e8d..69838f3 100644 --- a/planetwars-server/src/routes/demo.rs +++ b/planetwars-server/src/routes/demo.rs @@ -46,7 +46,7 @@ pub async fn submit_bot( let opponent_bot_version = db::bots::active_bot_version(opponent_bot.id, &conn) .map_err(|_| StatusCode::BAD_REQUEST)?; - let player_bot_version = save_code_string(¶ms.code, None, &conn) + let player_bot_version = save_code_string(¶ms.code, None, &conn, &config) // TODO: can we recover from this? .expect("could not save bot code"); diff --git a/planetwars-server/src/routes/matches.rs b/planetwars-server/src/routes/matches.rs index f33a5f1..a980daa 100644 --- a/planetwars-server/src/routes/matches.rs +++ b/planetwars-server/src/routes/matches.rs @@ -1,11 +1,11 @@ -use axum::{extract::Path, Json}; +use axum::{extract::Path, Extension, Json}; use hyper::StatusCode; use serde::{Deserialize, Serialize}; -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; use crate::{ db::matches::{self, MatchState}, - DatabaseConnection, MATCHES_DIR, + DatabaseConnection, GlobalConfig, }; #[derive(Serialize, Deserialize)] @@ -59,10 +59,11 @@ pub async fn get_match_data( pub async fn get_match_log( Path(match_id): Path, conn: DatabaseConnection, + Extension(config): Extension>, ) -> Result, StatusCode> { let match_base = matches::find_match_base(match_id, &conn).map_err(|_| StatusCode::NOT_FOUND)?; - let log_path = PathBuf::from(MATCHES_DIR).join(&match_base.log_path); + let log_path = PathBuf::from(&config.match_logs_directory).join(&match_base.log_path); let log_contents = std::fs::read(log_path).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(log_contents) } From 0cf7b5299d1085e32760ae9843625724a09c8c29 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sat, 16 Jul 2022 21:47:22 +0200 Subject: [PATCH 44/51] integrate registry with GlobalConfig --- planetwars-server/src/lib.rs | 11 ++++- planetwars-server/src/modules/registry.rs | 57 ++++++++++++++++------- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index 7e14add..87495e9 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -40,6 +40,9 @@ pub struct GlobalConfig { pub bots_directory: String, pub match_logs_directory: String, pub maps_directory: String, + + pub registry_directory: String, + pub registry_admin_password: String, } pub async fn seed_simplebot(config: &GlobalConfig, pool: &ConnectionPool) { @@ -109,7 +112,7 @@ pub fn get_config() -> Result { .try_deserialize() } -async fn run_registry(db_pool: DbPool) { +async fn run_registry(config: Arc, db_pool: DbPool) { // TODO: put in config let addr = SocketAddr::from(([127, 0, 0, 1], 9001)); @@ -117,6 +120,7 @@ async fn run_registry(db_pool: DbPool) { .serve( registry_service() .layer(Extension(db_pool)) + .layer(Extension(config)) .into_make_service(), ) .await @@ -133,12 +137,15 @@ pub async fn run_app() { bots_directory: "./data/bots".to_string(), match_logs_directory: "./data/matches".to_string(), maps_directory: "./data/maps".to_string(), + + registry_directory: "./data/registry".to_string(), + registry_admin_password: "verysecretadminpassword".to_string(), }); let db_pool = prepare_db(&configuration.database_url, &global_config).await; tokio::spawn(run_ranker(global_config.clone(), db_pool.clone())); - tokio::spawn(run_registry(db_pool.clone())); + tokio::spawn(run_registry(global_config.clone(), db_pool.clone())); let api_service = Router::new() .nest("/api", api()) diff --git a/planetwars-server/src/modules/registry.rs b/planetwars-server/src/modules/registry.rs index 7198a61..3f6dad2 100644 --- a/planetwars-server/src/modules/registry.rs +++ b/planetwars-server/src/modules/registry.rs @@ -6,24 +6,22 @@ use axum::headers::authorization::Basic; use axum::headers::Authorization; use axum::response::{IntoResponse, Response}; use axum::routing::{get, head, post, put}; -use axum::{async_trait, Router}; +use axum::{async_trait, Extension, Router}; use futures::StreamExt; use hyper::StatusCode; use serde::Serialize; use sha2::{Digest, Sha256}; use std::path::PathBuf; +use std::sync::Arc; use tokio::io::AsyncWriteExt; use tokio_util::io::ReaderStream; use crate::db::bots::NewBotVersion; use crate::util::gen_alphanumeric; -use crate::{db, DatabaseConnection}; +use crate::{db, DatabaseConnection, GlobalConfig}; use crate::db::users::{authenticate_user, Credentials, User}; -// TODO: put this in a config file -const REGISTRY_PATH: &str = "./data/registry"; - pub fn registry_service() -> Router { Router::new() // The docker API requires this trailing slash @@ -49,8 +47,6 @@ fn registry_api_v2() -> Router { } const ADMIN_USERNAME: &str = "admin"; -// TODO: put this in some configuration -const ADMIN_PASSWORD: &str = "supersecretpassword"; type AuthorizationHeader = TypedHeader>; @@ -105,8 +101,12 @@ where password: basic.password(), }; + let Extension(config) = Extension::>::from_request(req) + .await + .unwrap(); + if credentials.username == ADMIN_USERNAME { - if credentials.password == ADMIN_PASSWORD { + if credentials.password == config.registry_admin_password { Ok(RegistryAuth::Admin) } else { Err(RegistryAuthError::InvalidCredentials) @@ -162,11 +162,14 @@ async fn check_blob_exists( db_conn: DatabaseConnection, auth: RegistryAuth, Path((repository_name, raw_digest)): Path<(String, String)>, + Extension(config): Extension>, ) -> Result { check_access(&repository_name, &auth, &db_conn)?; let digest = raw_digest.strip_prefix("sha256:").unwrap(); - let blob_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); + let blob_path = PathBuf::from(&config.registry_directory) + .join("sha256") + .join(&digest); if blob_path.exists() { let metadata = std::fs::metadata(&blob_path).unwrap(); Ok((StatusCode::OK, [("Content-Length", metadata.len())])) @@ -179,11 +182,14 @@ async fn get_blob( db_conn: DatabaseConnection, auth: RegistryAuth, Path((repository_name, raw_digest)): Path<(String, String)>, + Extension(config): Extension>, ) -> Result { check_access(&repository_name, &auth, &db_conn)?; let digest = raw_digest.strip_prefix("sha256:").unwrap(); - let blob_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); + let blob_path = PathBuf::from(&config.registry_directory) + .join("sha256") + .join(&digest); if !blob_path.exists() { return Err(StatusCode::NOT_FOUND); } @@ -197,13 +203,18 @@ async fn create_upload( db_conn: DatabaseConnection, auth: RegistryAuth, Path(repository_name): Path, + Extension(config): Extension>, ) -> Result { check_access(&repository_name, &auth, &db_conn)?; let uuid = gen_alphanumeric(16); - tokio::fs::File::create(PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid)) - .await - .unwrap(); + tokio::fs::File::create( + PathBuf::from(&config.registry_directory) + .join("uploads") + .join(&uuid), + ) + .await + .unwrap(); Ok(Response::builder() .status(StatusCode::ACCEPTED) @@ -222,11 +233,14 @@ async fn patch_upload( auth: RegistryAuth, Path((repository_name, uuid)): Path<(String, String)>, mut stream: BodyStream, + Extension(config): Extension>, ) -> Result { check_access(&repository_name, &auth, &db_conn)?; // TODO: support content range header in request - let upload_path = PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid); + let upload_path = PathBuf::from(&config.registry_directory) + .join("uploads") + .join(&uuid); let mut file = tokio::fs::OpenOptions::new() .read(false) .write(true) @@ -266,10 +280,13 @@ async fn put_upload( Path((repository_name, uuid)): Path<(String, String)>, Query(params): Query, mut stream: BodyStream, + Extension(config): Extension>, ) -> Result { check_access(&repository_name, &auth, &db_conn)?; - let upload_path = PathBuf::from(REGISTRY_PATH).join("uploads").join(&uuid); + let upload_path = PathBuf::from(&config.registry_directory) + .join("uploads") + .join(&uuid); let mut file = tokio::fs::OpenOptions::new() .read(false) .write(true) @@ -293,7 +310,9 @@ async fn put_upload( return Err(StatusCode::BAD_REQUEST); } - let target_path = PathBuf::from(REGISTRY_PATH).join("sha256").join(&digest); + let target_path = PathBuf::from(&config.registry_directory) + .join("sha256") + .join(&digest); tokio::fs::rename(&upload_path, &target_path).await.unwrap(); Ok(Response::builder() @@ -314,10 +333,11 @@ async fn get_manifest( db_conn: DatabaseConnection, auth: RegistryAuth, Path((repository_name, reference)): Path<(String, String)>, + Extension(config): Extension>, ) -> Result { check_access(&repository_name, &auth, &db_conn)?; - let manifest_path = PathBuf::from(REGISTRY_PATH) + let manifest_path = PathBuf::from(&config.registry_directory) .join("manifests") .join(&repository_name) .join(&reference) @@ -339,10 +359,11 @@ async fn put_manifest( auth: RegistryAuth, Path((repository_name, reference)): Path<(String, String)>, mut stream: BodyStream, + Extension(config): Extension>, ) -> Result { let bot = check_access(&repository_name, &auth, &db_conn)?; - let repository_dir = PathBuf::from(REGISTRY_PATH) + let repository_dir = PathBuf::from(&config.registry_directory) .join("manifests") .join(&repository_name); From dad19548d1e704c31800c5d6b132299ee5e88d45 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sat, 16 Jul 2022 21:57:12 +0200 Subject: [PATCH 45/51] read GlobalConfig from configuration.toml --- planetwars-server/configuration.toml | 10 +++++++ planetwars-server/src/lib.rs | 39 ++++++++++++++-------------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/planetwars-server/configuration.toml b/planetwars-server/configuration.toml index ee7002e..721aca1 100644 --- a/planetwars-server/configuration.toml +++ b/planetwars-server/configuration.toml @@ -1 +1,11 @@ database_url = "postgresql://planetwars:planetwars@localhost/planetwars" + +python_runner_image = "python:3.10-slim-buster" +container_registry_url = "localhost:9001" + +bots_directory = "./data/bots" +match_logs_directory = "./data/matches" +maps_directory = "./data/maps" + +registry_directory = "./data/registry" +registry_admin_password ="verysecretadminpassword" diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index 87495e9..ad7741c 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -18,7 +18,7 @@ use config::ConfigError; use diesel::{Connection, PgConnection}; use modules::ranking::run_ranker; use modules::registry::registry_service; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use axum::{ async_trait, @@ -33,15 +33,29 @@ const SIMPLEBOT_PATH: &str = "../simplebot/simplebot.py"; type ConnectionPool = bb8::Pool>; +#[derive(Serialize, Deserialize)] pub struct GlobalConfig { + /// url for the postgres database + pub database_url: String, + + /// which image to use for running python bots pub python_runner_image: String, + + /// url for the internal container registry + /// this will be used when running bots pub container_registry_url: String, + /// directory where bot code will be stored pub bots_directory: String, + /// directory where match logs will be stored pub match_logs_directory: String, + /// directory where map files will be stored pub maps_directory: String, + /// base directory for registry data pub registry_directory: String, + /// secret admin password for internal docker login + /// used to pull bots when running matches pub registry_admin_password: String, } @@ -71,8 +85,8 @@ pub async fn seed_simplebot(config: &GlobalConfig, pool: &ConnectionPool) { pub type DbPool = Pool>; -pub async fn prepare_db(database_url: &str, config: &GlobalConfig) -> DbPool { - let manager = DieselConnectionManager::::new(database_url); +pub async fn prepare_db(config: &GlobalConfig) -> DbPool { + let manager = DieselConnectionManager::::new(&config.database_url); let pool = bb8::Pool::builder().build(manager).await.unwrap(); seed_simplebot(&config, &pool).await; pool @@ -104,7 +118,7 @@ pub fn api() -> Router { .route("/save_bot", post(routes::bots::save_bot)) } -pub fn get_config() -> Result { +pub fn get_config() -> Result { config::Config::builder() .add_source(config::File::with_name("configuration.toml")) .add_source(config::Environment::with_prefix("PLANETWARS")) @@ -128,21 +142,8 @@ async fn run_registry(config: Arc, db_pool: DbPool) { } pub async fn run_app() { - let configuration = get_config().unwrap(); - - let global_config = Arc::new(GlobalConfig { - python_runner_image: "python:3.10-slim-buster".to_string(), - container_registry_url: "localhost:9001".to_string(), - - bots_directory: "./data/bots".to_string(), - match_logs_directory: "./data/matches".to_string(), - maps_directory: "./data/maps".to_string(), - - registry_directory: "./data/registry".to_string(), - registry_admin_password: "verysecretadminpassword".to_string(), - }); - - let db_pool = prepare_db(&configuration.database_url, &global_config).await; + let global_config = Arc::new(get_config().unwrap()); + let db_pool = prepare_db(&global_config).await; tokio::spawn(run_ranker(global_config.clone(), db_pool.clone())); tokio::spawn(run_registry(global_config.clone(), db_pool.clone())); From c16b068f8b3848e775f2490851fbc788139febe0 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sun, 17 Jul 2022 15:10:17 +0200 Subject: [PATCH 46/51] cleanup: remove old configuration code --- planetwars-server/src/lib.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index ad7741c..123fdab 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -28,11 +28,9 @@ use axum::{ Router, }; -// TODO: make these configurable -const SIMPLEBOT_PATH: &str = "../simplebot/simplebot.py"; - type ConnectionPool = bb8::Pool>; +// this should probably be modularized a bit as the config grows #[derive(Serialize, Deserialize)] pub struct GlobalConfig { /// url for the postgres database @@ -59,6 +57,9 @@ pub struct GlobalConfig { pub registry_admin_password: String, } +// TODO: do we still need this? Is there a better way? +const SIMPLEBOT_PATH: &str = "../simplebot/simplebot.py"; + pub async fn seed_simplebot(config: &GlobalConfig, pool: &ConnectionPool) { let conn = pool.get().await.expect("could not get database connection"); // This transaction is expected to fail when simplebot already exists. @@ -88,7 +89,7 @@ pub type DbPool = Pool>; pub async fn prepare_db(config: &GlobalConfig) -> DbPool { let manager = DieselConnectionManager::::new(&config.database_url); let pool = bb8::Pool::builder().build(manager).await.unwrap(); - seed_simplebot(&config, &pool).await; + seed_simplebot(config, &pool).await; pool } @@ -160,11 +161,6 @@ pub async fn run_app() { axum::Server::bind(&addr).serve(api_service).await.unwrap(); } -#[derive(Deserialize)] -pub struct Configuration { - pub database_url: String, -} - // we can also write a custom extractor that grabs a connection from the pool // which setup is appropriate depends on your application pub struct DatabaseConnection(PooledConnection<'static, DieselConnectionManager>); From 09c543eee3dbc361627e4ad6b5f5f55317ff334b Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sun, 17 Jul 2022 17:07:53 +0200 Subject: [PATCH 47/51] create all required directories on startup --- planetwars-server/src/lib.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index 123fdab..8962c3e 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -8,7 +8,8 @@ pub mod routes; pub mod schema; pub mod util; -use std::net::SocketAddr; +use std::path::PathBuf; +use std::{net::SocketAddr, fs}; use std::ops::Deref; use std::sync::Arc; @@ -93,6 +94,19 @@ pub async fn prepare_db(config: &GlobalConfig) -> DbPool { pool } +// create all directories required for further operation +fn init_directories(config: &GlobalConfig) -> std::io::Result<()> { + fs::create_dir_all(&config.bots_directory)?; + fs::create_dir_all(&config.maps_directory)?; + fs::create_dir_all(&config.match_logs_directory)?; + + let registry_path = PathBuf::from(&config.registry_directory); + fs::create_dir_all(registry_path.join("sha256"))?; + fs::create_dir_all(registry_path.join("manifests"))?; + fs::create_dir_all(registry_path.join("uploads"))?; + Ok(()) +} + pub fn api() -> Router { Router::new() .route("/register", post(routes::users::register)) @@ -145,6 +159,7 @@ async fn run_registry(config: Arc, db_pool: DbPool) { pub async fn run_app() { let global_config = Arc::new(get_config().unwrap()); let db_pool = prepare_db(&global_config).await; + init_directories(&global_config).unwrap(); tokio::spawn(run_ranker(global_config.clone(), db_pool.clone())); tokio::spawn(run_registry(global_config.clone(), db_pool.clone())); From e5cb04208f2287f5ba4f0f70367c0f8190e0082e Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sun, 17 Jul 2022 18:23:24 +0200 Subject: [PATCH 48/51] allow disabling ranker in develpoment --- planetwars-server/configuration.toml | 2 ++ planetwars-server/src/lib.rs | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/planetwars-server/configuration.toml b/planetwars-server/configuration.toml index 721aca1..13012f9 100644 --- a/planetwars-server/configuration.toml +++ b/planetwars-server/configuration.toml @@ -9,3 +9,5 @@ maps_directory = "./data/maps" registry_directory = "./data/registry" registry_admin_password ="verysecretadminpassword" + +ranker_enabled = false diff --git a/planetwars-server/src/lib.rs b/planetwars-server/src/lib.rs index 8962c3e..7bc50f3 100644 --- a/planetwars-server/src/lib.rs +++ b/planetwars-server/src/lib.rs @@ -8,10 +8,10 @@ pub mod routes; pub mod schema; pub mod util; -use std::path::PathBuf; -use std::{net::SocketAddr, fs}; use std::ops::Deref; +use std::path::PathBuf; use std::sync::Arc; +use std::{fs, net::SocketAddr}; use bb8::{Pool, PooledConnection}; use bb8_diesel::{self, DieselConnectionManager}; @@ -56,6 +56,9 @@ pub struct GlobalConfig { /// secret admin password for internal docker login /// used to pull bots when running matches pub registry_admin_password: String, + + /// Whether to run the ranker + pub ranker_enabled: bool, } // TODO: do we still need this? Is there a better way? @@ -161,7 +164,9 @@ pub async fn run_app() { let db_pool = prepare_db(&global_config).await; init_directories(&global_config).unwrap(); - tokio::spawn(run_ranker(global_config.clone(), db_pool.clone())); + if global_config.ranker_enabled { + tokio::spawn(run_ranker(global_config.clone(), db_pool.clone())); + } tokio::spawn(run_registry(global_config.clone(), db_pool.clone())); let api_service = Router::new() From 270476e0387201c85aa686254eb2f4a33205b7d1 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 18 Jul 2022 07:40:01 +0200 Subject: [PATCH 49/51] use texture for rendering planets --- web/pw-visualizer/assets/res/earth.png | Bin 0 -> 12873 bytes .../assets/shaders/frag/image.glsl | 1 - .../assets/shaders/frag/masked_image.glsl | 21 +++ web/pw-visualizer/src/assets.ts | 4 + web/pw-visualizer/src/index.ts | 164 ++++++++++-------- web/pw-visualizer/src/webgl/index.ts | 2 +- 6 files changed, 113 insertions(+), 79 deletions(-) create mode 100644 web/pw-visualizer/assets/res/earth.png create mode 100644 web/pw-visualizer/assets/shaders/frag/masked_image.glsl diff --git a/web/pw-visualizer/assets/res/earth.png b/web/pw-visualizer/assets/res/earth.png new file mode 100644 index 0000000000000000000000000000000000000000..7897e3249f1c5895adfc7a698434875fcd353d7b GIT binary patch literal 12873 zcmV-PGPcc$P)py;U|9^)9!~6e#KCq?_%K90qy1f|Go&! z1ZDsWy`NDe7+qxx5OHEKunF+i{kV+>C51A|(+gfba|gwg&bA_6D}0v+u)`z*WGNz~5P>i--iElOMd0tckqrQ-EEO zuYP1jUI%Ufu0= zcqec=@HBp|{l9>Hi4#G@kv$QmTiyrndEf_#UwK^{=S$0wAz;Sfm6u0D(C|)K>QgZ9m;UvWb$dDZvpm4l!dOW zj=jGn(8h#`d`4ZtUVDowgWCF z`zr1!#9|O>bJjtEbFw@ni-9wcEIe6xx5GH+=IFoE*#cSGXZ6*%9C2-qv>9E%F-Rax z+J~YSQOa%PK@Q9QKMzUw70JhV#FUuD>u@#_ftY4n7pX9tB75~LFa8Bcpomm0r%hq5T{_cIG1iFAvkxj{W0}{~Ui$O@>iZ_9dX?|k4BCCD4e z$v7mVOxhd%IHFnh%@M}9OOgF+q&T_|>&i^hKNnX}Za#!a3id+c;?vyVIfxy06YDq)gsoM^}~ z7g!^J9IeJg;BS=wqHFCX#IGY#BFB+#Irt3W_HnBsb@ViOP6{MTt1tpMhx`FQq1s|( zix&yZP$XtP&CR+L$>SG_ZyWNwOso%3&vrGTZUFjle9gOY{Rum%z41EEGvEyiuI9fKblX$Cy2Ix5clYd zN#_CiU*NAus;@_p72a&5m`N8>mth#AjA}M;w@f`*Lh4@Y!<21mM?Ag<1=`zOUh;FG7 zUBKsYS2)i@(z-`HZ~&rhb-C~_Q?6-dQ|#sJVWmk|9me3E^z&w9DV~FQAzxva=VmHy zE3!3^ZHMawP&;b-I+(%FNLEJw^Y*aq7TIToxCKYZ(6E(G3}V~iUe#z>HoVuKv| zTbp#s=la$Qd@RSgbR?f8y-)uGl&jRdwuxM|c6)iERfczBujqOZvtU*RBg=2=dHD@) zFaP>TlxUWhzt;MI&*V6#j^Y@y=XRS?g1i7bJIL1QZlmLHbJ;%L!&uhdzYn+ju@RCl zFwe_hKW1DB7#vPMH_#ixiwxKqhyGfUTvcHZa&N5rcNuAM!PER44c+aCLW0c0PH zUf?r9+>tw#!ANFZ>)sY!2=(I*G9*Z}sHy{#mUYLxG#H zFV0dLJI@RPo-ls<#RBC&9Q)?}tibWB&R*E2+y#iQeh9j;y)<3gk4R^w*blcY$n|ts ze27ajk3MwozGIEwKZBcpH%MsiddQw^JCn@F>JT;S@w^WXV1cSkrOrc4Ix@ z73^yi+9tjd>GMX`w{K+}_ek9QT7I3e|CZx8+%EW+jeb_bj`aFQliv+=vY0hh(bxUEoX&yTOraw|zpf@~{y@-$xMSR1$4 z?Af?SfQ_QqmpN@)g7~|4BD-O)q~=i#v*=`OEqyO=w8t^KkqSfBGxQv7xP2Uo`~K_c zlzqD{uZnIxNFKRP#U)SU<5Z`*&Y?7)u$Cfcvh5c5#rWc6zk%)5B1`jofZqXI1{iZkY%_8n@ZJEj6v{Z#7wWtz<5=$`%p|Y?+0L8{ zY?x8jcI3lkzou1~2ts4kj7*S~Ld#Yq?jl=>?kjVQxdG|FM6V$;|B1-Ha6iNcU>C$M zU?TFOZ$OxGVy(1TaQJ^vhNVY zpCICojgbdCPp<`M>)nJ4nM~MUq{S3Q-~E@Lf~s6 zX!o$$&9H-z<^!9QC16^*u`}do?Z@FojuglQqzFo$t-&3%x0U4>>}xQM7TF4XlI%A+ zL5gZd+7BNqf-7iWFH4tTrH_{;og?cap}5*fSZ@;Sx5s%u+ zawYZ+Jj(ccOrP8tWSs~W)4rP#P!lYzMBvUg7MASh*_2a3ua-o4KAwf_JttjypK8?2&M+f~)}VA-f)6Hj-|v zT^Wg8H*qEvD=&XC_Kn-C|JZjJ$L$$X@JJ9g1%85iu}-ahDX{EiurKy?_?y{vJBJcHQYb@^Y)UucC*jH*{8SBU6)r`;9`k^8N|UaRKVsjwdO;5|j^8Gf z;E_UD3#sbsTK?N=|L*T^bjXva0Bd8%w*0O%>H3+3J-dH?z;8Zc9N#h?Pmwq1LOk{^ zBcGkW))pZT_oR>~Q9(YBex3S&x0faz;;N|$_%JIfY%E8iH-`{B;*EpImS?X~dmPE; zJSONFRF|RHwI?jAT4A`@EE_5GEI)v_0%#q6NdA|;2U$7nfnT$9yRbL*H4Gjv(S>cw zxvt4?-*5cxlSK+1@rJcLwc2xt^{5hxtcr6T`n9|>v_ShA0`&k}H2Lj@#&2JxjWeUR z2`-kOMy(Gif^i`2ZOiw~me>VWg9S@<1NR%koNM&)l=1te$b<3?#KUkS-zO2t#rH_} z;=hX2YuvaDVcUe?qhI6u2ox=7nei4QA)p#xG5ToKW+0`Rs*qN?(|G{5dv*tsDl8^l z$8F>X^k~ceG0 zI}HIA;%>$}(CEu*OMxFyF`wuTU@*Gx$JG>*(|%RpgGkZUZzAr+oyNM@*@Pz-DZrnN z!GDpmzf+94}{|)?0fPe@U9GfuZrjmuEhS@CCI+`OghJbY$J4U%U%jn zww|R7nvl;gc^llS%4+R-V0;c4BJS*g`!?;1*xKpZOMew^FMXr70I@W5f*0Wh=-1** z-vh3OUQDr%=63%yY>Sjcx&=`trF{tgMY)w{I|<7Qq_?#tPGq-`7gTFoU_4R5&h8N*v@ELdkFDCX^A2m1}C7O>)Ck+7=>Q8 zXc3|y49!rajJS64XO$pVr?VY)oJW?Ien*ynVY`r*_@Nwq-%ab+A6ygNJ$NoKBF_LH zQ#wz`U&u2L6Vd$=UM9wum%-}5U&#JqJ_DSNc(hqx>2C5TB)N@j{t8vW${qOOJe^yn zp0a1qE70ZJx;)W3nnA#~aCh)(_N8}aD5B_l&T+g(?P^*xe!)u2b9L5^$D-fBrz!sn zdl>r3A5orY5hP&iS{gq`d0+ZQtb&;StnzcsS|3uQua)8q1|d#x&CY2F8Mo;lS2|6A zQ*#RLgEi1)Jf%F*B6W;L5>_0FEZK|p-yexM1I;1ZM9_m&v0A@iIeqbC^z%6?OQ$y( zooBzZ}e>_xS&=dTcdhbprj`g!~{ZHH^32OK=&LuyaFL~}XY zwUTTk9dAbbKnD#(UNYAq|j zUbFTZ@HwjKOk{=Det-xgiwKe=`^ z1bAKPXogRzb8phk0hVXG6`Ho`Bt!cy3i|h&%$x#@=m3b1WCzM6|5Wr zG3R{5g?D3DxoXsoE<;#_pQFp`RBFvi`5e3oNfX|KF6YDLi56*5_Q6hhU*Bd@tyO@P zsKrTlr(J+Kz%buZXP^r^N5ILYNf&8lx)FZ`O{-qtN-fSgfR)kB;`dk`j73+BTv>$F z8L*s`dg;7mkrBTPMyz30ne$JSC14XYoklMMOqm1AGwp=#mU)vmF|)G{@O|T$%YY{W z$P#H^dVq6)YmgX&W~`o}??J$BNb=4_$jZ9bUa$NA*D~~dJ+Nz&zjY(;_#5#711A7W zNl=9e;Fn04ax;37K$J+t2UdpNc`5r_7gvZ(tP>w z3gs(2K7hR^bE!-eiMV1Z#owK>znhHyif+@I2{sHpqs!$fPt-SKfd8VO-1Ai1T{E&g zn=x+G^oU(ytZ)g-_3E2(oO=mA^L`CNFkOdkn5}E?9kc*1_TCeX8hnw zIZie!DBYS*)8Ercc*DlQ@a4qRp zx|ZVnp!$QqV;mehtacmhl+w3TzGG4+vm0;&`Cg)4MA5cwf1e`}bM&#um-r5{Rf!JK zp1_aKNt#LE!zuc;d=8cimb*|naCWBC%FbZyf|HAoZA&C9gMdTO6~-=YA@YK6U35{ z_Qg+-?}d2{Nhd$fgG^OfEu84#T)=%2_+W}YH7~|n>koWUIdE#m6UxFS*h-}7NZPVU z@vH`X4L$R+OM4bc-&V9QNu$-mlN7sJ=)s?pTsNZa*Tzdz>F1A1KW{`hu7ZAn@|Zi) z&<)(8$Wgo4K9nyJopq6_J$b*V4Q`@KZ$q{`d%%(h-l({usX8sZZ_6@ z+&_TdSNiD%Rswj7a!`UKMF@eFiaY^+6iAj%Wpl**l{J5Ut=$g1k7_GvYpaF#BddkW zu{}<;_IFkcug}V9##pXdxj$|6vkBP61w#}7T@`zo`8cv#iTGzSqR`5-RPIBTukQ#f zyH6&NZ{qu*CpoNlav#Tj-e>glUXZ$BK4zOd{KSx<57;Gu9G%8!;9~MU^h<#AkzD)j zz~khb1o!tK!`8+)hdK7MjnU6XLA?C5gVykHLKVv>&{Sf6YU_IY^IXaEx`M+ zzw*{K`WNXEg$(2~M*ptctS)qiy6=K+d(@I4`d}n_6{3XdWnFC-b|Kx1|0M7!@;%tC zz-s6LX!ju1v8?O$n~dXkAsLdi2`Eh>(wCLK&IZwbHap*yE8v<4!k@OAwvOc!x2M#rD7GL3B51xK`=!0np7Je@Ho4 z8Z5`3SlX+224Va6A5xlh<*^d_692CL;9xk3W_tT4`8bbY~1W{%GU)TKhUOxmXErYORm->BsB& zxzhJ6km#aj2mD`-fi?D4#!L_mYa|>K&{KLXMU=LkL^tVIsoK{zOIu9-jMrLV(fn?t zBv>z$24DD;OgnsR0Bq%>N{(dx)fwq&l!T9^YGTe(1)9NER$Gr#1lX$CjpaW*g zGHna+98zlADv_L)OO!F6_B>X^oqN%jq@OHLv{JCbgPZ9_CIBxOzirljOMjvcPiXp` z`JgKS>)Rd31K=YFMDsrUhll zR#|Hnq|f)Z7|FG^AK5+i?os;xOU6DU`5B6MU~2b&uhOI|2g^z4ZOU)A!L35lzgd5e^N!@;Z1j74PI;o0f)yU@ zsV%=CVHkb}H^XZ4r_v@*tK^Wvr#5idKN~*&IFd%_961_h=Pd!h@@d z77l!KH=?O5ieZS?-&7=E;>XCc=OyJgYmqMr$Y><4`fI=y{o^h|!k#}%EGpu{L+E?o zVZbqgWT^sq^Z;d}We=JKx|VP@5|*2{B-FrWB|TWKbg{xpNOqRb0>4M}5LV8%hmpdT z5gjWvbtmO(a19TxBAS{1Y|xW(_OR@0JP`@f(fp%*wGTHrnGJv=`sKKle6{GW?C&qq zm@Z_p)UM$XrAZea*YMyfF$DdCcq-Uoj{{2{4MPeOOeKE>nP%x>g%Wq9fnx%y<{rnV? z4mz?t!?7pcolBE0ELM2%#fbOcn)oRh?t?ASW&SAmD?_vyqAkr+U#;wRBccJ1EaQ9V z*X*+LL<@=)9$ep+B5BkwMMB#{n*Hk8SCKKHzciRt95k)Aw;M5woCnM?`mSPIij*o_ zHRXA?E9;_f^?K+WUCN)x_Dns2zY^kUu+qJ40(N=uQwR2%f>nSIBL4ka!il2VB4m4# zMOpX~vOS8f$#5b1wfaeUq6Nh@>rbOr3l$EDR&(=>^57u3!I~X-ktbo+LS`f@eY{F(%xBh3gOectg&n3W#$nt8W!?^E7d17B4-n?UoBMj!kSQc-mk<(Q|CT!M2wjkz}B1oc7E zn{A#!^5`u=Jgt@@H+FuGf2%r!ku8>14*UQ*!zwTd@lTmVwqmL8qQ5Lxfoy=>uV$Q# zoC6@mAGKL)_oAe%X^!z0kkglYS7pFE<-aCf60oU8nxk8pP0F7(K& znmi$5un)zp@MFjpLLXycec_t5C5ScWGQ`FBNFXq(EUqZAk2hzvLZu5T^8^9i#iPX>Nbm9x`yyM;jc2rOpw{mO4$^e zQHaCQ2*kHxSc-ppGx|Ayj3!ICz#K_fIgSH<8gRTU?1enD%{UAAcmO$WKvrpuScn9= zHLMwUE6rz(*9ou0SsObZeI{YH0-;z3{d}uZq;m-3ycJ>kl5d9Jx*DcqDoilp=T@aS+m2ANWxKSwgYE_+yGa55HJhlu)XwV(jHrZ zF7$iwz4Am$%@9PBzb~TMKL^=nX7#xIMOg7up~d1(vortz5z0wKK~&x+giAShqas5u z4vB8Gece3DOK)_LB3G@2vOgHvrCjK1X>b{c~mpxcq009c|{;O z-e<`1t${yiWRtYYbMKu`#$0>~y{b?bl8dm_^Z99dfT}0~i{hWk--aVwimGvd@%KxE zNFlj3G_G~&WC*$!-qV!pl=1#1a@DspOOwtITGpaa4^vrKU$Mi=6Rjx^(Oll8=;gV5 zGESv45K}{$SlL0nULUo*O zF2&voR7F-7&Y=0Z%q~%`P;{dwAN_NAqP@ZD=pV?30?DzrB13{6o@^#3m}>m}Xg3)P z=acA~<1WMxy;ZCNniZhk9f3e5?E96!Zxc!og}u?k<3nlxKW6;hdWTg;qR+&OkwBd) z@ImaYK$U|@>iIVMd2~W@O)LBG8s&)w$P2E?7eYzCHu@@L4jy67}N0)zmX_AFw7;kHLlERkSq zPs_xLc&yU3E)q@u)Jr+KkUbQc62fFk-ocPz0gkqR*S+v8`s;}+geV~D%3X*pUE_GF zs|&F50(OEa5Oj~gZpw*DtwQYo@y6fJLrTwkWfSz8a6L#(@~SWr@mADk$|~tD>UlT1 z+*K$K7J|v>*TKpd?VE1I6SJ8-b0|J#-UjHa+g>C@*@-k?!2h6o6%7KukfOh$nS&&1 z)TolY^;p?sJ7ooMKjIYBjL%~4pk&yLU+Nxf&!y;cSfI)7ZN7?(G{0^ z_HQjsy2cDf4~I>tMj%!wv?{k=HQf&UNRcH$;!=ua0`hUtWSCU24BTl5P_@jwzeJa} zYTkiAMVB{0av^jI7t;JXEFr8l=W#`rMHC-(dag2lLZGb4mJ`GhV8alCIPw_+cO~FSW9*_=A1XVdYuFR2akwE*f&^l9 zI%QT1S`&T$eQ$wde_0tfLEb%YA-`F%LL*-X)+$Z9<*^)@9wwY7br`l@;)a0oRGBiX z1zDllUoU9fb%43X*yk5Cb}H7v_O`pYH0f64L3CN)6-bWH8FI`+9{h-)RZK3j{FY@x z-lCv!|7?sqmkQ~HzJa}z-M=*Hmgm#xa{e)Z3@am5jx{;T%y?5pH1}vNA#5I2Mb{e@ zm6PCLWz1UpNApY-FQJz+XGO7hxiDtiF zz?iE6FB;=MNd-;9E@iu`5+`sLx|~HOIvs@WQ=VWKdhA3#K=>8r#m1O#FJMe732%ZZ zMJvf==z>?JK*vOMIcHE@vJlHF@n1y6Un8t)LErK`-pc5@t!MFnpZ37Es$JxHOs~8d zsa&I-z_UnIm9n`TUCyGCg;-ULs-i2Y8r|qyw<|r5XPF*-yC9E69KN(MXHgxCT2m~u zNkX`yi5f?v%XmcrV~#`j`g@9M)LX17TD-)wp2s?087l$a<9R&I+p*S;@;qiCbRphJ zRX>^Gh>t=u`FikghWm^%UP9FTTZF0>G#IfgG#lqt{03!NO5p^F2WXkC;l}8dQxar3 zw{ckl_VWvL8GY|7Z&b!wj4T5q&9SNl?cr&h_ZZ{&bijEF_Pa8V6nXxtg3GKn9r+TM z#g$eQDAOx1-B< zpr>&TGsdZMor0<`l;W!ebs_o$?Q6V>hjBnpcUEgv(3fRq47O%7LA>+I;v-FDspl{xr4)&nmRW}-(OeyO%NxytPH{q@l6n&S>?U`va%Dp7Pr=ZR)TCFHDvl)j=oP-`dvf| zIe1irqN)Ymqx9QLz8BL9K&w3qWhSQ$!l$q=L_+wweWQ@lX3gZullti*rQauV^xLTv zQ7veF^ljj6IpjT2>DSi^qor^OdF&&+yRhb|*GKj|0XHlC{ymq(okdctA;!u&e-pLN}qqoCvrz(RSU}ES7PaaHpH*XY{bb+YYQCuIXR_l zyQ1rpYOPG)m&z~^JF9Sl+$dksUK&=yjx1@QwMp)r{33TGr(i!bK3Xs7sxj}1+w(jZ z32ur2yCWtL>!K~9_#%i_Zo?NzpOEnhrN7^Jk-GC(13Qe?N4q42O#9&WOhdjDy~c5$MJ9n}Y(jBmraF!B$UPa4TawVFz{`lU)-+&_!*50)?%gg{ zsb0jz_ty@`Mh0*gZi>MK@zwWjQ@z$<+;2McV-*8Az_Z+u;#da>IC6O=mk{^-x!Dgl z6Jdfl+~iFNQO}mx6HpJ5;YDLZqmLrXTSxq{Gj7Q|5@aot@jGvfzgYyVnnS*bAI9QV(5AkfMH?pZYz01rTaLl{{aK!1qLr{K z3DsI3l4HI;(ZlzR-}qEsM$)qf>GVvC=%6%SRb^YGtah3idn!_^FAJm5lSdA0(~(3Y zV7nMUM75o>+Lcuir!!aY(*$whetVvR$*q@W5_ksKGvjz;kQdu^tX{+_u`YT> z+XV4IjD+K0B*Z(-vvU*vN|8rAF$AeVlx7xB5aru?8O9rf`*xt#79#1Yv}3NzAWI}Y z>)@8BzrOV$+qk&juQ0nJd;K&wc0O4S2HA=Qo3jOEfMrIaJ$&QQc8`#S0F$%}-` zO&Q6_Sklk#3Gi>yryc7;vI)C-iMh163MhsH5i@Pt`*9i3QLE+1V#}?y8xVzE8g?Yx z!_}YZV!UmEe`FYIVEb^k)9rXBAVos6Y|CaLo}OMqTj04c!a<$fj zX#Tfy9IG2iHSSUueqhU6kG@mb0dekfNzB=(Jqf%oxYUtSSs5wv;hJZlQM(z@EW73; zoQ%`~nCsa8iMh_Oz3B$_CA|Y!eVYq>ne=rYB5lMv$QM1W15tgOiBzn0H5H9OGQ?g* zb1TI^@srP>-B=sR2JMogp;_w#&hPIl5&^`o+o}+2wZ+K3eIML4_$wnOzh5AUDenfp zjf6c%gh5DN#mmX>Z!rznKCJkWR$_PHPVzlu3F0|+8nAU(@!N(jWKup23B1XBKkgu= z>58OaSJFlOX>A6Q7JM?%ykxOC;+CzIR8P~U0|ybWdn5y0!2ZbB!^gvzAW?`vMV9?> zj}LD`H1KDT-7?dt-QM4CeDNb*n2h)k_$)!S2)Gf^_D{sE2uiCn7&+H3BIlkap^Mde zk?77COcg1NjS**)1^9b#Ywb0p7|A!0prm$45jFzRtDKC)Sh$)%)3gQ13L?7hrX((p zvNIYu9B~`nvY;{gkOU(4BiU&mMlv1G=>J=l=t5j@w*=nW|KBZ*^gMr;xJ|SpmX7#sM`~N!+@sXe3|Mz)_O&ugj#_<0C zAA!8=t00a|wU@q3zl0@7D)_0$9)DhmvP2?Ij0TPXt|WVKN+-7-;8rAiPt3np5OD<% zhOtOG@+rv5AWwaQc4ijxrN0)q8hE8#aoV3a0fc7|;^8(0DIW4RI>Y?eh(5&q`7g*z ze7wOlh0b zdXvIJq|nLq{&pvDFUu5ANFdVd?uvbckh$b>S4Wp?|TNw61~)BkVt5$Bp$5oKJh ny^2IS&Om$*78i70kpTQZ+G#sBj=5TD00000NkvXXu0mjfzX1Z4 literal 0 HcmV?d00001 diff --git a/web/pw-visualizer/assets/shaders/frag/image.glsl b/web/pw-visualizer/assets/shaders/frag/image.glsl index 69c8b91..f8d62c9 100644 --- a/web/pw-visualizer/assets/shaders/frag/image.glsl +++ b/web/pw-visualizer/assets/shaders/frag/image.glsl @@ -10,5 +10,4 @@ uniform sampler2D u_texture; void main() { gl_FragColor = texture2D(u_texture, v_texCoord); -// gl_FragColor = vec4(0.7, 0.7, 0.0, 1.0); } diff --git a/web/pw-visualizer/assets/shaders/frag/masked_image.glsl b/web/pw-visualizer/assets/shaders/frag/masked_image.glsl new file mode 100644 index 0000000..da0c787 --- /dev/null +++ b/web/pw-visualizer/assets/shaders/frag/masked_image.glsl @@ -0,0 +1,21 @@ +#ifdef GL_ES +precision mediump float; +#endif + +// Passed in from the vertex shader. +varying vec2 v_texCoord; + +uniform float u_step_interval; +uniform float u_time; +uniform vec3 u_color; +uniform vec3 u_color_next; + + +// The texture. +uniform sampler2D u_texture; + +void main() { + float alpha = texture2D(u_texture, v_texCoord).a; + vec3 color = mix(u_color, u_color_next, u_time); + gl_FragColor = vec4(color, alpha); +} diff --git a/web/pw-visualizer/src/assets.ts b/web/pw-visualizer/src/assets.ts index e04f2c1..be155c5 100644 --- a/web/pw-visualizer/src/assets.ts +++ b/web/pw-visualizer/src/assets.ts @@ -4,9 +4,13 @@ export {default as earthSvg} from "../assets/res/earth.svg"; export {default as marsSvg} from "../assets/res/mars.svg"; export {default as venusSvg} from "../assets/res/venus.svg"; +export {default as earthPng} from "../assets/res/earth.png"; + export {default as fontPng} from "../assets/res/font.png"; export {default as imageFragmentShader} from "../assets/shaders/frag/image.glsl?url"; +export {default as maskedImageFragmentShader} from "../assets/shaders/frag/masked_image.glsl?url"; + export {default as simpleFragmentShader} from "../assets/shaders/frag/simple.glsl?url"; export {default as vorFragmentShader} from "../assets/shaders/frag/vor.glsl?url"; diff --git a/web/pw-visualizer/src/index.ts b/web/pw-visualizer/src/index.ts index cd58aa7..6f2c1b4 100644 --- a/web/pw-visualizer/src/index.ts +++ b/web/pw-visualizer/src/index.ts @@ -22,12 +22,13 @@ import { UniformMatrix3fv, UniformBool, } from "./webgl/shader"; -import { Renderer } from "./webgl/renderer"; +import { DefaultRenderable, Renderer } from "./webgl/renderer"; import { VertexBuffer, IndexBuffer } from "./webgl/buffer"; import { VertexBufferLayout, VertexArray } from "./webgl/vertexBufferLayout"; import { defaultLabelFactory, LabelFactory, Align, Label } from "./webgl/text"; import { VoronoiBuilder } from "./voronoi/voronoi"; import * as assets from "./assets"; +import { Texture } from "./webgl/texture"; function to_bbox(box: number[]): BBox { @@ -133,6 +134,7 @@ export class GameInstance { shader: Shader; vor_shader: Shader; image_shader: Shader; + masked_image_shader: Shader; text_factory: LabelFactory; planet_labels: Label[]; @@ -174,6 +176,7 @@ export class GameInstance { this.vor_shader = shaders["vor"].create_shader(GL, { PLANETS: "" + planets.length, }); + this.masked_image_shader = shaders["masked_image"].create_shader(GL); this.text_factory = defaultLabelFactory(GL, this.image_shader); this.planet_labels = []; @@ -234,45 +237,53 @@ export class GameInstance { } _create_planets(planets: Float32Array, meshes: Mesh[]) { + const earth = Texture.fromImage(GL, assets.earthPng, 'earth'); + for (let i = 0; i < this.planet_count; i++) { { const transform = new UniformMatrix3fv([ - 1, - 0, - 0, - 0, - 1, - 0, - -planets[i * 3], - -planets[i * 3 + 1], - 1, + 1, 0, 0, + 0, 1, 0, + -planets[i * 3], -planets[i * 3 + 1], 1, // TODO: why are negations needed? ]); - const indexBuffer = new IndexBuffer( - GL, - meshes[i % meshes.length].cells - ); - const positionBuffer = new VertexBuffer( - GL, - meshes[i % meshes.length].positions - ); - - const layout = new VertexBufferLayout(); - layout.push(GL.FLOAT, 3, 4, "a_position"); + const gl = GL; + const ib = new IndexBuffer(gl, [ + 0, 1, 2, + 1, 2, 3 + ]); + const vb_pos = new VertexBuffer(gl, [ + -1, 1, + 1, 1, + -1, -1, + 1, -1 + ]); + const vb_tex = new VertexBuffer(gl, [ + 0, 0, + 1, 0, + 0, 1, + 1, 1]); + + const layout_pos = new VertexBufferLayout(); + // 2? + layout_pos.push(gl.FLOAT, 2, 4, "a_position"); + + const layout_tex = new VertexBufferLayout(); + layout_tex.push(gl.FLOAT, 2, 4, "a_texCoord"); + const vao = new VertexArray(); - vao.addBuffer(positionBuffer, layout); - - this.renderer.addToDraw( - indexBuffer, - vao, - this.shader, - { - u_trans: transform, - u_trans_next: transform, - }, - [], - LAYERS.planet - ); + vao.addBuffer(vb_pos, layout_pos); + vao.addBuffer(vb_tex, layout_tex); + + const uniforms = { + u_trans: transform, + u_trans_next: transform, + }; + + const renderable = new DefaultRenderable(ib, vao, this.masked_image_shader, [earth], uniforms); + + this.renderer.addRenderable(renderable, LAYERS.planet); + } { @@ -430,25 +441,30 @@ export class GameInstance { this.use_vor = false; } + const shaders_to_update = [ + this.shader, + this.image_shader, + this.masked_image_shader, + ]; + + // If not playing, still reder with different viewbox, so people can still pan etc. if (!this.playing) { this.last_time = time; - this.shader.uniform( - GL, - "u_viewbox", - new Uniform4f(this.resizer.get_viewbox()) - ); + shaders_to_update.forEach((shader) => { + shader.uniform( + GL, + "u_viewbox", + new Uniform4f(this.resizer.get_viewbox()) + ); + }) + this.vor_shader.uniform( GL, "u_viewbox", new Uniform4f(this.resizer.get_viewbox()) ); - this.image_shader.uniform( - GL, - "u_viewbox", - new Uniform4f(this.resizer.get_viewbox()) - ); this.renderer.render(GL); return; @@ -481,39 +497,24 @@ export class GameInstance { this.vor_shader.uniform(GL, "u_resolution", new Uniform2f(RESOLUTION)); this.vor_shader.uniform(GL, "u_vor", new UniformBool(this.use_vor)); - this.shader.uniform( - GL, - "u_time", - new Uniform1f((time - this.last_time) / ms_per_frame) - ); - this.shader.uniform( - GL, - "u_mouse", - new Uniform2f(this.resizer.get_mouse_pos()) - ); - this.shader.uniform( - GL, - "u_viewbox", - new Uniform4f(this.resizer.get_viewbox()) - ); - this.shader.uniform(GL, "u_resolution", new Uniform2f(RESOLUTION)); - - this.image_shader.uniform( - GL, - "u_time", - new Uniform1f((time - this.last_time) / ms_per_frame) - ); - this.image_shader.uniform( - GL, - "u_mouse", - new Uniform2f(this.resizer.get_mouse_pos()) - ); - this.image_shader.uniform( - GL, - "u_viewbox", - new Uniform4f(this.resizer.get_viewbox()) - ); - this.image_shader.uniform(GL, "u_resolution", new Uniform2f(RESOLUTION)); + shaders_to_update.forEach((shader) => { + shader.uniform( + GL, + "u_time", + new Uniform1f((time - this.last_time) / ms_per_frame) + ); + shader.uniform( + GL, + "u_mouse", + new Uniform2f(this.resizer.get_mouse_pos()) + ); + shader.uniform( + GL, + "u_viewbox", + new Uniform4f(this.resizer.get_viewbox()) + ); + shader.uniform(GL, "u_resolution", new Uniform2f(RESOLUTION)); + }); // Render this.renderer.render(GL); @@ -616,6 +617,15 @@ export async function set_instance(source: string): Promise { assets.simpleVertexShader, ), ])(), + (async () => + <[string, ShaderFactory]>[ + "masked_image", + await ShaderFactory.create_factory( + assets.maskedImageFragmentShader, + assets.simpleVertexShader, + ), + ])(), + ]; let shaders_array: [string, ShaderFactory][]; [meshes, shaders_array] = await Promise.all([ diff --git a/web/pw-visualizer/src/webgl/index.ts b/web/pw-visualizer/src/webgl/index.ts index 1742713..8d785ef 100644 --- a/web/pw-visualizer/src/webgl/index.ts +++ b/web/pw-visualizer/src/webgl/index.ts @@ -57,8 +57,8 @@ async function main() { return; } + // TODO: do we still need this? const mesh = await url_to_mesh("static/res/images/earth.svg"); - console.log(Math.max(...mesh.positions), Math.min(...mesh.positions)); const renderer = new Renderer(); const factory = await ShaderFactory.create_factory(assets.simpleFragmentShader, assets.simpleVertexShader); From 6e494aca463b8632fe2b78ee89b8651187f8061a Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 18 Jul 2022 07:54:18 +0200 Subject: [PATCH 50/51] add half-pixel correction to text --- web/pw-visualizer/src/webgl/text.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/web/pw-visualizer/src/webgl/text.ts b/web/pw-visualizer/src/webgl/text.ts index fdfbc55..cb72a42 100644 --- a/web/pw-visualizer/src/webgl/text.ts +++ b/web/pw-visualizer/src/webgl/text.ts @@ -79,7 +79,6 @@ export class Label { const verts_pos = []; const verts_tex = []; - const letterHeight = this.font.letterHeight / this.font.textureHeight; let xPos = 0; switch (h_align) { @@ -108,10 +107,17 @@ export class Label { for (let i = 0; i < text.length; i++) { const info = this.font.glyphInfos[text[i]]; if (info) { + const dx = info.width / this.font.letterHeight; - const letterWidth = info.width / this.font.textureWidth; - const x0 = info.x / this.font.textureWidth; - const y0 = info.y / this.font.textureHeight; + + // apply half-pixel correction to prevent texture bleeding + // we should address the center of each texel, not the border + // https://gamedev.stackexchange.com/questions/46963/how-to-avoid-texture-bleeding-in-a-texture-atlas + const x0 = (info.x + 0.5) / this.font.textureWidth; + const y0 = (info.y + 0.5) / this.font.textureHeight; + const letterWidth = (info.width - 1) / this.font.textureWidth; + const letterHeight = (this.font.letterHeight - 1) / this.font.textureHeight; + verts_pos.push(xPos, yStart); verts_pos.push(xPos + dx, yStart); verts_pos.push(xPos, yStart-1); From d092f5d89c0fda5cc67349d5489b4ef1b294e053 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 18 Jul 2022 21:02:27 +0200 Subject: [PATCH 51/51] use texture for rendering ships --- web/pw-visualizer/assets/res/ship.png | Bin 0 -> 4963 bytes web/pw-visualizer/assets/res/ship.svg | 51 +++++++------ web/pw-visualizer/src/assets.ts | 1 + web/pw-visualizer/src/index.ts | 100 +++++++++++++------------ web/pw-visualizer/src/webgl/text.ts | 8 +- web/pw-visualizer/src/webgl/texture.ts | 24 +++--- 6 files changed, 96 insertions(+), 88 deletions(-) create mode 100644 web/pw-visualizer/assets/res/ship.png diff --git a/web/pw-visualizer/assets/res/ship.png b/web/pw-visualizer/assets/res/ship.png new file mode 100644 index 0000000000000000000000000000000000000000..fe289aa665f8de494bf67e657fb2b5eba0b8fbdd GIT binary patch literal 4963 zcmV-p6P)acP)-Jg zn2s$Bq*IvAR4q`o&@$B42OB$EQ`l8J=Vffc~ukgNbs3#w5jI3EUH z=Ri<2We;#gK)o`77zNw{?B~#+Xq7vLew(QQ+AqA4qZ6MZY4amy*ddhCOuDeHhao^{VSLMtemvI#iVs|6W#<^!A2 z3yP*}1HP z(Q`bo3V%V-l(op~a3(^IqTD=1Sy>4j?cBf3o;!GjFMEGDBE_XY%I5Q&G6Fvw!$sakiJJW?rfPI7yfKJj6T;|OBOj}MyW*$R# z{z$R|nCr*~nPE&rUJXNcdQtfWaHJDYWZt9!{DAPCZ7Pop>7H@K&4lk{TlrH5{>T95 z5%+S%JRp!bKWlryGC>>u)2+2 zT5`QVQ~WX>cW)vCz(xK{>B#xGOG(^7Mc+_d&Rg8<(TcEX0kYw&G@HvnV%#|CN}om;@=4s-{C%HBxMUt9?>%NC zj}iB$=^msLT?9uZc6q!9eaGwq-saN?OHQQyE{$d7AAK6(g2}jXOiJ(cH{j;ND3f#) zoNLf`%2UMk7BUi}kvvmvKeSu+kPJUX*BSdsvs}>v^q;f-*Q*7JEI|J%x!3S`fv!(>kl6PV?dmGu8(eqcdZzmIx2P+LD_XDT+RX+%)q4y1C z7m>cbeE_}akN#mwktfje+d?g+G+$TWr`T5Vi6{nqMX@c-K2T#VI#ItF-FcAM8IRsH z`D!htv__1%=vCDH8uBHzF6>1fLN%BPoTSl^R$#nGsU6UBQcC=@Vp|t0wpE&k&>0G! z^R0M9JdchKzp0^&#)vT%S+l9ZeM;@5$iFDHa}iSCEI8+*_qvsGhE`c-qvywWmQ_Y$ z#LyZomLkQ}>dH!_8lVQHH^qtJ6b-_>Jw{v(0d1;?^*X4ISakHprFe) z=d@AEM%EDbI#gfupjTXGb{R#=-b%C<|2&EqQ+Ec@`%YBq0kmvjXx+;yv$S6UEnftE z&iEPShr9vu<}Ixpjn1d|U`rSeLnn>zS>K8$z*5TlwGWwmw4%(dWG*_@@qeh$M%7rN z(8d@_Bs-M}46UjzPb#%jCA55E-A6G5#*dWRsS;W~Nj5OnAqhU2gLxU*HM%T80%Kl@ zW1IX~p^d8Yv_c!+1LIh9R){TCXrroVylzedj)LJ%dOdXjzgAj29K!s5%-x@)j69 z=#*<-snAB%(FhEyi`XVGjz(uY>^x$OjK;;&i^bP#o50Z8A0!9H$Y}Yh8Xp9&dUk>Yn zhOdk-=8z*srWhL(+DPVBvPGedx}f1JFM**^?0vgJ8+BoaLK{;}53mW0$ri^9N5j~< zpm7ly_nB#6G|<_kEsq$*(YS~vW4Hv=G%$KCj%i08F^Z$%tI-&K#xU(bp=QMh=YPX!*+Nh%w3V_npXrQJg-dc6v>IY#JEW zi)4q91EV-vzA~!hXBrq9ib0wc-|S0Y90B1+;`l1Gf<)M~`Yfl(YSUpWm7t!*qcmLJpwEnkf?`LSt7 zYrQ&6S8AsY9Ie!j(R_$$U}#OYXk>xa1ub7W4UB%n-*+MhMsbn|j67oaCyBt=Yxw(4 z?(QrFUCyVYTT2Hi5Cq;+RgH2~1FIuPRIcPFHNtOJKZgaZD%1pgUw%g>%r& zq+exvK|m z^{v&ObGFI7&siSV3GJaXWwd=^)mO0u#`~4qOBron{M_cCSOVj8V7_vjDRLI@YszhU z9WnmT^0;pNfrB9jgLXBx0XOD!bnk~vdDN#7k?z%um8N5;q~ zU@7kWH_x9bhth-W+2zVO7q4)!a1HMKwjRAiVr#yJI|H|%v!haMOaV6I&Tn^@RZ!!p z^$~?OIx!VkgK4e6Sf=6&`>^5Mg zH*4H+9CCYf>8tN~v!(!7;nvte;6DjCVWYq(;Bnmh$;GF8Yo;TG+gux+^7F=>xc5&# z5v~VzHQzs6uJ&b+BA>;NZ=(FpbR~X_?OLLP58OZm-v$sJzJQ-Mv{qT>!??2edIBFV-zV&@21f!9wb9FmB=NCvGX5U8 zhRYhl<;8r)_M?5xlW z{0g$CO3XQ@|Ax$$B*|Q4C=Wg4R!#cREkJ(=uJap~1OMvy9ChSgEDNH<#0i9ddash~ zLEieDVzf!CCEJi~u5uM%9c~5w!}Upgy)pD6^)&J(dF9vy{3^N|3bn<@hygMYvz1(( zuPoQ#ru;-udVn7g{vMsGEGDiwAysDMH+iKhvKu(j!r#{Tx|KB9ip*f7PKduDypvOvFIxCJ2|h!3=cX>7GxmDqTux5d z(U1WoceN6{3&|k}?bNEs8_3&A(wu_3veI>ytvF8YbICMdCE=Y|Rrv)GR?iuUG5D*n zQ)hVs-5s5s^JT(2tuAs;!9EGjb%b|T-Q?OrJrj_#abFBJ%c0VTtdxtA>A*U|JE^X+ z0XGY{OEH?9d=rB#Mrm24R%XyGz(6WX2j>!CKjE*IEFuHId%E_R2c`oX z3Ga+n*^A%rG6Tf71am#IV&9rF*+D(PUBp#00^q-qDl^v&{o57K0e^|)Wd6wEd2Bca z*U1lcs;;5cLz9UQ5FE#n;v8u7;}KX zFg=N9LUdAyU^L;cklSZrOah|^`Dz;Db44q=YI05!#v^@K_?glqFg`)VA8K0I7T}!Q z1Dw;8^MNah{r$0JLQDWPvDdB3f( zV@WUle?MWerpJ-i?RF5EChzF0Znu(8kzO^IElB>OZ{8rS+X={rK;!Mc#}RHTr!unC z)KEp1chM_WSx1Kn%T6Q49|FgjcqI}x)b!l8%s?hC520OPj0gV2sXFe+fnm-~v>7$p z1;)pbJ-oOk+2CfJAhXMp1&*&T2>Rb212*Y>J$ih8)*`Q4rO>Vk*66LXCMzCDHFpz zer0F{N-`JOrd1y*~ zvy@rD>cBd@jr2#2e&EOkl0BSyP`pa`pa6K8Fd1Z;j0VnYF!ZpII6n!kzz~DLGoh7G z;M4{%FSHg4#1a@#jlh`OK*~C&&aFk;lngb5AcmC#z0B*spNQ702#X#i_NYf*wdz$2lR zXv6oAN`9%bx`EV}N|i^6vuMdK(jGNdG=PUgYf*v+LMzaPm~|V8he4L9ttI^87gO0n z+z-x&k><8vYk{#1WJ%vgfPLDYam>BMeIMTg+!tC2L-rvFTL&%kT!7>>gpSxE(g(a% z&mYb>0daO{e-)8cbSb5EIPho47sAkrc9Uebdjh^ncz<=1Ex^qjD1h6Q!+~pnry@{% zxaW0un>r^?k@jH#Y(&X&X2h zxD5DfU}2xcT=a^efveL4tCp za=X6h1s+1bR%#zINx2+YHuU#r$cMZ^+eTd#T^obc*O-B + inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:dc="http://purl.org/dc/elements/1.1/"> + inkscape:window-y="32" + inkscape:window-maximized="1" + gridtolerance="10" + inkscape:pagecheckerboard="0" + units="cm"> + id="grid894" + originx="-160.50747" + originy="118.75037" /> @@ -53,7 +57,6 @@ image/svg+xml - @@ -61,17 +64,17 @@ inkscape:label="Layer 1" inkscape:groupmode="layer" id="layer1" - transform="translate(229.05372,-117.27915)"> + transform="translate(68.546255,1.4712222)"> + style="opacity:1;fill:#00ffff;fill-opacity:1;stroke:none;stroke-width:6.61458;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" /> diff --git a/web/pw-visualizer/src/assets.ts b/web/pw-visualizer/src/assets.ts index be155c5..5fa5215 100644 --- a/web/pw-visualizer/src/assets.ts +++ b/web/pw-visualizer/src/assets.ts @@ -5,6 +5,7 @@ export {default as marsSvg} from "../assets/res/mars.svg"; export {default as venusSvg} from "../assets/res/venus.svg"; export {default as earthPng} from "../assets/res/earth.png"; +export {default as shipPng} from "../assets/res/ship.png"; export {default as fontPng} from "../assets/res/font.png"; diff --git a/web/pw-visualizer/src/index.ts b/web/pw-visualizer/src/index.ts index 6f2c1b4..bee1bab 100644 --- a/web/pw-visualizer/src/index.ts +++ b/web/pw-visualizer/src/index.ts @@ -1,6 +1,4 @@ import { Game } from "planetwars-rs"; -// import { memory } from "planetwars-rs/planetwars_rs_bg"; -// const memory = planetwars_bg.memory; import type { Dictionary } from './webgl/util'; import type { BBox } from "./voronoi/voronoi-core"; @@ -8,8 +6,6 @@ import { Resizer, resizeCanvasToDisplaySize, FPSCounter, - url_to_mesh, - Mesh, } from "./webgl/util"; import { Shader, @@ -40,14 +36,6 @@ function to_bbox(box: number[]): BBox { }; } -// function f32v(ptr: number, size: number): Float32Array { -// return new Float32Array(memory.buffer, ptr, size); -// } - -// function i32v(ptr: number, size: number): Int32Array { -// return new Int32Array(memory.buffer, ptr, size); -// } - export function set_game_name(name: string) { ELEMENTS["name"].innerHTML = name; } @@ -142,6 +130,7 @@ export class GameInstance { ship_ibo: IndexBuffer; ship_vao: VertexArray; + ship_texture: Texture; // TODO: find a better way max_num_ships: number; @@ -161,8 +150,9 @@ export class GameInstance { constructor( game: Game, - meshes: Mesh[], - ship_mesh: Mesh, + planets_textures: Texture[], + ship_texture: Texture, + font_texture: Texture, shaders: Dictionary ) { this.game = game; @@ -178,10 +168,12 @@ export class GameInstance { }); this.masked_image_shader = shaders["masked_image"].create_shader(GL); - this.text_factory = defaultLabelFactory(GL, this.image_shader); + this.text_factory = defaultLabelFactory(GL, font_texture, this.image_shader); this.planet_labels = []; this.ship_labels = []; + this.ship_texture = ship_texture + this.resizer = new Resizer(CANVAS, [...game.get_viewbox()], true); this.renderer = new Renderer(); this.game.update_turn(0); @@ -191,15 +183,8 @@ export class GameInstance { // List of [(x, y, r)] for all planets this._create_voronoi(planets); - this._create_planets(planets, meshes); + this._create_planets(planets, planets_textures); - // create_shipes - this.ship_ibo = new IndexBuffer(GL, ship_mesh.cells); - const ship_positions = new VertexBuffer(GL, ship_mesh.positions); - const ship_layout = new VertexBufferLayout(); - ship_layout.push(GL.FLOAT, 3, 4, "a_position"); - this.ship_vao = new VertexArray(); - this.ship_vao.addBuffer(ship_positions, ship_layout); this.max_num_ships = 0; // Set slider correctly @@ -236,9 +221,7 @@ export class GameInstance { this.renderer.addRenderable(this.vor_builder.getRenderable(), LAYERS.vor); } - _create_planets(planets: Float32Array, meshes: Mesh[]) { - const earth = Texture.fromImage(GL, assets.earthPng, 'earth'); - + _create_planets(planets: Float32Array, planets_textures: Texture[]) { for (let i = 0; i < this.planet_count; i++) { { const transform = new UniformMatrix3fv([ @@ -280,7 +263,7 @@ export class GameInstance { u_trans_next: transform, }; - const renderable = new DefaultRenderable(ib, vao, this.masked_image_shader, [earth], uniforms); + const renderable = new DefaultRenderable(ib, vao, this.masked_image_shader, [planets_textures[0]], uniforms); this.renderer.addRenderable(renderable, LAYERS.planet); @@ -361,16 +344,39 @@ export class GameInstance { const ship_colours = this.game.get_ship_colours(); for (let i = this.max_num_ships; i < ship_counts.length; i++) { - this.renderer.addToDraw( - this.ship_ibo, - this.ship_vao, - this.shader, - {}, - [], - LAYERS.ship - ); + const gl = GL; + const ib = new IndexBuffer(gl, [ + 0, 1, 2, + 1, 2, 3 + ]); + const ratio = this.ship_texture.getWidth() / this.ship_texture.getHeight(); + const vb_pos = new VertexBuffer(gl, [ + -ratio, 1, + ratio, 1, + -ratio, -1, + ratio, -1 + ]); + const vb_tex = new VertexBuffer(gl, [ + 0, 0, + 1, 0, + 0, 1, + 1, 1, + ]); + + const layout_pos = new VertexBufferLayout(); + layout_pos.push(gl.FLOAT, 2, 4, "a_position"); + + const layout_tex = new VertexBufferLayout(); + layout_tex.push(gl.FLOAT, 2, 4, "a_texCoord"); + + const vao = new VertexArray(); + vao.addBuffer(vb_pos, layout_pos); + vao.addBuffer(vb_tex, layout_tex); + const renderable = new DefaultRenderable(ib, vao, this.masked_image_shader, [this.ship_texture], {}); + this.renderer.addRenderable(renderable, LAYERS.ship); const label = this.text_factory.build(GL); + this.ship_labels.push(label); this.renderer.addRenderable(label.getRenderable(), LAYERS.ship_label); } @@ -579,18 +585,17 @@ export class GameInstance { } var game_instance: GameInstance; -var meshes: Mesh[]; +var textures: Texture[]; var shaders: Dictionary; export async function set_instance(source: string): Promise { // TODO: embed shader programs - if (!meshes || !shaders) { - const mesh_promises = [ - assets.shipSvg, - assets.earthSvg, - assets.marsSvg, - assets.venusSvg, - ].map(url_to_mesh); + if (!textures || !shaders) { + const texture_promises = [ + Texture.fromImage(GL, assets.fontPng, "font"), + Texture.fromImage(GL, assets.shipPng, "ship"), + Texture.fromImage(GL, assets.earthPng, "earth") + ]; const shader_promies = [ (async () => @@ -628,8 +633,8 @@ export async function set_instance(source: string): Promise { ]; let shaders_array: [string, ShaderFactory][]; - [meshes, shaders_array] = await Promise.all([ - Promise.all(mesh_promises), + [textures, shaders_array] = await Promise.all([ + Promise.all(texture_promises), Promise.all(shader_promies), ]); @@ -641,8 +646,9 @@ export async function set_instance(source: string): Promise { game_instance = new GameInstance( Game.new(source), - meshes.slice(1), - meshes[0], + textures.slice(2), + textures[1], + textures[0], shaders ); diff --git a/web/pw-visualizer/src/webgl/text.ts b/web/pw-visualizer/src/webgl/text.ts index cb72a42..1ae6f37 100644 --- a/web/pw-visualizer/src/webgl/text.ts +++ b/web/pw-visualizer/src/webgl/text.ts @@ -33,8 +33,8 @@ export class LabelFactory { font: FontInfo; shader: Shader; - constructor(gl: WebGLRenderingContext, loc: string, font: FontInfo, shader: Shader) { - this.texture = Texture.fromImage(gl, loc, 'font'); + constructor(gl: WebGLRenderingContext, fontTexture: Texture, font: FontInfo, shader: Shader) { + this.texture = fontTexture; this.font = font; this.shader = shader; } @@ -144,7 +144,7 @@ export class Label { } } -export function defaultLabelFactory(gl: WebGLRenderingContext, shader: Shader): LabelFactory { +export function defaultLabelFactory(gl: WebGLRenderingContext, fontTexture: Texture, shader: Shader): LabelFactory { const fontInfo = { letterHeight: 8, spaceWidth: 8, @@ -195,5 +195,5 @@ export function defaultLabelFactory(gl: WebGLRenderingContext, shader: Shader): }, }; - return new LabelFactory(gl, fontPng, fontInfo, shader); + return new LabelFactory(gl, fontTexture, fontInfo, shader); } diff --git a/web/pw-visualizer/src/webgl/texture.ts b/web/pw-visualizer/src/webgl/texture.ts index 9d6adcf..faafe76 100644 --- a/web/pw-visualizer/src/webgl/texture.ts +++ b/web/pw-visualizer/src/webgl/texture.ts @@ -11,15 +11,18 @@ export class Texture { gl: WebGLRenderingContext, path: string, name: string, - ): Texture { - const out = new Texture(gl, name); + ): Promise { + return new Promise((resolve, reject) => { + const out = new Texture(gl, name); - const image = new Image(); - image.onload = out.setImage.bind(out, gl, image); - image.onerror = error; - image.src = path; - - return out; + const image = new Image(); + image.onload = () => { + out.setImage(gl, image); + resolve(out); + } + image.onerror = reject; + image.src = path; + }) } static fromRenderer( @@ -99,8 +102,3 @@ export class Texture { return this.height; } } - -function error(e: any) { - console.error("IMAGE LOAD ERROR"); - console.error(e); -}