diff --git a/scripts/network/init.sh b/scripts/network/init.sh index eb9c30b..3a7416f 100755 --- a/scripts/network/init.sh +++ b/scripts/network/init.sh @@ -10,6 +10,8 @@ berserker -c /etc/berserker/network-server.toml & SERVER_PID=$! +sleep 1 + berserker -c /etc/berserker/network-client.toml & CLIENT_PID=$! @@ -17,12 +19,12 @@ CLIENT_PID=$! cleanup() { echo "Killing client ($CLIENT_PID) and server ($SERVER_PID)" - kill -9 "$CLIENT_PID" - kill -9 "$SERVER_PID" + kill -9 "$CLIENT_PID" 2>/dev/null + kill -9 "$SERVER_PID" 2>/dev/null exit } -trap cleanup SIGINT SIGABRT +trap cleanup SIGINT SIGABRT SIGTERM -wait -n "$SERVER_PID" "$CLIENT_PID" +wait "$SERVER_PID" "$CLIENT_PID" diff --git a/src/worker/network.rs b/src/worker/network.rs index 25b9149..01b8617 100644 --- a/src/worker/network.rs +++ b/src/worker/network.rs @@ -8,9 +8,11 @@ use std::str; use std::time::{SystemTime, UNIX_EPOCH}; use std::{ fmt::Display, + io, io::{BufReader, prelude::*}, net::TcpListener, thread, + time::Duration, }; use crate::{BaseConfig, Worker, WorkerError, Workload, WorkloadConfig}; @@ -37,6 +39,29 @@ impl NetworkWorker { } } + fn bind_port( + &self, + addr: Ipv4Address, + target_port: u16, + ) -> Result { + const MAX_BIND_RETRIES: usize = 30; + let addr = addr.to_string(); + for _ in 0..MAX_BIND_RETRIES { + match TcpListener::bind((addr.as_str(), target_port)) { + Ok(l) => return Ok(l), + Err(e) if e.kind() == io::ErrorKind::AddrInUse => { + thread::sleep(Duration::from_secs(1)); + } + Err(e) => panic!("Failed to bind: {}", e), + } + } + + Err(WorkerError::InternalWithMessage(format!( + "Failed to bind {}:{} after {} retries: address in use", + addr, target_port, MAX_BIND_RETRIES + ))) + } + /// Start a simple server. The client side is going to be a networking /// worker as well, so for convenience of troubleshooting do not error /// out if something unexpected happened, log and proceed instead. @@ -47,8 +72,7 @@ impl NetworkWorker { ) -> Result<(), WorkerError> { debug!("Starting server at {:?}:{:?}", addr, target_port); - let listener = - TcpListener::bind((addr.to_string(), target_port)).unwrap(); + let listener = self.bind_port(addr, target_port)?; for stream in listener.incoming() { let mut stream = stream.unwrap();