From a96c82dea43fa51c0971557f0d232acafaf38fc5 Mon Sep 17 00:00:00 2001 From: Trisha Earley Date: Fri, 6 Nov 2020 18:47:04 +0000 Subject: [PATCH 001/208] :bug: Add content-length header when POSTing a response to the relay --- src/http_serde.rs | 3 +++ src/server/file_server.rs | 31 ++++++++++++++++++++----------- src/server/mod.rs | 13 +++++++++++-- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/http_serde.rs b/src/http_serde.rs index bdb40fd..0d94ad3 100644 --- a/src/http_serde.rs +++ b/src/http_serde.rs @@ -126,6 +126,9 @@ type Body = Receiver , std::convert::Infallible>>; pub struct Response { pub parts: ResponseParts, pub body: Option , + + // Needed to make Nginx happy + pub content_length: Option , } impl Response { diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 7798121..b150296 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -120,11 +120,14 @@ async fn serve_dir ( path, entries, }).unwrap (); + let body = s.into_bytes (); let mut resp = http_serde::Response::default (); + resp.content_length = Some (body.len ().try_into ().unwrap ()); resp .header ("content-type".to_string (), "text/html".to_string ().into_bytes ()) - .body_bytes (s.into_bytes ()) + .header ("content-length".to_string (), body.len ().to_string ().into_bytes ()) + .body_bytes (body) ; resp } @@ -161,7 +164,7 @@ async fn serve_file ( //println! ("Opening file {:?}", path); let mut tx = tx; - //let mut bytes_sent = 0; + let mut bytes_sent = 0; let mut bytes_left = end - start; loop { @@ -177,16 +180,18 @@ async fn serve_file ( } if tx.send (Ok::<_, Infallible> (buffer)).await.is_err () { + eprintln! ("Send failed while streaming file ({} bytes sent)", bytes_sent); break; } bytes_left -= bytes_read; if bytes_left == 0 { + eprintln! ("Finished streaming file"); break; } - //bytes_sent += bytes_read; - //println! ("Sent {} bytes", bytes_sent); + bytes_sent += bytes_read; + println! ("Sent {} bytes", bytes_sent); //delay_for (Duration::from_millis (50)).await; } @@ -197,13 +202,17 @@ async fn serve_file ( response.header (String::from ("accept-ranges"), b"bytes".to_vec ()); - if range_start.is_none () && range_end.is_none () { - response.status_code (http_serde::StatusCode::Ok); - response.header (String::from ("content-length"), end.to_string ().into_bytes ()); - } - else { - response.status_code (http_serde::StatusCode::PartialContent); - response.header (String::from ("content-range"), format! ("bytes {}-{}/{}", start, end - 1, end).into_bytes ()); + if should_send_body { + if range_start.is_none () && range_end.is_none () { + response.status_code (http_serde::StatusCode::Ok); + response.header (String::from ("content-length"), end.to_string ().into_bytes ()); + } + else { + response.status_code (http_serde::StatusCode::PartialContent); + response.header (String::from ("content-range"), format! ("bytes {}-{}/{}", start, end - 1, end).into_bytes ()); + } + + response.content_length = Some (end - start); } if let Some (body) = body { diff --git a/src/server/mod.rs b/src/server/mod.rs index dfba1df..cf0b305 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -83,14 +83,23 @@ async fn handle_req_resp <'a> ( .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) .header (crate::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).unwrap ())); + if let Some (length) = response.content_length { + resp_req = resp_req.header ("Content-Length", length.to_string ()); + } if let Some (body) = response.body { resp_req = resp_req.body (reqwest::Body::wrap_stream (body)); } + let req = resp_req.build ().unwrap (); + + eprintln! ("{:?}", req.headers ()); + //println! ("Step 6"); - if let Err (e) = resp_req.send ().await { - println! ("Err: {:?}", e); + match state.client.execute (req).await { + Ok (r) => eprintln! ("{:?} {:?}", r.status (), r.text ().await.unwrap ()), + Err (e) => eprintln! ("Err: {:?}", e), } + }); } } From 95a038f7af36af3c7b68c2398babf62d911abaad Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 3 Nov 2020 16:00:50 +0000 Subject: [PATCH 002/208] Running for real. Lots of todos added --- Dockerfile | 2 +- todo.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 52b3318..a53934e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update \ # Make sure the dependencies are all cached so we won't hammer crates.io ADD old-git.tar.gz . -RUN git checkout 16984ddcd3c9cdc04b2c4c3625eb83176c1b2dda \ +RUN git checkout 690f07dab67111a75fe190f014c8c23ef1753598 \ && git reset --hard \ && cargo check diff --git a/todo.md b/todo.md index 9987cc8..f76f183 100644 --- a/todo.md +++ b/todo.md @@ -1,3 +1,9 @@ +- Add file size in directory listing +- Large text files not streaming as expected? +- Allow spaces in server names +- Make file_server_root mandatory +- Deny unused HTTP methods for endpoints +- Hide ptth_server.toml from file server - ETag cache - Server-side hash? - Log / audit log? From 6b772ad512638ccff1ed03e0c0d0962f5a47a967 Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Mon, 2 Nov 2020 19:41:36 -0600 Subject: [PATCH 003/208] :recycle: Remove some unused code --- src/bin/ptth_server.rs | 8 +---- src/lib.rs | 6 +--- src/relay/mod.rs | 2 -- src/relay/watcher.rs | 79 ------------------------------------------ src/server/mod.rs | 9 +---- 5 files changed, 3 insertions(+), 101 deletions(-) delete mode 100644 src/relay/watcher.rs diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 3d972c1..1577ae0 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -27,11 +27,5 @@ async fn main () -> Result <(), Box > { toml::from_str (&config_s).expect (&format! ("Can't parse {:?} as TOML", config_file_path)) }; - let opt = Opt::from_args (); - - let opt = ptth::server::Opt { - - }; - - ptth::server::main (config_file, opt).await + ptth::server::main (config_file).await } diff --git a/src/lib.rs b/src/lib.rs index 7d53291..d610009 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,11 +114,7 @@ mod tests { file_server_root: None, }; spawn (async move { - let opt = server::Opt { - - }; - - server::main (config_file, opt).await.unwrap (); + server::main (config_file).await.unwrap (); }); tokio::time::delay_for (std::time::Duration::from_millis (500)).await; diff --git a/src/relay/mod.rs b/src/relay/mod.rs index cb84e4f..30844b3 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -1,5 +1,3 @@ -pub mod watcher; - use std::{ error::Error, collections::*, diff --git a/src/relay/watcher.rs b/src/relay/watcher.rs deleted file mode 100644 index 69820ed..0000000 --- a/src/relay/watcher.rs +++ /dev/null @@ -1,79 +0,0 @@ -// Watchers on the wall was the last good episode of Game of Thrones - -use std::{ - collections::*, - sync::Arc, - time::Duration, -}; - -use futures::channel::oneshot; -use tokio::{ - sync::Mutex, - time::delay_for, -}; - -pub struct Watchers { - pub senders: HashMap >, -} - -impl Default for Watchers { - fn default () -> Self { - Self { - senders: Default::default (), - } - } -} - -impl Watchers { - pub fn add_watcher_with_id (&mut self, s: oneshot::Sender , id: String) { - self.senders.insert (id, s); - } - - pub fn remove_watcher (&mut self, id: &str) { - self.senders.remove (id); - } - - pub fn wake_one (&mut self, msg: T, id: &str) -> bool { - //println! ("wake_one {}", id); - - if let Some (waiter) = self.senders.remove (id) { - waiter.send (msg).ok (); - true - } - else { - false - } - } - - pub fn num_watchers (&self) -> usize { - self.senders.len () - } - - pub async fn long_poll (that: Arc >, id: String) -> Option { - //println! ("long_poll {}", id); - - let (s, r) = oneshot::channel (); - let timeout = Duration::from_secs (5); - - let id_2 = id.clone (); - { - let mut that = that.lock ().await; - that.add_watcher_with_id (s, id_2); - //println! ("Added server {}", id); - } - - tokio::spawn (async move { - delay_for (timeout).await; - let mut that = that.lock ().await; - that.remove_watcher (&id); - //println! ("Removed server {}", id); - }); - - if let Ok (message) = r.await { - Some (message) - } - else { - None - } - } -} diff --git a/src/server/mod.rs b/src/server/mod.rs index cf0b305..c5986ed 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -24,7 +24,6 @@ pub mod file_server; struct ServerState { config: Config, - opt: Opt, handlebars: Handlebars <'static>, client: Client, } @@ -118,12 +117,7 @@ pub struct Config { pub file_server_root: Option , } -#[derive (Clone)] -pub struct Opt { - -} - -pub async fn main (config_file: ConfigFile, opt: Opt) +pub async fn main (config_file: ConfigFile) -> Result <(), Box > { use std::convert::TryInto; @@ -149,7 +143,6 @@ pub async fn main (config_file: ConfigFile, opt: Opt) relay_url: config_file.relay_url, file_server_root: config_file.file_server_root, }, - opt, handlebars, client, }); From ec4e0e633519d71bf366f82560522af68ee4a761 Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Thu, 5 Nov 2020 21:35:55 -0600 Subject: [PATCH 004/208] :bug: Fix graceful shutdown of relay and server in end_to_end test --- Cargo.toml | 3 ++ src/bin/ptth_relay.rs | 8 ++--- src/bin/ptth_server.rs | 10 +++--- src/lib.rs | 48 +++++++++++++++++++++++------ src/relay/mod.rs | 70 ++++++++++++++++++++++++++++++++++-------- src/server/mod.rs | 31 ++++++++++++++++++- 6 files changed, 139 insertions(+), 31 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 04faa6b..6972530 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,5 +29,8 @@ rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} structopt = "0.3.20" tokio = { version = "0.2.22", features = ["full"] } +tracing = "0.1.21" +tracing-futures = "0.2.4" +tracing-subscriber = "0.2.15" toml = "0.5.7" ulid = "0.4.1" diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index 4c57502..4f5ad9a 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -16,13 +16,13 @@ async fn main () -> Result <(), Box > { let config_file = { let config_file_path = "config/ptth_relay.toml"; - let mut f = File::open (config_file_path).expect (&format! ("Can't open {:?}", config_file_path)); + let mut f = File::open (config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); let mut buffer = vec! [0u8; 4096]; - let bytes_read = f.read (&mut buffer).expect (&format! ("Can't read {:?}", config_file_path)); + let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read {:?}", config_file_path)); buffer.truncate (bytes_read); - let config_s = String::from_utf8 (buffer).expect (&format! ("Can't parse {:?} as UTF-8", config_file_path)); - toml::from_str (&config_s).expect (&format! ("Can't parse {:?} as TOML", config_file_path)) + let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse {:?} as UTF-8", config_file_path)); + toml::from_str (&config_s).unwrap_or_else (|_| panic! ("Can't parse {:?} as TOML", config_file_path)) }; eprintln! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 1577ae0..1ec88a0 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -18,14 +18,14 @@ async fn main () -> Result <(), Box > { let config_file = { let config_file_path = "config/ptth_server.toml"; - let mut f = std::fs::File::open (config_file_path).expect (&format! ("Can't open {:?}", config_file_path)); + let mut f = std::fs::File::open (config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); let mut buffer = vec! [0u8; 4096]; - let bytes_read = f.read (&mut buffer).expect (&format! ("Can't read {:?}", config_file_path)); + let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read {:?}", config_file_path)); buffer.truncate (bytes_read); - let config_s = String::from_utf8 (buffer).expect (&format! ("Can't parse {:?} as UTF-8", config_file_path)); - toml::from_str (&config_s).expect (&format! ("Can't parse {:?} as TOML", config_file_path)) + let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse {:?} as UTF-8", config_file_path)); + toml::from_str (&config_s).unwrap_or_else (|_| panic! ("Can't parse {:?} as TOML", config_file_path)) }; - ptth::server::main (config_file).await + ptth::server::main (config_file, None).await } diff --git a/src/lib.rs b/src/lib.rs index d610009..a4ceb98 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,12 +41,21 @@ pub fn password_is_bad (mut password: String) -> bool { #[cfg (test)] mod tests { use std::{ - sync::Arc, + sync::{ + Arc, + atomic::{ + AtomicBool, + Ordering, + }, + }, + time::Duration, }; use tokio::{ runtime::Runtime, spawn, + sync::oneshot, + time::delay_for, }; use super::{ @@ -58,7 +67,7 @@ mod tests { fn check_bad_passwords () { use crate::password_is_bad; - for pw in vec! [ + for pw in &[ "password", "pAsSwOrD", "secret", @@ -80,6 +89,9 @@ mod tests { fn end_to_end () { use maplit::*; use reqwest::Client; + use tracing::{info}; + + tracing_subscriber::fmt::init (); let mut rt = Runtime::new ().unwrap (); @@ -99,8 +111,9 @@ mod tests { let relay_state = Arc::new (relay::RelayState::from (&config_file)); let relay_state_2 = relay_state.clone (); - spawn (async move { - relay::run_relay (relay_state_2, None).await.unwrap (); + let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); + let task_relay = spawn (async move { + relay::run_relay (relay_state_2, Some (stop_relay_rx)).await.unwrap (); }); assert! (relay_state.list_servers ().await.is_empty ()); @@ -113,18 +126,22 @@ mod tests { relay_url: "http://127.0.0.1:4000/7ZSFUKGV".into (), file_server_root: None, }; - spawn (async move { - server::main (config_file).await.unwrap (); - }); + let stop_server_atomic = Arc::new (AtomicBool::from (false)); + let task_server = { + let stop_server_atomic = stop_server_atomic.clone (); + spawn (async move { + server::main (config_file, Some (stop_server_atomic)).await.unwrap (); + }) + }; - tokio::time::delay_for (std::time::Duration::from_millis (500)).await; + delay_for (Duration::from_millis (500)).await; assert_eq! (relay_state.list_servers ().await, vec! [ server_name.to_string (), ]); let client = Client::builder () - .timeout (std::time::Duration::from_secs (2)) + .timeout (Duration::from_secs (2)) .build ().unwrap (); let resp = client.get (&format! ("{}/frontend/relay_up_check", relay_url)) @@ -156,6 +173,19 @@ mod tests { .send ().await.unwrap (); assert_eq! (resp.status (), reqwest::StatusCode::NOT_FOUND); + + info! ("Shutting down end-to-end test"); + + stop_server_atomic.store (true, Ordering::Relaxed); + stop_relay_tx.send (()).unwrap (); + + info! ("Sent stop messages"); + + task_relay.await.unwrap (); + info! ("Relay stopped"); + + task_server.await.unwrap (); + info! ("Server stopped"); }); } } diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 30844b3..18a7127 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -10,7 +10,6 @@ use std::{ }; use dashmap::DashMap; -use futures::channel::oneshot; use handlebars::Handlebars; use hyper::{ Body, @@ -26,7 +25,10 @@ use serde::{ Serialize, }; use tokio::{ - sync::Mutex, + sync::{ + Mutex, + oneshot, + }, }; use crate::{ @@ -183,17 +185,23 @@ async fn handle_http_listen ( if let Some (ParkedClients (v)) = request_rendezvous.remove (&watcher_code) { + // 1 or more clients were parked - Make the server + // handle them immediately + return status_reply (StatusCode::OK, rmp_serde::to_vec (&v).unwrap ()); } request_rendezvous.insert (watcher_code, ParkedServer (tx)); } - let one_req = vec! [ - rx.await.unwrap (), - ]; + // No clients were parked - make the server long-poll - return status_reply (StatusCode::OK, rmp_serde::to_vec (&one_req).unwrap ()); + let one_req = match rx.await { + Ok (r) => r, + Err (_) => return status_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon"), + }; + + status_reply (StatusCode::OK, rmp_serde::to_vec (&vec! [one_req]).unwrap ()) } async fn handle_http_response ( @@ -385,11 +393,13 @@ pub fn load_templates () Ok (handlebars) } +use tracing::info; + pub async fn run_relay ( state: Arc , - shutdown_oneshot: Option > + shutdown_oneshot: Option > ) - -> Result <(), Box > +-> Result <(), Box > { let addr = SocketAddr::from (( [0, 0, 0, 0], @@ -406,7 +416,7 @@ pub async fn run_relay ( } } - eprintln! ("Loaded {} server tripcodes", state.config.server_tripcodes.len ()); + info! ("Loaded {} server tripcodes", state.config.server_tripcodes.len ()); let make_svc = make_service_fn (|_conn| { let state = state.clone (); @@ -424,16 +434,52 @@ pub async fn run_relay ( .serve (make_svc); match shutdown_oneshot { - Some (rx) => server.with_graceful_shutdown (async { - rx.await.ok (); - }).await?, + Some (rx) => { + info! ("Configured for graceful shutdown"); + server.with_graceful_shutdown (async { + rx.await.ok (); + + state.response_rendezvous.clear (); + + let mut request_rendezvoux = state.request_rendezvous.lock ().await; + request_rendezvoux.clear (); + + info! ("Received graceful shutdown"); + }).await? + }, None => server.await?, }; + info! ("Exiting"); Ok (()) } #[cfg (test)] mod tests { + use std::time::Duration; + use tokio::{ + runtime::Runtime, + spawn, + sync::oneshot, + time::delay_for, + }; + + #[test] + fn so_crazy_it_might_work () { + let mut rt = Runtime::new ().unwrap (); + + rt.block_on (async { + let (tx, rx) = oneshot::channel (); + + let task_1 = spawn (async move { + delay_for (Duration::from_secs (1)).await; + tx.send (()).unwrap (); + }); + + rx.await.unwrap (); + + task_1.await.unwrap (); + }); + } } diff --git a/src/server/mod.rs b/src/server/mod.rs index c5986ed..6de6158 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -117,7 +117,17 @@ pub struct Config { pub file_server_root: Option , } -pub async fn main (config_file: ConfigFile) +use std::sync::atomic::{ + AtomicBool, + Ordering, +}; + +use tracing::info; + +pub async fn main ( + config_file: ConfigFile, + shutdown_atomic: Option > +) -> Result <(), Box > { use std::convert::TryInto; @@ -135,6 +145,7 @@ pub async fn main (config_file: ConfigFile) let client = Client::builder () .default_headers (headers) + .timeout (Duration::from_secs (30)) .build ().unwrap (); let handlebars = file_server::load_templates ()?; @@ -150,10 +161,24 @@ pub async fn main (config_file: ConfigFile) let mut backoff_delay = 0; loop { + if let Some (a) = &shutdown_atomic { + if a.load (Ordering::Relaxed) { + break; + } + } + if backoff_delay > 0 { delay_for (Duration::from_millis (backoff_delay)).await; } + if let Some (a) = &shutdown_atomic { + if a.load (Ordering::Relaxed) { + break; + } + } + + info! ("http_listen"); + let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, config_file.name)); let err_backoff_delay = std::cmp::min (30_000, backoff_delay * 2 + 500); @@ -186,4 +211,8 @@ pub async fn main (config_file: ConfigFile) handle_req_resp (state, req_resp).await; }); } + + info! ("Exiting"); + + Ok (()) } From 7d5a491c986464aee5504c2d8b283949b896bea8 Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Thu, 5 Nov 2020 21:44:32 -0600 Subject: [PATCH 005/208] :recycle: Remove Option<> --- src/bin/ptth_relay.rs | 2 +- src/bin/ptth_server.rs | 2 +- src/lib.rs | 4 ++-- src/relay/mod.rs | 29 ++++++++++++----------------- src/server/mod.rs | 2 +- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index 4f5ad9a..45683c7 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -50,6 +50,6 @@ async fn main () -> Result <(), Box > { relay::run_relay ( Arc::new (RelayState::from (&config_file)), - Some (rx) + rx ).await } diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 1ec88a0..63370de 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -27,5 +27,5 @@ async fn main () -> Result <(), Box > { toml::from_str (&config_s).unwrap_or_else (|_| panic! ("Can't parse {:?} as TOML", config_file_path)) }; - ptth::server::main (config_file, None).await + ptth::server::run_server (config_file, None).await } diff --git a/src/lib.rs b/src/lib.rs index a4ceb98..4350775 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -113,7 +113,7 @@ mod tests { let relay_state_2 = relay_state.clone (); let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); let task_relay = spawn (async move { - relay::run_relay (relay_state_2, Some (stop_relay_rx)).await.unwrap (); + relay::run_relay (relay_state_2, stop_relay_rx).await.unwrap (); }); assert! (relay_state.list_servers ().await.is_empty ()); @@ -130,7 +130,7 @@ mod tests { let task_server = { let stop_server_atomic = stop_server_atomic.clone (); spawn (async move { - server::main (config_file, Some (stop_server_atomic)).await.unwrap (); + server::run_server (config_file, Some (stop_server_atomic)).await.unwrap (); }) }; diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 18a7127..ef95378 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -397,7 +397,7 @@ use tracing::info; pub async fn run_relay ( state: Arc , - shutdown_oneshot: Option > + shutdown_oneshot: oneshot::Receiver <()> ) -> Result <(), Box > { @@ -433,22 +433,17 @@ pub async fn run_relay ( let server = Server::bind (&addr) .serve (make_svc); - match shutdown_oneshot { - Some (rx) => { - info! ("Configured for graceful shutdown"); - server.with_graceful_shutdown (async { - rx.await.ok (); - - state.response_rendezvous.clear (); - - let mut request_rendezvoux = state.request_rendezvous.lock ().await; - request_rendezvoux.clear (); - - info! ("Received graceful shutdown"); - }).await? - }, - None => server.await?, - }; + info! ("Configured for graceful shutdown"); + server.with_graceful_shutdown (async { + shutdown_oneshot.await.ok (); + + state.response_rendezvous.clear (); + + let mut request_rendezvoux = state.request_rendezvous.lock ().await; + request_rendezvoux.clear (); + + info! ("Received graceful shutdown"); + }).await?; info! ("Exiting"); Ok (()) diff --git a/src/server/mod.rs b/src/server/mod.rs index 6de6158..6cce392 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -124,7 +124,7 @@ use std::sync::atomic::{ use tracing::info; -pub async fn main ( +pub async fn run_server ( config_file: ConfigFile, shutdown_atomic: Option > ) From 50393e60a0ac33ea80fb7e228c07051ff3828286 Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Thu, 5 Nov 2020 22:18:47 -0600 Subject: [PATCH 006/208] :recycle: --- src/bin/load_toml.rs | 25 +++++++++++++++++++++++++ src/bin/ptth_relay.rs | 16 +++------------- src/bin/ptth_server.rs | 21 +++++++-------------- src/relay/mod.rs | 2 +- 4 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 src/bin/load_toml.rs diff --git a/src/bin/load_toml.rs b/src/bin/load_toml.rs new file mode 100644 index 0000000..acbb64d --- /dev/null +++ b/src/bin/load_toml.rs @@ -0,0 +1,25 @@ +use std::{ + fmt::Debug, + fs::File, + io::Read, + path::Path, +}; + +use serde::de::DeserializeOwned; + +pub fn load < + T: DeserializeOwned, + P: AsRef + Debug +> ( + config_file_path: P +) -> T { + let mut f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); + let mut buffer = vec! [0u8; 4096]; + let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read {:?}", config_file_path)); + buffer.truncate (bytes_read); + + { + let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse {:?} as UTF-8", config_file_path)); + toml::from_str (&config_s).unwrap_or_else (|_| panic! ("Can't parse {:?} as TOML", config_file_path)) + } +} diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index 45683c7..92f851e 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -6,24 +6,14 @@ use std::{ use tokio::sync::oneshot; +mod load_toml; + use ptth::relay; use ptth::relay::RelayState; #[tokio::main] async fn main () -> Result <(), Box > { - use std::io::Read; - - let config_file = { - let config_file_path = "config/ptth_relay.toml"; - - let mut f = File::open (config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); - let mut buffer = vec! [0u8; 4096]; - let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read {:?}", config_file_path)); - buffer.truncate (bytes_read); - - let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse {:?} as UTF-8", config_file_path)); - toml::from_str (&config_s).unwrap_or_else (|_| panic! ("Can't parse {:?} as TOML", config_file_path)) - }; + let config_file = load_toml::load ("config/ptth_relay.toml"); eprintln! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 63370de..fa98799 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -5,6 +5,8 @@ use std::{ use structopt::StructOpt; +mod load_toml; + #[derive (Debug, StructOpt)] struct Opt { #[structopt (long)] @@ -13,19 +15,10 @@ struct Opt { #[tokio::main] async fn main () -> Result <(), Box > { - use std::io::Read; + let config_file = load_toml::load ("config/ptth_server.toml"); - let config_file = { - let config_file_path = "config/ptth_server.toml"; - - let mut f = std::fs::File::open (config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); - let mut buffer = vec! [0u8; 4096]; - let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read {:?}", config_file_path)); - buffer.truncate (bytes_read); - - let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse {:?} as UTF-8", config_file_path)); - toml::from_str (&config_s).unwrap_or_else (|_| panic! ("Can't parse {:?} as TOML", config_file_path)) - }; - - ptth::server::run_server (config_file, None).await + ptth::server::run_server ( + config_file, + None + ).await } diff --git a/src/relay/mod.rs b/src/relay/mod.rs index ef95378..d615a80 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -21,7 +21,7 @@ use hyper::{ }; use hyper::service::{make_service_fn, service_fn}; use serde::{ - Deserialize, + Deserialize, Serialize, }; use tokio::{ From d8010cf33c5d6ff1980e84bd40adf65908fba4f8 Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Thu, 5 Nov 2020 23:03:33 -0600 Subject: [PATCH 007/208] :recycle: Graceful shutdown is now a oneshot for both relays and servers --- src/bin/load_toml.rs | 2 +- src/bin/ptth_relay.rs | 1 - src/bin/ptth_server.rs | 26 ++++++++++++++++++++++++- src/lib.rs | 16 ++++++++-------- src/server/mod.rs | 43 +++++++++++++++++++++--------------------- 5 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/bin/load_toml.rs b/src/bin/load_toml.rs index acbb64d..99acc89 100644 --- a/src/bin/load_toml.rs +++ b/src/bin/load_toml.rs @@ -20,6 +20,6 @@ pub fn load < { let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse {:?} as UTF-8", config_file_path)); - toml::from_str (&config_s).unwrap_or_else (|_| panic! ("Can't parse {:?} as TOML", config_file_path)) + toml::from_str (&config_s).unwrap_or_else (|e| panic! ("Can't parse {:?} as TOML: {}", config_file_path, e)) } } diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index 92f851e..a0ab66c 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -1,6 +1,5 @@ use std::{ error::Error, - fs::File, sync::Arc, }; diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index fa98799..495c4b6 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -4,6 +4,7 @@ use std::{ }; use structopt::StructOpt; +use tokio::sync::oneshot; mod load_toml; @@ -15,10 +16,33 @@ struct Opt { #[tokio::main] async fn main () -> Result <(), Box > { + tracing_subscriber::fmt::init (); + let config_file = load_toml::load ("config/ptth_server.toml"); + let rx = { + let (tx, rx) = oneshot::channel::<()> (); + + // I have to put the tx into a Cell here so that if Ctrl-C gets + // called multiple times, we won't send multiple shutdowns to the + // oneshot channel. (Which would be a compile error) + + let tx = Some (tx); + let tx = std::cell::Cell::new (tx); + + ctrlc::set_handler (move ||{ + let tx = tx.replace (None); + + if let Some (tx) = tx { + tx.send (()).unwrap (); + } + }).expect ("Error setting Ctrl-C handler"); + + rx + }; + ptth::server::run_server ( config_file, - None + rx ).await } diff --git a/src/lib.rs b/src/lib.rs index 4350775..82724e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,10 +43,6 @@ mod tests { use std::{ sync::{ Arc, - atomic::{ - AtomicBool, - Ordering, - }, }, time::Duration, }; @@ -91,6 +87,10 @@ mod tests { use reqwest::Client; use tracing::{info}; + // This should be the first line of the `tracing` + // crate documentation. Their docs are awful, but you + // didn't hear it from me. + tracing_subscriber::fmt::init (); let mut rt = Runtime::new ().unwrap (); @@ -126,11 +126,11 @@ mod tests { relay_url: "http://127.0.0.1:4000/7ZSFUKGV".into (), file_server_root: None, }; - let stop_server_atomic = Arc::new (AtomicBool::from (false)); + + let (stop_server_tx, stop_server_rx) = oneshot::channel (); let task_server = { - let stop_server_atomic = stop_server_atomic.clone (); spawn (async move { - server::run_server (config_file, Some (stop_server_atomic)).await.unwrap (); + server::run_server (config_file, stop_server_rx).await.unwrap (); }) }; @@ -176,7 +176,7 @@ mod tests { info! ("Shutting down end-to-end test"); - stop_server_atomic.store (true, Ordering::Relaxed); + stop_server_tx.send (()).unwrap (); stop_relay_tx.send (()).unwrap (); info! ("Sent stop messages"); diff --git a/src/server/mod.rs b/src/server/mod.rs index 6cce392..901b485 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -5,6 +5,7 @@ use std::{ time::Duration, }; +use futures::FutureExt; use handlebars::Handlebars; use hyper::{ StatusCode, @@ -12,8 +13,10 @@ use hyper::{ use reqwest::Client; use serde::Deserialize; use tokio::{ + sync::oneshot, time::delay_for, }; +use tracing::info; use crate::{ http_serde, @@ -117,16 +120,9 @@ pub struct Config { pub file_server_root: Option , } -use std::sync::atomic::{ - AtomicBool, - Ordering, -}; - -use tracing::info; - pub async fn run_server ( config_file: ConfigFile, - shutdown_atomic: Option > + shutdown_oneshot: oneshot::Receiver <()> ) -> Result <(), Box > { @@ -159,31 +155,36 @@ pub async fn run_server ( }); let mut backoff_delay = 0; + let mut shutdown_oneshot = shutdown_oneshot.fuse (); loop { - if let Some (a) = &shutdown_atomic { - if a.load (Ordering::Relaxed) { - break; - } - } - if backoff_delay > 0 { - delay_for (Duration::from_millis (backoff_delay)).await; - } - - if let Some (a) = &shutdown_atomic { - if a.load (Ordering::Relaxed) { + let mut delay = delay_for (Duration::from_millis (backoff_delay)).fuse (); + + if futures::select! ( + _ = delay => false, + _ = shutdown_oneshot => true, + ) { + info! ("Received graceful shutdown"); break; } } info! ("http_listen"); - let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, config_file.name)); + let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, config_file.name)).send (); let err_backoff_delay = std::cmp::min (30_000, backoff_delay * 2 + 500); - let req_resp = match req_req.send ().await { + let req_req = futures::select! { + r = req_req.fuse () => r, + _ = shutdown_oneshot => { + info! ("Received graceful shutdown"); + break; + }, + }; + + let req_resp = match req_req { Err (e) => { eprintln! ("Err: {:?}", e); backoff_delay = err_backoff_delay; From 851b895873e06a9bcc4f2fea4d925bede111eccc Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Thu, 5 Nov 2020 23:11:07 -0600 Subject: [PATCH 008/208] :bug: Turns out I had the modules all wrong. This one works good --- src/bin/ptth_relay.rs | 29 ++--------------------------- src/bin/ptth_server.rs | 28 ++-------------------------- src/graceful_shutdown.rs | 23 +++++++++++++++++++++++ src/lib.rs | 2 ++ src/{bin => }/load_toml.rs | 0 5 files changed, 29 insertions(+), 53 deletions(-) create mode 100644 src/graceful_shutdown.rs rename src/{bin => }/load_toml.rs (100%) diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index a0ab66c..dd3ebd2 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -3,42 +3,17 @@ use std::{ sync::Arc, }; -use tokio::sync::oneshot; - -mod load_toml; - use ptth::relay; use ptth::relay::RelayState; #[tokio::main] async fn main () -> Result <(), Box > { - let config_file = load_toml::load ("config/ptth_relay.toml"); + let config_file = ptth::load_toml::load ("config/ptth_relay.toml"); eprintln! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); - let rx = { - let (tx, rx) = oneshot::channel::<()> (); - - // I have to put the tx into a Cell here so that if Ctrl-C gets - // called multiple times, we won't send multiple shutdowns to the - // oneshot channel. (Which would be a compile error) - - let tx = Some (tx); - let tx = std::cell::Cell::new (tx); - - ctrlc::set_handler (move ||{ - let tx = tx.replace (None); - - if let Some (tx) = tx { - tx.send (()).unwrap (); - } - }).expect ("Error setting Ctrl-C handler"); - - rx - }; - relay::run_relay ( Arc::new (RelayState::from (&config_file)), - rx + ptth::graceful_shutdown::init () ).await } diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 495c4b6..790dc97 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -4,9 +4,6 @@ use std::{ }; use structopt::StructOpt; -use tokio::sync::oneshot; - -mod load_toml; #[derive (Debug, StructOpt)] struct Opt { @@ -18,31 +15,10 @@ struct Opt { async fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); - let config_file = load_toml::load ("config/ptth_server.toml"); - - let rx = { - let (tx, rx) = oneshot::channel::<()> (); - - // I have to put the tx into a Cell here so that if Ctrl-C gets - // called multiple times, we won't send multiple shutdowns to the - // oneshot channel. (Which would be a compile error) - - let tx = Some (tx); - let tx = std::cell::Cell::new (tx); - - ctrlc::set_handler (move ||{ - let tx = tx.replace (None); - - if let Some (tx) = tx { - tx.send (()).unwrap (); - } - }).expect ("Error setting Ctrl-C handler"); - - rx - }; + let config_file = ptth::load_toml::load ("config/ptth_server.toml"); ptth::server::run_server ( config_file, - rx + ptth::graceful_shutdown::init () ).await } diff --git a/src/graceful_shutdown.rs b/src/graceful_shutdown.rs new file mode 100644 index 0000000..63de1b3 --- /dev/null +++ b/src/graceful_shutdown.rs @@ -0,0 +1,23 @@ +use std::cell::Cell; +use tokio::sync::oneshot; + +pub fn init () -> oneshot::Receiver <()> { + let (tx, rx) = oneshot::channel::<()> (); + + // I have to put the tx into a Cell here so that if Ctrl-C gets + // called multiple times, we won't send multiple shutdowns to the + // oneshot channel. (Which would be a compile error) + + let tx = Some (tx); + let tx = Cell::new (tx); + + ctrlc::set_handler (move ||{ + let tx = tx.replace (None); + + if let Some (tx) = tx { + tx.send (()).unwrap (); + } + }).expect ("Error setting Ctrl-C handler"); + + rx +} diff --git a/src/lib.rs b/src/lib.rs index 82724e3..16d239c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,8 @@ pub const PTTH_MAGIC_HEADER: &str = "X-PTTH-2LJYXWC4"; // test stuff like spawn them both in the same process pub mod git_version; +pub mod graceful_shutdown; +pub mod load_toml; pub mod relay; pub mod server; diff --git a/src/bin/load_toml.rs b/src/load_toml.rs similarity index 100% rename from src/bin/load_toml.rs rename to src/load_toml.rs From b6f6987eec2c322fe4c6d7dc8d6836ab744d464a Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 6 Nov 2020 05:17:29 +0000 Subject: [PATCH 009/208] Update todo --- .rgignore | 1 + todo.md | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .rgignore diff --git a/.rgignore b/.rgignore new file mode 100644 index 0000000..ebbb1bf --- /dev/null +++ b/.rgignore @@ -0,0 +1 @@ +src/bad_passwords.txt diff --git a/todo.md b/todo.md index f76f183..a1e81d5 100644 --- a/todo.md +++ b/todo.md @@ -1,5 +1,12 @@ +- Not working behind Nginx +- Try sending the http_response "OK" _after_ the request body is received +- Add relay / server debug flags in config +- Tracing / structured logging, maybe? + +- Sort directory listings +- ".." from server to server list is broken +- Redirect to add trailing slashes - Add file size in directory listing -- Large text files not streaming as expected? - Allow spaces in server names - Make file_server_root mandatory - Deny unused HTTP methods for endpoints From 5626ecc05cff41aaa4009f919d3337d47f64ddca Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 6 Nov 2020 18:49:41 +0000 Subject: [PATCH 010/208] Update todo --- todo.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/todo.md b/todo.md index a1e81d5..3f9447b 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,5 @@ - Not working behind Nginx - Try sending the http_response "OK" _after_ the request body is received -- Add relay / server debug flags in config -- Tracing / structured logging, maybe? - Sort directory listings - ".." from server to server list is broken @@ -11,7 +9,7 @@ - Make file_server_root mandatory - Deny unused HTTP methods for endpoints - Hide ptth_server.toml from file server -- ETag cache +- ETag cache based on mtime - Server-side hash? - Log / audit log? From 3786cec8abd4686b1bb0bd5837272c2ea4c84296 Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 6 Nov 2020 20:55:55 +0000 Subject: [PATCH 011/208] :construction: Add a bunch more logging / tracing --- src/bin/ptth_relay.rs | 6 ++- src/http_serde.rs | 24 ++++++---- src/relay/mod.rs | 92 ++++++++++++++++++++++++--------------- src/server/file_server.rs | 30 ++++++++++--- src/server/mod.rs | 47 ++++++++++++-------- todo.md | 6 ++- 6 files changed, 136 insertions(+), 69 deletions(-) diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index dd3ebd2..8641805 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -3,14 +3,18 @@ use std::{ sync::Arc, }; +use tracing::{info}; + use ptth::relay; use ptth::relay::RelayState; #[tokio::main] async fn main () -> Result <(), Box > { + tracing_subscriber::fmt::init (); + let config_file = ptth::load_toml::load ("config/ptth_relay.toml"); - eprintln! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); + info! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); relay::run_relay ( Arc::new (RelayState::from (&config_file)), diff --git a/src/http_serde.rs b/src/http_serde.rs index 0d94ad3..bc52547 100644 --- a/src/http_serde.rs +++ b/src/http_serde.rs @@ -4,7 +4,7 @@ use std::{ }; use serde::{Deserialize, Serialize}; -use tokio::sync::mpsc::Receiver; +use tokio::sync::mpsc; // Hyper doesn't seem to make it easy to de/ser requests // and responses and stuff like that, so I do it by hand here. @@ -20,7 +20,7 @@ impl From for Error { } } -#[derive (Deserialize, Serialize)] +#[derive (Debug, Deserialize, Serialize)] pub enum Method { Get, Head, @@ -83,11 +83,15 @@ pub struct WrappedRequest { pub req: RequestParts, } -#[derive (Deserialize, Serialize)] +#[derive (Debug, Deserialize, Serialize)] pub enum StatusCode { - Ok, - NotFound, - PartialContent, + Ok, // 200 + PartialContent, // 206 + + BadRequest, // 400 + Forbidden, // 403 + NotFound, // 404 + MethodNotAllowed, // 405 } impl Default for StatusCode { @@ -100,8 +104,12 @@ impl From for hyper::StatusCode { fn from (x: StatusCode) -> Self { match x { StatusCode::Ok => Self::OK, - StatusCode::NotFound => Self::NOT_FOUND, StatusCode::PartialContent => Self::PARTIAL_CONTENT, + + StatusCode::BadRequest => Self::BAD_REQUEST, + StatusCode::Forbidden => Self::FORBIDDEN, + StatusCode::NotFound => Self::NOT_FOUND, + StatusCode::MethodNotAllowed => Self::METHOD_NOT_ALLOWED, } } } @@ -120,7 +128,7 @@ pub struct ResponseParts { // reqwest and hyper have different Body types for _some reason_ // so I have to do everything myself -type Body = Receiver , std::convert::Infallible>>; +type Body = mpsc::Receiver , std::convert::Infallible>>; #[derive (Default)] pub struct Response { diff --git a/src/relay/mod.rs b/src/relay/mod.rs index d615a80..803e30f 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -10,6 +10,7 @@ use std::{ }; use dashmap::DashMap; +use futures::stream::StreamExt; use handlebars::Handlebars; use hyper::{ Body, @@ -25,11 +26,14 @@ use serde::{ Serialize, }; use tokio::{ + spawn, sync::{ Mutex, + mpsc, oneshot, }, }; +use tracing::{debug, error, info, trace, warn}; use crate::{ http_serde, @@ -153,6 +157,9 @@ fn status_reply > (status: StatusCode, b: B) Response::builder ().status (status).body (b.into ()).unwrap () } +// Servers will come here and either handle queued requests from parked clients, +// or park themselves until a request comes in. + async fn handle_http_listen ( state: Arc , watcher_code: String, @@ -164,7 +171,7 @@ async fn handle_http_listen ( let expected_tripcode = match state.config.server_tripcodes.get (&watcher_code) { None => { - eprintln! ("Denied http_listen for non-existent server name {}", watcher_code); + error! ("Denied http_listen for non-existent server name {}", watcher_code); return trip_error; }, Some (x) => x, @@ -172,7 +179,7 @@ async fn handle_http_listen ( let actual_tripcode = blake3::hash (api_key); if expected_tripcode != &actual_tripcode { - eprintln! ("Denied http_listen for bad tripcode {}", base64::encode (actual_tripcode.as_bytes ())); + error! ("Denied http_listen for bad tripcode {}", base64::encode (actual_tripcode.as_bytes ())); return trip_error; } @@ -204,6 +211,8 @@ async fn handle_http_listen ( status_reply (StatusCode::OK, rmp_serde::to_vec (&vec! [one_req]).unwrap ()) } +// Servers will come here to stream responses to clients + async fn handle_http_response ( req: Request , state: Arc , @@ -211,21 +220,63 @@ async fn handle_http_response ( ) -> Response { - let (parts, body) = req.into_parts (); + let (parts, mut body) = req.into_parts (); let resp_parts: http_serde::ResponseParts = rmp_serde::from_read_ref (&base64::decode (parts.headers.get (crate::PTTH_MAGIC_HEADER).unwrap ()).unwrap ()).unwrap (); + // Intercept the body packets here so we can check when the stream + // ends or errors out + + let (mut body_tx, body_rx) = mpsc::channel (2); + + spawn (async move { + loop { + let item = body.next ().await; + + if let Some (item) = item { + if let Ok (bytes) = &item { + trace! ("Relaying {} bytes", bytes.len ()); + } + + if body_tx.send (item).await.is_err () { + error! ("Error relaying bytes"); + break; + } + } + else { + debug! ("Finished relaying bytes"); + break; + } + } + }); + + let body = Body::wrap_stream (body_rx); + match state.response_rendezvous.remove (&req_id) { Some ((_, tx)) => { + // UKAUFFY4 (Send half) match tx.send ((resp_parts, body)) { - Ok (()) => status_reply (StatusCode::OK, "Connected to remote client...\n"), - _ => status_reply (StatusCode::BAD_GATEWAY, "Failed to connect to client"), + Ok (()) => { + debug! ("Responding to server"); + status_reply (StatusCode::OK, "http_response completed.") + }, + _ => { + let msg = "Failed to connect to client"; + error! (msg); + status_reply (StatusCode::BAD_GATEWAY, msg) + }, } }, - None => status_reply (StatusCode::BAD_REQUEST, "Request ID not found in response_rendezvous"), + None => { + error! ("Server tried to respond to non-existent request"); + status_reply (StatusCode::BAD_REQUEST, "Request ID not found in response_rendezvous") + }, } } +// Clients will come here to start requests, and always park for at least +// a short amount of time. + async fn handle_http_request ( req: http::request::Parts, uri: String, @@ -286,6 +337,7 @@ async fn handle_http_request ( }, }; + // UKAUFFY4 (Receive half) match received { Ok ((parts, body)) => { let mut resp = Response::builder () @@ -393,8 +445,6 @@ pub fn load_templates () Ok (handlebars) } -use tracing::info; - pub async fn run_relay ( state: Arc , shutdown_oneshot: oneshot::Receiver <()> @@ -433,7 +483,6 @@ pub async fn run_relay ( let server = Server::bind (&addr) .serve (make_svc); - info! ("Configured for graceful shutdown"); server.with_graceful_shutdown (async { shutdown_oneshot.await.ok (); @@ -451,30 +500,5 @@ pub async fn run_relay ( #[cfg (test)] mod tests { - use std::time::Duration; - use tokio::{ - runtime::Runtime, - spawn, - sync::oneshot, - time::delay_for, - }; - - #[test] - fn so_crazy_it_might_work () { - let mut rt = Runtime::new ().unwrap (); - - rt.block_on (async { - let (tx, rx) = oneshot::channel (); - - let task_1 = spawn (async move { - delay_for (Duration::from_secs (1)).await; - tx.send (()).unwrap (); - }); - - rx.await.unwrap (); - - task_1.await.unwrap (); - }); - } } diff --git a/src/server/file_server.rs b/src/server/file_server.rs index b150296..1105993 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -21,6 +21,10 @@ use tokio::{ channel, }, }; +use tracing::{ + debug, error, info, + instrument, +}; use regex::Regex; @@ -33,7 +37,7 @@ fn parse_range_header (range_str: &str) -> (Option , Option ) { static ref RE: Regex = Regex::new (r"^bytes=(\d*)-(\d*)$").expect ("Couldn't compile regex for Range header"); } - println! ("{}", range_str); + debug! ("{}", range_str); let caps = match RE.captures (range_str) { Some (x) => x, @@ -98,6 +102,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry use std::borrow::Cow; +#[instrument (level = "debug", skip (handlebars, dir))] async fn serve_dir ( handlebars: &Handlebars <'static>, path: Cow <'_, str>, @@ -110,6 +115,8 @@ async fn serve_dir ( entries.push (read_dir_entry (entry).await); } + entries.sort_unstable_by (|a, b| a.file_name.partial_cmp (&b.file_name).unwrap ()); + #[derive (Serialize)] struct TemplateDirPage <'a> { path: Cow <'a, str>, @@ -132,6 +139,7 @@ async fn serve_dir ( resp } +#[instrument (level = "debug", skip (f))] async fn serve_file ( mut f: File, should_send_body: bool, @@ -157,7 +165,7 @@ async fn serve_file ( f.seek (SeekFrom::Start (start)).await.unwrap (); - println! ("Serving range {}-{}", start, end); + info! ("Serving range {}-{}", start, end); if should_send_body { tokio::spawn (async move { @@ -180,18 +188,18 @@ async fn serve_file ( } if tx.send (Ok::<_, Infallible> (buffer)).await.is_err () { - eprintln! ("Send failed while streaming file ({} bytes sent)", bytes_sent); + error! ("Send failed while streaming file ({} bytes sent)", bytes_sent); break; } bytes_left -= bytes_read; if bytes_left == 0 { - eprintln! ("Finished streaming file"); + info! ("Finished"); break; } bytes_sent += bytes_read; - println! ("Sent {} bytes", bytes_sent); + debug! ("Sent {} bytes", bytes_sent); //delay_for (Duration::from_millis (50)).await; } @@ -234,6 +242,7 @@ async fn serve_error ( resp } +#[instrument (level = "debug", skip (handlebars))] pub async fn serve_all ( handlebars: &Handlebars <'static>, root: &Path, @@ -243,7 +252,7 @@ pub async fn serve_all ( ) -> http_serde::Response { - println! ("Client requested {}", uri); + info! ("Client requested {}", uri); let mut range_start = None; let mut range_end = None; @@ -255,7 +264,14 @@ pub async fn serve_all ( range_end = end; } - let should_send_body = matches! (&method, http_serde::Method::Get); + let should_send_body = match &method { + http_serde::Method::Get => true, + http_serde::Method::Head => false, + m => { + debug! ("Unsupported method {:?}", m); + return serve_error (http_serde::StatusCode::MethodNotAllowed, "Unsupported method".into ()).await; + } + }; use percent_encoding::*; diff --git a/src/server/mod.rs b/src/server/mod.rs index 901b485..59050ca 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -16,7 +16,7 @@ use tokio::{ sync::oneshot, time::delay_for, }; -use tracing::info; +use tracing::{debug, error, info}; use crate::{ http_serde, @@ -40,13 +40,14 @@ fn status_reply (c: http_serde::StatusCode, body: &str) -> http_serde::Response } async fn handle_req_resp <'a> ( - state: Arc , + state: &Arc , req_resp: reqwest::Response ) { //println! ("Step 1"); if req_resp.status () != StatusCode::OK { // TODO: Error handling + error! ("http_listen didn't respond with 200 OK"); return; } @@ -54,15 +55,22 @@ async fn handle_req_resp <'a> ( let wrapped_reqs: Vec = match rmp_serde::from_read_ref (&body) { Ok (x) => x, - _ => return, + Err (e) => { + error! ("Can't parse wrapped requests: {:?}", e); + return; + }, }; + debug! ("Unwrapped {} requests", wrapped_reqs.len ()); + for wrapped_req in wrapped_reqs.into_iter () { let state = state.clone (); tokio::spawn (async move { let (req_id, parts) = (wrapped_req.id, wrapped_req.req); + debug! ("Handling request {}", req_id); + let response = if let Some (uri) = prefix_match (&parts.uri, "/files") { let default_root = PathBuf::from ("./"); let file_server_root: &std::path::Path = state.config.file_server_root @@ -78,6 +86,7 @@ async fn handle_req_resp <'a> ( ).await } else { + debug! ("404 not found"); status_reply (http_serde::StatusCode::NotFound, "404 Not Found") }; @@ -94,12 +103,16 @@ async fn handle_req_resp <'a> ( let req = resp_req.build ().unwrap (); - eprintln! ("{:?}", req.headers ()); + debug! ("{:?}", req.headers ()); //println! ("Step 6"); match state.client.execute (req).await { - Ok (r) => eprintln! ("{:?} {:?}", r.status (), r.text ().await.unwrap ()), - Err (e) => eprintln! ("Err: {:?}", e), + Ok (r) => { + let status = r.status (); + let text = r.text ().await.unwrap (); + debug! ("{:?} {:?}", status, text); + }, + Err (e) => error! ("Err: {:?}", e), } }); @@ -134,7 +147,7 @@ pub async fn run_server ( let tripcode = base64::encode (blake3::hash (config_file.api_key.as_bytes ()).as_bytes ()); - println! ("Our tripcode is {}", tripcode); + info! ("Our tripcode is {}", tripcode); let mut headers = reqwest::header::HeaderMap::new (); headers.insert ("X-ApiKey", config_file.api_key.try_into ().unwrap ()); @@ -170,7 +183,7 @@ pub async fn run_server ( } } - info! ("http_listen"); + debug! ("http_listen"); let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, config_file.name)).send (); @@ -186,7 +199,7 @@ pub async fn run_server ( let req_resp = match req_req { Err (e) => { - eprintln! ("Err: {:?}", e); + error! ("Err: {:?}", e); backoff_delay = err_backoff_delay; continue; }, @@ -197,20 +210,18 @@ pub async fn run_server ( }; if req_resp.status () != StatusCode::OK { - eprintln! ("{}", req_resp.status ()); - eprintln! ("{}", String::from_utf8 (req_resp.bytes ().await.unwrap ().to_vec ()).unwrap ()); + error! ("{}", req_resp.status ()); + let body = req_resp.bytes ().await.unwrap (); + let body = String::from_utf8 (body.to_vec ()).unwrap (); + error! ("{}", body); backoff_delay = err_backoff_delay; continue; } - // Spawn another task for each request so we can - // immediately listen for the next connection + // Unpack the requests, spawn them into new tasks, then loop back + // around. - let state = state.clone (); - - tokio::spawn (async move { - handle_req_resp (state, req_resp).await; - }); + handle_req_resp (&state, req_resp).await; } info! ("Exiting"); diff --git a/todo.md b/todo.md index 3f9447b..259d4f9 100644 --- a/todo.md +++ b/todo.md @@ -13,7 +13,11 @@ - Server-side hash? - Log / audit log? -- Prevent directory traversal attacks +- Prevent directory traversal attacks in file_server.rs - Error handling - Reverse proxy to other local servers + +Off-project stuff: + +- Benchmark directory entry sorting From e0298a5289f83f4b0ce34247db6a679e80383b15 Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 6 Nov 2020 23:43:52 +0000 Subject: [PATCH 012/208] :bug: Working on a bunch of bugs and error handling --- src/bin/ptth_relay.rs | 11 +- src/http_serde.rs | 4 +- src/relay/mod.rs | 236 +++++++++++++++++++++++++++----------- src/server/file_server.rs | 8 +- src/server/mod.rs | 53 ++++++--- todo.md | 1 + 6 files changed, 226 insertions(+), 87 deletions(-) diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index 8641805..9c6912b 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -4,13 +4,22 @@ use std::{ }; use tracing::{info}; +use tracing_subscriber::{ + fmt, + fmt::format::FmtSpan, + EnvFilter, +}; use ptth::relay; use ptth::relay::RelayState; #[tokio::main] async fn main () -> Result <(), Box > { - tracing_subscriber::fmt::init (); + fmt () + .with_env_filter (EnvFilter::from_default_env ()) + .with_span_events (FmtSpan::FULL) + .init () + ; let config_file = ptth::load_toml::load ("config/ptth_relay.toml"); diff --git a/src/http_serde.rs b/src/http_serde.rs index bc52547..2ef2fc5 100644 --- a/src/http_serde.rs +++ b/src/http_serde.rs @@ -24,6 +24,8 @@ impl From for Error { pub enum Method { Get, Head, + Post, + Put, } impl TryFrom for Method { @@ -42,7 +44,7 @@ impl TryFrom for Method { pub struct RequestParts { pub method: Method, - // Technically URIs are subtle and complex but I don't care + // Technically URIs are subtle and complex, but I don't care pub uri: String, // Technically Hyper has headers in a multi-map diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 803e30f..6b2c1cd 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -5,12 +5,16 @@ use std::{ iter::FromIterator, net::SocketAddr, sync::{ - Arc + Arc, }, + time::Duration, }; use dashmap::DashMap; -use futures::stream::StreamExt; +use futures::{ + FutureExt, + stream::StreamExt, +}; use handlebars::Handlebars; use hyper::{ Body, @@ -31,9 +35,14 @@ use tokio::{ Mutex, mpsc, oneshot, + RwLock, }, + time::delay_for, +}; +use tracing::{ + debug, error, info, trace, + instrument, }; -use tracing::{debug, error, info, trace, warn}; use crate::{ http_serde, @@ -69,12 +78,17 @@ can be parked */ -enum RequestRendezvous { - ParkedClients (Vec ), - ParkedServer (oneshot::Sender ), +#[derive (Debug)] +enum RelayError { + RelayShuttingDown, } -type ResponseRendezvous = oneshot::Sender <(http_serde::ResponseParts, Body)>; +enum RequestRendezvous { + ParkedClients (Vec ), + ParkedServer (oneshot::Sender >), +} + +type ResponseRendezvous = oneshot::Sender >; // Stuff we need to load from the config file and use to // set up the HTTP server @@ -118,7 +132,7 @@ pub struct RelayState { request_rendezvous: Mutex >, // Key: Request ID - response_rendezvous: DashMap , + response_rendezvous: RwLock >, } impl Default for RelayState { @@ -151,10 +165,19 @@ impl RelayState { } } -fn status_reply > (status: StatusCode, b: B) +fn ok_reply > (b: B) -> Response { - Response::builder ().status (status).body (b.into ()).unwrap () + Response::builder ().status (StatusCode::OK).body (b.into ()).unwrap () +} + +fn error_reply (status: StatusCode, b: &str) +-> Response +{ + Response::builder () + .status (status) + .header ("content-type", "text/plain") + .body (format! ("{}\n", b).into ()).unwrap () } // Servers will come here and either handle queued requests from parked clients, @@ -167,7 +190,7 @@ async fn handle_http_listen ( ) -> Response { - let trip_error = status_reply (StatusCode::UNAUTHORIZED, "Bad X-ApiKey"); + let trip_error = error_reply (StatusCode::UNAUTHORIZED, "Bad X-ApiKey"); let expected_tripcode = match state.config.server_tripcodes.get (&watcher_code) { None => { @@ -192,23 +215,35 @@ async fn handle_http_listen ( if let Some (ParkedClients (v)) = request_rendezvous.remove (&watcher_code) { - // 1 or more clients were parked - Make the server - // handle them immediately - - return status_reply (StatusCode::OK, rmp_serde::to_vec (&v).unwrap ()); + if ! v.is_empty () { + // 1 or more clients were parked - Make the server + // handle them immediately + + debug! ("Sending {} parked requests to server {}", v.len (), watcher_code); + return ok_reply (rmp_serde::to_vec (&v).unwrap ()); + } } - request_rendezvous.insert (watcher_code, ParkedServer (tx)); + debug! ("Parking server {}", watcher_code); + request_rendezvous.insert (watcher_code.clone (), ParkedServer (tx)); } // No clients were parked - make the server long-poll - let one_req = match rx.await { - Ok (r) => r, - Err (_) => return status_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon"), - }; - - status_reply (StatusCode::OK, rmp_serde::to_vec (&vec! [one_req]).unwrap ()) + futures::select! { + x = rx.fuse () => match x { + Ok (Ok (one_req)) => { + debug! ("Unparking server {}", watcher_code); + ok_reply (rmp_serde::to_vec (&vec! [one_req]).unwrap ()) + }, + Ok (Err (RelayError::RelayShuttingDown)) => error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon"), + Err (_) => error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error"), + }, + _ = delay_for (Duration::from_secs (30)).fuse () => { + debug! ("Timed out http_listen for server {}", watcher_code); + return error_reply (StatusCode::NO_CONTENT, "No requests now, long-poll again") + } + } } // Servers will come here to stream responses to clients @@ -226,7 +261,15 @@ async fn handle_http_response ( // Intercept the body packets here so we can check when the stream // ends or errors out + #[derive (Debug)] + enum BodyFinishedReason { + StreamFinished, + ClientDisconnected, + } + use BodyFinishedReason::*; + let (mut body_tx, body_rx) = mpsc::channel (2); + let (body_finished_tx, body_finished_rx) = oneshot::channel (); spawn (async move { loop { @@ -237,13 +280,15 @@ async fn handle_http_response ( trace! ("Relaying {} bytes", bytes.len ()); } - if body_tx.send (item).await.is_err () { - error! ("Error relaying bytes"); + if let Err (_e) = body_tx.send (item).await { + info! ("Body closed while relaying. (Client hung up?)"); + body_finished_tx.send (ClientDisconnected).unwrap (); break; } } else { debug! ("Finished relaying bytes"); + body_finished_tx.send (StreamFinished).unwrap (); break; } } @@ -251,26 +296,32 @@ async fn handle_http_response ( let body = Body::wrap_stream (body_rx); - match state.response_rendezvous.remove (&req_id) { - Some ((_, tx)) => { - // UKAUFFY4 (Send half) - match tx.send ((resp_parts, body)) { - Ok (()) => { - debug! ("Responding to server"); - status_reply (StatusCode::OK, "http_response completed.") - }, - _ => { - let msg = "Failed to connect to client"; - error! (msg); - status_reply (StatusCode::BAD_GATEWAY, msg) - }, - } - - }, - None => { - error! ("Server tried to respond to non-existent request"); - status_reply (StatusCode::BAD_REQUEST, "Request ID not found in response_rendezvous") + let tx = { + let response_rendezvous = state.response_rendezvous.read ().await; + match response_rendezvous.remove (&req_id) { + None => { + error! ("Server tried to respond to non-existent request"); + return error_reply (StatusCode::BAD_REQUEST, "Request ID not found in response_rendezvous"); + }, + Some ((_, x)) => x, + } + }; + + // UKAUFFY4 (Send half) + if tx.send (Ok ((resp_parts, body))).is_err () { + let msg = "Failed to connect to client"; + error! (msg); + return error_reply (StatusCode::BAD_GATEWAY, msg); + } + + debug! ("Connected server to client for streaming."); + match body_finished_rx.await.unwrap () { + StreamFinished => { + error_reply (StatusCode::OK, "StreamFinished") }, + ClientDisconnected => { + error_reply (StatusCode::OK, "ClientDisconnected") + } } } @@ -286,24 +337,29 @@ async fn handle_http_request ( -> Response { if ! state.config.server_tripcodes.contains_key (&watcher_code) { - return status_reply (StatusCode::NOT_FOUND, "Unknown server"); + return error_reply (StatusCode::NOT_FOUND, "Unknown server"); } let req = match http_serde::RequestParts::from_hyper (req.method, uri, req.headers) { Ok (x) => x, - _ => return status_reply (StatusCode::BAD_REQUEST, "Bad request"), + _ => return error_reply (StatusCode::BAD_REQUEST, "Bad request"), }; let (tx, rx) = oneshot::channel (); - let id = ulid::Ulid::new ().to_string (); - state.response_rendezvous.insert (id.clone (), tx); + let req_id = ulid::Ulid::new ().to_string (); + { + let response_rendezvous = state.response_rendezvous.read ().await; + response_rendezvous.insert (req_id.clone (), tx); + } + + trace! ("Created request {}", req_id); { let mut request_rendezvous = state.request_rendezvous.lock ().await; let wrapped = http_serde::WrappedRequest { - id, + id: req_id.clone (), req, }; @@ -311,18 +367,38 @@ async fn handle_http_request ( let new_rendezvous = match request_rendezvous.remove (&watcher_code) { Some (ParkedClients (mut v)) => { + debug! ("Parking request {} ({} already queued)", req_id, v.len ()); v.push (wrapped); ParkedClients (v) }, Some (ParkedServer (s)) => { // If sending to the server fails, queue it - match s.send (wrapped) { - Ok (()) => ParkedClients (vec! []), - Err (wrapped) => ParkedClients (vec! [wrapped]), + match s.send (Ok (wrapped)) { + Ok (()) => { + // TODO: This can actually still fail, if the server + // disconnects right as we're sending this. + // Then what? + + debug! ( + "Sending request {} directly to server {}", + req_id, + watcher_code, + ); + + ParkedClients (vec! []) + }, + Err (Ok (wrapped)) => { + debug! ("Parking request {}", req_id); + ParkedClients (vec! [wrapped]) + }, + Err (_) => unreachable! (), } }, - None => ParkedClients (vec! [wrapped]), + None => { + debug! ("Parking request {}", req_id); + ParkedClients (vec! [wrapped]) + }, }; request_rendezvous.insert (watcher_code, new_rendezvous); @@ -333,13 +409,14 @@ async fn handle_http_request ( let received = tokio::select! { val = rx => val, () = timeout => { - return status_reply (StatusCode::GATEWAY_TIMEOUT, "Remote server never responded") + debug! ("Timed out request {}", req_id); + return error_reply (StatusCode::GATEWAY_TIMEOUT, "Remote server never responded") }, }; // UKAUFFY4 (Receive half) match received { - Ok ((parts, body)) => { + Ok (Ok ((parts, body))) => { let mut resp = Response::builder () .status (hyper::StatusCode::from (parts.status_code)); @@ -347,13 +424,22 @@ async fn handle_http_request ( resp = resp.header (&k, v); } + debug! ("Unparked request {}", req_id); + resp.body (body) .unwrap () }, - _ => status_reply (StatusCode::GATEWAY_TIMEOUT, "Remote server timed out"), + Ok (Err (RelayError::RelayShuttingDown)) => { + error_reply (StatusCode::GATEWAY_TIMEOUT, "Relay shutting down") + }, + Err (_) => { + debug! ("Responder sender dropped for request {}", req_id); + error_reply (StatusCode::GATEWAY_TIMEOUT, "Remote server timed out") + }, } } +#[instrument (level = "trace", skip (req, state))] async fn handle_all (req: Request , state: Arc ) -> Result , Infallible> { @@ -371,13 +457,13 @@ async fn handle_all (req: Request , state: Arc ) handle_http_response (req, state, request_code).await } else { - status_reply (StatusCode::BAD_REQUEST, "Can't POST this\n") + error_reply (StatusCode::BAD_REQUEST, "Can't POST this") }); } Ok (if let Some (listen_code) = prefix_match (path, "/7ZSFUKGV/http_listen/") { let api_key = match api_key { - None => return Ok (status_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key")), + None => return Ok (error_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key")), Some (x) => x, }; handle_http_listen (state, listen_code.into (), api_key.as_bytes ()).await @@ -409,7 +495,7 @@ async fn handle_all (req: Request , state: Arc ) }; let s = state.handlebars.render ("relay_server_list", &page).unwrap (); - status_reply (StatusCode::OK, s) + ok_reply (s) } else if let Some (idx) = rest.find ('/') { let listen_code = String::from (&rest [0..idx]); @@ -419,14 +505,14 @@ async fn handle_all (req: Request , state: Arc ) handle_http_request (parts, path, state, listen_code).await } else { - status_reply (StatusCode::BAD_REQUEST, "Bad URI format") + error_reply (StatusCode::BAD_REQUEST, "Bad URI format") } } else if path == "/frontend/relay_up_check" { - status_reply (StatusCode::OK, "Relay is up\n") + error_reply (StatusCode::OK, "Relay is up") } else { - status_reply (StatusCode::OK, "Hi\n") + error_reply (StatusCode::OK, "Hi") }) } @@ -485,13 +571,29 @@ pub async fn run_relay ( server.with_graceful_shutdown (async { shutdown_oneshot.await.ok (); - - state.response_rendezvous.clear (); - - let mut request_rendezvoux = state.request_rendezvous.lock ().await; - request_rendezvoux.clear (); - info! ("Received graceful shutdown"); + + use RelayError::*; + + let mut response_rendezvous = state.response_rendezvous.write ().await; + let mut swapped = DashMap::default (); + + std::mem::swap (&mut swapped, &mut response_rendezvous); + + for (_, sender) in swapped.into_iter () { + sender.send (Err (RelayShuttingDown)).ok (); + } + + let mut request_rendezvous = state.request_rendezvous.lock ().await; + + for (_, x) in request_rendezvous.drain () { + use RequestRendezvous::*; + + match x { + ParkedClients (_) => (), + ParkedServer (sender) => drop (sender.send (Err (RelayShuttingDown))), + } + } }).await?; info! ("Exiting"); diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 1105993..11e4cdd 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -22,7 +22,7 @@ use tokio::{ }, }; use tracing::{ - debug, error, info, + debug, error, info, trace, warn, instrument, }; @@ -188,7 +188,7 @@ async fn serve_file ( } if tx.send (Ok::<_, Infallible> (buffer)).await.is_err () { - error! ("Send failed while streaming file ({} bytes sent)", bytes_sent); + warn! ("Cancelling file stream (Sent {} out of {} bytes)", bytes_sent, end - start); break; } @@ -199,7 +199,7 @@ async fn serve_file ( } bytes_sent += bytes_read; - debug! ("Sent {} bytes", bytes_sent); + trace! ("Sent {} bytes", bytes_sent); //delay_for (Duration::from_millis (50)).await; } @@ -242,7 +242,7 @@ async fn serve_error ( resp } -#[instrument (level = "debug", skip (handlebars))] +#[instrument (level = "debug", skip (handlebars, headers))] pub async fn serve_all ( handlebars: &Handlebars <'static>, root: &Path, diff --git a/src/server/mod.rs b/src/server/mod.rs index 59050ca..4c74381 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -16,7 +16,7 @@ use tokio::{ sync::oneshot, time::delay_for, }; -use tracing::{debug, error, info}; +use tracing::{debug, error, info, warn}; use crate::{ http_serde, @@ -45,12 +45,6 @@ async fn handle_req_resp <'a> ( ) { //println! ("Step 1"); - if req_resp.status () != StatusCode::OK { - // TODO: Error handling - error! ("http_listen didn't respond with 200 OK"); - return; - } - let body = req_resp.bytes ().await.unwrap (); let wrapped_reqs: Vec = match rmp_serde::from_read_ref (&body) { @@ -112,7 +106,14 @@ async fn handle_req_resp <'a> ( let text = r.text ().await.unwrap (); debug! ("{:?} {:?}", status, text); }, - Err (e) => error! ("Err: {:?}", e), + Err (e) => { + if e.is_request () { + warn! ("Error while POSTing response. Client probably hung up."); + } + else { + error! ("Err: {:?}", e); + } + }, } }); @@ -154,7 +155,7 @@ pub async fn run_server ( let client = Client::builder () .default_headers (headers) - .timeout (Duration::from_secs (30)) + .timeout (Duration::from_secs (40)) .build ().unwrap (); let handlebars = file_server::load_templates ()?; @@ -171,6 +172,8 @@ pub async fn run_server ( let mut shutdown_oneshot = shutdown_oneshot.fuse (); loop { + // TODO: Extract loop body to function? + if backoff_delay > 0 { let mut delay = delay_for (Duration::from_millis (backoff_delay)).fuse (); @@ -199,22 +202,44 @@ pub async fn run_server ( let req_resp = match req_req { Err (e) => { - error! ("Err: {:?}", e); - backoff_delay = err_backoff_delay; + if e.is_timeout () { + error! ("Client-side timeout. Is an overly-aggressive firewall closing long-lived connections? Is the network flakey?"); + if backoff_delay != 0 { + debug! ("backoff_delay = 0"); + backoff_delay = 0; + } + } + else { + error! ("Err: {:?}", e); + if backoff_delay != err_backoff_delay { + error! ("Non-timeout issue, increasing backoff_delay"); + backoff_delay = err_backoff_delay; + } + } continue; }, Ok (r) => { - backoff_delay = 0; + if backoff_delay != 0 { + debug! ("backoff_delay = 0"); + backoff_delay = 0; + } r }, }; - if req_resp.status () != StatusCode::OK { + if req_resp.status () == StatusCode::NO_CONTENT { + debug! ("http_listen long poll timed out on the server, good."); + continue; + } + else if req_resp.status () != StatusCode::OK { error! ("{}", req_resp.status ()); let body = req_resp.bytes ().await.unwrap (); let body = String::from_utf8 (body.to_vec ()).unwrap (); error! ("{}", body); - backoff_delay = err_backoff_delay; + if backoff_delay != err_backoff_delay { + error! ("Non-timeout issue, increasing backoff_delay"); + backoff_delay = err_backoff_delay; + } continue; } diff --git a/todo.md b/todo.md index 259d4f9..37fed23 100644 --- a/todo.md +++ b/todo.md @@ -1,3 +1,4 @@ +- Relay doesn't always shut down _if_ accessed by Firefox? - Not working behind Nginx - Try sending the http_response "OK" _after_ the request body is received From 32798e82509c5130bb5bfb1e5a237fce062eeff0 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 7 Nov 2020 00:30:56 +0000 Subject: [PATCH 013/208] :construction: Still hunting a bug where the relay can't shut down if Firefox is connected --- README.md | 3 +- src/bin/ptth_file_server.rs | 41 ++++++++++++++++--- src/bin/ptth_server.rs | 1 - src/relay/mod.rs | 79 +++++++++++++++++++++++-------------- 4 files changed, 86 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index cdf44b4..f08902f 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,6 @@ For now, either email me (if you know me personally) or make a pull request to a ## License PTTH is licensed under the -[GNU AGPLv3](https://www.gnu.org/licenses/agpl-3.0.html), -with an exception for my current employer. +[GNU AGPLv3](https://www.gnu.org/licenses/agpl-3.0.html) Copyright 2020 "Trish" diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index fb3fabf..5636d06 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -17,6 +17,10 @@ use hyper::{ }, StatusCode, }; +use serde::Deserialize; +use tracing::{ + debug, info, trace, warn, +}; use ptth::{ http_serde::RequestParts, @@ -24,8 +28,14 @@ use ptth::{ server::file_server, }; +#[derive (Default)] +pub struct Config { + pub file_server_root: Option , +} + struct ServerState <'a> { - handlebars: Arc >, + config: Config, + handlebars: handlebars::Handlebars <'a>, } fn status_reply > (status: StatusCode, b: B) @@ -41,8 +51,6 @@ async fn handle_all (req: Request , state: Arc >) //println! ("{}", path); if let Some (path) = prefix_match (path, "/files") { - let root = PathBuf::from ("./"); - let path = path.into (); let (parts, _) = req.into_parts (); @@ -52,7 +60,18 @@ async fn handle_all (req: Request , state: Arc >) _ => return Ok (status_reply (StatusCode::BAD_REQUEST, "Bad request")), }; - let ptth_resp = file_server::serve_all (&state.handlebars, &root, ptth_req.method, &ptth_req.uri, &ptth_req.headers).await; + let default_root = PathBuf::from ("./"); + let file_server_root: &std::path::Path = state.config.file_server_root + .as_ref () + .unwrap_or (&default_root); + + let ptth_resp = file_server::serve_all ( + &state.handlebars, + file_server_root, + ptth_req.method, + &ptth_req.uri, + &ptth_req.headers + ).await; let mut resp = Response::builder () .status (StatusCode::from (ptth_resp.parts.status_code)); @@ -77,14 +96,26 @@ async fn handle_all (req: Request , state: Arc >) } } +#[derive (Deserialize)] +pub struct ConfigFile { + pub file_server_root: Option , +} + #[tokio::main] async fn main () -> Result <(), Box > { + tracing_subscriber::fmt::init (); + let config_file: ConfigFile = ptth::load_toml::load ("config/ptth_server.toml"); + info! ("file_server_root: {:?}", config_file.file_server_root); + let addr = SocketAddr::from(([0, 0, 0, 0], 4000)); - let handlebars = Arc::new (file_server::load_templates ()?); + let handlebars = file_server::load_templates ()?; let state = Arc::new (ServerState { handlebars, + config: Config { + file_server_root: config_file.file_server_root, + }, }); let make_svc = make_service_fn (|_conn| { diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 790dc97..aad61c7 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -14,7 +14,6 @@ struct Opt { #[tokio::main] async fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); - let config_file = ptth::load_toml::load ("config/ptth_server.toml"); ptth::server::run_server ( diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 6b2c1cd..add443b 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -36,6 +36,7 @@ use tokio::{ mpsc, oneshot, RwLock, + watch, }, time::delay_for, }; @@ -133,26 +134,22 @@ pub struct RelayState { // Key: Request ID response_rendezvous: RwLock >, -} - -impl Default for RelayState { - fn default () -> Self { - Self { - config: Config::from (&ConfigFile::default ()), - handlebars: Arc::new (load_templates ().unwrap ()), - request_rendezvous: Default::default (), - response_rendezvous: Default::default (), - } - } + + shutdown_watch_tx: watch::Sender , + shutdown_watch_rx: watch::Receiver , } impl From <&ConfigFile> for RelayState { fn from (config_file: &ConfigFile) -> Self { + let (shutdown_watch_tx, shutdown_watch_rx) = watch::channel (false); + Self { config: Config::from (config_file), handlebars: Arc::new (load_templates ().unwrap ()), request_rendezvous: Default::default (), response_rendezvous: Default::default (), + shutdown_watch_tx, + shutdown_watch_rx, } } } @@ -270,27 +267,39 @@ async fn handle_http_response ( let (mut body_tx, body_rx) = mpsc::channel (2); let (body_finished_tx, body_finished_rx) = oneshot::channel (); + let mut shutdown_watch_rx = state.shutdown_watch_rx.clone (); spawn (async move { - loop { - let item = body.next ().await; - - if let Some (item) = item { - if let Ok (bytes) = &item { - trace! ("Relaying {} bytes", bytes.len ()); - } + if shutdown_watch_rx.recv ().await == Some (false) { + loop { + let item = body.next ().await; - if let Err (_e) = body_tx.send (item).await { - info! ("Body closed while relaying. (Client hung up?)"); - body_finished_tx.send (ClientDisconnected).unwrap (); + if let Some (item) = item { + if let Ok (bytes) = &item { + trace! ("Relaying {} bytes", bytes.len ()); + } + + futures::select! { + x = body_tx.send (item).fuse () => if let Err (_) = x { + info! ("Body closed while relaying. (Client hung up?)"); + body_finished_tx.send (ClientDisconnected).unwrap (); + break; + }, + _ = shutdown_watch_rx.recv ().fuse () => { + debug! ("Closing stream: relay is shutting down"); + break; + }, + } + } + else { + debug! ("Finished relaying bytes"); + body_finished_tx.send (StreamFinished).unwrap (); break; } } - else { - debug! ("Finished relaying bytes"); - body_finished_tx.send (StreamFinished).unwrap (); - break; - } + } + else { + debug! ("Can't relay bytes, relay is shutting down"); } }); @@ -315,13 +324,17 @@ async fn handle_http_response ( } debug! ("Connected server to client for streaming."); - match body_finished_rx.await.unwrap () { - StreamFinished => { + match body_finished_rx.await { + Ok (StreamFinished) => { error_reply (StatusCode::OK, "StreamFinished") }, - ClientDisconnected => { + Ok (ClientDisconnected) => { error_reply (StatusCode::OK, "ClientDisconnected") - } + }, + Err (e) => { + debug! ("body_finished_rx {}", e); + error_reply (StatusCode::OK, "body_finished_rx Err") + }, } } @@ -569,10 +582,14 @@ pub async fn run_relay ( let server = Server::bind (&addr) .serve (make_svc); + + server.with_graceful_shutdown (async { shutdown_oneshot.await.ok (); info! ("Received graceful shutdown"); + state.shutdown_watch_tx.broadcast (true).unwrap (); + use RelayError::*; let mut response_rendezvous = state.response_rendezvous.write ().await; @@ -594,6 +611,8 @@ pub async fn run_relay ( ParkedServer (sender) => drop (sender.send (Err (RelayShuttingDown))), } } + + info! ("Performed all cleanup"); }).await?; info! ("Exiting"); From 75177cec804ff71bfa81bab28501f66c8ada9781 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 7 Nov 2020 01:34:58 +0000 Subject: [PATCH 014/208] :construction: Guess it's a bug in hyper. You can't gracefully shutdown while a client is connected. --- src/bin/ptth_file_server.rs | 59 ++++++++++++++++++++++++++++++++----- src/server/file_server.rs | 31 +++++++++++++++---- src/server/mod.rs | 3 +- 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index 5636d06..aec5855 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -1,11 +1,13 @@ use std::{ convert::Infallible, error::Error, + net::SocketAddr, path::PathBuf, sync::Arc, - net::SocketAddr, + time::Duration, }; +use futures::FutureExt; use hyper::{ Body, Request, @@ -18,8 +20,15 @@ use hyper::{ StatusCode, }; use serde::Deserialize; +use tokio::{ + sync::{ + oneshot, + watch, + }, + time::delay_for, +}; use tracing::{ - debug, info, trace, warn, + debug, error, info, trace, warn, }; use ptth::{ @@ -36,6 +45,8 @@ pub struct Config { struct ServerState <'a> { config: Config, handlebars: handlebars::Handlebars <'a>, + + shutdown_watch_rx: watch::Receiver , } fn status_reply > (status: StatusCode, b: B) @@ -45,7 +56,7 @@ fn status_reply > (status: StatusCode, b: B) } async fn handle_all (req: Request , state: Arc >) --> Result , Infallible> +-> Result , String> { let path = req.uri ().path (); //println! ("{}", path); @@ -65,12 +76,19 @@ async fn handle_all (req: Request , state: Arc >) .as_ref () .unwrap_or (&default_root); + let mut shutdown_watch_rx = state.shutdown_watch_rx.clone (); + if shutdown_watch_rx.recv ().await != Some (false) { + error! ("Can't serve, I'm shutting down"); + panic! ("Can't serve, I'm shutting down"); + } + let ptth_resp = file_server::serve_all ( &state.handlebars, file_server_root, ptth_req.method, &ptth_req.uri, - &ptth_req.headers + &ptth_req.headers, + Some (shutdown_watch_rx) ).await; let mut resp = Response::builder () @@ -111,18 +129,22 @@ async fn main () -> Result <(), Box > { let handlebars = file_server::load_templates ()?; + let (shutdown_watch_tx, shutdown_watch_rx) = watch::channel (false); + let state = Arc::new (ServerState { - handlebars, config: Config { file_server_root: config_file.file_server_root, }, + handlebars, + + shutdown_watch_rx, }); let make_svc = make_service_fn (|_conn| { let state = state.clone (); async { - Ok::<_, Infallible> (service_fn (move |req| { + Ok::<_, String> (service_fn (move |req| { let state = state.clone (); handle_all (req, state) @@ -130,9 +152,30 @@ async fn main () -> Result <(), Box > { } }); - let server = Server::bind (&addr).serve (make_svc); + let shutdown_oneshot = ptth::graceful_shutdown::init (); + let (force_shutdown_tx, force_shutdown_rx) = oneshot::channel (); - server.await?; + let server = Server::bind (&addr) + .serve (make_svc) + .with_graceful_shutdown (async move { + shutdown_oneshot.await.ok (); + info! ("Received graceful shutdown"); + + shutdown_watch_tx.broadcast (true).unwrap (); + force_shutdown_tx.send (()).unwrap (); + }); + + let force_shutdown_fut = async move { + force_shutdown_rx.await.unwrap (); + delay_for (Duration::from_secs (5)).await; + + error! ("Forcing shutdown"); + }; + + futures::select! { + x = server.fuse () => x?, + _ = force_shutdown_fut.fuse () => (), + }; Ok (()) } diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 11e4cdd..4155ebe 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -9,6 +9,7 @@ use std::{ path::{Path, PathBuf}, }; +use futures::FutureExt; use handlebars::Handlebars; use tokio::{ fs::{ @@ -20,6 +21,7 @@ use tokio::{ sync::mpsc::{ channel, }, + sync::watch, }; use tracing::{ debug, error, info, trace, warn, @@ -139,14 +141,15 @@ async fn serve_dir ( resp } -#[instrument (level = "debug", skip (f))] +#[instrument (level = "debug", skip (f, cancel_rx))] async fn serve_file ( mut f: File, should_send_body: bool, range_start: Option , - range_end: Option + range_end: Option , + mut cancel_rx: Option > ) -> http_serde::Response { - let (tx, rx) = channel (2); + let (tx, rx) = channel (1); let body = if should_send_body { Some (rx) } @@ -169,6 +172,7 @@ async fn serve_file ( if should_send_body { tokio::spawn (async move { + { //println! ("Opening file {:?}", path); let mut tx = tx; @@ -187,7 +191,20 @@ async fn serve_file ( break; } - if tx.send (Ok::<_, Infallible> (buffer)).await.is_err () { + let send_fut = tx.send (Ok::<_, Infallible> (buffer)); + + let send_result = match &mut cancel_rx { + Some (cancel_rx) => futures::select! { + x = send_fut.fuse () => x, + _ = cancel_rx.recv ().fuse () => { + error! ("Cancelled"); + break; + }, + }, + None => send_fut.await, + }; + + if send_result.is_err () { warn! ("Cancelling file stream (Sent {} out of {} bytes)", bytes_sent, end - start); break; } @@ -203,6 +220,8 @@ async fn serve_file ( //delay_for (Duration::from_millis (50)).await; } + } + debug! ("Exited stream scope"); }); } @@ -249,6 +268,7 @@ pub async fn serve_all ( method: http_serde::Method, uri: &str, headers: &HashMap >, + cancel_rx: Option > ) -> http_serde::Response { @@ -297,7 +317,8 @@ pub async fn serve_all ( file, should_send_body, range_start, - range_end + range_end, + cancel_rx ).await } else { diff --git a/src/server/mod.rs b/src/server/mod.rs index 4c74381..86a1bc3 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -76,7 +76,8 @@ async fn handle_req_resp <'a> ( file_server_root, parts.method, uri, - &parts.headers + &parts.headers, + None ).await } else { From f02e12aeccdff621929d93c6d800f8872c803ebb Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 7 Nov 2020 01:40:29 +0000 Subject: [PATCH 015/208] :bug: Add forced shutdown to ptth_file_server. --- src/bin/ptth_file_server.rs | 21 +++++---------------- src/server/file_server.rs | 29 +++++------------------------ src/server/mod.rs | 3 +-- 3 files changed, 11 insertions(+), 42 deletions(-) diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index aec5855..c36bf84 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -45,8 +45,6 @@ pub struct Config { struct ServerState <'a> { config: Config, handlebars: handlebars::Handlebars <'a>, - - shutdown_watch_rx: watch::Receiver , } fn status_reply > (status: StatusCode, b: B) @@ -76,19 +74,12 @@ async fn handle_all (req: Request , state: Arc >) .as_ref () .unwrap_or (&default_root); - let mut shutdown_watch_rx = state.shutdown_watch_rx.clone (); - if shutdown_watch_rx.recv ().await != Some (false) { - error! ("Can't serve, I'm shutting down"); - panic! ("Can't serve, I'm shutting down"); - } - let ptth_resp = file_server::serve_all ( &state.handlebars, file_server_root, ptth_req.method, &ptth_req.uri, - &ptth_req.headers, - Some (shutdown_watch_rx) + &ptth_req.headers ).await; let mut resp = Response::builder () @@ -129,15 +120,11 @@ async fn main () -> Result <(), Box > { let handlebars = file_server::load_templates ()?; - let (shutdown_watch_tx, shutdown_watch_rx) = watch::channel (false); - let state = Arc::new (ServerState { config: Config { file_server_root: config_file.file_server_root, }, handlebars, - - shutdown_watch_rx, }); let make_svc = make_service_fn (|_conn| { @@ -161,13 +148,15 @@ async fn main () -> Result <(), Box > { shutdown_oneshot.await.ok (); info! ("Received graceful shutdown"); - shutdown_watch_tx.broadcast (true).unwrap (); + // Kick off a timer for a forced shutdown force_shutdown_tx.send (()).unwrap (); }); let force_shutdown_fut = async move { force_shutdown_rx.await.unwrap (); - delay_for (Duration::from_secs (5)).await; + let timeout = 10; + info! ("Forced shutdown in {} seconds", timeout); + delay_for (Duration::from_secs (timeout)).await; error! ("Forcing shutdown"); }; diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 4155ebe..fd7e922 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -141,13 +141,12 @@ async fn serve_dir ( resp } -#[instrument (level = "debug", skip (f, cancel_rx))] +#[instrument (level = "debug", skip (f))] async fn serve_file ( mut f: File, should_send_body: bool, range_start: Option , - range_end: Option , - mut cancel_rx: Option > + range_end: Option ) -> http_serde::Response { let (tx, rx) = channel (1); let body = if should_send_body { @@ -172,7 +171,6 @@ async fn serve_file ( if should_send_body { tokio::spawn (async move { - { //println! ("Opening file {:?}", path); let mut tx = tx; @@ -191,20 +189,7 @@ async fn serve_file ( break; } - let send_fut = tx.send (Ok::<_, Infallible> (buffer)); - - let send_result = match &mut cancel_rx { - Some (cancel_rx) => futures::select! { - x = send_fut.fuse () => x, - _ = cancel_rx.recv ().fuse () => { - error! ("Cancelled"); - break; - }, - }, - None => send_fut.await, - }; - - if send_result.is_err () { + if tx.send (Ok::<_, Infallible> (buffer)).await.is_err () { warn! ("Cancelling file stream (Sent {} out of {} bytes)", bytes_sent, end - start); break; } @@ -220,8 +205,6 @@ async fn serve_file ( //delay_for (Duration::from_millis (50)).await; } - } - debug! ("Exited stream scope"); }); } @@ -267,8 +250,7 @@ pub async fn serve_all ( root: &Path, method: http_serde::Method, uri: &str, - headers: &HashMap >, - cancel_rx: Option > + headers: &HashMap > ) -> http_serde::Response { @@ -317,8 +299,7 @@ pub async fn serve_all ( file, should_send_body, range_start, - range_end, - cancel_rx + range_end ).await } else { diff --git a/src/server/mod.rs b/src/server/mod.rs index 86a1bc3..4c74381 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -76,8 +76,7 @@ async fn handle_req_resp <'a> ( file_server_root, parts.method, uri, - &parts.headers, - None + &parts.headers ).await } else { From 9e134d55aa99697cfe656ac7276da87c249c74e3 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 7 Nov 2020 02:26:34 +0000 Subject: [PATCH 016/208] :tada: Add forced shutdown fallback to graceful_shutdown module --- src/bin/ptth_file_server.rs | 23 ++-------- src/graceful_shutdown.rs | 85 +++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 23 deletions(-) diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index c36bf84..7834934 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -139,32 +139,15 @@ async fn main () -> Result <(), Box > { } }); - let shutdown_oneshot = ptth::graceful_shutdown::init (); - let (force_shutdown_tx, force_shutdown_rx) = oneshot::channel (); + let (shutdown_rx, forced_shutdown) = ptth::graceful_shutdown::init_with_force (); let server = Server::bind (&addr) .serve (make_svc) .with_graceful_shutdown (async move { - shutdown_oneshot.await.ok (); - info! ("Received graceful shutdown"); - - // Kick off a timer for a forced shutdown - force_shutdown_tx.send (()).unwrap (); + shutdown_rx.await.ok (); }); - let force_shutdown_fut = async move { - force_shutdown_rx.await.unwrap (); - let timeout = 10; - info! ("Forced shutdown in {} seconds", timeout); - delay_for (Duration::from_secs (timeout)).await; - - error! ("Forcing shutdown"); - }; - - futures::select! { - x = server.fuse () => x?, - _ = force_shutdown_fut.fuse () => (), - }; + forced_shutdown.wrap_server (server).await??; Ok (()) } diff --git a/src/graceful_shutdown.rs b/src/graceful_shutdown.rs index 63de1b3..15cb518 100644 --- a/src/graceful_shutdown.rs +++ b/src/graceful_shutdown.rs @@ -1,5 +1,14 @@ -use std::cell::Cell; -use tokio::sync::oneshot; +use std::{ + cell::Cell, + time::Duration, +}; + +use futures::prelude::*; +use tokio::{ + sync::oneshot, + time::delay_for, +}; +use tracing::{debug, error, info, trace, warn}; pub fn init () -> oneshot::Receiver <()> { let (tx, rx) = oneshot::channel::<()> (); @@ -11,7 +20,7 @@ pub fn init () -> oneshot::Receiver <()> { let tx = Some (tx); let tx = Cell::new (tx); - ctrlc::set_handler (move ||{ + ctrlc::set_handler (move || { let tx = tx.replace (None); if let Some (tx) = tx { @@ -21,3 +30,73 @@ pub fn init () -> oneshot::Receiver <()> { rx } + +#[derive (Debug)] +pub enum ShutdownError { + ForcedShutdown, +} + +use std::{ + error, + fmt, +}; + +impl fmt::Display for ShutdownError { + fn fmt (&self, f: &mut fmt::Formatter <'_>) -> fmt::Result { + use ShutdownError::*; + + let desc = match self { + ForcedShutdown => "Shutdown was forced after a timeout", + }; + + write! (f, "{}", desc) + } +} + +impl error::Error for ShutdownError { + +} + +pub struct ForcedShutdown { + rx: oneshot::Receiver <()>, + tx: oneshot::Sender <()>, +} + +impl ForcedShutdown { + pub async fn wrap_server < + T, + F: Future + > ( + self, + server: F + ) -> Result { + let fut = async move { + self.rx.await.unwrap (); + self.tx.send (()).unwrap (); + let timeout = 10; + debug! ("Starting graceful shutdown. Forcing shutdown in {} seconds", timeout); + delay_for (Duration::from_secs (timeout)).await; + + error! ("Forcing shutdown"); + }; + + futures::select! { + x = server.fuse () => { + info! ("Shut down gracefully"); + Ok (x) + }, + _ = fut.fuse () => Err (ShutdownError::ForcedShutdown), + } + } +} + +pub fn init_with_force () -> (oneshot::Receiver <()>, ForcedShutdown) { + let (tx, rx) = oneshot::channel (); + + let f = ForcedShutdown { + rx: init (), + tx, + }; + + (rx, f) +} From e0b8c8cb580d2cbab8d479e91897c1118a9378cb Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 7 Nov 2020 02:29:45 +0000 Subject: [PATCH 017/208] Add forced shutdown to ptth_relay. --- src/bin/ptth_relay.rs | 14 ++++++++++---- src/relay/mod.rs | 4 +--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index 9c6912b..77bbb56 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -25,8 +25,14 @@ async fn main () -> Result <(), Box > { info! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); - relay::run_relay ( - Arc::new (RelayState::from (&config_file)), - ptth::graceful_shutdown::init () - ).await + let (shutdown_rx, forced_shutdown) = ptth::graceful_shutdown::init_with_force (); + + forced_shutdown.wrap_server ( + relay::run_relay ( + Arc::new (RelayState::from (&config_file)), + shutdown_rx + ) + ).await??; + + Ok (()) } diff --git a/src/relay/mod.rs b/src/relay/mod.rs index add443b..10c642d 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -586,7 +586,6 @@ pub async fn run_relay ( server.with_graceful_shutdown (async { shutdown_oneshot.await.ok (); - info! ("Received graceful shutdown"); state.shutdown_watch_tx.broadcast (true).unwrap (); @@ -612,10 +611,9 @@ pub async fn run_relay ( } } - info! ("Performed all cleanup"); + debug! ("Performed all cleanup"); }).await?; - info! ("Exiting"); Ok (()) } From 151f236a0b8de510b1896deeaa9c571f845040af Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 7 Nov 2020 23:10:01 +0000 Subject: [PATCH 018/208] :lipstick: Tweak a few things and update todo --- src/graceful_shutdown.rs | 2 +- src/server/file_server.rs | 2 +- todo.md | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/graceful_shutdown.rs b/src/graceful_shutdown.rs index 15cb518..c1917e6 100644 --- a/src/graceful_shutdown.rs +++ b/src/graceful_shutdown.rs @@ -73,7 +73,7 @@ impl ForcedShutdown { let fut = async move { self.rx.await.unwrap (); self.tx.send (()).unwrap (); - let timeout = 10; + let timeout = 5; debug! ("Starting graceful shutdown. Forcing shutdown in {} seconds", timeout); delay_for (Duration::from_secs (timeout)).await; diff --git a/src/server/file_server.rs b/src/server/file_server.rs index fd7e922..bc04821 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -196,7 +196,7 @@ async fn serve_file ( bytes_left -= bytes_read; if bytes_left == 0 { - info! ("Finished"); + debug! ("Finished"); break; } diff --git a/todo.md b/todo.md index 37fed23..ed1393c 100644 --- a/todo.md +++ b/todo.md @@ -1,4 +1,3 @@ -- Relay doesn't always shut down _if_ accessed by Firefox? - Not working behind Nginx - Try sending the http_response "OK" _after_ the request body is received @@ -22,3 +21,10 @@ Off-project stuff: - Benchmark directory entry sorting + +Known issues: + +Relay can't shut down gracefully if Firefox is connected to it, e.g. if Firefox +kept a connection open while watching a video. +I'm pretty sure this is a bug in Hyper, so for now I've worked around it with a +forced shutdown timer. From 27b877e80e53f26e2ed8cc7be7d606d2f6918372 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 7 Nov 2020 23:10:39 +0000 Subject: [PATCH 019/208] :whale: Update cached version in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a53934e..4056e35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update \ # Make sure the dependencies are all cached so we won't hammer crates.io ADD old-git.tar.gz . -RUN git checkout 690f07dab67111a75fe190f014c8c23ef1753598 \ +RUN git checkout 151f236a0b8de510b1896deeaa9c571f845040af \ && git reset --hard \ && cargo check From 70522f851a1a65847b07dfd4d31079d75c56aa63 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 02:36:27 +0000 Subject: [PATCH 020/208] Add module diagram --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index f08902f..ca0e78d 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,35 @@ WireGuard can also pierce firewalls, but it requires root permissions, and the client must be a WireGuard node. PTTH allows any normal HTTP client such as curl, Firefox, etc. +## Module overview + +``` ++------------+ +-------------+ +------------------+ +| ptth_relay | | ptth_server | | ptth_file_server | ++------------+ +-------------+ +------------------+ + | | | | + \ / \ / + V V V V + +------------+ +-------------+ + | http_serde | | file_server | + +------------+ +-------------+ +``` + +The top-level binaries are ptth_relay, ptth_server, and ptth_file_server. + +ptth_relay should run on a well-known public server, behind an HTTPS proxy +such as Caddy, Nginx, or Apache. + +ptth_file_server is a standalone HTTP file server, similar to Python2's +`-m SimpleHTTPServer` command, or Python3's `-m http.server`. + +ptth_server and ptth_file_server use the `file_server` module. ptth_server +will connect out to a ptth_relay instance and serve files through the reverse +HTTP tunnel. + +The http_serde module is shared by ptth_relay and ptth_server so that they +can communicate with each other easily. + ## Why are GitHub issues disabled? Because they are not part of Git. From 8c7f4684b4a121ff17e7a93ef79c0620f7f66bab Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 02:37:11 +0000 Subject: [PATCH 021/208] Log request paths --- src/relay/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 10c642d..ef8aee9 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -459,6 +459,8 @@ async fn handle_all (req: Request , state: Arc ) let path = req.uri ().path (); //println! ("{}", path); + debug! ("Request path: {}", path); + let api_key = req.headers ().get ("X-ApiKey"); if req.method () == Method::POST { From 5fa0bd8584e5be19c18b89a73bf2816e447eb0c7 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 02:38:27 +0000 Subject: [PATCH 022/208] :recycle: Remove unused `use` statements --- src/bin/ptth_file_server.rs | 10 ---------- src/server/file_server.rs | 2 -- 2 files changed, 12 deletions(-) diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index 7834934..d765e9e 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -1,13 +1,10 @@ use std::{ - convert::Infallible, error::Error, net::SocketAddr, path::PathBuf, sync::Arc, - time::Duration, }; -use futures::FutureExt; use hyper::{ Body, Request, @@ -20,13 +17,6 @@ use hyper::{ StatusCode, }; use serde::Deserialize; -use tokio::{ - sync::{ - oneshot, - watch, - }, - time::delay_for, -}; use tracing::{ debug, error, info, trace, warn, }; diff --git a/src/server/file_server.rs b/src/server/file_server.rs index bc04821..76511ed 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -9,7 +9,6 @@ use std::{ path::{Path, PathBuf}, }; -use futures::FutureExt; use handlebars::Handlebars; use tokio::{ fs::{ @@ -21,7 +20,6 @@ use tokio::{ sync::mpsc::{ channel, }, - sync::watch, }; use tracing::{ debug, error, info, trace, warn, From f42068db899f3fc4669f81e84703c018d9335265 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 03:16:13 +0000 Subject: [PATCH 023/208] :bug: Fix a bug in the backoff and update todo. Video streaming and seeking is working pretty well now behind Caddy, but I'm still seeing some lag when Firefox first starts a request. --- build.bash | 3 +++ src/server/mod.rs | 24 +++++++++++++----------- todo.md | 5 ++--- 3 files changed, 18 insertions(+), 14 deletions(-) create mode 100755 build.bash diff --git a/build.bash b/build.bash new file mode 100755 index 0000000..68d5058 --- /dev/null +++ b/build.bash @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +sudo docker build -t ptth . diff --git a/src/server/mod.rs b/src/server/mod.rs index 4c74381..0d23170 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -42,7 +42,7 @@ fn status_reply (c: http_serde::StatusCode, body: &str) -> http_serde::Response async fn handle_req_resp <'a> ( state: &Arc , req_resp: reqwest::Response -) { +) -> Result <(), ()> { //println! ("Step 1"); let body = req_resp.bytes ().await.unwrap (); @@ -51,7 +51,7 @@ async fn handle_req_resp <'a> ( Ok (x) => x, Err (e) => { error! ("Can't parse wrapped requests: {:?}", e); - return; + return Err (()); }, }; @@ -118,6 +118,8 @@ async fn handle_req_resp <'a> ( }); } + + Ok (()) } #[derive (Default, Deserialize)] @@ -204,10 +206,6 @@ pub async fn run_server ( Err (e) => { if e.is_timeout () { error! ("Client-side timeout. Is an overly-aggressive firewall closing long-lived connections? Is the network flakey?"); - if backoff_delay != 0 { - debug! ("backoff_delay = 0"); - backoff_delay = 0; - } } else { error! ("Err: {:?}", e); @@ -219,10 +217,6 @@ pub async fn run_server ( continue; }, Ok (r) => { - if backoff_delay != 0 { - debug! ("backoff_delay = 0"); - backoff_delay = 0; - } r }, }; @@ -246,7 +240,15 @@ pub async fn run_server ( // Unpack the requests, spawn them into new tasks, then loop back // around. - handle_req_resp (&state, req_resp).await; + if handle_req_resp (&state, req_resp).await.is_err () { + backoff_delay = err_backoff_delay; + continue; + } + + if backoff_delay != 0 { + debug! ("backoff_delay = 0"); + backoff_delay = 0; + } } info! ("Exiting"); diff --git a/todo.md b/todo.md index ed1393c..e91173c 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,6 @@ -- Not working behind Nginx -- Try sending the http_response "OK" _after_ the request body is received +- Not working behind Nginx (Works okay behind Caddy) +- Still getting the slow request turtle in FF - 500-900 ms wait time -- Sort directory listings - ".." from server to server list is broken - Redirect to add trailing slashes - Add file size in directory listing From 345fa64ad04587e290c7343f8159512a589c3b5b Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 15:01:15 +0000 Subject: [PATCH 024/208] Hide ptth_server.toml with 403 Forbidden --- src/bin/ptth_file_server.rs | 3 ++- src/bin/ptth_server.rs | 6 ++++-- src/lib.rs | 2 +- src/server/file_server.rs | 11 ++++++++++- src/server/mod.rs | 8 ++++++-- todo.md | 10 +++++++--- 6 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index d765e9e..18b1865 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -69,7 +69,8 @@ async fn handle_all (req: Request , state: Arc >) file_server_root, ptth_req.method, &ptth_req.uri, - &ptth_req.headers + &ptth_req.headers, + None ).await; let mut resp = Response::builder () diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index aad61c7..d335867 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -14,10 +14,12 @@ struct Opt { #[tokio::main] async fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); - let config_file = ptth::load_toml::load ("config/ptth_server.toml"); + let path = PathBuf::from ("./config/ptth_server.toml"); + let config_file = ptth::load_toml::load (&path); ptth::server::run_server ( config_file, - ptth::graceful_shutdown::init () + ptth::graceful_shutdown::init (), + Some (path) ).await } diff --git a/src/lib.rs b/src/lib.rs index 16d239c..12fc7b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -132,7 +132,7 @@ mod tests { let (stop_server_tx, stop_server_rx) = oneshot::channel (); let task_server = { spawn (async move { - server::run_server (config_file, stop_server_rx).await.unwrap (); + server::run_server (config_file, stop_server_rx, None).await.unwrap (); }) }; diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 76511ed..e25b7e3 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -248,7 +248,8 @@ pub async fn serve_all ( root: &Path, method: http_serde::Method, uri: &str, - headers: &HashMap > + headers: &HashMap >, + hidden_path: Option <&Path> ) -> http_serde::Response { @@ -285,6 +286,14 @@ pub async fn serve_all ( let mut full_path = PathBuf::from (root); full_path.push (path); + debug! ("full_path = {:?}", full_path); + + if let Some (hidden_path) = hidden_path { + if full_path == hidden_path { + return serve_error (http_serde::StatusCode::Forbidden, "403 Forbidden".into ()).await; + } + } + if let Ok (dir) = read_dir (&full_path).await { serve_dir ( handlebars, diff --git a/src/server/mod.rs b/src/server/mod.rs index 0d23170..aa3e041 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -29,6 +29,7 @@ struct ServerState { config: Config, handlebars: Handlebars <'static>, client: Client, + hidden_path: Option , } fn status_reply (c: http_serde::StatusCode, body: &str) -> http_serde::Response @@ -76,7 +77,8 @@ async fn handle_req_resp <'a> ( file_server_root, parts.method, uri, - &parts.headers + &parts.headers, + state.hidden_path.as_ref ().map (|p| p.as_path ()) ).await } else { @@ -138,7 +140,8 @@ pub struct Config { pub async fn run_server ( config_file: ConfigFile, - shutdown_oneshot: oneshot::Receiver <()> + shutdown_oneshot: oneshot::Receiver <()>, + hidden_path: Option ) -> Result <(), Box > { @@ -168,6 +171,7 @@ pub async fn run_server ( }, handlebars, client, + hidden_path, }); let mut backoff_delay = 0; diff --git a/todo.md b/todo.md index e91173c..b62398a 100644 --- a/todo.md +++ b/todo.md @@ -1,16 +1,16 @@ - Not working behind Nginx (Works okay behind Caddy) -- Still getting the slow request turtle in FF - 500-900 ms wait time +- Reduce idle memory use? +- Folder icons in dir list - ".." from server to server list is broken - Redirect to add trailing slashes - Add file size in directory listing - Allow spaces in server names -- Make file_server_root mandatory - Deny unused HTTP methods for endpoints -- Hide ptth_server.toml from file server - ETag cache based on mtime - Server-side hash? - Log / audit log? +- Add "Last check-in time" to server list - Prevent directory traversal attacks in file_server.rs - Error handling @@ -27,3 +27,7 @@ Relay can't shut down gracefully if Firefox is connected to it, e.g. if Firefox kept a connection open while watching a video. I'm pretty sure this is a bug in Hyper, so for now I've worked around it with a forced shutdown timer. + +Sometimes I get the turtle icon in Firefox's network debugger. But this happens +even with Caddy running a static file server, so I can't prove that it's on my +side. The VPS is cheap, and the datacenter is far away. From c5691d9d05e581b454f495314d086f5512d1f0d3 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 15:53:09 +0000 Subject: [PATCH 025/208] :bug: Fix the backlinks from servers up to the relay --- .gitignore | 1 + ptth_handlebars/file_server_dir.html | 11 +- ptth_handlebars/file_server_root.html | 38 +++++ ptth_handlebars/relay_root.html | 33 +++++ src/bin/ptth_file_server.rs | 89 ++++++------ src/http_serde.rs | 6 + src/relay/mod.rs | 5 + src/server/file_server.rs | 193 ++++++++++++++++++-------- src/server/mod.rs | 32 ++--- todo.md | 3 +- 10 files changed, 283 insertions(+), 128 deletions(-) create mode 100644 ptth_handlebars/file_server_root.html create mode 100644 ptth_handlebars/relay_root.html diff --git a/.gitignore b/.gitignore index 690f90c..d4996e0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /ptth_server.toml /ptth_relay.toml /target +/test diff --git a/ptth_handlebars/file_server_dir.html b/ptth_handlebars/file_server_dir.html index 675d58f..552fb24 100644 --- a/ptth_handlebars/file_server_dir.html +++ b/ptth_handlebars/file_server_dir.html @@ -10,25 +10,30 @@ display: inline-block; padding: 10px; min-width: 50%; + text-decoration: none; } .entry_list div:nth-child(odd) { background-color: #ddd; } -{{path}} +{{path}} {{server_name}} +

{{server_name}}

+ +

{{path}}

+
{{#each entries}} {{/each}} diff --git a/ptth_handlebars/file_server_root.html b/ptth_handlebars/file_server_root.html new file mode 100644 index 0000000..00a88be --- /dev/null +++ b/ptth_handlebars/file_server_root.html @@ -0,0 +1,38 @@ + + + + + +{{server_name}} + + + +
+ + + + + +
+ + + diff --git a/ptth_handlebars/relay_root.html b/ptth_handlebars/relay_root.html new file mode 100644 index 0000000..79c7d30 --- /dev/null +++ b/ptth_handlebars/relay_root.html @@ -0,0 +1,33 @@ + + + + + +PTTH relay + + + +

PTTH relay

+ +
+ + + +
+ + + diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index 18b1865..2f638bc 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -23,7 +23,6 @@ use tracing::{ use ptth::{ http_serde::RequestParts, - prefix_match, server::file_server, }; @@ -46,54 +45,50 @@ fn status_reply > (status: StatusCode, b: B) async fn handle_all (req: Request , state: Arc >) -> Result , String> { - let path = req.uri ().path (); - //println! ("{}", path); + debug! ("req.uri () = {:?}", req.uri ()); - if let Some (path) = prefix_match (path, "/files") { - let path = path.into (); - - let (parts, _) = req.into_parts (); - - let ptth_req = match RequestParts::from_hyper (parts.method, path, parts.headers) { - Ok (x) => x, - _ => return Ok (status_reply (StatusCode::BAD_REQUEST, "Bad request")), - }; - - let default_root = PathBuf::from ("./"); - let file_server_root: &std::path::Path = state.config.file_server_root - .as_ref () - .unwrap_or (&default_root); - - let ptth_resp = file_server::serve_all ( - &state.handlebars, - file_server_root, - ptth_req.method, - &ptth_req.uri, - &ptth_req.headers, - None - ).await; - - let mut resp = Response::builder () - .status (StatusCode::from (ptth_resp.parts.status_code)); - - use std::str::FromStr; - - for (k, v) in ptth_resp.parts.headers.into_iter () { - resp = resp.header (hyper::header::HeaderName::from_str (&k).unwrap (), v); - } - - let body = ptth_resp.body - .map (Body::wrap_stream) - .unwrap_or_else (Body::empty) - ; - - let resp = resp.body (body).unwrap (); - - Ok (resp) - } - else { - Ok (status_reply (StatusCode::NOT_FOUND, "404 Not Found\n")) + let path = req.uri ().path (); + + let path = path.into (); + + let (parts, _) = req.into_parts (); + + let ptth_req = match RequestParts::from_hyper (parts.method, path, parts.headers) { + Ok (x) => x, + _ => return Ok (status_reply (StatusCode::BAD_REQUEST, "Bad request")), + }; + + let default_root = PathBuf::from ("./"); + let file_server_root: &std::path::Path = state.config.file_server_root + .as_ref () + .unwrap_or (&default_root); + + let ptth_resp = file_server::serve_all ( + &state.handlebars, + file_server_root, + ptth_req.method, + &ptth_req.uri, + &ptth_req.headers, + None + ).await; + + let mut resp = Response::builder () + .status (StatusCode::from (ptth_resp.parts.status_code)); + + use std::str::FromStr; + + for (k, v) in ptth_resp.parts.headers.into_iter () { + resp = resp.header (hyper::header::HeaderName::from_str (&k).unwrap (), v); } + + let body = ptth_resp.body + .map (Body::wrap_stream) + .unwrap_or_else (Body::empty) + ; + + let resp = resp.body (body).unwrap (); + + Ok (resp) } #[derive (Deserialize)] diff --git a/src/http_serde.rs b/src/http_serde.rs index 2ef2fc5..056ceb4 100644 --- a/src/http_serde.rs +++ b/src/http_serde.rs @@ -158,11 +158,17 @@ impl Response { } pub fn body_bytes (&mut self, b: Vec ) -> &mut Self { + use std::convert::TryInto; + + self.content_length = b.len ().try_into ().ok (); + self.header ("content-length".to_string (), b.len ().to_string ().into_bytes ()); + let (mut tx, rx) = tokio::sync::mpsc::channel (1); tokio::spawn (async move { tx.send (Ok (b)).await.unwrap (); }); self.body = Some (rx); + self } } diff --git a/src/relay/mod.rs b/src/relay/mod.rs index ef8aee9..6d017fb 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -523,6 +523,10 @@ async fn handle_all (req: Request , state: Arc ) error_reply (StatusCode::BAD_REQUEST, "Bad URI format") } } + else if path == "/" { + let s = state.handlebars.render ("relay_root", &()).unwrap (); + ok_reply (s) + } else if path == "/frontend/relay_up_check" { error_reply (StatusCode::OK, "Relay is up") } @@ -539,6 +543,7 @@ pub fn load_templates () for (k, v) in vec! [ ("relay_server_list", "relay_server_list.html"), + ("relay_root", "relay_root.html"), ].into_iter () { handlebars.register_template_file (k, format! ("ptth_handlebars/{}", v))?; } diff --git a/src/server/file_server.rs b/src/server/file_server.rs index e25b7e3..35d9ac2 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -10,8 +10,10 @@ use std::{ }; use handlebars::Handlebars; +use serde::Serialize; use tokio::{ fs::{ + DirEntry, File, read_dir, ReadDir, @@ -28,7 +30,43 @@ use tracing::{ use regex::Regex; -use crate::http_serde; +use crate::{ + http_serde, + prefix_match, +}; + +#[derive (Serialize)] +struct ServerInfo <'a> { + server_name: &'a str, +} + +#[derive (Serialize)] +struct TemplateDirEntry { + icon: &'static str, + trailing_slash: &'static str, + + // Unfortunately file_name will allocate as long as some platforms + // (Windows!) aren't UTF-8. Cause I don't want to write separate code + // for such a small problem. + + file_name: String, + + // This could be a Cow with file_name if no encoding was done but + // it's simpler to allocate. + + encoded_file_name: String, + + error: bool, +} + +#[derive (Serialize)] +struct TemplateDirPage <'a> { + #[serde (flatten)] + server_info: ServerInfo <'a>, + + path: Cow <'a, str>, + entries: Vec , +} fn parse_range_header (range_str: &str) -> (Option , Option ) { use lazy_static::*; @@ -52,47 +90,60 @@ fn parse_range_header (range_str: &str) -> (Option , Option ) { (start, end) } -use serde::Serialize; -use tokio::fs::DirEntry; - -// This could probably be done with borrows, if I owned the -// tokio::fs::DirEntry instead of consuming it - -#[derive (Serialize)] -struct TemplateDirEntry { - trailing_slash: &'static str, - file_name: String, - encoded_file_name: String, - error: bool, -} - async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry { - let trailing_slash = match entry.file_type ().await { - Ok (t) => if t.is_dir () { - "/" - } - else { - "" - }, - Err (_) => "", - }; - let file_name = match entry.file_name ().into_string () { Ok (x) => x, Err (_) => return TemplateDirEntry { + icon: "⚠️", trailing_slash: "", - file_name: "".into (), + file_name: "File / directory name is not UTF-8".into (), encoded_file_name: "".into (), error: true, }, }; + let (trailing_slash, icon) = match entry.file_type ().await { + Ok (t) => if t.is_dir () { + ("/", "📁") + } + else { + let icon = if file_name.ends_with (".mp4") { + "🎞️" + } + else if file_name.ends_with (".avi") { + "🎞️" + } + else if file_name.ends_with (".mkv") { + "🎞️" + } + else if file_name.ends_with (".jpg") { + "📷" + } + else if file_name.ends_with (".jpeg") { + "📷" + } + else if file_name.ends_with (".png") { + "📷" + } + else if file_name.ends_with (".bmp") { + "📷" + } + else { + "📄" + }; + + ("", icon) + }, + Err (_) => ("", "⚠️"), + }; + use percent_encoding::*; let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string (); TemplateDirEntry { + icon, trailing_slash: &trailing_slash, file_name, encoded_file_name, @@ -100,6 +151,25 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry } } +async fn serve_root ( + handlebars: &Handlebars <'static>, +) -> http_serde::Response +{ + let server_info = ServerInfo { + server_name: "PTTH file server", + }; + + let s = handlebars.render ("file_server_root", &server_info).unwrap (); + let body = s.into_bytes (); + + let mut resp = http_serde::Response::default (); + resp + .header ("content-type".to_string (), "text/html".to_string ().into_bytes ()) + .body_bytes (body) + ; + resp +} + use std::borrow::Cow; #[instrument (level = "debug", skip (handlebars, dir))] @@ -109,6 +179,10 @@ async fn serve_dir ( mut dir: ReadDir ) -> http_serde::Response { + let server_info = ServerInfo { + server_name: "PTTH file server", + }; + let mut entries = vec! []; while let Ok (Some (entry)) = dir.next_entry ().await { @@ -117,23 +191,16 @@ async fn serve_dir ( entries.sort_unstable_by (|a, b| a.file_name.partial_cmp (&b.file_name).unwrap ()); - #[derive (Serialize)] - struct TemplateDirPage <'a> { - path: Cow <'a, str>, - entries: Vec , - } - let s = handlebars.render ("file_server_dir", &TemplateDirPage { path, entries, + server_info, }).unwrap (); let body = s.into_bytes (); let mut resp = http_serde::Response::default (); - resp.content_length = Some (body.len ().try_into ().unwrap ()); resp .header ("content-type".to_string (), "text/html".to_string ().into_bytes ()) - .header ("content-length".to_string (), body.len ().to_string ().into_bytes ()) .body_bytes (body) ; resp @@ -237,8 +304,8 @@ async fn serve_error ( -> http_serde::Response { let mut resp = http_serde::Response::default (); - resp.status_code (status_code) - .body_bytes (msg.into_bytes ()); + resp.status_code (status_code); + resp.body_bytes (msg.into_bytes ()); resp } @@ -254,31 +321,19 @@ pub async fn serve_all ( -> http_serde::Response { info! ("Client requested {}", uri); - let mut range_start = None; - let mut range_end = None; - - if let Some (v) = headers.get ("range") { - let v = std::str::from_utf8 (v).unwrap (); - - let (start, end) = parse_range_header (v); - range_start = start; - range_end = end; - } - - let should_send_body = match &method { - http_serde::Method::Get => true, - http_serde::Method::Head => false, - m => { - debug! ("Unsupported method {:?}", m); - return serve_error (http_serde::StatusCode::MethodNotAllowed, "Unsupported method".into ()).await; - } - }; use percent_encoding::*; + let uri = match prefix_match (uri, "/files/") { + Some (x) => x, + None => { + return serve_root (handlebars).await; + }, + }; + // TODO: There is totally a dir traversal attack in here somewhere - let encoded_path = &uri [1..]; + let encoded_path = &uri [..]; let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().unwrap (); let path = Path::new (&*path_s); @@ -294,7 +349,10 @@ pub async fn serve_all ( } } - if let Ok (dir) = read_dir (&full_path).await { + if uri == "/" { + serve_root (handlebars).await + } + else if let Ok (dir) = read_dir (&full_path).await { serve_dir ( handlebars, full_path.to_string_lossy (), @@ -302,6 +360,26 @@ pub async fn serve_all ( ).await } else if let Ok (file) = File::open (&full_path).await { + let mut range_start = None; + let mut range_end = None; + + if let Some (v) = headers.get ("range") { + let v = std::str::from_utf8 (v).unwrap (); + + let (start, end) = parse_range_header (v); + range_start = start; + range_end = end; + } + + let should_send_body = match &method { + http_serde::Method::Get => true, + http_serde::Method::Head => false, + m => { + debug! ("Unsupported method {:?}", m); + return serve_error (http_serde::StatusCode::MethodNotAllowed, "Unsupported method".into ()).await; + } + }; + serve_file ( file, should_send_body, @@ -322,6 +400,7 @@ pub fn load_templates () for (k, v) in vec! [ ("file_server_dir", "file_server_dir.html"), + ("file_server_root", "file_server_root.html"), ].into_iter () { handlebars.register_template_file (k, format! ("ptth_handlebars/{}", v))?; } diff --git a/src/server/mod.rs b/src/server/mod.rs index aa3e041..6b9f06b 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -66,25 +66,19 @@ async fn handle_req_resp <'a> ( debug! ("Handling request {}", req_id); - let response = if let Some (uri) = prefix_match (&parts.uri, "/files") { - let default_root = PathBuf::from ("./"); - let file_server_root: &std::path::Path = state.config.file_server_root - .as_ref () - .unwrap_or (&default_root); - - file_server::serve_all ( - &state.handlebars, - file_server_root, - parts.method, - uri, - &parts.headers, - state.hidden_path.as_ref ().map (|p| p.as_path ()) - ).await - } - else { - debug! ("404 not found"); - status_reply (http_serde::StatusCode::NotFound, "404 Not Found") - }; + let default_root = PathBuf::from ("./"); + let file_server_root: &std::path::Path = state.config.file_server_root + .as_ref () + .unwrap_or (&default_root); + + let response = file_server::serve_all ( + &state.handlebars, + file_server_root, + parts.method, + &parts.uri, + &parts.headers, + state.hidden_path.as_ref ().map (|p| p.as_path ()) + ).await; let mut resp_req = state.client .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) diff --git a/todo.md b/todo.md index b62398a..0064c5b 100644 --- a/todo.md +++ b/todo.md @@ -1,8 +1,7 @@ - Not working behind Nginx (Works okay behind Caddy) - Reduce idle memory use? -- Folder icons in dir list -- ".." from server to server list is broken +- Package templates into exe for release - Redirect to add trailing slashes - Add file size in directory listing - Allow spaces in server names From 435232bf6cbe5847ce2487a87c64ca5558fb2455 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 16:00:31 +0000 Subject: [PATCH 026/208] :recycle: Use a prelude to clean up the unused imports for `tracing` --- src/bin/ptth_file_server.rs | 4 +--- src/graceful_shutdown.rs | 3 ++- src/lib.rs | 7 ++----- src/prelude.rs | 1 + src/server/file_server.rs | 6 ++---- src/server/mod.rs | 11 +---------- todo.md | 1 + 7 files changed, 10 insertions(+), 23 deletions(-) create mode 100644 src/prelude.rs diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index 2f638bc..8291428 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -17,12 +17,10 @@ use hyper::{ StatusCode, }; use serde::Deserialize; -use tracing::{ - debug, error, info, trace, warn, -}; use ptth::{ http_serde::RequestParts, + prelude::*, server::file_server, }; diff --git a/src/graceful_shutdown.rs b/src/graceful_shutdown.rs index c1917e6..992414e 100644 --- a/src/graceful_shutdown.rs +++ b/src/graceful_shutdown.rs @@ -8,7 +8,8 @@ use tokio::{ sync::oneshot, time::delay_for, }; -use tracing::{debug, error, info, trace, warn}; + +use crate::prelude::*; pub fn init () -> oneshot::Receiver <()> { let (tx, rx) = oneshot::channel::<()> (); diff --git a/src/lib.rs b/src/lib.rs index 12fc7b4..435bae3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,3 @@ -pub mod http_serde; - // It's easier if the server can stream its response body // back to the relay un-changed inside its request body // So we wrap the server's actual response head @@ -7,12 +5,11 @@ pub mod http_serde; pub const PTTH_MAGIC_HEADER: &str = "X-PTTH-2LJYXWC4"; -// Basically binaries, but in the lib we can do experimental -// test stuff like spawn them both in the same process - pub mod git_version; pub mod graceful_shutdown; +pub mod http_serde; pub mod load_toml; +pub mod prelude; pub mod relay; pub mod server; diff --git a/src/prelude.rs b/src/prelude.rs new file mode 100644 index 0000000..f4af411 --- /dev/null +++ b/src/prelude.rs @@ -0,0 +1 @@ +pub use tracing::{debug, error, info, trace, warn}; diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 35d9ac2..ec1b136 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -23,15 +23,13 @@ use tokio::{ channel, }, }; -use tracing::{ - debug, error, info, trace, warn, - instrument, -}; +use tracing::instrument; use regex::Regex; use crate::{ http_serde, + prelude::*, prefix_match, }; diff --git a/src/server/mod.rs b/src/server/mod.rs index 6b9f06b..7b5cdcd 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -16,11 +16,10 @@ use tokio::{ sync::oneshot, time::delay_for, }; -use tracing::{debug, error, info, warn}; use crate::{ http_serde, - prefix_match, + prelude::*, }; pub mod file_server; @@ -32,14 +31,6 @@ struct ServerState { hidden_path: Option , } -fn status_reply (c: http_serde::StatusCode, body: &str) -> http_serde::Response -{ - let mut r = http_serde::Response::default (); - r.status_code (c) - .body_bytes (body.as_bytes ().to_vec ()); - r -} - async fn handle_req_resp <'a> ( state: &Arc , req_resp: reqwest::Response diff --git a/todo.md b/todo.md index 0064c5b..398278c 100644 --- a/todo.md +++ b/todo.md @@ -1,6 +1,7 @@ - Not working behind Nginx (Works okay behind Caddy) - Reduce idle memory use? +- Compress bad passwords file - Package templates into exe for release - Redirect to add trailing slashes - Add file size in directory listing From 02da0ff0fc067c0292a3bdfb71833559f479ce9b Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 17:58:14 +0000 Subject: [PATCH 027/208] :bug: Redirect to add trailing slashes for directories --- ptth_handlebars/file_server_dir.html | 39 +++-- ptth_handlebars/file_server_root.html | 4 +- src/bin/ptth_file_server.rs | 8 +- src/http_serde.rs | 30 +++- src/lib.rs | 8 +- src/server/file_server.rs | 198 ++++++++++++++++++++------ todo.md | 24 +++- 7 files changed, 238 insertions(+), 73 deletions(-) diff --git a/ptth_handlebars/file_server_dir.html b/ptth_handlebars/file_server_dir.html index 552fb24..4d4cdc0 100644 --- a/ptth_handlebars/file_server_dir.html +++ b/ptth_handlebars/file_server_dir.html @@ -9,10 +9,16 @@ .entry { display: inline-block; padding: 10px; - min-width: 50%; + width: 100%; text-decoration: none; } - .entry_list div:nth-child(odd) { + .grey { + color: #888; + } + .entry_list { + width: 100%; + } + .entry_list tr:nth-child(even) { background-color: #ddd; } @@ -24,21 +30,30 @@

{{path}}

-
+ + + + + + + + - + + + + {{#each entries}} - + + + + {{/each}} - + +
NameSize
📁 ../
+{{this.icon}} {{this.file_name}}{{this.trailing_slash}}{{this.size}}
diff --git a/ptth_handlebars/file_server_root.html b/ptth_handlebars/file_server_root.html index 00a88be..a234e11 100644 --- a/ptth_handlebars/file_server_root.html +++ b/ptth_handlebars/file_server_root.html @@ -9,10 +9,10 @@ .entry { display: inline-block; padding: 10px; - min-width: 50%; + width: 100%; text-decoration: none; } - .entry_list div:nth-child(odd) { + .entry_list div:nth-child(even) { background-color: #ddd; } diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index 8291428..184c762 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -32,6 +32,7 @@ pub struct Config { struct ServerState <'a> { config: Config, handlebars: handlebars::Handlebars <'a>, + hidden_path: Option , } fn status_reply > (status: StatusCode, b: B) @@ -67,7 +68,7 @@ async fn handle_all (req: Request , state: Arc >) ptth_req.method, &ptth_req.uri, &ptth_req.headers, - None + state.hidden_path.as_ref ().map (|p| p.as_path ()) ).await; let mut resp = Response::builder () @@ -97,7 +98,9 @@ pub struct ConfigFile { #[tokio::main] async fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); - let config_file: ConfigFile = ptth::load_toml::load ("config/ptth_server.toml"); + + let path = PathBuf::from ("./config/ptth_server.toml"); + let config_file: ConfigFile = ptth::load_toml::load (&path); info! ("file_server_root: {:?}", config_file.file_server_root); let addr = SocketAddr::from(([0, 0, 0, 0], 4000)); @@ -109,6 +112,7 @@ async fn main () -> Result <(), Box > { file_server_root: config_file.file_server_root, }, handlebars, + hidden_path: Some (path), }); let make_svc = make_service_fn (|_conn| { diff --git a/src/http_serde.rs b/src/http_serde.rs index 056ceb4..52aae59 100644 --- a/src/http_serde.rs +++ b/src/http_serde.rs @@ -1,6 +1,6 @@ use std::{ collections::*, - convert::{TryFrom}, + convert::{TryFrom, TryInto}, }; use serde::{Deserialize, Serialize}; @@ -85,11 +85,13 @@ pub struct WrappedRequest { pub req: RequestParts, } -#[derive (Debug, Deserialize, Serialize)] +#[derive (Debug, Deserialize, Serialize, PartialEq)] pub enum StatusCode { Ok, // 200 PartialContent, // 206 + TemporaryRedirect, // 307 + BadRequest, // 400 Forbidden, // 403 NotFound, // 404 @@ -108,6 +110,8 @@ impl From for hyper::StatusCode { StatusCode::Ok => Self::OK, StatusCode::PartialContent => Self::PARTIAL_CONTENT, + StatusCode::TemporaryRedirect => Self::TEMPORARY_REDIRECT, + StatusCode::BadRequest => Self::BAD_REQUEST, StatusCode::Forbidden => Self::FORBIDDEN, StatusCode::NotFound => Self::NOT_FOUND, @@ -142,6 +146,24 @@ pub struct Response { } impl Response { + pub async fn into_bytes (self) -> Vec { + let mut body = match self.body { + None => return Vec::new (), + Some (x) => x, + }; + + let mut result = match self.content_length { + None => Vec::new (), + Some (x) => Vec::with_capacity (x.try_into ().unwrap ()), + }; + + while let Some (Ok (mut chunk)) = body.recv ().await { + result.append (&mut chunk); + } + + result + } + pub fn status_code (&mut self, c: StatusCode) -> &mut Self { self.parts.status_code = c; self @@ -158,14 +180,12 @@ impl Response { } pub fn body_bytes (&mut self, b: Vec ) -> &mut Self { - use std::convert::TryInto; - self.content_length = b.len ().try_into ().ok (); self.header ("content-length".to_string (), b.len ().to_string ().into_bytes ()); let (mut tx, rx) = tokio::sync::mpsc::channel (1); tokio::spawn (async move { - tx.send (Ok (b)).await.unwrap (); + tx.send (Ok (b)).await.ok (); }); self.body = Some (rx); diff --git a/src/lib.rs b/src/lib.rs index 435bae3..7270684 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -86,12 +86,10 @@ mod tests { use reqwest::Client; use tracing::{info}; - // This should be the first line of the `tracing` - // crate documentation. Their docs are awful, but you - // didn't hear it from me. - - tracing_subscriber::fmt::init (); + // Prefer this form for tests, since all tests share one process + // and we don't care if another test already installed a subscriber. + tracing_subscriber::fmt ().try_init ().ok (); let mut rt = Runtime::new ().unwrap (); // Spawn the root task diff --git a/src/server/file_server.rs b/src/server/file_server.rs index ec1b136..65f18ed 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -1,6 +1,7 @@ // Static file server that can plug into the PTTH reverse server use std::{ + borrow::Cow, cmp::{min, max}, collections::*, convert::{Infallible, TryInto}, @@ -28,7 +29,11 @@ use tracing::instrument; use regex::Regex; use crate::{ - http_serde, + http_serde::{ + Method, + Response, + StatusCode, + }, prelude::*, prefix_match, }; @@ -54,6 +59,8 @@ struct TemplateDirEntry { encoded_file_name: String, + size: Cow <'static, str>, + error: bool, } @@ -97,13 +104,28 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry trailing_slash: "", file_name: "File / directory name is not UTF-8".into (), encoded_file_name: "".into (), + size: "".into (), error: true, }, }; - let (trailing_slash, icon) = match entry.file_type ().await { - Ok (t) => if t.is_dir () { - ("/", "📁") + let metadata = match entry.metadata ().await { + Ok (x) => x, + Err (_) => return TemplateDirEntry { + icon: "⚠️", + trailing_slash: "", + file_name: "Could not fetch metadata".into (), + encoded_file_name: "".into (), + size: "".into (), + error: true, + }, + }; + + let (trailing_slash, icon, size) = { + let t = metadata.file_type (); + + if t.is_dir () { + ("/", "📁", "".into ()) } else { let icon = if file_name.ends_with (".mp4") { @@ -131,9 +153,8 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry "📄" }; - ("", icon) - }, - Err (_) => ("", "⚠️"), + ("", icon, pretty_print_bytes (metadata.len ()).into ()) + } }; use percent_encoding::*; @@ -145,13 +166,14 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry trailing_slash: &trailing_slash, file_name, encoded_file_name, + size, error: false, } } async fn serve_root ( handlebars: &Handlebars <'static>, -) -> http_serde::Response +) -> Response { let server_info = ServerInfo { server_name: "PTTH file server", @@ -160,7 +182,7 @@ async fn serve_root ( let s = handlebars.render ("file_server_root", &server_info).unwrap (); let body = s.into_bytes (); - let mut resp = http_serde::Response::default (); + let mut resp = Response::default (); resp .header ("content-type".to_string (), "text/html".to_string ().into_bytes ()) .body_bytes (body) @@ -168,14 +190,12 @@ async fn serve_root ( resp } -use std::borrow::Cow; - #[instrument (level = "debug", skip (handlebars, dir))] async fn serve_dir ( handlebars: &Handlebars <'static>, path: Cow <'_, str>, mut dir: ReadDir -) -> http_serde::Response +) -> Response { let server_info = ServerInfo { server_name: "PTTH file server", @@ -196,7 +216,7 @@ async fn serve_dir ( }).unwrap (); let body = s.into_bytes (); - let mut resp = http_serde::Response::default (); + let mut resp = Response::default (); resp .header ("content-type".to_string (), "text/html".to_string ().into_bytes ()) .body_bytes (body) @@ -210,7 +230,9 @@ async fn serve_file ( should_send_body: bool, range_start: Option , range_end: Option -) -> http_serde::Response { +) + -> Response +{ let (tx, rx) = channel (1); let body = if should_send_body { Some (rx) @@ -271,17 +293,17 @@ async fn serve_file ( }); } - let mut response = http_serde::Response::default (); + let mut response = Response::default (); response.header (String::from ("accept-ranges"), b"bytes".to_vec ()); if should_send_body { if range_start.is_none () && range_end.is_none () { - response.status_code (http_serde::StatusCode::Ok); + response.status_code (StatusCode::Ok); response.header (String::from ("content-length"), end.to_string ().into_bytes ()); } else { - response.status_code (http_serde::StatusCode::PartialContent); + response.status_code (StatusCode::PartialContent); response.header (String::from ("content-range"), format! ("bytes {}-{}/{}", start, end - 1, end).into_bytes ()); } @@ -295,15 +317,23 @@ async fn serve_file ( response } -async fn serve_error ( - status_code: http_serde::StatusCode, - msg: String +fn serve_error ( + status_code: StatusCode, + msg: &str ) --> http_serde::Response +-> Response { - let mut resp = http_serde::Response::default (); + let mut resp = Response::default (); resp.status_code (status_code); - resp.body_bytes (msg.into_bytes ()); + resp.body_bytes (msg.as_bytes ().to_vec ()); + resp +} + +fn serve_307 (location: String) -> Response { + let mut resp = Response::default (); + resp.status_code (StatusCode::TemporaryRedirect); + resp.header ("location".to_string (), location.into_bytes ()); + resp.body_bytes (b"Redirecting...".to_vec ()); resp } @@ -311,18 +341,22 @@ async fn serve_error ( pub async fn serve_all ( handlebars: &Handlebars <'static>, root: &Path, - method: http_serde::Method, + method: Method, uri: &str, headers: &HashMap >, hidden_path: Option <&Path> ) --> http_serde::Response +-> Response { info! ("Client requested {}", uri); use percent_encoding::*; - let uri = match prefix_match (uri, "/files/") { + if uri == "/favicon.ico" { + return serve_error (StatusCode::NotFound, ""); + } + + let uri = match prefix_match (uri, "/files") { Some (x) => x, None => { return serve_root (handlebars).await; @@ -331,7 +365,7 @@ pub async fn serve_all ( // TODO: There is totally a dir traversal attack in here somewhere - let encoded_path = &uri [..]; + let encoded_path = &uri [1..]; let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().unwrap (); let path = Path::new (&*path_s); @@ -343,14 +377,17 @@ pub async fn serve_all ( if let Some (hidden_path) = hidden_path { if full_path == hidden_path { - return serve_error (http_serde::StatusCode::Forbidden, "403 Forbidden".into ()).await; + return serve_error (StatusCode::Forbidden, "403 Forbidden"); } } - if uri == "/" { - serve_root (handlebars).await - } - else if let Ok (dir) = read_dir (&full_path).await { + let has_trailing_slash = path_s.is_empty () || path_s.ends_with ("/"); + + if let Ok (dir) = read_dir (&full_path).await { + if ! has_trailing_slash { + return serve_307 (format! ("{}/", path.file_name ().unwrap ().to_str ().unwrap ())); + } + serve_dir ( handlebars, full_path.to_string_lossy (), @@ -370,11 +407,11 @@ pub async fn serve_all ( } let should_send_body = match &method { - http_serde::Method::Get => true, - http_serde::Method::Head => false, + Method::Get => true, + Method::Head => false, m => { debug! ("Unsupported method {:?}", m); - return serve_error (http_serde::StatusCode::MethodNotAllowed, "Unsupported method".into ()).await; + return serve_error (StatusCode::MethodNotAllowed, "Unsupported method"); } }; @@ -386,7 +423,7 @@ pub async fn serve_all ( ).await } else { - serve_error (http_serde::StatusCode::NotFound, "404 Not Found".into ()).await + serve_error (StatusCode::NotFound, "404 Not Found") } } @@ -406,15 +443,60 @@ pub fn load_templates () Ok (handlebars) } +fn pretty_print_bytes (b: u64) -> String { + if b < 1024 { + format! ("{} B", b) + } + else if (b + 512) < 1024 * 1024 { + format! ("{} KiB", (b + 512) / 1024) + } + else if (b + 512 * 1024) < 1024 * 1024 * 1024 { + format! ("{} MiB", (b + 512 * 1024) / 1024 / 1024) + } + else { + format! ("{} GiB", (b + 512 * 1024 * 1024) / 1024 / 1024 / 1024) + } +} + #[cfg (test)] mod tests { + use std::{ + ffi::OsStr, + path::{ + Component, + Path, + PathBuf + }, + }; + + use tokio::runtime::Runtime; + + use crate::http_serde::{ + StatusCode, + }; + + #[test] + fn pretty_print_bytes () { + for (input_after, expected_before, expected_after) in vec! [ + (1, "0 B", "1 B"), + (1024, "1023 B", "1 KiB"), + (1024 + 512, "1 KiB", "2 KiB"), + (1023 * 1024 + 512, "1023 KiB", "1 MiB"), + ((1024 + 512) * 1024, "1 MiB", "2 MiB"), + (1023 * 1024 * 1024 + 512 * 1024, "1023 MiB", "1 GiB"), + ((1024 + 512) * 1024 * 1024, "1 GiB", "2 GiB"), + + ].into_iter () { + let actual = super::pretty_print_bytes (input_after - 1); + assert_eq! (&actual, expected_before); + + let actual = super::pretty_print_bytes (input_after); + assert_eq! (&actual, expected_after); + } + } + #[test] fn i_hate_paths () { - use std::{ - ffi::OsStr, - path::{Component, Path} - }; - let mut components = Path::new ("/home/user").components (); assert_eq! (components.next (), Some (Component::RootDir)); @@ -434,4 +516,38 @@ mod tests { assert_eq! (components.next (), Some (Component::CurDir)); assert_eq! (components.next (), None); } + + #[test] + fn file_server () { + use crate::{ + http_serde::Method, + prelude::*, + }; + + tracing_subscriber::fmt ().try_init ().ok (); + let mut rt = Runtime::new ().unwrap (); + + rt.block_on (async { + let handlebars = super::load_templates ().unwrap (); + let file_server_root = PathBuf::from ("./"); + let headers = Default::default (); + + for (uri_path, expected_status) in vec! [ + ("/", StatusCode::Ok), + ("/files/src", StatusCode::TemporaryRedirect), + ("/files/src/", StatusCode::Ok), + ].into_iter () { + let resp = super::serve_all ( + &handlebars, + &file_server_root, + Method::Get, + uri_path, + &headers, + None + ).await; + + assert_eq! (resp.parts.status_code, expected_status); + } + }); + } } diff --git a/todo.md b/todo.md index 398278c..5f4fe46 100644 --- a/todo.md +++ b/todo.md @@ -1,10 +1,6 @@ - Not working behind Nginx (Works okay behind Caddy) - Reduce idle memory use? -- Compress bad passwords file -- Package templates into exe for release -- Redirect to add trailing slashes -- Add file size in directory listing - Allow spaces in server names - Deny unused HTTP methods for endpoints - ETag cache based on mtime @@ -17,11 +13,13 @@ - Reverse proxy to other local servers -Off-project stuff: +# Off-project stuff: - Benchmark directory entry sorting -Known issues: +# Known issues: + +## Graceful shutdown Relay can't shut down gracefully if Firefox is connected to it, e.g. if Firefox kept a connection open while watching a video. @@ -31,3 +29,17 @@ forced shutdown timer. Sometimes I get the turtle icon in Firefox's network debugger. But this happens even with Caddy running a static file server, so I can't prove that it's on my side. The VPS is cheap, and the datacenter is far away. + +## Embedded asssets + +The bad_passwords file is huge. Since it's static, it should only be in physical +RAM when the server first launches, and then the kernel will let it be paged +out. + +Rust has some open issues with compiling assets into the exe, so I'm not +going to push on this for now, for neither bad_passwords nor the HTML assets: + +https://github.com/rust-lang/rust/issues/65818 + +I also considered compressing the passwords file, but I couldn't even get +brotli to give it a decent ratio. From 2b93aa8b8370a9115103b3de18c5156de2e00a57 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 18:12:45 +0000 Subject: [PATCH 028/208] :bug: Implement 416 Range Not Satisfiable I had a lot of trouble getting AlwaysEqual to compile, so I tested it in a completely separate crate and vendored it back into PTTH. It's also AGPLv3. --- Cargo.toml | 6 + crates/always_equal/Cargo.toml | 9 + crates/always_equal/src/lib.rs | 146 ++++++++++++++ src/http_serde.rs | 4 + src/server/file_server.rs | 344 ++++++++++++++++++++++++++------- todo.md | 2 + 6 files changed, 445 insertions(+), 66 deletions(-) create mode 100644 crates/always_equal/Cargo.toml create mode 100644 crates/always_equal/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 6972530..6cf0f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,3 +34,9 @@ tracing-futures = "0.2.4" tracing-subscriber = "0.2.15" toml = "0.5.7" ulid = "0.4.1" + +always_equal = { path = "crates/always_equal" } + +[workspace] + +members = ["crates/*"] diff --git a/crates/always_equal/Cargo.toml b/crates/always_equal/Cargo.toml new file mode 100644 index 0000000..fdf764a --- /dev/null +++ b/crates/always_equal/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "always_equal" +version = "0.1.0" +authors = ["_"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/always_equal/src/lib.rs b/crates/always_equal/src/lib.rs new file mode 100644 index 0000000..2e15641 --- /dev/null +++ b/crates/always_equal/src/lib.rs @@ -0,0 +1,146 @@ +pub mod test { + #[derive (Debug)] + pub struct AlwaysEqual { + inner: Option , + } + + impl AlwaysEqual { + pub fn into_inner (self) -> T { + match self.inner { + Some (x) => x, + None => unreachable! (), + } + } + + pub fn testing_blank () -> Self { + Self { + inner: None, + } + } + } + + impl Default for AlwaysEqual { + fn default () -> Self { + Self::from (T::default ()) + } + } + + impl From for AlwaysEqual { + fn from (inner: T) -> Self { + Self { + inner: Some (inner), + } + } + } + + impl PartialEq for AlwaysEqual { + fn eq (&self, other: &Self) -> bool { + self.inner.is_none () || other.inner.is_none () + } + } +} + +pub mod prod { + use std::fmt; + + pub struct AlwaysEqual { + inner: T, + } + + impl fmt::Debug for AlwaysEqual { + fn fmt (&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt (f) + } + } + + impl fmt::Display for AlwaysEqual { + fn fmt (&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt (f) + } + } + + impl AlwaysEqual { + pub fn into_inner (self) -> T { + self.inner + } + } + + impl From for AlwaysEqual { + fn from (inner: T) -> Self { + Self { + inner, + } + } + } + + impl PartialEq for AlwaysEqual { + fn eq (&self, _other: &Self) -> bool { + false + } + } +} + +#[cfg (test)] +mod tests { + use std::fs::File; + + use super::test::*; + + // Can't impl Clone or PartialEq because of the File + type CantCompare = Option ; + + #[derive (Debug, Default, PartialEq)] + struct MyStruct + { + file: AlwaysEqual , + name: &'static str, + } + + #[test] + fn test_1 () { + let concrete_1 = MyStruct { + file: None.into (), + name: "my_struct", + }; + let concrete_2 = MyStruct { + file: None.into (), + name: "my_struct", + }; + let concrete_bad = MyStruct { + file: None.into (), + name: "not_my_struct", + }; + + assert_ne! (concrete_1, concrete_2); + assert_ne! (concrete_2, concrete_bad); + assert_ne! (concrete_bad, concrete_1); + + let dummy_1 = MyStruct { + file: AlwaysEqual::testing_blank (), + name: "my_struct", + }; + let dummy_2 = MyStruct { + file: AlwaysEqual::testing_blank (), + name: "my_struct", + }; + let dummy_bad = MyStruct { + file: AlwaysEqual::testing_blank (), + name: "not_my_struct", + }; + + assert_eq! (dummy_1, dummy_2); + assert_ne! (dummy_2, dummy_bad); + assert_ne! (dummy_bad, dummy_1); + + assert_eq! (concrete_1, dummy_1); + assert_eq! (concrete_bad, dummy_bad); + } + + #[test] + fn test_2 () { + let v1 = Vec::>::new (); + let v2 = Vec::>::new (); + + assert_eq! (v1, v2); + } +} diff --git a/src/http_serde.rs b/src/http_serde.rs index 52aae59..a39e131 100644 --- a/src/http_serde.rs +++ b/src/http_serde.rs @@ -88,6 +88,7 @@ pub struct WrappedRequest { #[derive (Debug, Deserialize, Serialize, PartialEq)] pub enum StatusCode { Ok, // 200 + NoContent, // 204 PartialContent, // 206 TemporaryRedirect, // 307 @@ -96,6 +97,7 @@ pub enum StatusCode { Forbidden, // 403 NotFound, // 404 MethodNotAllowed, // 405 + RangeNotSatisfiable, // 416 } impl Default for StatusCode { @@ -108,6 +110,7 @@ impl From for hyper::StatusCode { fn from (x: StatusCode) -> Self { match x { StatusCode::Ok => Self::OK, + StatusCode::NoContent => Self::NO_CONTENT, StatusCode::PartialContent => Self::PARTIAL_CONTENT, StatusCode::TemporaryRedirect => Self::TEMPORARY_REDIRECT, @@ -116,6 +119,7 @@ impl From for hyper::StatusCode { StatusCode::Forbidden => Self::FORBIDDEN, StatusCode::NotFound => Self::NOT_FOUND, StatusCode::MethodNotAllowed => Self::METHOD_NOT_ALLOWED, + StatusCode::RangeNotSatisfiable => Self::RANGE_NOT_SATISFIABLE, } } } diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 65f18ed..d602c19 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -2,15 +2,17 @@ use std::{ borrow::Cow, - cmp::{min, max}, + cmp::min, collections::*, convert::{Infallible, TryInto}, error::Error, + fmt::Debug, io::SeekFrom, path::{Path, PathBuf}, }; use handlebars::Handlebars; +use percent_encoding::*; use serde::Serialize; use tokio::{ fs::{ @@ -28,6 +30,12 @@ use tracing::instrument; use regex::Regex; +#[cfg (test)] +use always_equal::test::AlwaysEqual; + +#[cfg (not (test))] +use always_equal::prod::AlwaysEqual; + use crate::{ http_serde::{ Method, @@ -90,11 +98,54 @@ fn parse_range_header (range_str: &str) -> (Option , Option ) { let end = caps.get (2).map (|x| x.as_str ()); let start = start.map (|x| u64::from_str_radix (x, 10).ok ()).flatten (); - let end = end.map (|x| u64::from_str_radix (x, 10).ok ()).flatten (); + + // HTTP specifies ranges as [start inclusive, end inclusive] + // But that's dumb and [start inclusive, end exclusive) is better + + let end = end.map (|x| u64::from_str_radix (x, 10).ok ().map (|x| x + 1)).flatten (); (start, end) } +use std::ops::Range; + +#[derive (Debug, PartialEq)] +enum ParsedRange { + Ok (Range ), + PartialContent (Range ), + RangeNotSatisfiable (u64), +} + +fn check_range (range_str: Option <&str>, file_len: u64) + -> ParsedRange +{ + use ParsedRange::*; + + let not_satisfiable = RangeNotSatisfiable (file_len); + + let range_str = match range_str { + None => return Ok (0..file_len), + Some (x) => x, + }; + + let (start, end) = parse_range_header (range_str); + + let start = start.unwrap_or (0); + if start >= file_len { + return not_satisfiable; + } + + let end = end.unwrap_or (file_len); + if end > file_len { + return not_satisfiable; + } + if end < start { + return not_satisfiable; + } + + PartialContent (start..end) +} + async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry { let file_name = match entry.file_name ().into_string () { @@ -228,8 +279,8 @@ async fn serve_dir ( async fn serve_file ( mut f: File, should_send_body: bool, - range_start: Option , - range_end: Option + range: Range , + range_requested: bool ) -> Response { @@ -241,18 +292,11 @@ async fn serve_file ( None }; - let file_md = f.metadata ().await.unwrap (); - let file_len = file_md.len (); + f.seek (SeekFrom::Start (range.start)).await.unwrap (); - let start = range_start.unwrap_or (0); - let end = range_end.unwrap_or (file_len); + info! ("Serving range {}-{}", range.start, range.end); - let start = max (0, min (start, file_len)); - let end = max (0, min (end, file_len)); - - f.seek (SeekFrom::Start (start)).await.unwrap (); - - info! ("Serving range {}-{}", start, end); + let content_length = range.end - range.start; if should_send_body { tokio::spawn (async move { @@ -260,7 +304,7 @@ async fn serve_file ( let mut tx = tx; let mut bytes_sent = 0; - let mut bytes_left = end - start; + let mut bytes_left = content_length; loop { let mut buffer = vec! [0u8; 65_536]; @@ -275,7 +319,7 @@ async fn serve_file ( } if tx.send (Ok::<_, Infallible> (buffer)).await.is_err () { - warn! ("Cancelling file stream (Sent {} out of {} bytes)", bytes_sent, end - start); + warn! ("Cancelling file stream (Sent {} out of {} bytes)", bytes_sent, content_length); break; } @@ -298,16 +342,19 @@ async fn serve_file ( response.header (String::from ("accept-ranges"), b"bytes".to_vec ()); if should_send_body { - if range_start.is_none () && range_end.is_none () { - response.status_code (StatusCode::Ok); - response.header (String::from ("content-length"), end.to_string ().into_bytes ()); + if range_requested { + response.status_code (StatusCode::PartialContent); + response.header (String::from ("content-range"), format! ("bytes {}-{}/{}", range.start, range.end - 1, range.end).into_bytes ()); } else { - response.status_code (StatusCode::PartialContent); - response.header (String::from ("content-range"), format! ("bytes {}-{}/{}", start, end - 1, end).into_bytes ()); + response.status_code (StatusCode::Ok); + response.header (String::from ("content-length"), range.end.to_string ().into_bytes ()); } - response.content_length = Some (end - start); + response.content_length = Some (content_length); + } + else { + response.status_code (StatusCode::NoContent); } if let Some (body) = body { @@ -337,32 +384,73 @@ fn serve_307 (location: String) -> Response { resp } -#[instrument (level = "debug", skip (handlebars, headers))] -pub async fn serve_all ( - handlebars: &Handlebars <'static>, +// Sort of an internal API endpoint to make testing work better. +// Eventually we could expose this as JSON or Msgpack or whatever. For now +// it's just a Rust struct that we can test on without caring about +// human-readable HTML + +#[derive (Debug, PartialEq)] +struct ServeDirParams { + path: PathBuf, + dir: AlwaysEqual , +} + +#[derive (Debug, PartialEq)] +struct ServeFileParams { + send_body: bool, + range: Range , + range_requested: bool, + file: AlwaysEqual , +} + +#[derive (Debug, PartialEq)] +enum InternalResponse { + Favicon, + Forbidden, + MethodNotAllowed, + NotFound, + RangeNotSatisfiable (u64), + Redirect (String), + Root, + ServeDir (ServeDirParams), + ServeFile (ServeFileParams), +} + +async fn internal_serve_all ( root: &Path, method: Method, uri: &str, headers: &HashMap >, hidden_path: Option <&Path> ) --> Response + -> InternalResponse { + use InternalResponse::*; + info! ("Client requested {}", uri); - use percent_encoding::*; + let send_body = match &method { + Method::Get => true, + Method::Head => false, + m => { + debug! ("Unsupported method {:?}", m); + return MethodNotAllowed; + } + }; if uri == "/favicon.ico" { - return serve_error (StatusCode::NotFound, ""); + return Favicon; } let uri = match prefix_match (uri, "/files") { Some (x) => x, - None => { - return serve_root (handlebars).await; - }, + None => return Root, }; + if uri == "" { + return Redirect ("files/".to_string ()); + } + // TODO: There is totally a dir traversal attack in here somewhere let encoded_path = &uri [1..]; @@ -377,7 +465,7 @@ pub async fn serve_all ( if let Some (hidden_path) = hidden_path { if full_path == hidden_path { - return serve_error (StatusCode::Forbidden, "403 Forbidden"); + return Forbidden; } } @@ -385,45 +473,82 @@ pub async fn serve_all ( if let Ok (dir) = read_dir (&full_path).await { if ! has_trailing_slash { - return serve_307 (format! ("{}/", path.file_name ().unwrap ().to_str ().unwrap ())); + return Redirect (format! ("{}/", path.file_name ().unwrap ().to_str ().unwrap ())); } - serve_dir ( - handlebars, - full_path.to_string_lossy (), - dir - ).await + let dir = dir.into (); + + ServeDir (ServeDirParams { + dir, + path: full_path, + }) } else if let Ok (file) = File::open (&full_path).await { - let mut range_start = None; - let mut range_end = None; + let file_md = file.metadata ().await.unwrap (); + let file_len = file_md.len (); - if let Some (v) = headers.get ("range") { - let v = std::str::from_utf8 (v).unwrap (); - - let (start, end) = parse_range_header (v); - range_start = start; - range_end = end; + let range_header = headers.get ("range").map (|v| std::str::from_utf8 (v).ok ()).flatten (); + + let file = file.into (); + + match check_range (range_header, file_len) { + ParsedRange::RangeNotSatisfiable (file_len) => RangeNotSatisfiable (file_len), + ParsedRange::Ok (range) => ServeFile (ServeFileParams { + file, + send_body, + range, + range_requested: false, + }), + ParsedRange::PartialContent (range) => ServeFile (ServeFileParams { + file, + send_body, + range, + range_requested: true, + }), } - - let should_send_body = match &method { - Method::Get => true, - Method::Head => false, - m => { - debug! ("Unsupported method {:?}", m); - return serve_error (StatusCode::MethodNotAllowed, "Unsupported method"); - } - }; - - serve_file ( - file, - should_send_body, - range_start, - range_end - ).await } else { - serve_error (StatusCode::NotFound, "404 Not Found") + NotFound + } +} + +#[instrument (level = "debug", skip (handlebars, headers))] +pub async fn serve_all ( + handlebars: &Handlebars <'static>, + root: &Path, + method: Method, + uri: &str, + headers: &HashMap >, + hidden_path: Option <&Path> +) + -> Response +{ + use InternalResponse::*; + + match internal_serve_all (root, method, uri, headers, hidden_path).await { + Favicon => serve_error (StatusCode::NotFound, ""), + Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), + MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method"), + NotFound => serve_error (StatusCode::NotFound, "404 Not Found"), + RangeNotSatisfiable (file_len) => { + let mut resp = Response::default (); + resp.status_code (StatusCode::RangeNotSatisfiable) + .header ("content-range".to_string (), format! ("bytes */{}", file_len).into_bytes ()); + resp + }, + Redirect (location) => serve_307 (location), + + Root => serve_root (handlebars).await, + ServeDir (ServeDirParams { + path, + dir, + }) => serve_dir (handlebars, path.to_string_lossy (), dir.into_inner ()).await, + ServeFile (ServeFileParams { + file, + send_body, + range, + range_requested, + }) => serve_file (file.into_inner (), send_body, range, range_requested).await, } } @@ -469,12 +594,52 @@ mod tests { }, }; + use maplit::*; use tokio::runtime::Runtime; + use always_equal::test::AlwaysEqual; + use crate::http_serde::{ StatusCode, }; + #[test] + fn parse_range_header () { + for (input, expected) in vec! [ + ("", (None, None)), + ("bytes=0-", (Some (0), None)), + ("bytes=0-999", (Some (0), Some (1000))), + ("bytes=111-999", (Some (111), Some (1000))), + ].into_iter () { + let actual = super::parse_range_header (input); + assert_eq! (actual, expected); + } + + use super::ParsedRange::*; + + for (header, file_len, expected) in vec! [ + (None, 0, Ok (0..0)), + (None, 1024, Ok (0..1024)), + + (Some (""), 0, RangeNotSatisfiable (0)), + (Some (""), 1024, PartialContent (0..1024)), + + (Some ("bytes=0-"), 1024, PartialContent (0..1024)), + (Some ("bytes=0-999"), 1024, PartialContent (0..1000)), + (Some ("bytes=0-1023"), 1024, PartialContent (0..1024)), + (Some ("bytes=111-999"), 1024, PartialContent (111..1000)), + (Some ("bytes=111-1023"), 1024, PartialContent (111..1024)), + (Some ("bytes=200-100"), 1024, RangeNotSatisfiable (1024)), + + (Some ("bytes=0-"), 512, PartialContent (0..512)), + (Some ("bytes=0-1023"), 512, RangeNotSatisfiable (512)), + (Some ("bytes=1000-1023"), 512, RangeNotSatisfiable (512)), + ].into_iter () { + let actual = super::check_range (header, file_len); + assert_eq! (actual, expected); + } + } + #[test] fn pretty_print_bytes () { for (input_after, expected_before, expected_after) in vec! [ @@ -521,23 +686,32 @@ mod tests { fn file_server () { use crate::{ http_serde::Method, - prelude::*, + //prelude::*, + }; + use super::{ + InternalResponse, + internal_serve_all, + load_templates, + serve_all, + ServeDirParams, + ServeFileParams, }; tracing_subscriber::fmt ().try_init ().ok (); let mut rt = Runtime::new ().unwrap (); rt.block_on (async { - let handlebars = super::load_templates ().unwrap (); + let handlebars = load_templates ().unwrap (); let file_server_root = PathBuf::from ("./"); let headers = Default::default (); for (uri_path, expected_status) in vec! [ ("/", StatusCode::Ok), + ("/files", StatusCode::TemporaryRedirect), ("/files/src", StatusCode::TemporaryRedirect), ("/files/src/", StatusCode::Ok), ].into_iter () { - let resp = super::serve_all ( + let resp = serve_all ( &handlebars, &file_server_root, Method::Get, @@ -548,6 +722,44 @@ mod tests { assert_eq! (resp.parts.status_code, expected_status); } + + { + use InternalResponse::*; + + for (uri_path, expected) in vec! [ + ("/", Root), + ("/files", Redirect ("files/".to_string ())), + ("/files/src", Redirect ("src/".to_string ())), + ("/files/src/bad_passwords.txt", ServeFile (ServeFileParams { + send_body: true, + range: 0..1_048_576, + range_requested: false, + file: AlwaysEqual::testing_blank (), + })), + ].into_iter () { + let resp = internal_serve_all ( + &file_server_root, + Method::Get, + uri_path, + &headers, + None + ).await; + + assert_eq! (resp, expected); + } + + let resp = internal_serve_all ( + &file_server_root, + Method::Get, + "/files/src/bad_passwords.txt", + &hashmap! { + "range".into () => b"bytes=0-2000000".to_vec (), + }, + None + ).await; + + assert_eq! (resp, RangeNotSatisfiable (1_048_576)); + } }); } } diff --git a/todo.md b/todo.md index 5f4fe46..1bdcf0e 100644 --- a/todo.md +++ b/todo.md @@ -1,6 +1,8 @@ - Not working behind Nginx (Works okay behind Caddy) - Reduce idle memory use? +- Flip match_prefix args +- Impl multi-range / multi-part byte serving - Allow spaces in server names - Deny unused HTTP methods for endpoints - ETag cache based on mtime From f81d819c3187d79972c6fc36b6e5977c5df162d4 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 8 Nov 2020 23:53:31 +0000 Subject: [PATCH 029/208] Flip match_prefix args --- src/lib.rs | 9 ++++++--- src/relay/mod.rs | 6 +++--- src/server/file_server.rs | 2 +- todo.md | 1 - 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7270684..db08c33 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,10 +13,13 @@ pub mod prelude; pub mod relay; pub mod server; -pub fn prefix_match <'a> (hay: &'a str, needle: &str) -> Option <&'a str> +// The arguments are in order so they are in order overall: +// e.g. prefix_match ("/prefix", "/prefix/middle/suffix") -> "/middle/suffix" + +pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str> { - if hay.starts_with (needle) { - Some (&hay [needle.len ()..]) + if hay.starts_with (prefix) { + Some (&hay [prefix.len ()..]) } else { None diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 6d017fb..db1f130 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -467,7 +467,7 @@ async fn handle_all (req: Request , state: Arc ) // This is stuff the server can use. Clients can't // POST right now - return Ok (if let Some (request_code) = prefix_match (path, "/7ZSFUKGV/http_response/") { + return Ok (if let Some (request_code) = prefix_match ("/7ZSFUKGV/http_response/", path) { let request_code = request_code.into (); handle_http_response (req, state, request_code).await } @@ -476,14 +476,14 @@ async fn handle_all (req: Request , state: Arc ) }); } - Ok (if let Some (listen_code) = prefix_match (path, "/7ZSFUKGV/http_listen/") { + Ok (if let Some (listen_code) = prefix_match ("/7ZSFUKGV/http_listen/", path) { let api_key = match api_key { None => return Ok (error_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key")), Some (x) => x, }; handle_http_listen (state, listen_code.into (), api_key.as_bytes ()).await } - else if let Some (rest) = prefix_match (path, "/frontend/servers/") { + else if let Some (rest) = prefix_match ("/frontend/servers/", path) { if rest == "" { #[derive (Serialize)] struct ServerEntry <'a> { diff --git a/src/server/file_server.rs b/src/server/file_server.rs index d602c19..3a01bb3 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -442,7 +442,7 @@ async fn internal_serve_all ( return Favicon; } - let uri = match prefix_match (uri, "/files") { + let uri = match prefix_match ("/files", uri) { Some (x) => x, None => return Root, }; diff --git a/todo.md b/todo.md index 1bdcf0e..afc6fb9 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,6 @@ - Not working behind Nginx (Works okay behind Caddy) - Reduce idle memory use? -- Flip match_prefix args - Impl multi-range / multi-part byte serving - Allow spaces in server names - Deny unused HTTP methods for endpoints From 49cd2921157fe63997f664933f609f682575a2cd Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 9 Nov 2020 00:04:35 +0000 Subject: [PATCH 030/208] :bug: Allowing spaces in server names --- src/relay/mod.rs | 16 +++++++++++----- todo.md | 1 - 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/relay/mod.rs b/src/relay/mod.rs index db1f130..47a64d4 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -108,7 +108,7 @@ struct Config { impl From <&ConfigFile> for Config { fn from (f: &ConfigFile) -> Self { - let trips = HashMap::from_iter (f.server_tripcodes.iter () + let server_tripcodes = HashMap::from_iter (f.server_tripcodes.iter () .map (|(k, v)| { use std::convert::TryInto; let bytes: Vec = base64::decode (v).unwrap (); @@ -116,11 +116,15 @@ impl From <&ConfigFile> for Config { let v = blake3::Hash::from (bytes); - (k.clone (), v) + let k = percent_encoding::percent_encode (k.as_bytes (), percent_encoding::NON_ALPHANUMERIC).to_string (); + + debug! ("Tripcode {} => {}", k, v.to_hex ()); + + (k, v) })); Self { - server_tripcodes: trips, + server_tripcodes, } } } @@ -485,10 +489,12 @@ async fn handle_all (req: Request , state: Arc ) } else if let Some (rest) = prefix_match ("/frontend/servers/", path) { if rest == "" { + use std::borrow::Cow; + #[derive (Serialize)] struct ServerEntry <'a> { path: &'a str, - name: &'a str, + name: Cow <'a, str>, } #[derive (Serialize)] @@ -503,7 +509,7 @@ async fn handle_all (req: Request , state: Arc ) let page = ServerListPage { servers: names.iter () .map (|name| ServerEntry { - name: &name, + name: percent_encoding::percent_decode_str (name).decode_utf8 ().unwrap_or_else (|_| "Server name isn't UTF-8".into ()), path: &name, }) .collect (), diff --git a/todo.md b/todo.md index afc6fb9..3f50221 100644 --- a/todo.md +++ b/todo.md @@ -2,7 +2,6 @@ - Reduce idle memory use? - Impl multi-range / multi-part byte serving -- Allow spaces in server names - Deny unused HTTP methods for endpoints - ETag cache based on mtime - Server-side hash? From 116b3b4900a8f9a2f4eaef00bdbb1668f3b9912b Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 9 Nov 2020 00:53:07 +0000 Subject: [PATCH 031/208] Marking known issues and wrapping up dev for the night --- src/bin/ptth_server.rs | 35 +++++++++++++++++++++++-------- src/lib.rs | 4 ++-- src/relay/mod.rs | 4 +--- src/server/file_server.rs | 43 ++++++++++++++++++++++++++------------- todo.md | 17 +++++++++++++++- 5 files changed, 75 insertions(+), 28 deletions(-) diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index d335867..9013f10 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -4,6 +4,8 @@ use std::{ }; use structopt::StructOpt; +use tokio::runtime; +use tracing::debug; #[derive (Debug, StructOpt)] struct Opt { @@ -11,15 +13,32 @@ struct Opt { file_server_root: Option , } -#[tokio::main] -async fn main () -> Result <(), Box > { +fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); let path = PathBuf::from ("./config/ptth_server.toml"); - let config_file = ptth::load_toml::load (&path); + let config_file: ptth::server::ConfigFile = ptth::load_toml::load (&path); - ptth::server::run_server ( - config_file, - ptth::graceful_shutdown::init (), - Some (path) - ).await + let mut rt = if false { + debug! ("Trying to use less RAM"); + + runtime::Builder::new () + .threaded_scheduler () + .enable_all () + .core_threads (1) + .thread_stack_size (1024 * 1024) + .build ()? + } + else { + runtime::Runtime::new ()? + }; + + rt.block_on (async { + ptth::server::run_server ( + config_file, + ptth::graceful_shutdown::init (), + Some (path) + ).await + })?; + + Ok (()) } diff --git a/src/lib.rs b/src/lib.rs index db08c33..c20ef88 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -87,7 +87,7 @@ mod tests { fn end_to_end () { use maplit::*; use reqwest::Client; - use tracing::{info}; + use tracing::{debug, info}; // Prefer this form for tests, since all tests share one process // and we don't care if another test already installed a subscriber. @@ -100,7 +100,7 @@ mod tests { let server_name = "aliens_wildland"; let api_key = "AnacondaHardcoverGrannyUnlatchLankinessMutate"; let tripcode = base64::encode (blake3::hash (api_key.as_bytes ()).as_bytes ()); - println! ("Relay is expecting tripcode {}", tripcode); + debug! ("Relay is expecting tripcode {}", tripcode); let config_file = relay::ConfigFile { port: None, server_tripcodes: hashmap! { diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 47a64d4..4c3dad7 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -116,11 +116,9 @@ impl From <&ConfigFile> for Config { let v = blake3::Hash::from (bytes); - let k = percent_encoding::percent_encode (k.as_bytes (), percent_encoding::NON_ALPHANUMERIC).to_string (); - debug! ("Tripcode {} => {}", k, v.to_hex ()); - (k, v) + (k.clone (), v) })); Self { diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 3a01bb3..00a96d4 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -292,15 +292,15 @@ async fn serve_file ( None }; - f.seek (SeekFrom::Start (range.start)).await.unwrap (); - info! ("Serving range {}-{}", range.start, range.end); let content_length = range.end - range.start; + let seek = SeekFrom::Start (range.start); + if should_send_body { tokio::spawn (async move { - //println! ("Opening file {:?}", path); + f.seek (seek).await.unwrap (); let mut tx = tx; let mut bytes_sent = 0; @@ -341,21 +341,21 @@ async fn serve_file ( response.header (String::from ("accept-ranges"), b"bytes".to_vec ()); - if should_send_body { - if range_requested { - response.status_code (StatusCode::PartialContent); - response.header (String::from ("content-range"), format! ("bytes {}-{}/{}", range.start, range.end - 1, range.end).into_bytes ()); - } - else { - response.status_code (StatusCode::Ok); - response.header (String::from ("content-length"), range.end.to_string ().into_bytes ()); - } - - response.content_length = Some (content_length); + if range_requested { + response.status_code (StatusCode::PartialContent); + response.header (String::from ("content-range"), format! ("bytes {}-{}/{}", range.start, range.end - 1, range.end).into_bytes ()); } else { + response.status_code (StatusCode::Ok); + response.header (String::from ("content-length"), range.end.to_string ().into_bytes ()); + } + + if ! should_send_body { response.status_code (StatusCode::NoContent); } + else { + response.content_length = Some (content_length); + } if let Some (body) = body { response.body (body); @@ -759,6 +759,21 @@ mod tests { ).await; assert_eq! (resp, RangeNotSatisfiable (1_048_576)); + + let resp = internal_serve_all ( + &file_server_root, + Method::Head, + "/files/src/bad_passwords.txt", + &headers, + None + ).await; + + assert_eq! (resp, ServeFile (ServeFileParams { + send_body: false, + range: 0..1_048_576, + range_requested: false, + file: AlwaysEqual::testing_blank (), + })); } }); } diff --git a/todo.md b/todo.md index 3f50221..558f2f2 100644 --- a/todo.md +++ b/todo.md @@ -1,5 +1,5 @@ - Not working behind Nginx (Works okay behind Caddy) -- Reduce idle memory use? +- Show file server names in HTML - Impl multi-range / multi-part byte serving - Deny unused HTTP methods for endpoints @@ -43,3 +43,18 @@ https://github.com/rust-lang/rust/issues/65818 I also considered compressing the passwords file, but I couldn't even get brotli to give it a decent ratio. + +## RAM use is kinda high + +I tried to reduce the thread count in Tokio, but it's still around 12 or 13 +MiB even when the server is doing nothing. + +I'll leave in the minimize_ram setting for now, but it doesn't actually +reduce RAM use. + +## Server names can't have spaces + +I tried to figure out the percent encoding and it didn't work. + +Maybe Base64 would be better or something? At least it's unambiguous and it +can go straight from UTF-8 to bytes to ASCII-armored. From 4d4a46f1679bf7cac4dcfbc0221ae01b6e4a59ff Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 9 Nov 2020 01:06:53 +0000 Subject: [PATCH 032/208] :whale: --- .dockerignore | 1 + .gitignore | 2 ++ build-and-minimize.bash | 27 +++++++++++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100755 build-and-minimize.bash diff --git a/.dockerignore b/.dockerignore index a26b144..16e1d8d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ /config +/ptth_latest.tar.gz /target diff --git a/.gitignore b/.gitignore index d4996e0..932b8d8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,7 @@ /*.tar.gz /ptth_server.toml /ptth_relay.toml +/ptth_build_L6KLMVS6/ /target /test + diff --git a/build-and-minimize.bash b/build-and-minimize.bash new file mode 100755 index 0000000..155ec58 --- /dev/null +++ b/build-and-minimize.bash @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Use: ./build-and-minimize.bash + +# Makes a docker image, saves it as .tar, unpacks the tar, removes +# the base Debian layer, and repacks it so I can upload to servers a little +# faster. + +TEMP_GIBBERISH="ptth_build_L6KLMVS6" +TEMP_TAR="$TEMP_GIBBERISH/ptth.tar" +UPLOADABLE_TAR="$PWD/ptth_latest.tar.gz" + +# This is magic and will need to be updated whenever we update the +# Debian layer. + +BOTTOM_LAYER="cec906613726ec32de92af0ec1cd6692c34df78782227f4415cd12c47a264dd4" + +mkdir -p "$TEMP_GIBBERISH/ptth" + +sudo docker build -t ptth:latest . +sudo docker image save ptth:latest | pv > "$TEMP_TAR" +tar -C "$TEMP_GIBBERISH/ptth" -xf "$TEMP_TAR" + +rm -rf "$TEMP_GIBBERISH/ptth/$BOTTOM_LAYER" + +pushd "$TEMP_GIBBERISH/ptth" + tar -czf "$UPLOADABLE_TAR" . +popd From 97147941221ea651c1ca2b63f5600c06dd922a7f Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 9 Nov 2020 16:33:13 +0000 Subject: [PATCH 033/208] Add trace log for streaming files --- src/server/file_server.rs | 8 +++++++- todo.md | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 00a96d4..3b7de13 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -306,6 +306,9 @@ async fn serve_file ( let mut bytes_sent = 0; let mut bytes_left = content_length; + let mark_interval = 200_000; + let mut next_mark = mark_interval; + loop { let mut buffer = vec! [0u8; 65_536]; let bytes_read: u64 = f.read (&mut buffer).await.unwrap ().try_into ().unwrap (); @@ -330,7 +333,10 @@ async fn serve_file ( } bytes_sent += bytes_read; - trace! ("Sent {} bytes", bytes_sent); + while next_mark <= bytes_sent { + trace! ("Sent {} bytes", next_mark); + next_mark += mark_interval; + } //delay_for (Duration::from_millis (50)).await; } diff --git a/todo.md b/todo.md index 558f2f2..4f10a06 100644 --- a/todo.md +++ b/todo.md @@ -1,4 +1,5 @@ -- Not working behind Nginx (Works okay behind Caddy) +- Add byte counter in `trace!()` macro in the file server +- Not working great behind reverse proxies - Show file server names in HTML - Impl multi-range / multi-part byte serving @@ -58,3 +59,16 @@ I tried to figure out the percent encoding and it didn't work. Maybe Base64 would be better or something? At least it's unambiguous and it can go straight from UTF-8 to bytes to ASCII-armored. + +## Turtle in Firefox's network debugger + +The turtle shows up if Firefox has to wait more than 500 ms till first byte. +Curl says we can download a small file (950 bytes) end-to-end in about 250 ms. + +So I think somewhere between Firefox and Caddy, something is getting confused. +Firefox, probably the same as Chromium, doesn't try to buffer entire videos +at once, so I think it purposely hangs the download, and then I'm not sure +what happens. + +I might have to build a client that imitates this behavior, since it's hard +to control. From 1d469c8dff6e6c326e1293e923469045af960868 Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 9 Nov 2020 17:02:36 +0000 Subject: [PATCH 034/208] Update todo / readme --- README.md | 14 ++++++++++++-- todo.md | 3 ++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ca0e78d..3b51c2f 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,16 @@ If you only have pre-built binaries: "ptth_file_server" is not needed for production. It's a standalone build of the file server module, for testing. +To run the relay behind Nginx, these directives improve time-to-first-byte +when streaming: + +``` +client_max_body_size 0; +proxy_http_version 1.1; +proxy_request_buffering off; +proxy_buffering off; +``` + ## Comparison with normal HTTP Normal HTTP: @@ -176,14 +186,14 @@ ptth_server and ptth_file_server use the `file_server` module. ptth_server will connect out to a ptth_relay instance and serve files through the reverse HTTP tunnel. -The http_serde module is shared by ptth_relay and ptth_server so that they +The `http_serde` module is shared by ptth_relay and ptth_server so that they can communicate with each other easily. ## Why are GitHub issues disabled? Because they are not part of Git. -For now, either email me (if you know me personally) or make a pull request to add an item to todo.md. +For now, either email me (if you know me personally) or make a pull request to add an item to [todo.md](todo.md). ## License diff --git a/todo.md b/todo.md index 4f10a06..d09be5b 100644 --- a/todo.md +++ b/todo.md @@ -1,4 +1,5 @@ -- Add byte counter in `trace!()` macro in the file server +- "Preview as" feature for Markdown / pretty-printed logs +- Add Prometheus metrics - Not working great behind reverse proxies - Show file server names in HTML From 63abdc3a1607b7f983709e34d27adcdeeacd85e8 Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 10 Nov 2020 00:44:21 +0000 Subject: [PATCH 035/208] :lipstick: Show file server name in directory pages --- src/bin/ptth_file_server.rs | 8 +++++++- src/lib.rs | 3 +++ src/server/file_server.rs | 39 +++++++++++++++---------------------- src/server/mod.rs | 10 +++++++++- todo.md | 2 +- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index 184c762..c6e263f 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -32,6 +32,7 @@ pub struct Config { struct ServerState <'a> { config: Config, handlebars: handlebars::Handlebars <'a>, + server_info: file_server::ServerInfo, hidden_path: Option , } @@ -63,7 +64,8 @@ async fn handle_all (req: Request , state: Arc >) .unwrap_or (&default_root); let ptth_resp = file_server::serve_all ( - &state.handlebars, + &state.handlebars, + &state.server_info, file_server_root, ptth_req.method, &ptth_req.uri, @@ -93,6 +95,7 @@ async fn handle_all (req: Request , state: Arc >) #[derive (Deserialize)] pub struct ConfigFile { pub file_server_root: Option , + pub name: Option , } #[tokio::main] @@ -112,6 +115,9 @@ async fn main () -> Result <(), Box > { file_server_root: config_file.file_server_root, }, handlebars, + server_info: crate::file_server::ServerInfo { + server_name: config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()).clone (), + }, hidden_path: Some (path), }); diff --git a/src/lib.rs b/src/lib.rs index c20ef88..790562a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,6 +66,9 @@ mod tests { use crate::password_is_bad; for pw in &[ + "", + " ", + "user", "password", "pAsSwOrD", "secret", diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 3b7de13..ff38e3f 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -46,9 +46,9 @@ use crate::{ prefix_match, }; -#[derive (Serialize)] -struct ServerInfo <'a> { - server_name: &'a str, +#[derive (Debug, Serialize)] +pub struct ServerInfo { + pub server_name: String, } #[derive (Serialize)] @@ -75,7 +75,7 @@ struct TemplateDirEntry { #[derive (Serialize)] struct TemplateDirPage <'a> { #[serde (flatten)] - server_info: ServerInfo <'a>, + server_info: &'a ServerInfo, path: Cow <'a, str>, entries: Vec , @@ -224,12 +224,9 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry async fn serve_root ( handlebars: &Handlebars <'static>, + server_info: &ServerInfo ) -> Response { - let server_info = ServerInfo { - server_name: "PTTH file server", - }; - let s = handlebars.render ("file_server_root", &server_info).unwrap (); let body = s.into_bytes (); @@ -244,14 +241,11 @@ async fn serve_root ( #[instrument (level = "debug", skip (handlebars, dir))] async fn serve_dir ( handlebars: &Handlebars <'static>, + server_info: &ServerInfo, path: Cow <'_, str>, mut dir: ReadDir ) -> Response { - let server_info = ServerInfo { - server_name: "PTTH file server", - }; - let mut entries = vec! []; while let Ok (Some (entry)) = dir.next_entry ().await { @@ -521,6 +515,7 @@ async fn internal_serve_all ( #[instrument (level = "debug", skip (handlebars, headers))] pub async fn serve_all ( handlebars: &Handlebars <'static>, + server_info: &ServerInfo, root: &Path, method: Method, uri: &str, @@ -544,11 +539,11 @@ pub async fn serve_all ( }, Redirect (location) => serve_307 (location), - Root => serve_root (handlebars).await, + Root => serve_root (handlebars, server_info).await, ServeDir (ServeDirParams { path, dir, - }) => serve_dir (handlebars, path.to_string_lossy (), dir.into_inner ()).await, + }) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await, ServeFile (ServeFileParams { file, send_body, @@ -694,20 +689,17 @@ mod tests { http_serde::Method, //prelude::*, }; - use super::{ - InternalResponse, - internal_serve_all, - load_templates, - serve_all, - ServeDirParams, - ServeFileParams, - }; + use super::*; tracing_subscriber::fmt ().try_init ().ok (); let mut rt = Runtime::new ().unwrap (); rt.block_on (async { let handlebars = load_templates ().unwrap (); + let server_info = ServerInfo { + server_name: "PTTH File Server".to_string (), + }; + let file_server_root = PathBuf::from ("./"); let headers = Default::default (); @@ -718,7 +710,8 @@ mod tests { ("/files/src/", StatusCode::Ok), ].into_iter () { let resp = serve_all ( - &handlebars, + &handlebars, + &server_info, &file_server_root, Method::Get, uri_path, diff --git a/src/server/mod.rs b/src/server/mod.rs index 7b5cdcd..b6bc761 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -27,6 +27,7 @@ pub mod file_server; struct ServerState { config: Config, handlebars: Handlebars <'static>, + server_info: file_server::ServerInfo, client: Client, hidden_path: Option , } @@ -64,6 +65,7 @@ async fn handle_req_resp <'a> ( let response = file_server::serve_all ( &state.handlebars, + &state.server_info, file_server_root, parts.method, &parts.uri, @@ -136,9 +138,14 @@ pub async fn run_server ( panic! ("API key is too weak, server can't use it"); } + let server_info = file_server::ServerInfo { + server_name: config_file.name.clone (), + }; + let tripcode = base64::encode (blake3::hash (config_file.api_key.as_bytes ()).as_bytes ()); - info! ("Our tripcode is {}", tripcode); + info! ("Server name is {}", config_file.name); + info! ("Tripcode is {}", tripcode); let mut headers = reqwest::header::HeaderMap::new (); headers.insert ("X-ApiKey", config_file.api_key.try_into ().unwrap ()); @@ -155,6 +162,7 @@ pub async fn run_server ( file_server_root: config_file.file_server_root, }, handlebars, + server_info, client, hidden_path, }); diff --git a/todo.md b/todo.md index d09be5b..c28e579 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,7 @@ - "Preview as" feature for Markdown / pretty-printed logs +- Make a debug client to replicate the issue Firefox is having with turtling - Add Prometheus metrics - Not working great behind reverse proxies -- Show file server names in HTML - Impl multi-range / multi-part byte serving - Deny unused HTTP methods for endpoints From 13b816fd6edef1cc75e65e20ab896276f5986b21 Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 10 Nov 2020 01:02:59 +0000 Subject: [PATCH 036/208] :recycle: Adding space for a Markdown preview --- src/http_serde.rs | 4 ++++ src/server/file_server.rs | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/http_serde.rs b/src/http_serde.rs index a39e131..5276cfe 100644 --- a/src/http_serde.rs +++ b/src/http_serde.rs @@ -98,6 +98,8 @@ pub enum StatusCode { NotFound, // 404 MethodNotAllowed, // 405 RangeNotSatisfiable, // 416 + + InternalServerError, // 500 } impl Default for StatusCode { @@ -120,6 +122,8 @@ impl From for hyper::StatusCode { StatusCode::NotFound => Self::NOT_FOUND, StatusCode::MethodNotAllowed => Self::METHOD_NOT_ALLOWED, StatusCode::RangeNotSatisfiable => Self::RANGE_NOT_SATISFIABLE, + + StatusCode::InternalServerError => Self::INTERNAL_SERVER_ERROR, } } } diff --git a/src/server/file_server.rs b/src/server/file_server.rs index ff38e3f..cd8594f 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -228,12 +228,15 @@ async fn serve_root ( ) -> Response { let s = handlebars.render ("file_server_root", &server_info).unwrap (); - let body = s.into_bytes (); + serve_html (s) +} + +fn serve_html (s: String) -> Response { let mut resp = Response::default (); resp .header ("content-type".to_string (), "text/html".to_string ().into_bytes ()) - .body_bytes (body) + .body_bytes (s.into_bytes ()) ; resp } @@ -259,14 +262,8 @@ async fn serve_dir ( entries, server_info, }).unwrap (); - let body = s.into_bytes (); - let mut resp = Response::default (); - resp - .header ("content-type".to_string (), "text/html".to_string ().into_bytes ()) - .body_bytes (body) - ; - resp + serve_html (s) } #[instrument (level = "debug", skip (f))] @@ -414,6 +411,9 @@ enum InternalResponse { Root, ServeDir (ServeDirParams), ServeFile (ServeFileParams), + + MarkdownError, + ServeMarkdownPreview (String), } async fn internal_serve_all ( @@ -550,6 +550,8 @@ pub async fn serve_all ( range, range_requested, }) => serve_file (file.into_inner (), send_body, range, range_requested).await, + MarkdownError => serve_error (StatusCode::InternalServerError, "Error while rendering Markdown preview"), + ServeMarkdownPreview (s) => serve_html (s), } } From ff6e841e0bf7a97d49e8d882906dcff48c2f73af Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 10 Nov 2020 02:39:20 +0000 Subject: [PATCH 037/208] Markdown preview added to the standalone server, not linked in yet --- Cargo.toml | 2 + src/bin/ptth_file_server.rs | 6 +- src/server/file_server.rs | 148 ++++++++++++++++++++++++++++++------ test.md | 5 ++ 4 files changed, 135 insertions(+), 26 deletions(-) create mode 100644 test.md diff --git a/Cargo.toml b/Cargo.toml index 6cf0f5b..5bc3f40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ hyper = "0.13.8" lazy_static = "1.4.0" maplit = "1.0.2" percent-encoding = "2.1.0" +pulldown-cmark = "0.8.0" rand = "0.7.3" regex = "1.4.1" reqwest = { version = "0.10.8", features = ["stream"] } @@ -34,6 +35,7 @@ tracing-futures = "0.2.4" tracing-subscriber = "0.2.15" toml = "0.5.7" ulid = "0.4.1" +url = "2.2.0" always_equal = { path = "crates/always_equal" } diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index c6e263f..b287f05 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -47,13 +47,13 @@ async fn handle_all (req: Request , state: Arc >) { debug! ("req.uri () = {:?}", req.uri ()); - let path = req.uri ().path (); + let path_and_query = req.uri ().path_and_query ().map (|x| x.as_str ()).unwrap_or_else (|| req.uri ().path ()); - let path = path.into (); + let path_and_query = path_and_query.into (); let (parts, _) = req.into_parts (); - let ptth_req = match RequestParts::from_hyper (parts.method, path, parts.headers) { + let ptth_req = match RequestParts::from_hyper (parts.method, path_and_query, parts.headers) { Ok (x) => x, _ => return Ok (status_reply (StatusCode::BAD_REQUEST, "Bad request")), }; diff --git a/src/server/file_server.rs b/src/server/file_server.rs index cd8594f..c8d384c 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -381,6 +381,25 @@ fn serve_307 (location: String) -> Response { resp } +fn render_markdown (bytes: &[u8]) -> Result { + use pulldown_cmark::{Parser, Options, html}; + + let markdown_input = match std::str::from_utf8 (bytes) { + Err (_) => return Err (MarkdownError::FileIsNotUtf8), + Ok (x) => x, + }; + + let mut options = Options::empty (); + options.insert (Options::ENABLE_STRIKETHROUGH); + let parser = Parser::new_ext (markdown_input, options); + + // Write to String buffer. + let mut html_output = String::new (); + html::push_html (&mut html_output, parser); + + Ok (html_output) +} + // Sort of an internal API endpoint to make testing work better. // Eventually we could expose this as JSON or Msgpack or whatever. For now // it's just a Rust struct that we can test on without caring about @@ -400,10 +419,19 @@ struct ServeFileParams { file: AlwaysEqual , } +#[derive (Debug, PartialEq)] +enum MarkdownError { + FileIsTooBig, + FileIsNotMarkdown, + FileIsNotUtf8, +} + #[derive (Debug, PartialEq)] enum InternalResponse { Favicon, Forbidden, + InvalidUri, + InvalidQuery, MethodNotAllowed, NotFound, RangeNotSatisfiable (u64), @@ -412,8 +440,8 @@ enum InternalResponse { ServeDir (ServeDirParams), ServeFile (ServeFileParams), - MarkdownError, - ServeMarkdownPreview (String), + MarkdownErr (MarkdownError), + MarkdownPreview (String), } async fn internal_serve_all ( @@ -425,10 +453,16 @@ async fn internal_serve_all ( ) -> InternalResponse { + use std::str::FromStr; use InternalResponse::*; info! ("Client requested {}", uri); + let uri = match hyper::Uri::from_str (uri) { + Err (_) => return InvalidUri, + Ok (x) => x, + }; + let send_body = match &method { Method::Get => true, Method::Head => false, @@ -438,22 +472,22 @@ async fn internal_serve_all ( } }; - if uri == "/favicon.ico" { + if uri.path () == "/favicon.ico" { return Favicon; } - let uri = match prefix_match ("/files", uri) { + let path = match prefix_match ("/files", uri.path ()) { Some (x) => x, None => return Root, }; - if uri == "" { + if path == "" { return Redirect ("files/".to_string ()); } // TODO: There is totally a dir traversal attack in here somewhere - let encoded_path = &uri [1..]; + let encoded_path = &path [1..]; let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().unwrap (); let path = Path::new (&*path_s); @@ -476,6 +510,10 @@ async fn internal_serve_all ( return Redirect (format! ("{}/", path.file_name ().unwrap ().to_str ().unwrap ())); } + if uri.query ().is_some () { + return InvalidQuery; + } + let dir = dir.into (); ServeDir (ServeDirParams { @@ -483,28 +521,54 @@ async fn internal_serve_all ( path: full_path, }) } - else if let Ok (file) = File::open (&full_path).await { + else if let Ok (mut file) = File::open (&full_path).await { let file_md = file.metadata ().await.unwrap (); let file_len = file_md.len (); let range_header = headers.get ("range").map (|v| std::str::from_utf8 (v).ok ()).flatten (); - let file = file.into (); - match check_range (range_header, file_len) { ParsedRange::RangeNotSatisfiable (file_len) => RangeNotSatisfiable (file_len), - ParsedRange::Ok (range) => ServeFile (ServeFileParams { - file, - send_body, - range, - range_requested: false, - }), - ParsedRange::PartialContent (range) => ServeFile (ServeFileParams { - file, - send_body, - range, - range_requested: true, - }), + ParsedRange::Ok (range) => { + if uri.query () == Some ("as_markdown") { + const MAX_BUF_SIZE: u32 = 1_000_000; + if file_len > MAX_BUF_SIZE.try_into ().unwrap () { + MarkdownErr (MarkdownError::FileIsTooBig) + } + else { + let mut buffer = vec! [0u8; MAX_BUF_SIZE.try_into ().unwrap ()]; + let bytes_read = file.read (&mut buffer).await.unwrap (); + buffer.truncate (bytes_read); + + MarkdownPreview (render_markdown (&buffer).unwrap ()) + } + } + else { + let file = file.into (); + + ServeFile (ServeFileParams { + file, + send_body, + range, + range_requested: false, + }) + } + }, + ParsedRange::PartialContent (range) => { + if uri.query ().is_some () { + InvalidQuery + } + else { + let file = file.into (); + + ServeFile (ServeFileParams { + file, + send_body, + range, + range_requested: true, + }) + } + }, } } else { @@ -529,6 +593,8 @@ pub async fn serve_all ( match internal_serve_all (root, method, uri, headers, hidden_path).await { Favicon => serve_error (StatusCode::NotFound, ""), Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), + InvalidUri => serve_error (StatusCode::BadRequest, "Invalid URI"), + InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object"), MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method"), NotFound => serve_error (StatusCode::NotFound, "404 Not Found"), RangeNotSatisfiable (file_len) => { @@ -550,8 +616,12 @@ pub async fn serve_all ( range, range_requested, }) => serve_file (file.into_inner (), send_body, range, range_requested).await, - MarkdownError => serve_error (StatusCode::InternalServerError, "Error while rendering Markdown preview"), - ServeMarkdownPreview (s) => serve_html (s), + MarkdownErr (e) => match e { + MarkdownError::FileIsTooBig => serve_error (StatusCode::InternalServerError, "File is too big to preview as Markdown"), + MarkdownError::FileIsNotMarkdown => serve_error (StatusCode::BadRequest, "File is not Markdown"), + MarkdownError::FileIsNotUtf8 => serve_error (StatusCode::BadRequest, "File is not UTF-8"), + }, + MarkdownPreview (s) => serve_html (s), } } @@ -730,13 +800,23 @@ mod tests { for (uri_path, expected) in vec! [ ("/", Root), ("/files", Redirect ("files/".to_string ())), + ("/files/?", InvalidQuery), ("/files/src", Redirect ("src/".to_string ())), + ("/files/src/?", InvalidQuery), ("/files/src/bad_passwords.txt", ServeFile (ServeFileParams { send_body: true, range: 0..1_048_576, range_requested: false, file: AlwaysEqual::testing_blank (), })), + ("/files/test.md", ServeFile (ServeFileParams { + send_body: true, + range: 0..117, + range_requested: false, + file: AlwaysEqual::testing_blank (), + })), + ("/files/test.md?as_markdown", MarkdownPreview ("

Markdown test

\n

This is a test file for the Markdown previewing feature.

\n

Don\'t change it, it will break the tests.

\n".into ())), + ("/ ", InvalidUri), ].into_iter () { let resp = internal_serve_all ( &file_server_root, @@ -778,4 +858,26 @@ mod tests { } }); } + + #[test] + fn parse_uri () { + use hyper::Uri; + + assert! (Uri::from_maybe_shared ("/").is_ok ()); + } + + #[test] + fn markdown () { + use super::*; + + for (input, expected) in vec! [ + ("", ""), + ( + "Hello world, this is a ~~complicated~~ *very simple* example.", + "

Hello world, this is a complicated very simple example.

\n" + ), + ].into_iter () { + assert_eq! (expected, &render_markdown (input.as_bytes ()).unwrap ()); + } + } } diff --git a/test.md b/test.md new file mode 100644 index 0000000..120b141 --- /dev/null +++ b/test.md @@ -0,0 +1,5 @@ +# Markdown test + +This is a test file for the Markdown previewing feature. + +Don't change it, it will break the tests. From b333b56e80e00d268a37af1e3541f08bdc11c166 Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 10 Nov 2020 03:01:00 +0000 Subject: [PATCH 038/208] Make Markdown previews sans-serif on principle --- .gitignore | 1 - src/server/file_server.rs | 16 +++++++++------- test/face.png | Bin 0 -> 2669 bytes test.md => test/test.md | 2 ++ todo.md | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 test/face.png rename test.md => test/test.md (81%) diff --git a/.gitignore b/.gitignore index 932b8d8..bb1ca12 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,4 @@ /ptth_relay.toml /ptth_build_L6KLMVS6/ /target -/test diff --git a/src/server/file_server.rs b/src/server/file_server.rs index c8d384c..7d9084b 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -235,7 +235,7 @@ async fn serve_root ( fn serve_html (s: String) -> Response { let mut resp = Response::default (); resp - .header ("content-type".to_string (), "text/html".to_string ().into_bytes ()) + .header ("content-type".to_string (), "text/html; charset=UTF-8".to_string ().into_bytes ()) .body_bytes (s.into_bytes ()) ; resp @@ -394,10 +394,13 @@ fn render_markdown (bytes: &[u8]) -> Result { let parser = Parser::new_ext (markdown_input, options); // Write to String buffer. - let mut html_output = String::new (); - html::push_html (&mut html_output, parser); + let mut out = String::new (); - Ok (html_output) + out.push_str (""); + html::push_html (&mut out, parser); + out.push_str (""); + + Ok (out) } // Sort of an internal API endpoint to make testing work better. @@ -809,13 +812,12 @@ mod tests { range_requested: false, file: AlwaysEqual::testing_blank (), })), - ("/files/test.md", ServeFile (ServeFileParams { + ("/files/test/test.md", ServeFile (ServeFileParams { send_body: true, - range: 0..117, + range: 0..144, range_requested: false, file: AlwaysEqual::testing_blank (), })), - ("/files/test.md?as_markdown", MarkdownPreview ("

Markdown test

\n

This is a test file for the Markdown previewing feature.

\n

Don\'t change it, it will break the tests.

\n".into ())), ("/ ", InvalidUri), ].into_iter () { let resp = internal_serve_all ( diff --git a/test/face.png b/test/face.png new file mode 100644 index 0000000000000000000000000000000000000000..18668bb6760b9f7811e532f2bea65337ec44c9fc GIT binary patch literal 2669 zcmV-z3X=7SP)0004mX+uL$Nkc;* zaB^>EX>4Tx04R}tkv&MmKpe$iQ$;NmK|6@3WT;LSL`5963Pq?;YK2xEOfLO`CJjl7 zi=*ILaPVWX>fqw6tAnc`2!4P#IXWr2NQwVT3N2zhIPS;0dyl(!fKV$j)odFFRLwGy zv8b5Ntco442w)h!7($oCOnokqh{JPy-NVP%y9m$nKKJM7Rb`6 zkz)a6sE`~#_#gc4*33;#xJkh%(EeiEA0t4=E>N%9_V=-E*G~ZdGjOFf{pA`k^GSNO zsfCY#o^9abx~a)~z~v6m|D;QXy3IZuBvJ#Ml000PlNkl8lG+p#RXt!N!VwWaOHBD2~o2CK+BZGo~IDo@Nl;Ji5@8Y~4F}~mDi-^|d z^DF^Re$P42@A5oHsM%~X{^Z4E{Nd{gz!iWi09OG1JOGVGqtR$|H;#_m5&KlXu2!qD zSS%O}(ll+gT8%~{pU;ON=q#aA6m@WLP%4!W1mW}f91iDP05qFTk|dXxm*2g6_x0;n z2!cjNMrLMa?%uttQmI%hRtKTeG+ihZyk76}@-l{DVzJomc27-B8I495hEH*K?W}US zytK45K0eN5{>?fjlj(3cUcGvSVHibG&C@S}AZBN0$HvBbdU_ZP1_VJuq44qJ$6l|u z(P*@}NxJ}2snm-XFIv%q1woKbr+fSMZ7!EPZS;*sBNz-mdGh4go+6Rx&6_vna=FcY zm~Gw?MN!dc^jPjTo6X(bT@1rYrBVl3*lael+3c}BOQjM)5EMmqvkE~Fgdm7SBKc>& za=BcYOokxHX;T@3Ah}#_Hk*&-k3=HTXf*Aoa=T@e&*zVij|+vuW1pMN=F!nnfk4nf z5=*7hnVFgK@$nXT7K=48G2wJNPf?57XKgeZefsoiYikICluG66*RL~~OtaHPlH}an z+{DC$NF-9JRQK-Po12>>2%`NRp!T^-QB)uhn4h0tU0tPVTBFfSPEJlwPxtrtcUDHh z%4V}GD=WcZkk9Aa?RJO5Ar_0<-w--g^3gP1EEW$A4qzDOa=E>|y)X=)v60a<4H_Ab z#}fzyZ1(ATE7V!_IAhRV+!MOe#8&{W0Q}jDeFU1p8qNP|ZS2c%-V{aU^LY%zd_EtFqTAcskw_#Sk5d$7 zv)P_MfBx{{L$O$V=>Sj^mCa^Dp^)G2-`Lm)1Onl3IFrfL>-E3I3WI@R7=~eohle7O z=)V$){ag55sZ=tVjNkA7{Q2{@Z{IdHHVzIBTCxs;AP9okY&L=*gM))kr}G5m&Av#U z?CIpaMfq~J{(c8Cg4-XGZ zrP6;r>}~>}X}VsoA08eS3I!a;W3kxo?k+(P1VQZW?WNP{WHQ+z35c7`X7hMFold9M z>kS5j-EMceTmu6GAoPxp@6-S)l}a+142Q$XWHJ_u#bUAT?d^CxP7nm3N6XRyhYiE9 zLZL7kjaI95U|_&zvt7G(O{rA&_4W1k_8#+8&JjSP(I80@$MNOmTsMiQ6H<#al&R;yO4mC0mWF1LMtcLxAX)45zO6bh}ZtbFnc@82&hEY$1uR_G#;$l-8|jg8&7 zal>dd_V@S8WU`~-p3`}Eas$BrrfC|4#pm)7Z{-x3dP9Ch}~|tSS)tCy)_zj zt6O(Xkx>+tN~IPT7w6~a*VfiDnM_N>u-R-5ha;EEtyZhcQ%X!Lsh(9n>}=ha*d9T4h{~AM4}TgT<8-Mu!(ZHTr!!&FbsGW z@pwFuNC4-eRlZ;tR;$(b@85s+?Ag%J5cp?`-|vlr?tvhPMx$}LTzBr=$z(FAR4N*c zVi<;^XgC~Br_;4styZh0)9FYglF#RX6uDgXI+Mxd@pwEQufM+^l;C_mABjXz6kT6m zM^QAJ%}OMa>FH^$R(o(y#CFzWsQ07sFuFRZWLiLU_s bzXSLS!+vC>9$T(q00000NkvXXu0mjfa`FHg literal 0 HcmV?d00001 diff --git a/test.md b/test/test.md similarity index 81% rename from test.md rename to test/test.md index 120b141..119ee02 100644 --- a/test.md +++ b/test/test.md @@ -3,3 +3,5 @@ This is a test file for the Markdown previewing feature. Don't change it, it will break the tests. + +![A silly face](face.png) diff --git a/todo.md b/todo.md index c28e579..b3d977b 100644 --- a/todo.md +++ b/todo.md @@ -1,4 +1,4 @@ -- "Preview as" feature for Markdown / pretty-printed logs +- "Preview as" feature for Markdown (It's not threaded through the relay yet) - Make a debug client to replicate the issue Firefox is having with turtling - Add Prometheus metrics - Not working great behind reverse proxies From d8173e61b534e76134982d40364cb1c3703d7777 Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 18 Nov 2020 21:42:07 +0000 Subject: [PATCH 039/208] Pin Rust toolchain --- rust-toolchain | 1 + 1 file changed, 1 insertion(+) create mode 100644 rust-toolchain diff --git a/rust-toolchain b/rust-toolchain new file mode 100644 index 0000000..21998d3 --- /dev/null +++ b/rust-toolchain @@ -0,0 +1 @@ +1.47.0 From 3293b3e7f9ba5cfe260dd0e916d0015eec45a02b Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 18 Nov 2020 22:48:15 +0000 Subject: [PATCH 040/208] Add config_path command-line option --- src/bin/ptth_server.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 9013f10..e3c0d70 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -9,13 +9,18 @@ use tracing::debug; #[derive (Debug, StructOpt)] struct Opt { + //#[structopt (long)] + //file_server_root: Option , + #[structopt (long)] - file_server_root: Option , + config_path: Option , } fn main () -> Result <(), Box > { + let opt = Opt::from_args (); + tracing_subscriber::fmt::init (); - let path = PathBuf::from ("./config/ptth_server.toml"); + let path = opt.config_path.unwrap_or_else (|| PathBuf::from ("./config/ptth_server.toml")); let config_file: ptth::server::ConfigFile = ptth::load_toml::load (&path); let mut rt = if false { From b32990a6bb167a8b3182ca29fda16459a6ed901c Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 18 Nov 2020 23:24:47 +0000 Subject: [PATCH 041/208] :heavy_plus_sign: Add asset_root option --- .../relay}/relay_root.html | 0 .../relay}/relay_server_list.html | 0 .../relay}/test_relay_server_list.html | 0 .../server}/file_server_dir.html | 0 .../server}/file_server_root.html | 0 .../server}/test_file_server_dir.html | 0 src/bin/ptth_file_server.rs | 2 +- src/bin/ptth_server.rs | 10 ++++--- src/lib.rs | 2 +- src/relay/mod.rs | 10 ++++--- src/server/file_server.rs | 26 ++++++++++++++----- src/server/mod.rs | 7 +++-- 12 files changed, 40 insertions(+), 17 deletions(-) rename {ptth_handlebars => handlebars/relay}/relay_root.html (100%) rename {ptth_handlebars => handlebars/relay}/relay_server_list.html (100%) rename {ptth_handlebars => handlebars/relay}/test_relay_server_list.html (100%) rename {ptth_handlebars => handlebars/server}/file_server_dir.html (100%) rename {ptth_handlebars => handlebars/server}/file_server_root.html (100%) rename {ptth_handlebars => handlebars/server}/test_file_server_dir.html (100%) diff --git a/ptth_handlebars/relay_root.html b/handlebars/relay/relay_root.html similarity index 100% rename from ptth_handlebars/relay_root.html rename to handlebars/relay/relay_root.html diff --git a/ptth_handlebars/relay_server_list.html b/handlebars/relay/relay_server_list.html similarity index 100% rename from ptth_handlebars/relay_server_list.html rename to handlebars/relay/relay_server_list.html diff --git a/ptth_handlebars/test_relay_server_list.html b/handlebars/relay/test_relay_server_list.html similarity index 100% rename from ptth_handlebars/test_relay_server_list.html rename to handlebars/relay/test_relay_server_list.html diff --git a/ptth_handlebars/file_server_dir.html b/handlebars/server/file_server_dir.html similarity index 100% rename from ptth_handlebars/file_server_dir.html rename to handlebars/server/file_server_dir.html diff --git a/ptth_handlebars/file_server_root.html b/handlebars/server/file_server_root.html similarity index 100% rename from ptth_handlebars/file_server_root.html rename to handlebars/server/file_server_root.html diff --git a/ptth_handlebars/test_file_server_dir.html b/handlebars/server/test_file_server_dir.html similarity index 100% rename from ptth_handlebars/test_file_server_dir.html rename to handlebars/server/test_file_server_dir.html diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index b287f05..32a43b5 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -108,7 +108,7 @@ async fn main () -> Result <(), Box > { let addr = SocketAddr::from(([0, 0, 0, 0], 4000)); - let handlebars = file_server::load_templates ()?; + let handlebars = file_server::load_templates (&PathBuf::new ())?; let state = Arc::new (ServerState { config: Config { diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index e3c0d70..65a8189 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -14,13 +14,16 @@ struct Opt { #[structopt (long)] config_path: Option , + + #[structopt (long)] + asset_root: Option , } fn main () -> Result <(), Box > { let opt = Opt::from_args (); tracing_subscriber::fmt::init (); - let path = opt.config_path.unwrap_or_else (|| PathBuf::from ("./config/ptth_server.toml")); + let path = opt.config_path.clone ().unwrap_or_else (|| PathBuf::from ("./config/ptth_server.toml")); let config_file: ptth::server::ConfigFile = ptth::load_toml::load (&path); let mut rt = if false { @@ -37,11 +40,12 @@ fn main () -> Result <(), Box > { runtime::Runtime::new ()? }; - rt.block_on (async { + rt.block_on (async move { ptth::server::run_server ( config_file, ptth::graceful_shutdown::init (), - Some (path) + Some (path), + opt.asset_root ).await })?; diff --git a/src/lib.rs b/src/lib.rs index 790562a..d0551bb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -133,7 +133,7 @@ mod tests { let (stop_server_tx, stop_server_rx) = oneshot::channel (); let task_server = { spawn (async move { - server::run_server (config_file, stop_server_rx, None).await.unwrap (); + server::run_server (config_file, stop_server_rx, None, None).await.unwrap (); }) }; diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 4c3dad7..ce04aaf 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -147,7 +147,7 @@ impl From <&ConfigFile> for RelayState { Self { config: Config::from (config_file), - handlebars: Arc::new (load_templates ().unwrap ()), + handlebars: Arc::new (load_templates (&PathBuf::new ()).unwrap ()), request_rendezvous: Default::default (), response_rendezvous: Default::default (), shutdown_watch_tx, @@ -539,17 +539,21 @@ async fn handle_all (req: Request , state: Arc ) }) } -pub fn load_templates () +use std::path::{Path, PathBuf}; + +pub fn load_templates (asset_root: &Path) -> Result , Box > { let mut handlebars = Handlebars::new (); handlebars.set_strict_mode (true); + let asset_root = asset_root.join ("handlebars/relay"); + for (k, v) in vec! [ ("relay_server_list", "relay_server_list.html"), ("relay_root", "relay_root.html"), ].into_iter () { - handlebars.register_template_file (k, format! ("ptth_handlebars/{}", v))?; + handlebars.register_template_file (k, &asset_root.join (v))?; } Ok (handlebars) diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 7d9084b..d89fc43 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -381,7 +381,7 @@ fn serve_307 (location: String) -> Response { resp } -fn render_markdown (bytes: &[u8]) -> Result { +fn render_markdown (bytes: &[u8], out: &mut String) -> Result <(), MarkdownError> { use pulldown_cmark::{Parser, Options, html}; let markdown_input = match std::str::from_utf8 (bytes) { @@ -393,11 +393,17 @@ fn render_markdown (bytes: &[u8]) -> Result { options.insert (Options::ENABLE_STRIKETHROUGH); let parser = Parser::new_ext (markdown_input, options); + html::push_html (out, parser); + + Ok (()) +} + +fn render_markdown_styled (bytes: &[u8]) -> Result { // Write to String buffer. let mut out = String::new (); out.push_str (""); - html::push_html (&mut out, parser); + render_markdown (bytes, &mut out)?; out.push_str (""); Ok (out) @@ -543,7 +549,7 @@ async fn internal_serve_all ( let bytes_read = file.read (&mut buffer).await.unwrap (); buffer.truncate (bytes_read); - MarkdownPreview (render_markdown (&buffer).unwrap ()) + MarkdownPreview (render_markdown_styled (&buffer).unwrap ()) } } else { @@ -628,17 +634,21 @@ pub async fn serve_all ( } } -pub fn load_templates () +pub fn load_templates ( + asset_root: &Path +) -> Result , Box > { let mut handlebars = Handlebars::new (); handlebars.set_strict_mode (true); + let asset_root = asset_root.join ("handlebars/server"); + for (k, v) in vec! [ ("file_server_dir", "file_server_dir.html"), ("file_server_root", "file_server_root.html"), ].into_iter () { - handlebars.register_template_file (k, format! ("ptth_handlebars/{}", v))?; + handlebars.register_template_file (k, asset_root.join (v))?; } Ok (handlebars) @@ -770,7 +780,7 @@ mod tests { let mut rt = Runtime::new ().unwrap (); rt.block_on (async { - let handlebars = load_templates ().unwrap (); + let handlebars = load_templates (&PathBuf::new ()).unwrap (); let server_info = ServerInfo { server_name: "PTTH File Server".to_string (), }; @@ -879,7 +889,9 @@ mod tests { "

Hello world, this is a complicated very simple example.

\n" ), ].into_iter () { - assert_eq! (expected, &render_markdown (input.as_bytes ()).unwrap ()); + let mut out = String::default (); + render_markdown (input.as_bytes (), &mut out).unwrap (); + assert_eq! (expected, &out); } } } diff --git a/src/server/mod.rs b/src/server/mod.rs index b6bc761..7171c68 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -128,10 +128,13 @@ pub struct Config { pub async fn run_server ( config_file: ConfigFile, shutdown_oneshot: oneshot::Receiver <()>, - hidden_path: Option + hidden_path: Option , + asset_root: Option ) -> Result <(), Box > { + let asset_root = asset_root.unwrap_or_else (|| PathBuf::new ()); + use std::convert::TryInto; if crate::password_is_bad (config_file.api_key.clone ()) { @@ -154,7 +157,7 @@ pub async fn run_server ( .default_headers (headers) .timeout (Duration::from_secs (40)) .build ().unwrap (); - let handlebars = file_server::load_templates ()?; + let handlebars = file_server::load_templates (&asset_root).expect ("Can't load Handlebars templates"); let state = Arc::new (ServerState { config: Config { From ed45e190ae2284bdf49de9279d2a80b96342d726 Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 19 Nov 2020 00:10:32 +0000 Subject: [PATCH 042/208] Add todo --- todo.md | 1 + 1 file changed, 1 insertion(+) diff --git a/todo.md b/todo.md index b3d977b..4d97b58 100644 --- a/todo.md +++ b/todo.md @@ -1,3 +1,4 @@ +- Reload relay config (or at least tripcodes) without downtime - "Preview as" feature for Markdown (It's not threaded through the relay yet) - Make a debug client to replicate the issue Firefox is having with turtling - Add Prometheus metrics From c726cb3456572ff96246605dcda5699bf1046606 Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 19 Nov 2020 00:12:00 +0000 Subject: [PATCH 043/208] Add todo --- todo.md | 1 + 1 file changed, 1 insertion(+) diff --git a/todo.md b/todo.md index 4d97b58..655cea4 100644 --- a/todo.md +++ b/todo.md @@ -1,3 +1,4 @@ +- Sort relays in relay list - Reload relay config (or at least tripcodes) without downtime - "Preview as" feature for Markdown (It's not threaded through the relay yet) - Make a debug client to replicate the issue Firefox is having with turtling From 563465c85bac6da72eb1270551dfd82c04e1d872 Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 19 Nov 2020 01:01:28 +0000 Subject: [PATCH 044/208] Add todo --- todo.md | 1 + 1 file changed, 1 insertion(+) diff --git a/todo.md b/todo.md index 655cea4..f0531fc 100644 --- a/todo.md +++ b/todo.md @@ -1,6 +1,7 @@ - Sort relays in relay list - Reload relay config (or at least tripcodes) without downtime - "Preview as" feature for Markdown (It's not threaded through the relay yet) +- Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) - Make a debug client to replicate the issue Firefox is having with turtling - Add Prometheus metrics - Not working great behind reverse proxies From 5378e66e395e7df9ec00d69b129d9da5c6318655 Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 19 Nov 2020 01:45:42 +0000 Subject: [PATCH 045/208] :bug: Use file permissions to forbid access to ptth_server.toml --- Cargo.toml | 1 + src/bin/ptth_relay.rs | 2 +- src/load_toml.rs | 47 +++++++++++++++++++++++++++++++-------- src/relay/mod.rs | 3 ++- src/server/file_server.rs | 10 +++++++-- todo.md | 5 +++++ 6 files changed, 55 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5bc3f40..2ced302 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ reqwest = { version = "0.10.8", features = ["stream"] } rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} structopt = "0.3.20" +# thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" tracing-futures = "0.2.4" diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index 77bbb56..a525580 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -21,7 +21,7 @@ async fn main () -> Result <(), Box > { .init () ; - let config_file = ptth::load_toml::load ("config/ptth_relay.toml"); + let config_file = ptth::load_toml::load_public ("config/ptth_relay.toml"); info! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); diff --git a/src/load_toml.rs b/src/load_toml.rs index 99acc89..58f7159 100644 --- a/src/load_toml.rs +++ b/src/load_toml.rs @@ -7,19 +7,48 @@ use std::{ use serde::de::DeserializeOwned; -pub fn load < +pub const CONFIG_PERMISSIONS_MODE: u32 = 33152; + +fn load_inner < + T: DeserializeOwned +> ( + mut f: File +) -> T { + let mut buffer = vec! [0u8; 4096]; + let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read config")); + buffer.truncate (bytes_read); + + let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse config as UTF-8")); + toml::from_str (&config_s).unwrap_or_else (|e| panic! ("Can't parse config as TOML: {}", e)) +} + +/// For files that contain public-viewable information + +pub fn load_public < T: DeserializeOwned, P: AsRef + Debug > ( config_file_path: P ) -> T { let mut f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); - let mut buffer = vec! [0u8; 4096]; - let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read {:?}", config_file_path)); - buffer.truncate (bytes_read); - - { - let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse {:?} as UTF-8", config_file_path)); - toml::from_str (&config_s).unwrap_or_else (|e| panic! ("Can't parse {:?} as TOML: {}", config_file_path, e)) - } + load_inner (f) +} + +/// For files that may contain secrets and should have permissions or other +/// safeties checked + +pub fn load < + T: DeserializeOwned, + P: AsRef + Debug +> ( + config_file_path: P +) -> T { + use std::os::unix::fs::PermissionsExt; + + let mut f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); + + let mode = f.metadata ().unwrap ().permissions ().mode (); + assert_eq! (mode, CONFIG_PERMISSIONS_MODE, "Config file has bad permissions mode, it should be octal 0600"); + + load_inner (f) } diff --git a/src/relay/mod.rs b/src/relay/mod.rs index ce04aaf..0b44369 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -500,7 +500,8 @@ async fn handle_all (req: Request , state: Arc ) servers: Vec >, } - let names = state.list_servers ().await; + let mut names = state.list_servers ().await; + names.sort (); //println! ("Found {} servers", names.len ()); diff --git a/src/server/file_server.rs b/src/server/file_server.rs index d89fc43..2e3a688 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -501,8 +501,7 @@ async fn internal_serve_all ( let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().unwrap (); let path = Path::new (&*path_s); - let mut full_path = PathBuf::from (root); - full_path.push (path); + let full_path = root.join (path); debug! ("full_path = {:?}", full_path); @@ -531,7 +530,14 @@ async fn internal_serve_all ( }) } else if let Ok (mut file) = File::open (&full_path).await { + use std::os::unix::fs::PermissionsExt; + let file_md = file.metadata ().await.unwrap (); + if file_md.permissions ().mode () == crate::load_toml::CONFIG_PERMISSIONS_MODE + { + return Forbidden; + } + let file_len = file_md.len (); let range_header = headers.get ("range").map (|v| std::str::from_utf8 (v).ok ()).flatten (); diff --git a/todo.md b/todo.md index f0531fc..53daba4 100644 --- a/todo.md +++ b/todo.md @@ -76,3 +76,8 @@ what happens. I might have to build a client that imitates this behavior, since it's hard to control. + +## Server won't work on Windows + +This is because I use Unix-specific file permissions to protect the server +config. From 1b9eabf45838a540c2d0ee69b8e83e1677e873c6 Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 24 Nov 2020 23:55:59 +0000 Subject: [PATCH 046/208] Relay sorting was fixed in the last commit --- todo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/todo.md b/todo.md index 53daba4..47e3694 100644 --- a/todo.md +++ b/todo.md @@ -1,4 +1,4 @@ -- Sort relays in relay list +- Print tripcode without running the server - Reload relay config (or at least tripcodes) without downtime - "Preview as" feature for Markdown (It's not threaded through the relay yet) - Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) From bfe07fddc3b23cd88c8eb3b327f8f428fbdbf9c4 Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 24 Nov 2020 23:56:43 +0000 Subject: [PATCH 047/208] :heavy_plus_sign: Add "--print-tripcode" option to ptth_server --- src/bin/ptth_server.rs | 8 ++++++++ src/server/mod.rs | 27 ++++++++++++++++++++++++--- todo.md | 1 - 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 65a8189..4b4e047 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -17,6 +17,9 @@ struct Opt { #[structopt (long)] asset_root: Option , + + #[structopt (long)] + print_tripcode: bool, } fn main () -> Result <(), Box > { @@ -26,6 +29,11 @@ fn main () -> Result <(), Box > { let path = opt.config_path.clone ().unwrap_or_else (|| PathBuf::from ("./config/ptth_server.toml")); let config_file: ptth::server::ConfigFile = ptth::load_toml::load (&path); + if opt.print_tripcode { + println! (r#""{}" = "{}""#, config_file.name, config_file.tripcode ()); + return Ok (()); + } + let mut rt = if false { debug! ("Trying to use less RAM"); diff --git a/src/server/mod.rs b/src/server/mod.rs index 7171c68..334e8c8 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -119,6 +119,12 @@ pub struct ConfigFile { pub file_server_root: Option , } +impl ConfigFile { + pub fn tripcode (&self) -> String { + base64::encode (blake3::hash (self.api_key.as_bytes ()).as_bytes ()) + } +} + #[derive (Default)] pub struct Config { pub relay_url: String, @@ -145,10 +151,8 @@ pub async fn run_server ( server_name: config_file.name.clone (), }; - let tripcode = base64::encode (blake3::hash (config_file.api_key.as_bytes ()).as_bytes ()); - info! ("Server name is {}", config_file.name); - info! ("Tripcode is {}", tripcode); + info! ("Tripcode is {}", config_file.tripcode ()); let mut headers = reqwest::header::HeaderMap::new (); headers.insert ("X-ApiKey", config_file.api_key.try_into ().unwrap ()); @@ -255,3 +259,20 @@ pub async fn run_server ( Ok (()) } + +#[cfg (test)] +mod tests { + use super::*; + + #[test] + fn tripcode_algo () { + let config = ConfigFile { + name: "TestName".into (), + api_key: "PlaypenCausalPlatformCommodeImproveCatalyze".into (), + relay_url: "".into (), + file_server_root: None, + }; + + assert_eq! (config.tripcode (), "A9rPwZyY89Ag4TJjMoyYA2NeGOm99Je6rq1s0rg8PfY=".to_string ()); + } +} diff --git a/todo.md b/todo.md index 47e3694..1327803 100644 --- a/todo.md +++ b/todo.md @@ -1,4 +1,3 @@ -- Print tripcode without running the server - Reload relay config (or at least tripcodes) without downtime - "Preview as" feature for Markdown (It's not threaded through the relay yet) - Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) From a3e76cf120f04c8bed3f3052af19b0c0e5a2d463 Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 25 Nov 2020 00:16:14 +0000 Subject: [PATCH 048/208] :recycle: Clippy pass --- src/bin/ptth_file_server.rs | 4 +- src/load_toml.rs | 4 +- src/server/file_server.rs | 99 +++++++++++++++++++++++-------------- src/server/mod.rs | 4 +- 4 files changed, 69 insertions(+), 42 deletions(-) diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index 32a43b5..bbcfcf9 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -70,7 +70,7 @@ async fn handle_all (req: Request , state: Arc >) ptth_req.method, &ptth_req.uri, &ptth_req.headers, - state.hidden_path.as_ref ().map (|p| p.as_path ()) + state.hidden_path.as_deref () ).await; let mut resp = Response::builder () @@ -116,7 +116,7 @@ async fn main () -> Result <(), Box > { }, handlebars, server_info: crate::file_server::ServerInfo { - server_name: config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()).clone (), + server_name: config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()), }, hidden_path: Some (path), }); diff --git a/src/load_toml.rs b/src/load_toml.rs index 58f7159..921ce1f 100644 --- a/src/load_toml.rs +++ b/src/load_toml.rs @@ -30,7 +30,7 @@ pub fn load_public < > ( config_file_path: P ) -> T { - let mut f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); + let f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); load_inner (f) } @@ -45,7 +45,7 @@ pub fn load < ) -> T { use std::os::unix::fs::PermissionsExt; - let mut f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); + let f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); let mode = f.metadata ().unwrap ().permissions ().mode (); assert_eq! (mode, CONFIG_PERMISSIONS_MODE, "Config file has bad permissions mode, it should be octal 0600"); diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 2e3a688..c5f6e9a 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -146,6 +146,34 @@ fn check_range (range_str: Option <&str>, file_len: u64) PartialContent (start..end) } +fn get_icon (file_name: &str) -> &'static str { + // Because my editor actually doesn't render these + + let video = "🎞️"; + let picture = "📷"; + let file = "📄"; + + if + file_name.ends_with (".mp4") || + file_name.ends_with (".avi") || + file_name.ends_with (".mkv") || + file_name.ends_with (".webm") + { + video + } + else if + file_name.ends_with (".jpg") || + file_name.ends_with (".jpeg") || + file_name.ends_with (".png") || + file_name.ends_with (".bmp") + { + picture + } + else { + file + } +} + async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry { let file_name = match entry.file_name ().into_string () { @@ -174,37 +202,13 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let (trailing_slash, icon, size) = { let t = metadata.file_type (); + let icon_folder = "📁"; if t.is_dir () { - ("/", "📁", "".into ()) + ("/", icon_folder, "".into ()) } else { - let icon = if file_name.ends_with (".mp4") { - "🎞️" - } - else if file_name.ends_with (".avi") { - "🎞️" - } - else if file_name.ends_with (".mkv") { - "🎞️" - } - else if file_name.ends_with (".jpg") { - "📷" - } - else if file_name.ends_with (".jpeg") { - "📷" - } - else if file_name.ends_with (".png") { - "📷" - } - else if file_name.ends_with (".bmp") { - "📷" - } - else { - "📄" - }; - - ("", icon, pretty_print_bytes (metadata.len ()).into ()) + ("", get_icon (&file_name), pretty_print_bytes (metadata.len ()).into ()) } }; @@ -385,7 +389,7 @@ fn render_markdown (bytes: &[u8], out: &mut String) -> Result <(), MarkdownError use pulldown_cmark::{Parser, Options, html}; let markdown_input = match std::str::from_utf8 (bytes) { - Err (_) => return Err (MarkdownError::FileIsNotUtf8), + Err (_) => return Err (MarkdownError::NotUtf8), Ok (x) => x, }; @@ -430,9 +434,9 @@ struct ServeFileParams { #[derive (Debug, PartialEq)] enum MarkdownError { - FileIsTooBig, - FileIsNotMarkdown, - FileIsNotUtf8, + TooBig, + // NotMarkdown, + NotUtf8, } #[derive (Debug, PartialEq)] @@ -511,7 +515,7 @@ async fn internal_serve_all ( } } - let has_trailing_slash = path_s.is_empty () || path_s.ends_with ("/"); + let has_trailing_slash = path_s.is_empty () || path_s.ends_with ('/'); if let Ok (dir) = read_dir (&full_path).await { if ! has_trailing_slash { @@ -548,7 +552,7 @@ async fn internal_serve_all ( if uri.query () == Some ("as_markdown") { const MAX_BUF_SIZE: u32 = 1_000_000; if file_len > MAX_BUF_SIZE.try_into ().unwrap () { - MarkdownErr (MarkdownError::FileIsTooBig) + MarkdownErr (MarkdownError::TooBig) } else { let mut buffer = vec! [0u8; MAX_BUF_SIZE.try_into ().unwrap ()]; @@ -632,9 +636,9 @@ pub async fn serve_all ( range_requested, }) => serve_file (file.into_inner (), send_body, range, range_requested).await, MarkdownErr (e) => match e { - MarkdownError::FileIsTooBig => serve_error (StatusCode::InternalServerError, "File is too big to preview as Markdown"), - MarkdownError::FileIsNotMarkdown => serve_error (StatusCode::BadRequest, "File is not Markdown"), - MarkdownError::FileIsNotUtf8 => serve_error (StatusCode::BadRequest, "File is not UTF-8"), + MarkdownError::TooBig => serve_error (StatusCode::InternalServerError, "File is too big to preview as Markdown"), + //MarkdownError::NotMarkdown => serve_error (StatusCode::BadRequest, "File is not Markdown"), + MarkdownError::NotUtf8 => serve_error (StatusCode::BadRequest, "File is not UTF-8"), }, MarkdownPreview (s) => serve_html (s), } @@ -695,6 +699,29 @@ mod tests { StatusCode, }; + #[test] + fn icons () { + let video = "🎞️"; + let picture = "📷"; + let file = "📄"; + + for (input, expected) in vec! [ + ("copying_is_not_theft.mp4", video), + ("copying_is_not_theft.avi", video), + ("copying_is_not_theft.mkv", video), + ("copying_is_not_theft.webm", video), + ("lolcats.jpg", picture), + ("lolcats.jpeg", picture), + ("lolcats.png", picture), + ("lolcats.bmp", picture), + ("ptth.log", file), + ("README.md", file), + ("todo.txt", file), + ].into_iter () { + assert_eq! (super::get_icon (input), expected); + } + } + #[test] fn parse_range_header () { for (input, expected) in vec! [ diff --git a/src/server/mod.rs b/src/server/mod.rs index 334e8c8..61b59d7 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -70,7 +70,7 @@ async fn handle_req_resp <'a> ( parts.method, &parts.uri, &parts.headers, - state.hidden_path.as_ref ().map (|p| p.as_path ()) + state.hidden_path.as_deref () ).await; let mut resp_req = state.client @@ -139,7 +139,7 @@ pub async fn run_server ( ) -> Result <(), Box > { - let asset_root = asset_root.unwrap_or_else (|| PathBuf::new ()); + let asset_root = asset_root.unwrap_or_else (PathBuf::new); use std::convert::TryInto; From 7aafbba4d9a5af4305479fb63c5c8a5c822935e0 Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 25 Nov 2020 02:17:08 +0000 Subject: [PATCH 049/208] :heavy_plus_sign: Add "last seen" to server list --- Cargo.toml | 1 + handlebars/relay/relay_server_list.html | 48 +++++-- handlebars/server/file_server_dir.html | 14 +- src/relay/mod.rs | 178 ++++++++++++++++++++---- src/server/file_server.rs | 9 +- todo.md | 1 - 6 files changed, 196 insertions(+), 55 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2ced302..bfb8f78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ license = "AGPL-3.0" aho-corasick = "0.7.14" base64 = "0.12.3" blake3 = "0.3.7" +chrono = "0.4.19" ctrlc = { version = "3.1.7", features = [ "termination" ] } dashmap = "3.11.10" futures = "0.3.7" diff --git a/handlebars/relay/relay_server_list.html b/handlebars/relay/relay_server_list.html index 2c9b6a9..08d536f 100644 --- a/handlebars/relay/relay_server_list.html +++ b/handlebars/relay/relay_server_list.html @@ -6,12 +6,25 @@ body { font-family: sans-serif; } - .entry { - display: inline-block; - padding: 20px; - min-width: 50%; + td { + padding: 0px; } - .entry_list div:nth-child(odd) { + td > * { + padding: 20px; + display: block; + } + .entry { + + } + .grey { + color: #888; + text-align: right; + } + .entry_list { + width: 100%; + border-collapse: collapse; + } + tbody tr:nth-child(odd) { background-color: #ddd; } @@ -21,19 +34,28 @@

Server list

-
- {{#if servers}} + + + + + + + + + {{#each servers}} - + + + + {{/each}} + + +
NameLast seen
{{this.name}}{{this.last_seen}}
{{else}} - (No servers are running) + (No servers have reported since this relay started) {{/if}} -
- diff --git a/handlebars/server/file_server_dir.html b/handlebars/server/file_server_dir.html index 4d4cdc0..b7b5666 100644 --- a/handlebars/server/file_server_dir.html +++ b/handlebars/server/file_server_dir.html @@ -6,19 +6,25 @@ body { font-family: sans-serif; } - .entry { - display: inline-block; + td { + padding: 0; + } + td > * { padding: 10px; - width: 100%; + display: block; + } + .entry { text-decoration: none; } .grey { color: #888; + text-align: right; } .entry_list { width: 100%; + border-collapse: collapse; } - .entry_list tr:nth-child(even) { + tbody tr:nth-child(odd) { background-color: #ddd; } diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 0b44369..90a90c3 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -127,12 +127,32 @@ impl From <&ConfigFile> for Config { } } +use chrono::{ + DateTime, + SecondsFormat, + Utc +}; + +#[derive (Clone)] +pub struct ServerStatus { + last_seen: DateTime , +} + +impl Default for ServerStatus { + fn default () -> Self { + Self { + last_seen: Utc::now (), + } + } +} + pub struct RelayState { config: Config, handlebars: Arc >, // Key: Server ID request_rendezvous: Mutex >, + server_status: Mutex >, // Key: Request ID response_rendezvous: RwLock >, @@ -149,6 +169,7 @@ impl From <&ConfigFile> for RelayState { config: Config::from (config_file), handlebars: Arc::new (load_templates (&PathBuf::new ()).unwrap ()), request_rendezvous: Default::default (), + server_status: Default::default (), response_rendezvous: Default::default (), shutdown_watch_tx, shutdown_watch_rx, @@ -205,6 +226,16 @@ async fn handle_http_listen ( return trip_error; } + // End of early returns + + { + let mut server_status = state.server_status.lock ().await; + + let mut status = server_status.entry (watcher_code.clone ()).or_insert_with (Default::default); + + status.last_seen = Utc::now (); + } + use RequestRendezvous::*; let (tx, rx) = oneshot::channel (); @@ -454,6 +485,99 @@ async fn handle_http_request ( } } +#[derive (Debug, PartialEq)] +enum LastSeen { + Negative, + Connected, + Description (String), +} + +// Mnemonic is "now - last_seen" + +fn pretty_print_last_seen ( + now: DateTime , + last_seen: DateTime +) -> LastSeen +{ + use LastSeen::*; + + let dur = now.signed_duration_since (last_seen); + + if dur < chrono::Duration::zero () { + return Negative; + } + + if dur.num_minutes () < 1 { + return Connected; + } + + if dur.num_hours () < 1 { + return Description (format! ("{} m ago", dur.num_minutes ())); + } + + if dur.num_days () < 1 { + return Description (format! ("{} h ago", dur.num_hours ())); + } + + Description (last_seen.to_rfc3339_opts (SecondsFormat::Secs, true)) +} + + +async fn handle_server_list ( + state: Arc +) -> Response +{ + use std::borrow::Cow; + + #[derive (Serialize)] + struct ServerEntry <'a> { + path: String, + name: String, + last_seen: Cow <'a, str>, + } + + #[derive (Serialize)] + struct ServerListPage <'a> { + servers: Vec >, + } + + let servers = { + let guard = state.server_status.lock ().await; + (*guard).clone () + }; + + let now = Utc::now (); + + let mut servers: Vec <_> = servers.into_iter () + .map (|(name, server)| { + let display_name = percent_encoding::percent_decode_str (&name).decode_utf8 ().unwrap_or_else (|_| "Server name isn't UTF-8".into ()).to_string (); + + use LastSeen::*; + + let last_seen = match pretty_print_last_seen (now, server.last_seen) { + Negative => "Error (negative time)".into (), + Connected => "Connected".into (), + Description (s) => s.into (), + }; + + ServerEntry { + name: display_name, + path: name, + last_seen: last_seen, + } + }) + .collect (); + + servers.sort_by (|a, b| a.name.cmp (&b.name)); + + let page = ServerListPage { + servers, + }; + + let s = state.handlebars.render ("relay_server_list", &page).unwrap (); + ok_reply (s) +} + #[instrument (level = "trace", skip (req, state))] async fn handle_all (req: Request , state: Arc ) -> Result , Infallible> @@ -487,35 +611,7 @@ async fn handle_all (req: Request , state: Arc ) } else if let Some (rest) = prefix_match ("/frontend/servers/", path) { if rest == "" { - use std::borrow::Cow; - - #[derive (Serialize)] - struct ServerEntry <'a> { - path: &'a str, - name: Cow <'a, str>, - } - - #[derive (Serialize)] - struct ServerListPage <'a> { - servers: Vec >, - } - - let mut names = state.list_servers ().await; - names.sort (); - - //println! ("Found {} servers", names.len ()); - - let page = ServerListPage { - servers: names.iter () - .map (|name| ServerEntry { - name: percent_encoding::percent_decode_str (name).decode_utf8 ().unwrap_or_else (|_| "Server name isn't UTF-8".into ()), - path: &name, - }) - .collect (), - }; - - let s = state.handlebars.render ("relay_server_list", &page).unwrap (); - ok_reply (s) + handle_server_list (state).await } else if let Some (idx) = rest.find ('/') { let listen_code = String::from (&rest [0..idx]); @@ -635,5 +731,29 @@ pub async fn run_relay ( #[cfg (test)] mod tests { + use super::*; + #[test] + fn test_pretty_print_last_seen () { + use LastSeen::*; + + let last_seen = DateTime::parse_from_rfc3339 ("2019-05-29T00:00:00+00:00").unwrap ().with_timezone (&Utc); + + for (input, expected) in vec! [ + ("2019-05-28T23:59:59+00:00", Negative), + ("2019-05-29T00:00:00+00:00", Connected), + ("2019-05-29T00:00:59+00:00", Connected), + ("2019-05-29T00:01:30+00:00", Description ("1 m ago".into ())), + ("2019-05-29T00:59:30+00:00", Description ("59 m ago".into ())), + ("2019-05-29T01:00:30+00:00", Description ("1 h ago".into ())), + ("2019-05-29T10:00:00+00:00", Description ("10 h ago".into ())), + ("2019-05-30T00:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), + ("2019-05-30T10:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), + ("2019-05-31T00:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), + ].into_iter () { + let now = DateTime::parse_from_rfc3339 (input).unwrap ().with_timezone (&Utc); + let actual = pretty_print_last_seen (now, last_seen); + assert_eq! (actual, expected); + } + } } diff --git a/src/server/file_server.rs b/src/server/file_server.rs index c5f6e9a..1e05bbc 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -685,20 +685,13 @@ mod tests { ffi::OsStr, path::{ Component, - Path, - PathBuf + Path, }, }; use maplit::*; use tokio::runtime::Runtime; - use always_equal::test::AlwaysEqual; - - use crate::http_serde::{ - StatusCode, - }; - #[test] fn icons () { let video = "🎞️"; diff --git a/todo.md b/todo.md index 1327803..eb775f8 100644 --- a/todo.md +++ b/todo.md @@ -10,7 +10,6 @@ - ETag cache based on mtime - Server-side hash? - Log / audit log? -- Add "Last check-in time" to server list - Prevent directory traversal attacks in file_server.rs - Error handling From b40233cc62d44673475623d99f8547f13a21bb48 Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 25 Nov 2020 02:30:57 +0000 Subject: [PATCH 050/208] :recycle: Wrap relay config in a RwLock --- src/relay/mod.rs | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 90a90c3..4af0314 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -147,7 +147,7 @@ impl Default for ServerStatus { } pub struct RelayState { - config: Config, + config: RwLock , handlebars: Arc >, // Key: Server ID @@ -166,7 +166,7 @@ impl From <&ConfigFile> for RelayState { let (shutdown_watch_tx, shutdown_watch_rx) = watch::channel (false); Self { - config: Config::from (config_file), + config: Config::from (config_file).into (), handlebars: Arc::new (load_templates (&PathBuf::new ()).unwrap ()), request_rendezvous: Default::default (), server_status: Default::default (), @@ -212,16 +212,20 @@ async fn handle_http_listen ( { let trip_error = error_reply (StatusCode::UNAUTHORIZED, "Bad X-ApiKey"); - let expected_tripcode = match state.config.server_tripcodes.get (&watcher_code) { - None => { - error! ("Denied http_listen for non-existent server name {}", watcher_code); - return trip_error; - }, - Some (x) => x, + let expected_tripcode = { + let config = state.config.read ().await; + + match config.server_tripcodes.get (&watcher_code) { + None => { + error! ("Denied http_listen for non-existent server name {}", watcher_code); + return trip_error; + }, + Some (x) => (*x).clone (), + } }; let actual_tripcode = blake3::hash (api_key); - if expected_tripcode != &actual_tripcode { + if expected_tripcode != actual_tripcode { error! ("Denied http_listen for bad tripcode {}", base64::encode (actual_tripcode.as_bytes ())); return trip_error; } @@ -382,8 +386,11 @@ async fn handle_http_request ( ) -> Response { - if ! state.config.server_tripcodes.contains_key (&watcher_code) { - return error_reply (StatusCode::NOT_FOUND, "Unknown server"); + { + let config = state.config.read ().await; + if ! config.server_tripcodes.contains_key (&watcher_code) { + return error_reply (StatusCode::NOT_FOUND, "Unknown server"); + } } let req = match http_serde::RequestParts::from_hyper (req.method, uri, req.headers) { @@ -669,16 +676,16 @@ pub async fn run_relay ( { let mut tripcode_set = HashSet::new (); - - for (_, v) in state.config.server_tripcodes.iter () { + let config = state.config.read ().await; + for (_, v) in config.server_tripcodes.iter () { if ! tripcode_set.insert (v) { panic! ("Two servers have the same tripcode. That is not allowed."); } } + + info! ("Loaded {} server tripcodes", config.server_tripcodes.len ()); } - info! ("Loaded {} server tripcodes", state.config.server_tripcodes.len ()); - let make_svc = make_service_fn (|_conn| { let state = state.clone (); From 8369dc86751d0476764086f50998369c60cde476 Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 25 Nov 2020 03:09:21 +0000 Subject: [PATCH 051/208] Reload relay config every minute --- src/bin/ptth_relay.rs | 7 +++++-- src/lib.rs | 2 +- src/relay/mod.rs | 48 ++++++++++++++++++++++++++++++++----------- todo.md | 1 - 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index a525580..cd78d21 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -1,5 +1,6 @@ use std::{ error::Error, + path::PathBuf, sync::Arc, }; @@ -21,7 +22,8 @@ async fn main () -> Result <(), Box > { .init () ; - let config_file = ptth::load_toml::load_public ("config/ptth_relay.toml"); + let config_path = PathBuf::from ("config/ptth_relay.toml"); + let config_file = ptth::load_toml::load_public (&config_path); info! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); @@ -30,7 +32,8 @@ async fn main () -> Result <(), Box > { forced_shutdown.wrap_server ( relay::run_relay ( Arc::new (RelayState::from (&config_file)), - shutdown_rx + shutdown_rx, + Some (config_path) ) ).await??; diff --git a/src/lib.rs b/src/lib.rs index d0551bb..bc89e25 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -116,7 +116,7 @@ mod tests { let relay_state_2 = relay_state.clone (); let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); let task_relay = spawn (async move { - relay::run_relay (relay_state_2, stop_relay_rx).await.unwrap (); + relay::run_relay (relay_state_2, stop_relay_rx, None).await.unwrap (); }); assert! (relay_state.list_servers ().await.is_empty ()); diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 4af0314..6fbe3ca 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -663,9 +663,35 @@ pub fn load_templates (asset_root: &Path) Ok (handlebars) } +async fn reload_config ( + state: &Arc , + config_reload_path: &Path +) -> Option <()> { + use tokio::prelude::*; + + let mut f = tokio::fs::File::open (config_reload_path).await.ok ()?; + + let mut buffer = vec! [0u8; 4096]; + let bytes_read = f.read (&mut buffer).await.ok ()?; + buffer.truncate (bytes_read); + + let config_s = String::from_utf8 (buffer).ok ()?; + let new_config: ConfigFile = toml::from_str (&config_s).ok ()?; + + let new_config = Config::from (&new_config); + + let mut config = state.config.write ().await; + (*config) = new_config; + + debug! ("Loaded {} server tripcodes", config.server_tripcodes.len ()); + + Some (()) +} + pub async fn run_relay ( state: Arc , - shutdown_oneshot: oneshot::Receiver <()> + shutdown_oneshot: oneshot::Receiver <()>, + config_reload_path: Option ) -> Result <(), Box > { @@ -674,16 +700,16 @@ pub async fn run_relay ( 4000, )); - { - let mut tripcode_set = HashSet::new (); - let config = state.config.read ().await; - for (_, v) in config.server_tripcodes.iter () { - if ! tripcode_set.insert (v) { - panic! ("Two servers have the same tripcode. That is not allowed."); + if let Some (config_reload_path) = config_reload_path { + let state_2 = state.clone (); + tokio::spawn (async move { + let mut reload_interval = tokio::time::interval (Duration::from_secs (60)); + + loop { + reload_interval.tick ().await; + reload_config (&state_2, &config_reload_path).await; } - } - - info! ("Loaded {} server tripcodes", config.server_tripcodes.len ()); + }); } let make_svc = make_service_fn (|_conn| { @@ -701,8 +727,6 @@ pub async fn run_relay ( let server = Server::bind (&addr) .serve (make_svc); - - server.with_graceful_shutdown (async { shutdown_oneshot.await.ok (); diff --git a/todo.md b/todo.md index eb775f8..e100cd8 100644 --- a/todo.md +++ b/todo.md @@ -1,4 +1,3 @@ -- Reload relay config (or at least tripcodes) without downtime - "Preview as" feature for Markdown (It's not threaded through the relay yet) - Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) - Make a debug client to replicate the issue Firefox is having with turtling From bf4e5c7a5be7d6a67a83dd5cb6cfbda8fcf07a9c Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 26 Nov 2020 21:50:55 +0000 Subject: [PATCH 052/208] :recycle: Planning changes to relay config --- src/relay/mod.rs | 59 ++++++++++++++++++++++++++++++------------------ todo.md | 3 +++ 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 6fbe3ca..9b9171e 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Cow, error::Error, collections::*, convert::Infallible, @@ -94,10 +95,16 @@ type ResponseRendezvous = oneshot::Sender , + pub display_names: HashMap , +} + #[derive (Default, Deserialize)] pub struct ConfigFile { pub port: Option , - pub server_tripcodes: HashMap , + pub servers: ConfigServers, } // Stuff we actually need at runtime @@ -108,7 +115,7 @@ struct Config { impl From <&ConfigFile> for Config { fn from (f: &ConfigFile) -> Self { - let server_tripcodes = HashMap::from_iter (f.server_tripcodes.iter () + let server_tripcodes = HashMap::from_iter (f.servers.tripcodes.iter () .map (|(k, v)| { use std::convert::TryInto; let bytes: Vec = base64::decode (v).unwrap (); @@ -529,33 +536,34 @@ fn pretty_print_last_seen ( Description (last_seen.to_rfc3339_opts (SecondsFormat::Secs, true)) } +#[derive (Serialize)] +struct ServerEntry <'a> { + path: String, + name: String, + last_seen: Cow <'a, str>, +} -async fn handle_server_list ( - state: Arc -) -> Response +#[derive (Serialize)] +struct ServerListPage <'a> { + servers: Vec >, +} + +async fn handle_server_list_internal (state: &Arc ) +-> ServerListPage <'static> { - use std::borrow::Cow; + let all_servers: Vec <_> = { + let guard = state.config.read ().await; + (*guard).server_tripcodes.keys ().cloned ().collect () + }; - #[derive (Serialize)] - struct ServerEntry <'a> { - path: String, - name: String, - last_seen: Cow <'a, str>, - } - - #[derive (Serialize)] - struct ServerListPage <'a> { - servers: Vec >, - } - - let servers = { + let server_status = { let guard = state.server_status.lock ().await; (*guard).clone () }; let now = Utc::now (); - let mut servers: Vec <_> = servers.into_iter () + let mut servers: Vec <_> = server_status.into_iter () .map (|(name, server)| { let display_name = percent_encoding::percent_decode_str (&name).decode_utf8 ().unwrap_or_else (|_| "Server name isn't UTF-8".into ()).to_string (); @@ -577,9 +585,16 @@ async fn handle_server_list ( servers.sort_by (|a, b| a.name.cmp (&b.name)); - let page = ServerListPage { + ServerListPage { servers, - }; + } +} + +async fn handle_server_list ( + state: Arc +) -> Response +{ + let page = handle_server_list_internal (&state).await; let s = state.handlebars.render ("relay_server_list", &page).unwrap (); ok_reply (s) diff --git a/todo.md b/todo.md index e100cd8..d2f7013 100644 --- a/todo.md +++ b/todo.md @@ -1,3 +1,6 @@ +- Server list should include offline or never-launched servers +- Allow relay to rename servers +- Estimate bandwidth per server? - "Preview as" feature for Markdown (It's not threaded through the relay yet) - Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) - Make a debug client to replicate the issue Firefox is having with turtling From 28ce6a32cd03d9c4961033da6bc2c78988294be2 Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 26 Nov 2020 23:30:33 +0000 Subject: [PATCH 053/208] :lipstick: Show servers in the server list even if they aren't connected yet --- Cargo.toml | 3 +- handlebars/relay/relay_server_list.html | 2 +- src/bin/ptth_relay.rs | 9 +- src/lib.rs | 12 ++- src/relay/config.rs | 108 ++++++++++++++++++++ src/relay/mod.rs | 125 +++++++++--------------- todo.md | 2 - 7 files changed, 171 insertions(+), 90 deletions(-) create mode 100644 src/relay/config.rs diff --git a/Cargo.toml b/Cargo.toml index bfb8f78..e85a51d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ futures = "0.3.7" handlebars = "3.5.1" http = "0.2.1" hyper = "0.13.8" +itertools = "0.9.0" lazy_static = "1.4.0" maplit = "1.0.2" percent-encoding = "2.1.0" @@ -30,7 +31,7 @@ reqwest = { version = "0.10.8", features = ["stream"] } rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} structopt = "0.3.20" -# thiserror = "1.0.22" +thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" tracing-futures = "0.2.4" diff --git a/handlebars/relay/relay_server_list.html b/handlebars/relay/relay_server_list.html index 08d536f..3fd8a8a 100644 --- a/handlebars/relay/relay_server_list.html +++ b/handlebars/relay/relay_server_list.html @@ -46,7 +46,7 @@ {{#each servers}} - {{this.name}} + {{this.display_name}} {{this.last_seen}} {{/each}} diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index cd78d21..ecf8867 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -12,7 +12,10 @@ use tracing_subscriber::{ }; use ptth::relay; -use ptth::relay::RelayState; +use ptth::relay::{ + Config, + RelayState, +}; #[tokio::main] async fn main () -> Result <(), Box > { @@ -23,7 +26,7 @@ async fn main () -> Result <(), Box > { ; let config_path = PathBuf::from ("config/ptth_relay.toml"); - let config_file = ptth::load_toml::load_public (&config_path); + let config = Config::from_file (&config_path).await?; info! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); @@ -31,7 +34,7 @@ async fn main () -> Result <(), Box > { forced_shutdown.wrap_server ( relay::run_relay ( - Arc::new (RelayState::from (&config_file)), + Arc::new (RelayState::from (config)), shutdown_rx, Some (config_path) ) diff --git a/src/lib.rs b/src/lib.rs index bc89e25..9877e04 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,6 +43,7 @@ pub fn password_is_bad (mut password: String) -> bool { #[cfg (test)] mod tests { use std::{ + convert::TryFrom, sync::{ Arc, }, @@ -104,14 +105,17 @@ mod tests { let api_key = "AnacondaHardcoverGrannyUnlatchLankinessMutate"; let tripcode = base64::encode (blake3::hash (api_key.as_bytes ()).as_bytes ()); debug! ("Relay is expecting tripcode {}", tripcode); - let config_file = relay::ConfigFile { + let config_file = relay::config::file::Config { port: None, - server_tripcodes: hashmap! { - server_name.into () => tripcode, + servers: hashmap! { + server_name.into () => relay::config::file::Server { + tripcode, + display_name: None, + }, }, }; - let relay_state = Arc::new (relay::RelayState::from (&config_file)); + let relay_state = Arc::new (relay::RelayState::from (relay::config::Config::try_from (config_file).unwrap ())); let relay_state_2 = relay_state.clone (); let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); diff --git a/src/relay/config.rs b/src/relay/config.rs new file mode 100644 index 0000000..d48b2df --- /dev/null +++ b/src/relay/config.rs @@ -0,0 +1,108 @@ +use std::{ + collections::*, + convert::{TryFrom, TryInto}, + iter::FromIterator, + path::Path, +}; + +use serde::Deserialize; +use thiserror::Error; + +#[derive (Error, Debug)] +pub enum ConfigError { + #[error ("I/O error")] + Io (#[from] std::io::Error), + + #[error ("UTF-8 decoding failed")] + Utf8 (#[from] std::string::FromUtf8Error), + + #[error ("TOML parsing failed")] + Toml (#[from] toml::de::Error), + + #[error ("base64 decoding failed")] + Base64Decode (#[from] base64::DecodeError), + + #[error ("tripcode not 32 bytes after decoding")] + TripcodeBadLength, + + #[error ("unknown config error")] + Unknown, +} + +// Stuff we need to load from the config file and use to +// set up the HTTP server + +pub mod file { + use super::*; + + #[derive (Deserialize)] + pub struct Server { + pub tripcode: String, + pub display_name: Option , + } + + #[derive (Deserialize)] + pub struct Config { + pub port: Option , + pub servers: HashMap , + } +} + +// Stuff we actually need at runtime + +pub struct Server { + pub tripcode: blake3::Hash, + pub display_name: Option , +} + +pub struct Config { + pub servers: HashMap , +} + +impl TryFrom for Server { + type Error = ConfigError; + + fn try_from (f: file::Server) -> Result { + let bytes: Vec = base64::decode (f.tripcode)?; + let bytes: [u8; 32] = (&bytes [..]).try_into ().map_err (|_| ConfigError::TripcodeBadLength)?; + + let tripcode = blake3::Hash::from (bytes); + + Ok (Self { + tripcode, + display_name: f.display_name, + }) + } +} + +impl TryFrom for Config { + type Error = ConfigError; + + fn try_from (f: file::Config) -> Result { + let servers = f.servers.into_iter () + .map (|(k, v)| Ok::<_, ConfigError> ((k, v.try_into ()?))); + + let servers = itertools::process_results (servers, |i| HashMap::from_iter (i))?; + + Ok (Self { + servers, + }) + } +} + +impl Config { + pub async fn from_file (path: &Path) -> Result { + use tokio::prelude::*; + + let mut f = tokio::fs::File::open (path).await?; + + let mut buffer = vec! [0u8; 4096]; + let bytes_read = f.read (&mut buffer).await?; + buffer.truncate (bytes_read); + + let config_s = String::from_utf8 (buffer)?; + let new_config: file::Config = toml::from_str (&config_s)?; + + Self::try_from (new_config) + } +} diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 9b9171e..11cfdd5 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -27,7 +27,6 @@ use hyper::{ }; use hyper::service::{make_service_fn, service_fn}; use serde::{ - Deserialize, Serialize, }; use tokio::{ @@ -51,6 +50,10 @@ use crate::{ prefix_match, }; +pub mod config; + +pub use config::{Config, ConfigError}; + /* Here's what we need to handle: @@ -92,48 +95,6 @@ enum RequestRendezvous { type ResponseRendezvous = oneshot::Sender >; -// Stuff we need to load from the config file and use to -// set up the HTTP server - -#[derive (Default, Deserialize)] -pub struct ConfigServers { - pub tripcodes: HashMap , - pub display_names: HashMap , -} - -#[derive (Default, Deserialize)] -pub struct ConfigFile { - pub port: Option , - pub servers: ConfigServers, -} - -// Stuff we actually need at runtime - -struct Config { - server_tripcodes: HashMap , -} - -impl From <&ConfigFile> for Config { - fn from (f: &ConfigFile) -> Self { - let server_tripcodes = HashMap::from_iter (f.servers.tripcodes.iter () - .map (|(k, v)| { - use std::convert::TryInto; - let bytes: Vec = base64::decode (v).unwrap (); - let bytes: [u8; 32] = (&bytes [..]).try_into ().unwrap (); - - let v = blake3::Hash::from (bytes); - - debug! ("Tripcode {} => {}", k, v.to_hex ()); - - (k.clone (), v) - })); - - Self { - server_tripcodes, - } - } -} - use chrono::{ DateTime, SecondsFormat, @@ -168,12 +129,12 @@ pub struct RelayState { shutdown_watch_rx: watch::Receiver , } -impl From <&ConfigFile> for RelayState { - fn from (config_file: &ConfigFile) -> Self { +impl From for RelayState { + fn from (config: Config) -> Self { let (shutdown_watch_tx, shutdown_watch_rx) = watch::channel (false); Self { - config: Config::from (config_file).into (), + config: config.into (), handlebars: Arc::new (load_templates (&PathBuf::new ()).unwrap ()), request_rendezvous: Default::default (), server_status: Default::default (), @@ -222,12 +183,12 @@ async fn handle_http_listen ( let expected_tripcode = { let config = state.config.read ().await; - match config.server_tripcodes.get (&watcher_code) { + match config.servers.get (&watcher_code) { None => { error! ("Denied http_listen for non-existent server name {}", watcher_code); return trip_error; }, - Some (x) => (*x).clone (), + Some (x) => (*x).tripcode.clone (), } }; let actual_tripcode = blake3::hash (api_key); @@ -395,7 +356,7 @@ async fn handle_http_request ( { { let config = state.config.read ().await; - if ! config.server_tripcodes.contains_key (&watcher_code) { + if ! config.servers.contains_key (&watcher_code) { return error_reply (StatusCode::NOT_FOUND, "Unknown server"); } } @@ -538,8 +499,8 @@ fn pretty_print_last_seen ( #[derive (Serialize)] struct ServerEntry <'a> { - path: String, - name: String, + id: String, + display_name: String, last_seen: Cow <'a, str>, } @@ -551,39 +512,56 @@ struct ServerListPage <'a> { async fn handle_server_list_internal (state: &Arc ) -> ServerListPage <'static> { - let all_servers: Vec <_> = { + let display_names: HashMap = { let guard = state.config.read ().await; - (*guard).server_tripcodes.keys ().cloned ().collect () + + let servers = (*guard).servers.iter () + .map (|(k, v)| { + let display_name = v.display_name + .clone () + .unwrap_or_else (|| k.clone ()); + + (k.clone (), display_name) + }); + + HashMap::from_iter (servers) }; - let server_status = { + let server_statuses = { let guard = state.server_status.lock ().await; (*guard).clone () }; let now = Utc::now (); - let mut servers: Vec <_> = server_status.into_iter () - .map (|(name, server)| { - let display_name = percent_encoding::percent_decode_str (&name).decode_utf8 ().unwrap_or_else (|_| "Server name isn't UTF-8".into ()).to_string (); - + let mut servers: Vec <_> = display_names.into_iter () + .map (|(id, display_name)| { use LastSeen::*; - let last_seen = match pretty_print_last_seen (now, server.last_seen) { + let status = match server_statuses.get (&id) { + None => return ServerEntry { + display_name, + id, + last_seen: "Never".into (), + }, + Some (x) => x, + }; + + let last_seen = match pretty_print_last_seen (now, status.last_seen) { Negative => "Error (negative time)".into (), Connected => "Connected".into (), Description (s) => s.into (), }; ServerEntry { - name: display_name, - path: name, - last_seen: last_seen, + display_name, + id, + last_seen, } }) .collect (); - servers.sort_by (|a, b| a.name.cmp (&b.name)); + servers.sort_by (|a, b| a.display_name.cmp (&b.display_name)); ServerListPage { servers, @@ -681,26 +659,15 @@ pub fn load_templates (asset_root: &Path) async fn reload_config ( state: &Arc , config_reload_path: &Path -) -> Option <()> { - use tokio::prelude::*; - - let mut f = tokio::fs::File::open (config_reload_path).await.ok ()?; - - let mut buffer = vec! [0u8; 4096]; - let bytes_read = f.read (&mut buffer).await.ok ()?; - buffer.truncate (bytes_read); - - let config_s = String::from_utf8 (buffer).ok ()?; - let new_config: ConfigFile = toml::from_str (&config_s).ok ()?; - - let new_config = Config::from (&new_config); +) -> Result <(), ConfigError> { + let new_config = Config::from_file (config_reload_path).await?; let mut config = state.config.write ().await; (*config) = new_config; - debug! ("Loaded {} server tripcodes", config.server_tripcodes.len ()); + debug! ("Loaded {} server configs", config.servers.len ()); - Some (()) + Ok (()) } pub async fn run_relay ( @@ -722,7 +689,7 @@ pub async fn run_relay ( loop { reload_interval.tick ().await; - reload_config (&state_2, &config_reload_path).await; + reload_config (&state_2, &config_reload_path).await.ok (); } }); } diff --git a/todo.md b/todo.md index d2f7013..02488b8 100644 --- a/todo.md +++ b/todo.md @@ -1,5 +1,3 @@ -- Server list should include offline or never-launched servers -- Allow relay to rename servers - Estimate bandwidth per server? - "Preview as" feature for Markdown (It's not threaded through the relay yet) - Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) From 7c2ce6586475a6fa7d432da047ef8ce669a66238 Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 26 Nov 2020 23:33:10 +0000 Subject: [PATCH 054/208] :recycle: Extract src/tests.rs --- src/lib.rs | 156 +-------------------------------------------------- src/tests.rs | 153 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 155 deletions(-) create mode 100644 src/tests.rs diff --git a/src/lib.rs b/src/lib.rs index 9877e04..da7e290 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,158 +41,4 @@ pub fn password_is_bad (mut password: String) -> bool { } #[cfg (test)] -mod tests { - use std::{ - convert::TryFrom, - sync::{ - Arc, - }, - time::Duration, - }; - - use tokio::{ - runtime::Runtime, - spawn, - sync::oneshot, - time::delay_for, - }; - - use super::{ - relay, - server, - }; - - #[test] - fn check_bad_passwords () { - use crate::password_is_bad; - - for pw in &[ - "", - " ", - "user", - "password", - "pAsSwOrD", - "secret", - "123123", - ] { - assert! (password_is_bad (pw.to_string ())); - } - - use rand::prelude::*; - - let mut entropy = [0u8; 32]; - thread_rng ().fill_bytes (&mut entropy); - let good_password = base64::encode (entropy); - - assert! (! password_is_bad (good_password)); - } - - #[test] - fn end_to_end () { - use maplit::*; - use reqwest::Client; - use tracing::{debug, info}; - - // Prefer this form for tests, since all tests share one process - // and we don't care if another test already installed a subscriber. - - tracing_subscriber::fmt ().try_init ().ok (); - let mut rt = Runtime::new ().unwrap (); - - // Spawn the root task - rt.block_on (async { - let server_name = "aliens_wildland"; - let api_key = "AnacondaHardcoverGrannyUnlatchLankinessMutate"; - let tripcode = base64::encode (blake3::hash (api_key.as_bytes ()).as_bytes ()); - debug! ("Relay is expecting tripcode {}", tripcode); - let config_file = relay::config::file::Config { - port: None, - servers: hashmap! { - server_name.into () => relay::config::file::Server { - tripcode, - display_name: None, - }, - }, - }; - - let relay_state = Arc::new (relay::RelayState::from (relay::config::Config::try_from (config_file).unwrap ())); - - let relay_state_2 = relay_state.clone (); - let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); - let task_relay = spawn (async move { - relay::run_relay (relay_state_2, stop_relay_rx, None).await.unwrap (); - }); - - assert! (relay_state.list_servers ().await.is_empty ()); - - let relay_url = "http://127.0.0.1:4000"; - - let config_file = server::ConfigFile { - name: server_name.into (), - api_key: api_key.into (), - relay_url: "http://127.0.0.1:4000/7ZSFUKGV".into (), - file_server_root: None, - }; - - let (stop_server_tx, stop_server_rx) = oneshot::channel (); - let task_server = { - spawn (async move { - server::run_server (config_file, stop_server_rx, None, None).await.unwrap (); - }) - }; - - delay_for (Duration::from_millis (500)).await; - - assert_eq! (relay_state.list_servers ().await, vec! [ - server_name.to_string (), - ]); - - let client = Client::builder () - .timeout (Duration::from_secs (2)) - .build ().unwrap (); - - let resp = client.get (&format! ("{}/frontend/relay_up_check", relay_url)) - .send ().await.unwrap ().bytes ().await.unwrap (); - - assert_eq! (resp, "Relay is up\n"); - - let resp = client.get (&format! ("{}/frontend/servers/{}/files/COPYING", relay_url, server_name)) - .send ().await.unwrap ().bytes ().await.unwrap (); - - if blake3::hash (&resp) != blake3::Hash::from ([ - 0xca, 0x02, 0x92, 0x78, - 0x9c, 0x0a, 0x0e, 0xcb, - 0xa7, 0x06, 0xf4, 0xb3, - 0xf3, 0x49, 0x30, 0x07, - - 0xa9, 0x95, 0x17, 0x31, - 0xc1, 0xd4, 0x32, 0xc5, - 0x2c, 0x4a, 0xac, 0x1f, - 0x1a, 0xbb, 0xa8, 0xef, - ]) { - panic! ("{}", String::from_utf8 (resp.to_vec ()).unwrap ()); - } - - // Requesting a file from a server that isn't registered - // will error out - - let resp = client.get (&format! ("{}/frontend/servers/obviously_this_server_does_not_exist/files/COPYING", relay_url)) - .send ().await.unwrap (); - - assert_eq! (resp.status (), reqwest::StatusCode::NOT_FOUND); - - info! ("Shutting down end-to-end test"); - - stop_server_tx.send (()).unwrap (); - stop_relay_tx.send (()).unwrap (); - - info! ("Sent stop messages"); - - task_relay.await.unwrap (); - info! ("Relay stopped"); - - task_server.await.unwrap (); - info! ("Server stopped"); - }); - } -} +mod tests; diff --git a/src/tests.rs b/src/tests.rs new file mode 100644 index 0000000..529a33d --- /dev/null +++ b/src/tests.rs @@ -0,0 +1,153 @@ +use std::{ + convert::TryFrom, + sync::{ + Arc, + }, + time::Duration, +}; + +use tokio::{ + runtime::Runtime, + spawn, + sync::oneshot, + time::delay_for, +}; + +use super::{ + relay, + server, +}; + +#[test] +fn check_bad_passwords () { + use crate::password_is_bad; + + for pw in &[ + "", + " ", + "user", + "password", + "pAsSwOrD", + "secret", + "123123", + ] { + assert! (password_is_bad (pw.to_string ())); + } + + use rand::prelude::*; + + let mut entropy = [0u8; 32]; + thread_rng ().fill_bytes (&mut entropy); + let good_password = base64::encode (entropy); + + assert! (! password_is_bad (good_password)); +} + +#[test] +fn end_to_end () { + use maplit::*; + use reqwest::Client; + use tracing::{debug, info}; + + // Prefer this form for tests, since all tests share one process + // and we don't care if another test already installed a subscriber. + + tracing_subscriber::fmt ().try_init ().ok (); + let mut rt = Runtime::new ().unwrap (); + + // Spawn the root task + rt.block_on (async { + let server_name = "aliens_wildland"; + let api_key = "AnacondaHardcoverGrannyUnlatchLankinessMutate"; + let tripcode = base64::encode (blake3::hash (api_key.as_bytes ()).as_bytes ()); + debug! ("Relay is expecting tripcode {}", tripcode); + let config_file = relay::config::file::Config { + port: None, + servers: hashmap! { + server_name.into () => relay::config::file::Server { + tripcode, + display_name: None, + }, + }, + }; + + let relay_state = Arc::new (relay::RelayState::from (relay::config::Config::try_from (config_file).unwrap ())); + + let relay_state_2 = relay_state.clone (); + let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); + let task_relay = spawn (async move { + relay::run_relay (relay_state_2, stop_relay_rx, None).await.unwrap (); + }); + + assert! (relay_state.list_servers ().await.is_empty ()); + + let relay_url = "http://127.0.0.1:4000"; + + let config_file = server::ConfigFile { + name: server_name.into (), + api_key: api_key.into (), + relay_url: "http://127.0.0.1:4000/7ZSFUKGV".into (), + file_server_root: None, + }; + + let (stop_server_tx, stop_server_rx) = oneshot::channel (); + let task_server = { + spawn (async move { + server::run_server (config_file, stop_server_rx, None, None).await.unwrap (); + }) + }; + + delay_for (Duration::from_millis (500)).await; + + assert_eq! (relay_state.list_servers ().await, vec! [ + server_name.to_string (), + ]); + + let client = Client::builder () + .timeout (Duration::from_secs (2)) + .build ().unwrap (); + + let resp = client.get (&format! ("{}/frontend/relay_up_check", relay_url)) + .send ().await.unwrap ().bytes ().await.unwrap (); + + assert_eq! (resp, "Relay is up\n"); + + let resp = client.get (&format! ("{}/frontend/servers/{}/files/COPYING", relay_url, server_name)) + .send ().await.unwrap ().bytes ().await.unwrap (); + + if blake3::hash (&resp) != blake3::Hash::from ([ + 0xca, 0x02, 0x92, 0x78, + 0x9c, 0x0a, 0x0e, 0xcb, + 0xa7, 0x06, 0xf4, 0xb3, + 0xf3, 0x49, 0x30, 0x07, + + 0xa9, 0x95, 0x17, 0x31, + 0xc1, 0xd4, 0x32, 0xc5, + 0x2c, 0x4a, 0xac, 0x1f, + 0x1a, 0xbb, 0xa8, 0xef, + ]) { + panic! ("{}", String::from_utf8 (resp.to_vec ()).unwrap ()); + } + + // Requesting a file from a server that isn't registered + // will error out + + let resp = client.get (&format! ("{}/frontend/servers/obviously_this_server_does_not_exist/files/COPYING", relay_url)) + .send ().await.unwrap (); + + assert_eq! (resp.status (), reqwest::StatusCode::NOT_FOUND); + + info! ("Shutting down end-to-end test"); + + stop_server_tx.send (()).unwrap (); + stop_relay_tx.send (()).unwrap (); + + info! ("Sent stop messages"); + + task_relay.await.unwrap (); + info! ("Relay stopped"); + + task_server.await.unwrap (); + info! ("Server stopped"); + }); +} From c4108f6f2f91246b21b4c46092afa4815abd429a Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 26 Nov 2020 23:41:32 +0000 Subject: [PATCH 055/208] Move the bad passwords code into the server module since the relay doesn't need it --- src/lib.rs | 14 ----------- src/{ => server}/bad_passwords.txt | 0 src/server/file_server.rs | 6 ++--- src/server/mod.rs | 39 +++++++++++++++++++++++++++++- src/tests.rs | 25 ------------------- 5 files changed, 41 insertions(+), 43 deletions(-) rename src/{ => server}/bad_passwords.txt (100%) diff --git a/src/lib.rs b/src/lib.rs index da7e290..e0e2cf3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,19 +26,5 @@ pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str> } } -// Thanks to https://github.com/robsheldon/bad-passwords-index - -const BAD_PASSWORDS: &[u8] = include_bytes! ("bad_passwords.txt"); - -pub fn password_is_bad (mut password: String) -> bool { - password.make_ascii_lowercase (); - - let ac = aho_corasick::AhoCorasick::new (&[ - password - ]); - - ac.find (BAD_PASSWORDS).is_some () -} - #[cfg (test)] mod tests; diff --git a/src/bad_passwords.txt b/src/server/bad_passwords.txt similarity index 100% rename from src/bad_passwords.txt rename to src/server/bad_passwords.txt diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 1e05bbc..1c2cf77 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -842,7 +842,7 @@ mod tests { ("/files/?", InvalidQuery), ("/files/src", Redirect ("src/".to_string ())), ("/files/src/?", InvalidQuery), - ("/files/src/bad_passwords.txt", ServeFile (ServeFileParams { + ("/files/src/server/bad_passwords.txt", ServeFile (ServeFileParams { send_body: true, range: 0..1_048_576, range_requested: false, @@ -870,7 +870,7 @@ mod tests { let resp = internal_serve_all ( &file_server_root, Method::Get, - "/files/src/bad_passwords.txt", + "/files/src/server/bad_passwords.txt", &hashmap! { "range".into () => b"bytes=0-2000000".to_vec (), }, @@ -882,7 +882,7 @@ mod tests { let resp = internal_serve_all ( &file_server_root, Method::Head, - "/files/src/bad_passwords.txt", + "/files/src/server/bad_passwords.txt", &headers, None ).await; diff --git a/src/server/mod.rs b/src/server/mod.rs index 61b59d7..784354f 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -24,6 +24,20 @@ use crate::{ pub mod file_server; +// Thanks to https://github.com/robsheldon/bad-passwords-index + +const BAD_PASSWORDS: &[u8] = include_bytes! ("bad_passwords.txt"); + +pub fn password_is_bad (mut password: String) -> bool { + password.make_ascii_lowercase (); + + let ac = aho_corasick::AhoCorasick::new (&[ + password + ]); + + ac.find (BAD_PASSWORDS).is_some () +} + struct ServerState { config: Config, handlebars: Handlebars <'static>, @@ -143,7 +157,7 @@ pub async fn run_server ( use std::convert::TryInto; - if crate::password_is_bad (config_file.api_key.clone ()) { + if password_is_bad (config_file.api_key.clone ()) { panic! ("API key is too weak, server can't use it"); } @@ -275,4 +289,27 @@ mod tests { assert_eq! (config.tripcode (), "A9rPwZyY89Ag4TJjMoyYA2NeGOm99Je6rq1s0rg8PfY=".to_string ()); } + + #[test] + fn check_bad_passwords () { + for pw in &[ + "", + " ", + "user", + "password", + "pAsSwOrD", + "secret", + "123123", + ] { + assert! (password_is_bad (pw.to_string ())); + } + + use rand::prelude::*; + + let mut entropy = [0u8; 32]; + thread_rng ().fill_bytes (&mut entropy); + let good_password = base64::encode (entropy); + + assert! (! password_is_bad (good_password)); + } } diff --git a/src/tests.rs b/src/tests.rs index 529a33d..959cf01 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -18,31 +18,6 @@ use super::{ server, }; -#[test] -fn check_bad_passwords () { - use crate::password_is_bad; - - for pw in &[ - "", - " ", - "user", - "password", - "pAsSwOrD", - "secret", - "123123", - ] { - assert! (password_is_bad (pw.to_string ())); - } - - use rand::prelude::*; - - let mut entropy = [0u8; 32]; - thread_rng ().fill_bytes (&mut entropy); - let good_password = base64::encode (entropy); - - assert! (! password_is_bad (good_password)); -} - #[test] fn end_to_end () { use maplit::*; From a6ecb1c6a898062001d3529b48e513ff80f2d01e Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 26 Nov 2020 23:51:10 +0000 Subject: [PATCH 056/208] :recycle: Move load_toml into server --- src/bin/ptth_file_server.rs | 2 +- src/bin/ptth_server.rs | 2 +- src/lib.rs | 1 - src/server/file_server.rs | 2 +- src/{ => server}/load_toml.rs | 0 src/server/mod.rs | 1 + 6 files changed, 4 insertions(+), 4 deletions(-) rename src/{ => server}/load_toml.rs (100%) diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index bbcfcf9..bb6ca9e 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -103,7 +103,7 @@ async fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); let path = PathBuf::from ("./config/ptth_server.toml"); - let config_file: ConfigFile = ptth::load_toml::load (&path); + let config_file: ConfigFile = ptth::server::load_toml::load (&path); info! ("file_server_root: {:?}", config_file.file_server_root); let addr = SocketAddr::from(([0, 0, 0, 0], 4000)); diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index 4b4e047..ee52100 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -27,7 +27,7 @@ fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); let path = opt.config_path.clone ().unwrap_or_else (|| PathBuf::from ("./config/ptth_server.toml")); - let config_file: ptth::server::ConfigFile = ptth::load_toml::load (&path); + let config_file: ptth::server::ConfigFile = ptth::server::load_toml::load (&path); if opt.print_tripcode { println! (r#""{}" = "{}""#, config_file.name, config_file.tripcode ()); diff --git a/src/lib.rs b/src/lib.rs index e0e2cf3..ec195a0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,6 @@ pub const PTTH_MAGIC_HEADER: &str = "X-PTTH-2LJYXWC4"; pub mod git_version; pub mod graceful_shutdown; pub mod http_serde; -pub mod load_toml; pub mod prelude; pub mod relay; pub mod server; diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 1c2cf77..4b519b7 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -537,7 +537,7 @@ async fn internal_serve_all ( use std::os::unix::fs::PermissionsExt; let file_md = file.metadata ().await.unwrap (); - if file_md.permissions ().mode () == crate::load_toml::CONFIG_PERMISSIONS_MODE + if file_md.permissions ().mode () == super::load_toml::CONFIG_PERMISSIONS_MODE { return Forbidden; } diff --git a/src/load_toml.rs b/src/server/load_toml.rs similarity index 100% rename from src/load_toml.rs rename to src/server/load_toml.rs diff --git a/src/server/mod.rs b/src/server/mod.rs index 784354f..f89fe55 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -23,6 +23,7 @@ use crate::{ }; pub mod file_server; +pub mod load_toml; // Thanks to https://github.com/robsheldon/bad-passwords-index From 64a0d90762973369fb3fd9bf030d6ebdfeb481da Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 26 Nov 2020 23:53:03 +0000 Subject: [PATCH 057/208] :recycle: Move git_version into relay --- src/bin/ptth_relay.rs | 2 +- src/lib.rs | 1 - src/{ => relay}/git_version.rs | 0 src/relay/mod.rs | 1 + 4 files changed, 2 insertions(+), 2 deletions(-) rename src/{ => relay}/git_version.rs (100%) diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index ecf8867..c0c878b 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -28,7 +28,7 @@ async fn main () -> Result <(), Box > { let config_path = PathBuf::from ("config/ptth_relay.toml"); let config = Config::from_file (&config_path).await?; - info! ("ptth_relay Git version: {:?}", ptth::git_version::GIT_VERSION); + info! ("ptth_relay Git version: {:?}", ptth::relay::git_version::GIT_VERSION); let (shutdown_rx, forced_shutdown) = ptth::graceful_shutdown::init_with_force (); diff --git a/src/lib.rs b/src/lib.rs index ec195a0..a541132 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,6 @@ pub const PTTH_MAGIC_HEADER: &str = "X-PTTH-2LJYXWC4"; -pub mod git_version; pub mod graceful_shutdown; pub mod http_serde; pub mod prelude; diff --git a/src/git_version.rs b/src/relay/git_version.rs similarity index 100% rename from src/git_version.rs rename to src/relay/git_version.rs diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 11cfdd5..5ba0fb8 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -51,6 +51,7 @@ use crate::{ }; pub mod config; +pub mod git_version; pub use config::{Config, ConfigError}; From 84bb326f37e1df31d49d9203a8706963f1bedf7a Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 27 Nov 2020 00:03:11 +0000 Subject: [PATCH 058/208] :recycle: Extract crate ptth_core --- Cargo.toml | 2 +- crates/ptth_core/Cargo.toml | 17 +++++++++++++++++ .../ptth_core/src}/graceful_shutdown.rs | 0 {src => crates/ptth_core/src}/http_serde.rs | 0 crates/ptth_core/src/lib.rs | 16 ++++++++++++++++ crates/ptth_core/src/prelude.rs | 4 ++++ src/bin/ptth_file_server.rs | 6 +++--- src/bin/ptth_relay.rs | 2 +- src/bin/ptth_server.rs | 2 +- src/lib.rs | 16 ---------------- src/prelude.rs | 1 - src/relay/mod.rs | 7 ++----- src/server/file_server.rs | 5 ++--- src/server/mod.rs | 2 +- 14 files changed, 48 insertions(+), 32 deletions(-) create mode 100644 crates/ptth_core/Cargo.toml rename {src => crates/ptth_core/src}/graceful_shutdown.rs (100%) rename {src => crates/ptth_core/src}/http_serde.rs (100%) create mode 100644 crates/ptth_core/src/lib.rs create mode 100644 crates/ptth_core/src/prelude.rs delete mode 100644 src/prelude.rs diff --git a/Cargo.toml b/Cargo.toml index e85a51d..440e284 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ aho-corasick = "0.7.14" base64 = "0.12.3" blake3 = "0.3.7" chrono = "0.4.19" -ctrlc = { version = "3.1.7", features = [ "termination" ] } dashmap = "3.11.10" futures = "0.3.7" handlebars = "3.5.1" @@ -41,6 +40,7 @@ ulid = "0.4.1" url = "2.2.0" always_equal = { path = "crates/always_equal" } +ptth_core = { path = "crates/ptth_core" } [workspace] diff --git a/crates/ptth_core/Cargo.toml b/crates/ptth_core/Cargo.toml new file mode 100644 index 0000000..7a584a0 --- /dev/null +++ b/crates/ptth_core/Cargo.toml @@ -0,0 +1,17 @@ +[package] + +name = "ptth_core" +version = "0.1.0" +authors = ["Trish"] +edition = "2018" +license = "AGPL-3.0" + +[dependencies] + +ctrlc = { version = "3.1.7", features = [ "termination" ] } +futures = "0.3.7" +hyper = "0.13.8" +serde = {version = "1.0.117", features = ["derive"]} +tokio = { version = "0.2.22", features = ["full"] } +tracing = "0.1.21" +tracing-futures = "0.2.4" diff --git a/src/graceful_shutdown.rs b/crates/ptth_core/src/graceful_shutdown.rs similarity index 100% rename from src/graceful_shutdown.rs rename to crates/ptth_core/src/graceful_shutdown.rs diff --git a/src/http_serde.rs b/crates/ptth_core/src/http_serde.rs similarity index 100% rename from src/http_serde.rs rename to crates/ptth_core/src/http_serde.rs diff --git a/crates/ptth_core/src/lib.rs b/crates/ptth_core/src/lib.rs new file mode 100644 index 0000000..2cc81b7 --- /dev/null +++ b/crates/ptth_core/src/lib.rs @@ -0,0 +1,16 @@ +pub mod graceful_shutdown; +pub mod http_serde; +pub mod prelude; + +// The arguments are in order so they are in order overall: +// e.g. prefix_match ("/prefix", "/prefix/middle/suffix") -> "/middle/suffix" + +pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str> +{ + if hay.starts_with (prefix) { + Some (&hay [prefix.len ()..]) + } + else { + None + } +} diff --git a/crates/ptth_core/src/prelude.rs b/crates/ptth_core/src/prelude.rs new file mode 100644 index 0000000..b66b273 --- /dev/null +++ b/crates/ptth_core/src/prelude.rs @@ -0,0 +1,4 @@ +pub use tracing::{ + debug, error, info, trace, warn, + instrument, +}; diff --git a/src/bin/ptth_file_server.rs b/src/bin/ptth_file_server.rs index bb6ca9e..47e825a 100644 --- a/src/bin/ptth_file_server.rs +++ b/src/bin/ptth_file_server.rs @@ -18,11 +18,11 @@ use hyper::{ }; use serde::Deserialize; -use ptth::{ +use ptth_core::{ http_serde::RequestParts, prelude::*, - server::file_server, }; +use ptth::server::file_server; #[derive (Default)] pub struct Config { @@ -133,7 +133,7 @@ async fn main () -> Result <(), Box > { } }); - let (shutdown_rx, forced_shutdown) = ptth::graceful_shutdown::init_with_force (); + let (shutdown_rx, forced_shutdown) = ptth_core::graceful_shutdown::init_with_force (); let server = Server::bind (&addr) .serve (make_svc) diff --git a/src/bin/ptth_relay.rs b/src/bin/ptth_relay.rs index c0c878b..112c665 100644 --- a/src/bin/ptth_relay.rs +++ b/src/bin/ptth_relay.rs @@ -30,7 +30,7 @@ async fn main () -> Result <(), Box > { info! ("ptth_relay Git version: {:?}", ptth::relay::git_version::GIT_VERSION); - let (shutdown_rx, forced_shutdown) = ptth::graceful_shutdown::init_with_force (); + let (shutdown_rx, forced_shutdown) = ptth_core::graceful_shutdown::init_with_force (); forced_shutdown.wrap_server ( relay::run_relay ( diff --git a/src/bin/ptth_server.rs b/src/bin/ptth_server.rs index ee52100..c4ae0cd 100644 --- a/src/bin/ptth_server.rs +++ b/src/bin/ptth_server.rs @@ -51,7 +51,7 @@ fn main () -> Result <(), Box > { rt.block_on (async move { ptth::server::run_server ( config_file, - ptth::graceful_shutdown::init (), + ptth_core::graceful_shutdown::init (), Some (path), opt.asset_root ).await diff --git a/src/lib.rs b/src/lib.rs index a541132..f806df2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,24 +5,8 @@ pub const PTTH_MAGIC_HEADER: &str = "X-PTTH-2LJYXWC4"; -pub mod graceful_shutdown; -pub mod http_serde; -pub mod prelude; pub mod relay; pub mod server; -// The arguments are in order so they are in order overall: -// e.g. prefix_match ("/prefix", "/prefix/middle/suffix") -> "/middle/suffix" - -pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str> -{ - if hay.starts_with (prefix) { - Some (&hay [prefix.len ()..]) - } - else { - None - } -} - #[cfg (test)] mod tests; diff --git a/src/prelude.rs b/src/prelude.rs deleted file mode 100644 index f4af411..0000000 --- a/src/prelude.rs +++ /dev/null @@ -1 +0,0 @@ -pub use tracing::{debug, error, info, trace, warn}; diff --git a/src/relay/mod.rs b/src/relay/mod.rs index 5ba0fb8..3dd53ac 100644 --- a/src/relay/mod.rs +++ b/src/relay/mod.rs @@ -40,14 +40,11 @@ use tokio::{ }, time::delay_for, }; -use tracing::{ - debug, error, info, trace, - instrument, -}; -use crate::{ +use ptth_core::{ http_serde, prefix_match, + prelude::*, }; pub mod config; diff --git a/src/server/file_server.rs b/src/server/file_server.rs index 4b519b7..07604a8 100644 --- a/src/server/file_server.rs +++ b/src/server/file_server.rs @@ -36,7 +36,7 @@ use always_equal::test::AlwaysEqual; #[cfg (not (test))] use always_equal::prod::AlwaysEqual; -use crate::{ +use ptth_core::{ http_serde::{ Method, Response, @@ -796,9 +796,8 @@ mod tests { #[test] fn file_server () { - use crate::{ + use ptth_core::{ http_serde::Method, - //prelude::*, }; use super::*; diff --git a/src/server/mod.rs b/src/server/mod.rs index f89fe55..8e969fd 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -17,7 +17,7 @@ use tokio::{ time::delay_for, }; -use crate::{ +use ptth_core::{ http_serde, prelude::*, }; From 4c9595ee2edd976db99f40c344e4f941d17d9e5f Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 27 Nov 2020 00:20:18 +0000 Subject: [PATCH 059/208] :recycle: Extract crate ptth_relay --- Cargo.toml | 3 +- crates/ptth_core/src/lib.rs | 7 +++++ crates/ptth_relay/Cargo.toml | 30 +++++++++++++++++++ .../relay => crates/ptth_relay/src}/config.rs | 0 .../ptth_relay/src}/git_version.rs | 0 .../mod.rs => crates/ptth_relay/src/lib.rs | 2 +- .../ptth_relay/src/main.rs | 9 +++--- src/lib.rs | 8 ----- src/server/mod.rs | 2 +- src/tests.rs | 9 +++--- 10 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 crates/ptth_relay/Cargo.toml rename {src/relay => crates/ptth_relay/src}/config.rs (100%) rename {src/relay => crates/ptth_relay/src}/git_version.rs (100%) rename src/relay/mod.rs => crates/ptth_relay/src/lib.rs (99%) rename src/bin/ptth_relay.rs => crates/ptth_relay/src/main.rs (83%) diff --git a/Cargo.toml b/Cargo.toml index 440e284..5ef4f3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ aho-corasick = "0.7.14" base64 = "0.12.3" blake3 = "0.3.7" chrono = "0.4.19" -dashmap = "3.11.10" futures = "0.3.7" handlebars = "3.5.1" http = "0.2.1" @@ -30,7 +29,6 @@ reqwest = { version = "0.10.8", features = ["stream"] } rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} structopt = "0.3.20" -thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" tracing-futures = "0.2.4" @@ -41,6 +39,7 @@ url = "2.2.0" always_equal = { path = "crates/always_equal" } ptth_core = { path = "crates/ptth_core" } +ptth_relay = { path = "crates/ptth_relay" } [workspace] diff --git a/crates/ptth_core/src/lib.rs b/crates/ptth_core/src/lib.rs index 2cc81b7..dd92c18 100644 --- a/crates/ptth_core/src/lib.rs +++ b/crates/ptth_core/src/lib.rs @@ -2,6 +2,13 @@ pub mod graceful_shutdown; pub mod http_serde; pub mod prelude; +// It's easier if the server can stream its response body +// back to the relay un-changed inside its request body +// So we wrap the server's actual response head +// (status code, headers, etc.) in this one header field. + +pub const PTTH_MAGIC_HEADER: &str = "X-PTTH-2LJYXWC4"; + // The arguments are in order so they are in order overall: // e.g. prefix_match ("/prefix", "/prefix/middle/suffix") -> "/middle/suffix" diff --git a/crates/ptth_relay/Cargo.toml b/crates/ptth_relay/Cargo.toml new file mode 100644 index 0000000..938364f --- /dev/null +++ b/crates/ptth_relay/Cargo.toml @@ -0,0 +1,30 @@ +[package] + +name = "ptth_relay" +version = "0.1.0" +authors = ["Trish"] +edition = "2018" +license = "AGPL-3.0" + +[dependencies] + +base64 = "0.12.3" +blake3 = "0.3.7" +chrono = "0.4.19" +dashmap = "3.11.10" +futures = "0.3.7" +handlebars = "3.5.1" +http = "0.2.1" +hyper = "0.13.8" +itertools = "0.9.0" +rmp-serde = "0.14.4" +serde = {version = "1.0.117", features = ["derive"]} +thiserror = "1.0.22" +tokio = { version = "0.2.22", features = ["full"] } +toml = "0.5.7" +tracing = "0.1.21" +tracing-futures = "0.2.4" +tracing-subscriber = "0.2.15" +ulid = "0.4.1" + +ptth_core = { path = "../ptth_core" } diff --git a/src/relay/config.rs b/crates/ptth_relay/src/config.rs similarity index 100% rename from src/relay/config.rs rename to crates/ptth_relay/src/config.rs diff --git a/src/relay/git_version.rs b/crates/ptth_relay/src/git_version.rs similarity index 100% rename from src/relay/git_version.rs rename to crates/ptth_relay/src/git_version.rs diff --git a/src/relay/mod.rs b/crates/ptth_relay/src/lib.rs similarity index 99% rename from src/relay/mod.rs rename to crates/ptth_relay/src/lib.rs index 3dd53ac..306ebff 100644 --- a/src/relay/mod.rs +++ b/crates/ptth_relay/src/lib.rs @@ -256,7 +256,7 @@ async fn handle_http_response ( -> Response { let (parts, mut body) = req.into_parts (); - let resp_parts: http_serde::ResponseParts = rmp_serde::from_read_ref (&base64::decode (parts.headers.get (crate::PTTH_MAGIC_HEADER).unwrap ()).unwrap ()).unwrap (); + let resp_parts: http_serde::ResponseParts = rmp_serde::from_read_ref (&base64::decode (parts.headers.get (ptth_core::PTTH_MAGIC_HEADER).unwrap ()).unwrap ()).unwrap (); // Intercept the body packets here so we can check when the stream // ends or errors out diff --git a/src/bin/ptth_relay.rs b/crates/ptth_relay/src/main.rs similarity index 83% rename from src/bin/ptth_relay.rs rename to crates/ptth_relay/src/main.rs index 112c665..663ae44 100644 --- a/src/bin/ptth_relay.rs +++ b/crates/ptth_relay/src/main.rs @@ -11,10 +11,11 @@ use tracing_subscriber::{ EnvFilter, }; -use ptth::relay; -use ptth::relay::{ +use ptth_relay::{ Config, + git_version::GIT_VERSION, RelayState, + run_relay, }; #[tokio::main] @@ -28,12 +29,12 @@ async fn main () -> Result <(), Box > { let config_path = PathBuf::from ("config/ptth_relay.toml"); let config = Config::from_file (&config_path).await?; - info! ("ptth_relay Git version: {:?}", ptth::relay::git_version::GIT_VERSION); + info! ("ptth_relay Git version: {:?}", GIT_VERSION); let (shutdown_rx, forced_shutdown) = ptth_core::graceful_shutdown::init_with_force (); forced_shutdown.wrap_server ( - relay::run_relay ( + run_relay ( Arc::new (RelayState::from (config)), shutdown_rx, Some (config_path) diff --git a/src/lib.rs b/src/lib.rs index f806df2..ede77e6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,3 @@ -// It's easier if the server can stream its response body -// back to the relay un-changed inside its request body -// So we wrap the server's actual response head -// (status code, headers, etc.) in this one header field. - -pub const PTTH_MAGIC_HEADER: &str = "X-PTTH-2LJYXWC4"; - -pub mod relay; pub mod server; #[cfg (test)] diff --git a/src/server/mod.rs b/src/server/mod.rs index 8e969fd..a6b8d69 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -90,7 +90,7 @@ async fn handle_req_resp <'a> ( let mut resp_req = state.client .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) - .header (crate::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).unwrap ())); + .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).unwrap ())); if let Some (length) = response.content_length { resp_req = resp_req.header ("Content-Length", length.to_string ()); diff --git a/src/tests.rs b/src/tests.rs index 959cf01..9c6bc98 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -14,7 +14,6 @@ use tokio::{ }; use super::{ - relay, server, }; @@ -36,22 +35,22 @@ fn end_to_end () { let api_key = "AnacondaHardcoverGrannyUnlatchLankinessMutate"; let tripcode = base64::encode (blake3::hash (api_key.as_bytes ()).as_bytes ()); debug! ("Relay is expecting tripcode {}", tripcode); - let config_file = relay::config::file::Config { + let config_file = ptth_relay::config::file::Config { port: None, servers: hashmap! { - server_name.into () => relay::config::file::Server { + server_name.into () => ptth_relay::config::file::Server { tripcode, display_name: None, }, }, }; - let relay_state = Arc::new (relay::RelayState::from (relay::config::Config::try_from (config_file).unwrap ())); + let relay_state = Arc::new (ptth_relay::RelayState::from (ptth_relay::config::Config::try_from (config_file).unwrap ())); let relay_state_2 = relay_state.clone (); let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); let task_relay = spawn (async move { - relay::run_relay (relay_state_2, stop_relay_rx, None).await.unwrap (); + ptth_relay::run_relay (relay_state_2, stop_relay_rx, None).await.unwrap (); }); assert! (relay_state.list_servers ().await.is_empty ()); From bbb88c01e8ad28d256c8faf316f46d7214184ec2 Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 27 Nov 2020 00:50:22 +0000 Subject: [PATCH 060/208] :recycle: Extract ptth_server crate. Docker still broken --- Cargo.toml | 24 ++---------- crates/ptth_server/Cargo.toml | 37 ++++++++++++++++++ .../ptth_server/src}/bad_passwords.txt | 0 .../ptth_server/src}/bin/ptth_file_server.rs | 7 +++- .../ptth_server/src}/bin/ptth_server.rs | 25 +++++------- .../ptth_server/src}/file_server.rs | 32 +++------------ .../mod.rs => crates/ptth_server/src/lib.rs | 0 .../ptth_server/src}/load_toml.rs | 0 {test => crates/ptth_server/test}/face.png | Bin {test => crates/ptth_server/test}/test.md | 0 src/lib.rs | 2 - src/tests.rs | 8 +--- 12 files changed, 61 insertions(+), 74 deletions(-) create mode 100644 crates/ptth_server/Cargo.toml rename {src/server => crates/ptth_server/src}/bad_passwords.txt (100%) rename {src => crates/ptth_server/src}/bin/ptth_file_server.rs (96%) rename {src => crates/ptth_server/src}/bin/ptth_server.rs (67%) rename {src/server => crates/ptth_server/src}/file_server.rs (96%) rename src/server/mod.rs => crates/ptth_server/src/lib.rs (100%) rename {src/server => crates/ptth_server/src}/load_toml.rs (100%) rename {test => crates/ptth_server/test}/face.png (100%) rename {test => crates/ptth_server/test}/test.md (100%) diff --git a/Cargo.toml b/Cargo.toml index 5ef4f3f..1c1eeca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,40 +6,22 @@ authors = ["Trish"] edition = "2018" license = "AGPL-3.0" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] -aho-corasick = "0.7.14" +[dev-dependencies] + base64 = "0.12.3" blake3 = "0.3.7" -chrono = "0.4.19" -futures = "0.3.7" -handlebars = "3.5.1" -http = "0.2.1" -hyper = "0.13.8" -itertools = "0.9.0" -lazy_static = "1.4.0" maplit = "1.0.2" -percent-encoding = "2.1.0" -pulldown-cmark = "0.8.0" -rand = "0.7.3" -regex = "1.4.1" reqwest = { version = "0.10.8", features = ["stream"] } -rmp-serde = "0.14.4" -serde = {version = "1.0.117", features = ["derive"]} -structopt = "0.3.20" tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" -tracing-futures = "0.2.4" tracing-subscriber = "0.2.15" -toml = "0.5.7" -ulid = "0.4.1" -url = "2.2.0" always_equal = { path = "crates/always_equal" } ptth_core = { path = "crates/ptth_core" } ptth_relay = { path = "crates/ptth_relay" } +ptth_server = { path = "crates/ptth_server" } [workspace] diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml new file mode 100644 index 0000000..a0a61af --- /dev/null +++ b/crates/ptth_server/Cargo.toml @@ -0,0 +1,37 @@ +[package] + +name = "ptth_server" +version = "0.1.0" +authors = ["Trish"] +edition = "2018" +license = "AGPL-3.0" + +[dependencies] + +aho-corasick = "0.7.14" +base64 = "0.12.3" +blake3 = "0.3.7" +futures = "0.3.7" +handlebars = "3.5.1" +hyper = "0.13.8" +lazy_static = "1.4.0" +percent-encoding = "2.1.0" +pulldown-cmark = "0.8.0" +regex = "1.4.1" +reqwest = { version = "0.10.8", features = ["stream"] } +rmp-serde = "0.14.4" +serde = {version = "1.0.117", features = ["derive"]} +structopt = "0.3.20" +tokio = { version = "0.2.22", features = ["full"] } +tracing = "0.1.21" +tracing-futures = "0.2.4" +tracing-subscriber = "0.2.15" +toml = "0.5.7" + +always_equal = { path = "../always_equal" } +ptth_core = { path = "../ptth_core" } + +[dev-dependencies] + +maplit = "1.0.2" +rand = "0.6.5" diff --git a/src/server/bad_passwords.txt b/crates/ptth_server/src/bad_passwords.txt similarity index 100% rename from src/server/bad_passwords.txt rename to crates/ptth_server/src/bad_passwords.txt diff --git a/src/bin/ptth_file_server.rs b/crates/ptth_server/src/bin/ptth_file_server.rs similarity index 96% rename from src/bin/ptth_file_server.rs rename to crates/ptth_server/src/bin/ptth_file_server.rs index 47e825a..cd517ae 100644 --- a/src/bin/ptth_file_server.rs +++ b/crates/ptth_server/src/bin/ptth_file_server.rs @@ -22,7 +22,10 @@ use ptth_core::{ http_serde::RequestParts, prelude::*, }; -use ptth::server::file_server; +use ptth_server::{ + file_server, + load_toml, +}; #[derive (Default)] pub struct Config { @@ -103,7 +106,7 @@ async fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); let path = PathBuf::from ("./config/ptth_server.toml"); - let config_file: ConfigFile = ptth::server::load_toml::load (&path); + let config_file: ConfigFile = load_toml::load (&path); info! ("file_server_root: {:?}", config_file.file_server_root); let addr = SocketAddr::from(([0, 0, 0, 0], 4000)); diff --git a/src/bin/ptth_server.rs b/crates/ptth_server/src/bin/ptth_server.rs similarity index 67% rename from src/bin/ptth_server.rs rename to crates/ptth_server/src/bin/ptth_server.rs index c4ae0cd..2718074 100644 --- a/src/bin/ptth_server.rs +++ b/crates/ptth_server/src/bin/ptth_server.rs @@ -5,7 +5,12 @@ use std::{ use structopt::StructOpt; use tokio::runtime; -use tracing::debug; + +use ptth_server::{ + ConfigFile, + load_toml, + run_server, +}; #[derive (Debug, StructOpt)] struct Opt { @@ -27,29 +32,17 @@ fn main () -> Result <(), Box > { tracing_subscriber::fmt::init (); let path = opt.config_path.clone ().unwrap_or_else (|| PathBuf::from ("./config/ptth_server.toml")); - let config_file: ptth::server::ConfigFile = ptth::server::load_toml::load (&path); + let config_file: ConfigFile = load_toml::load (&path); if opt.print_tripcode { println! (r#""{}" = "{}""#, config_file.name, config_file.tripcode ()); return Ok (()); } - let mut rt = if false { - debug! ("Trying to use less RAM"); - - runtime::Builder::new () - .threaded_scheduler () - .enable_all () - .core_threads (1) - .thread_stack_size (1024 * 1024) - .build ()? - } - else { - runtime::Runtime::new ()? - }; + let mut rt = runtime::Runtime::new ()?; rt.block_on (async move { - ptth::server::run_server ( + run_server ( config_file, ptth_core::graceful_shutdown::init (), Some (path), diff --git a/src/server/file_server.rs b/crates/ptth_server/src/file_server.rs similarity index 96% rename from src/server/file_server.rs rename to crates/ptth_server/src/file_server.rs index 07604a8..c111bbc 100644 --- a/src/server/file_server.rs +++ b/crates/ptth_server/src/file_server.rs @@ -805,43 +805,21 @@ mod tests { let mut rt = Runtime::new ().unwrap (); rt.block_on (async { - let handlebars = load_templates (&PathBuf::new ()).unwrap (); - let server_info = ServerInfo { - server_name: "PTTH File Server".to_string (), - }; - let file_server_root = PathBuf::from ("./"); let headers = Default::default (); - for (uri_path, expected_status) in vec! [ - ("/", StatusCode::Ok), - ("/files", StatusCode::TemporaryRedirect), - ("/files/src", StatusCode::TemporaryRedirect), - ("/files/src/", StatusCode::Ok), - ].into_iter () { - let resp = serve_all ( - &handlebars, - &server_info, - &file_server_root, - Method::Get, - uri_path, - &headers, - None - ).await; - - assert_eq! (resp.parts.status_code, expected_status); - } - { use InternalResponse::*; + let bad_passwords_path = "/files/src/bad_passwords.txt"; + for (uri_path, expected) in vec! [ ("/", Root), ("/files", Redirect ("files/".to_string ())), ("/files/?", InvalidQuery), ("/files/src", Redirect ("src/".to_string ())), ("/files/src/?", InvalidQuery), - ("/files/src/server/bad_passwords.txt", ServeFile (ServeFileParams { + (bad_passwords_path, ServeFile (ServeFileParams { send_body: true, range: 0..1_048_576, range_requested: false, @@ -869,7 +847,7 @@ mod tests { let resp = internal_serve_all ( &file_server_root, Method::Get, - "/files/src/server/bad_passwords.txt", + bad_passwords_path, &hashmap! { "range".into () => b"bytes=0-2000000".to_vec (), }, @@ -881,7 +859,7 @@ mod tests { let resp = internal_serve_all ( &file_server_root, Method::Head, - "/files/src/server/bad_passwords.txt", + bad_passwords_path, &headers, None ).await; diff --git a/src/server/mod.rs b/crates/ptth_server/src/lib.rs similarity index 100% rename from src/server/mod.rs rename to crates/ptth_server/src/lib.rs diff --git a/src/server/load_toml.rs b/crates/ptth_server/src/load_toml.rs similarity index 100% rename from src/server/load_toml.rs rename to crates/ptth_server/src/load_toml.rs diff --git a/test/face.png b/crates/ptth_server/test/face.png similarity index 100% rename from test/face.png rename to crates/ptth_server/test/face.png diff --git a/test/test.md b/crates/ptth_server/test/test.md similarity index 100% rename from test/test.md rename to crates/ptth_server/test/test.md diff --git a/src/lib.rs b/src/lib.rs index ede77e6..f18819e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,2 @@ -pub mod server; - #[cfg (test)] mod tests; diff --git a/src/tests.rs b/src/tests.rs index 9c6bc98..453a2ba 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -13,10 +13,6 @@ use tokio::{ time::delay_for, }; -use super::{ - server, -}; - #[test] fn end_to_end () { use maplit::*; @@ -57,7 +53,7 @@ fn end_to_end () { let relay_url = "http://127.0.0.1:4000"; - let config_file = server::ConfigFile { + let config_file = ptth_server::ConfigFile { name: server_name.into (), api_key: api_key.into (), relay_url: "http://127.0.0.1:4000/7ZSFUKGV".into (), @@ -67,7 +63,7 @@ fn end_to_end () { let (stop_server_tx, stop_server_rx) = oneshot::channel (); let task_server = { spawn (async move { - server::run_server (config_file, stop_server_rx, None, None).await.unwrap (); + ptth_server::run_server (config_file, stop_server_rx, None, None).await.unwrap (); }) }; From 5f9e84f7f62fdf0c0f8274540310b32d32baed6b Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 27 Nov 2020 01:07:39 +0000 Subject: [PATCH 061/208] :whale: Fix Docker --- Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4056e35..6406606 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,12 +8,12 @@ RUN apt-get update \ # Make sure the dependencies are all cached so we won't hammer crates.io ADD old-git.tar.gz . -RUN git checkout 151f236a0b8de510b1896deeaa9c571f845040af \ +RUN git checkout bbb88c01e8ad28d256c8faf316f46d7214184ec2 \ && git reset --hard \ -&& cargo check +&& cargo check -p ptth_relay -RUN cargo test --release \ -&& cargo build --release --bin ptth_relay +RUN cargo test --release --all \ +&& cargo build --release -p ptth_relay COPY .git .git @@ -22,8 +22,8 @@ ARG gitcommithash=HEAD RUN git checkout "$gitcommithash" \ && git reset --hard \ && echo "pub const GIT_VERSION: Option <&str> = Some (\"$(git rev-parse HEAD)\");" > src/git_version.rs \ -&& cargo test --release \ -&& cargo build --release --bin ptth_relay +&& cargo test --release --all \ +&& cargo build --release -p ptth_relay FROM debian:buster-slim as deploy @@ -33,7 +33,7 @@ RUN apt-get update \ COPY --from=build /usr/src/target/release/ptth_relay /root COPY --from=build /usr/src/src/git_version.rs /root/git_version.rs -COPY --from=build /usr/src/ptth_handlebars /root/ptth_handlebars +COPY --from=build /usr/src/handlebars /root/handlebars WORKDIR /root From 3e74f2b1efd26754350a9145e1b0525d89276db3 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 16:36:59 +0000 Subject: [PATCH 062/208] :recycle: Fix unwrap()s in ptth_core. Some were in an unused function, so I removed that function. Some were in graceful_shutdown and can't really be handled. So I made them into "expect"s --- crates/ptth_core/src/graceful_shutdown.rs | 6 +++--- crates/ptth_core/src/http_serde.rs | 18 ------------------ 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/crates/ptth_core/src/graceful_shutdown.rs b/crates/ptth_core/src/graceful_shutdown.rs index 992414e..82bd541 100644 --- a/crates/ptth_core/src/graceful_shutdown.rs +++ b/crates/ptth_core/src/graceful_shutdown.rs @@ -25,7 +25,7 @@ pub fn init () -> oneshot::Receiver <()> { let tx = tx.replace (None); if let Some (tx) = tx { - tx.send (()).unwrap (); + tx.send (()).expect ("Error sending Ctrl-C to program"); } }).expect ("Error setting Ctrl-C handler"); @@ -72,8 +72,8 @@ impl ForcedShutdown { server: F ) -> Result { let fut = async move { - self.rx.await.unwrap (); - self.tx.send (()).unwrap (); + self.rx.await.expect ("Error awaiting graceful shutdown signal"); + self.tx.send (()).expect ("Error forwarding graceful shutdown signal"); let timeout = 5; debug! ("Starting graceful shutdown. Forcing shutdown in {} seconds", timeout); delay_for (Duration::from_secs (timeout)).await; diff --git a/crates/ptth_core/src/http_serde.rs b/crates/ptth_core/src/http_serde.rs index 5276cfe..b2d4970 100644 --- a/crates/ptth_core/src/http_serde.rs +++ b/crates/ptth_core/src/http_serde.rs @@ -154,24 +154,6 @@ pub struct Response { } impl Response { - pub async fn into_bytes (self) -> Vec { - let mut body = match self.body { - None => return Vec::new (), - Some (x) => x, - }; - - let mut result = match self.content_length { - None => Vec::new (), - Some (x) => Vec::with_capacity (x.try_into ().unwrap ()), - }; - - while let Some (Ok (mut chunk)) = body.recv ().await { - result.append (&mut chunk); - } - - result - } - pub fn status_code (&mut self, c: StatusCode) -> &mut Self { self.parts.status_code = c; self From bf96d400b216d5d4a4bb41b4392f2709816fc82e Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 16:58:56 +0000 Subject: [PATCH 063/208] :recycle: Removing unwraps --- crates/ptth_relay/src/config.rs | 22 +---------------- crates/ptth_relay/src/errors.rs | 36 ++++++++++++++++++++++++++++ crates/ptth_relay/src/lib.rs | 42 ++++++++++++++++----------------- crates/ptth_relay/src/main.rs | 3 ++- src/tests.rs | 4 +++- 5 files changed, 62 insertions(+), 45 deletions(-) create mode 100644 crates/ptth_relay/src/errors.rs diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index d48b2df..471c69d 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -6,28 +6,8 @@ use std::{ }; use serde::Deserialize; -use thiserror::Error; -#[derive (Error, Debug)] -pub enum ConfigError { - #[error ("I/O error")] - Io (#[from] std::io::Error), - - #[error ("UTF-8 decoding failed")] - Utf8 (#[from] std::string::FromUtf8Error), - - #[error ("TOML parsing failed")] - Toml (#[from] toml::de::Error), - - #[error ("base64 decoding failed")] - Base64Decode (#[from] base64::DecodeError), - - #[error ("tripcode not 32 bytes after decoding")] - TripcodeBadLength, - - #[error ("unknown config error")] - Unknown, -} +use crate::errors::ConfigError; // Stuff we need to load from the config file and use to // set up the HTTP server diff --git a/crates/ptth_relay/src/errors.rs b/crates/ptth_relay/src/errors.rs new file mode 100644 index 0000000..28d58c3 --- /dev/null +++ b/crates/ptth_relay/src/errors.rs @@ -0,0 +1,36 @@ +use thiserror::Error; + +#[derive (Error, Debug)] +pub enum ConfigError { + #[error ("I/O error")] + Io (#[from] std::io::Error), + + #[error ("UTF-8 decoding failed")] + Utf8 (#[from] std::string::FromUtf8Error), + + #[error ("TOML parsing failed")] + Toml (#[from] toml::de::Error), + + #[error ("base64 decoding failed")] + Base64Decode (#[from] base64::DecodeError), + + #[error ("tripcode not 32 bytes after decoding")] + TripcodeBadLength, + + #[error ("unknown config error")] + Unknown, +} + +// I'm not sure how important this is, but it was already in the code + +#[derive (Error, Debug)] +pub enum ShuttingDownError { + #[error ("Relay is shutting down")] + ShuttingDown, +} + +#[derive (Error, Debug)] +pub enum RelayError { + #[error ("Handlebars template file error")] + TemplateFile (#[from] handlebars::TemplateFileError), +} diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 306ebff..8d04c84 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -2,9 +2,10 @@ use std::{ borrow::Cow, error::Error, collections::*, - convert::Infallible, + convert::{Infallible, TryFrom}, iter::FromIterator, net::SocketAddr, + path::{Path, PathBuf}, sync::{ Arc, }, @@ -48,9 +49,11 @@ use ptth_core::{ }; pub mod config; +pub mod errors; pub mod git_version; -pub use config::{Config, ConfigError}; +pub use config::Config; +pub use errors::*; /* @@ -81,17 +84,12 @@ can be parked */ -#[derive (Debug)] -enum RelayError { - RelayShuttingDown, -} - enum RequestRendezvous { ParkedClients (Vec ), - ParkedServer (oneshot::Sender >), + ParkedServer (oneshot::Sender >), } -type ResponseRendezvous = oneshot::Sender >; +type ResponseRendezvous = oneshot::Sender >; use chrono::{ DateTime, @@ -127,19 +125,21 @@ pub struct RelayState { shutdown_watch_rx: watch::Receiver , } -impl From for RelayState { - fn from (config: Config) -> Self { +impl TryFrom for RelayState { + type Error = RelayError; + + fn try_from (config: Config) -> Result { let (shutdown_watch_tx, shutdown_watch_rx) = watch::channel (false); - Self { + Ok (Self { config: config.into (), - handlebars: Arc::new (load_templates (&PathBuf::new ()).unwrap ()), + handlebars: Arc::new (load_templates (&PathBuf::new ())?), request_rendezvous: Default::default (), server_status: Default::default (), response_rendezvous: Default::default (), shutdown_watch_tx, shutdown_watch_rx, - } + }) } } @@ -236,7 +236,7 @@ async fn handle_http_listen ( debug! ("Unparking server {}", watcher_code); ok_reply (rmp_serde::to_vec (&vec! [one_req]).unwrap ()) }, - Ok (Err (RelayError::RelayShuttingDown)) => error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon"), + Ok (Err (ShuttingDownError::ShuttingDown)) => error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon"), Err (_) => error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error"), }, _ = delay_for (Duration::from_secs (30)).fuse () => { @@ -448,7 +448,7 @@ async fn handle_http_request ( resp.body (body) .unwrap () }, - Ok (Err (RelayError::RelayShuttingDown)) => { + Ok (Err (ShuttingDownError::ShuttingDown)) => { error_reply (StatusCode::GATEWAY_TIMEOUT, "Relay shutting down") }, Err (_) => { @@ -634,10 +634,8 @@ async fn handle_all (req: Request , state: Arc ) }) } -use std::path::{Path, PathBuf}; - pub fn load_templates (asset_root: &Path) --> Result , Box > +-> Result , RelayError> { let mut handlebars = Handlebars::new (); handlebars.set_strict_mode (true); @@ -712,7 +710,7 @@ pub async fn run_relay ( state.shutdown_watch_tx.broadcast (true).unwrap (); - use RelayError::*; + use ShuttingDownError::ShuttingDown; let mut response_rendezvous = state.response_rendezvous.write ().await; let mut swapped = DashMap::default (); @@ -720,7 +718,7 @@ pub async fn run_relay ( std::mem::swap (&mut swapped, &mut response_rendezvous); for (_, sender) in swapped.into_iter () { - sender.send (Err (RelayShuttingDown)).ok (); + sender.send (Err (ShuttingDown)).ok (); } let mut request_rendezvous = state.request_rendezvous.lock ().await; @@ -730,7 +728,7 @@ pub async fn run_relay ( match x { ParkedClients (_) => (), - ParkedServer (sender) => drop (sender.send (Err (RelayShuttingDown))), + ParkedServer (sender) => drop (sender.send (Err (ShuttingDown))), } } diff --git a/crates/ptth_relay/src/main.rs b/crates/ptth_relay/src/main.rs index 663ae44..009d704 100644 --- a/crates/ptth_relay/src/main.rs +++ b/crates/ptth_relay/src/main.rs @@ -1,4 +1,5 @@ use std::{ + convert::TryFrom, error::Error, path::PathBuf, sync::Arc, @@ -35,7 +36,7 @@ async fn main () -> Result <(), Box > { forced_shutdown.wrap_server ( run_relay ( - Arc::new (RelayState::from (config)), + Arc::new (RelayState::try_from (config)?), shutdown_rx, Some (config_path) ) diff --git a/src/tests.rs b/src/tests.rs index 453a2ba..a578242 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -41,7 +41,9 @@ fn end_to_end () { }, }; - let relay_state = Arc::new (ptth_relay::RelayState::from (ptth_relay::config::Config::try_from (config_file).unwrap ())); + let config = ptth_relay::config::Config::try_from (config_file).unwrap (); + + let relay_state = Arc::new (ptth_relay::RelayState::try_from (config).unwrap ()); let relay_state_2 = relay_state.clone (); let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); From 5c0d7ea99868387c9acf538894bc7ff6ff8f942a Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 17:08:33 +0000 Subject: [PATCH 064/208] :recycle: Removing unwraps --- crates/ptth_relay/src/lib.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 8d04c84..73bb72a 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -152,18 +152,18 @@ impl RelayState { } fn ok_reply > (b: B) --> Response +-> Result , http::Error> { - Response::builder ().status (StatusCode::OK).body (b.into ()).unwrap () + Response::builder ().status (StatusCode::OK).body (b.into ()) } fn error_reply (status: StatusCode, b: &str) --> Response +-> Result , http::Error> { Response::builder () .status (status) .header ("content-type", "text/plain") - .body (format! ("{}\n", b).into ()).unwrap () + .body (format! ("{}\n", b).into ()) } // Servers will come here and either handle queued requests from parked clients, @@ -174,7 +174,7 @@ async fn handle_http_listen ( watcher_code: String, api_key: &[u8], ) --> Response +-> Result , http::Error> { let trip_error = error_reply (StatusCode::UNAUTHORIZED, "Bad X-ApiKey"); @@ -253,7 +253,7 @@ async fn handle_http_response ( state: Arc , req_id: String, ) - -> Response +-> Result , http::Error> { let (parts, mut body) = req.into_parts (); let resp_parts: http_serde::ResponseParts = rmp_serde::from_read_ref (&base64::decode (parts.headers.get (ptth_core::PTTH_MAGIC_HEADER).unwrap ()).unwrap ()).unwrap (); @@ -350,7 +350,7 @@ async fn handle_http_request ( state: Arc , watcher_code: String ) - -> Response +-> Result , http::Error> { { let config = state.config.read ().await; @@ -446,7 +446,6 @@ async fn handle_http_request ( debug! ("Unparked request {}", req_id); resp.body (body) - .unwrap () }, Ok (Err (ShuttingDownError::ShuttingDown)) => { error_reply (StatusCode::GATEWAY_TIMEOUT, "Relay shutting down") @@ -568,7 +567,7 @@ async fn handle_server_list_internal (state: &Arc ) async fn handle_server_list ( state: Arc -) -> Response +) -> Result , http::Error> { let page = handle_server_list_internal (&state).await; @@ -578,7 +577,7 @@ async fn handle_server_list ( #[instrument (level = "trace", skip (req, state))] async fn handle_all (req: Request , state: Arc ) --> Result , Infallible> +-> Result , http::Error> { let path = req.uri ().path (); //println! ("{}", path); @@ -591,18 +590,18 @@ async fn handle_all (req: Request , state: Arc ) // This is stuff the server can use. Clients can't // POST right now - return Ok (if let Some (request_code) = prefix_match ("/7ZSFUKGV/http_response/", path) { + return if let Some (request_code) = prefix_match ("/7ZSFUKGV/http_response/", path) { let request_code = request_code.into (); handle_http_response (req, state, request_code).await } else { error_reply (StatusCode::BAD_REQUEST, "Can't POST this") - }); + }; } - Ok (if let Some (listen_code) = prefix_match ("/7ZSFUKGV/http_listen/", path) { + if let Some (listen_code) = prefix_match ("/7ZSFUKGV/http_listen/", path) { let api_key = match api_key { - None => return Ok (error_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key")), + None => return error_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key"), Some (x) => x, }; handle_http_listen (state, listen_code.into (), api_key.as_bytes ()).await @@ -631,7 +630,7 @@ async fn handle_all (req: Request , state: Arc ) } else { error_reply (StatusCode::OK, "Hi") - }) + } } pub fn load_templates (asset_root: &Path) From 687cffdf906b059006c32967aae54f0618dab386 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 18:37:33 +0000 Subject: [PATCH 065/208] :recycle: Fixing clippy lints --- crates/ptth_relay/src/config.rs | 12 +-- crates/ptth_relay/src/errors.rs | 42 +++++++++ crates/ptth_relay/src/lib.rs | 147 +++++++++++++++++++------------- 3 files changed, 135 insertions(+), 66 deletions(-) diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index 471c69d..9ea5d83 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -1,19 +1,21 @@ +// False positive with itertools::process_results +#![allow (clippy::redundant_closure)] + use std::{ - collections::*, + collections::HashMap, convert::{TryFrom, TryInto}, iter::FromIterator, path::Path, }; -use serde::Deserialize; - use crate::errors::ConfigError; // Stuff we need to load from the config file and use to // set up the HTTP server pub mod file { - use super::*; + use std::collections::HashMap; + use serde::Deserialize; #[derive (Deserialize)] pub struct Server { @@ -76,7 +78,7 @@ impl Config { let mut f = tokio::fs::File::open (path).await?; - let mut buffer = vec! [0u8; 4096]; + let mut buffer = vec! [0_u8; 4096]; let bytes_read = f.read (&mut buffer).await?; buffer.truncate (bytes_read); diff --git a/crates/ptth_relay/src/errors.rs b/crates/ptth_relay/src/errors.rs index 28d58c3..b0d52dc 100644 --- a/crates/ptth_relay/src/errors.rs +++ b/crates/ptth_relay/src/errors.rs @@ -29,8 +29,50 @@ pub enum ShuttingDownError { ShuttingDown, } +#[derive (Error, Debug)] +pub enum HandleHttpResponseError { + #[error ("HTTP error")] + Http (#[from] http::Error), + + #[error ("Missing PTTH magic header")] + MissingPtthMagicHeader, + + #[error ("PTTH magic header is not base64")] + PtthMagicHeaderNotBase64 (base64::DecodeError), + + #[error ("PTTH magic header could not be decoded as MessagePack")] + PtthMagicHeaderNotMsgPack (rmp_serde::decode::Error), + + #[error ("Couldn't tell server something")] + LostServer, + + #[error ("Relaying task panicked")] + RelayingTaskPanicked (#[from] tokio::task::JoinError), +} + +#[derive (Error, Debug)] +pub enum RequestError { + #[error ("HTTP error")] + Http (#[from] http::Error), + + #[error ("MessagePack encode error")] + MsgPack (#[from] rmp_serde::encode::Error), + + #[error ("Handlebars rendering error")] + Handlebars (#[from] handlebars::RenderError), + + #[error ("Error handling HTTP response")] + HandleHttpResponse (#[from] HandleHttpResponseError), + + #[error ("Error is mysterious!")] + Mysterious, +} + #[derive (Error, Debug)] pub enum RelayError { #[error ("Handlebars template file error")] TemplateFile (#[from] handlebars::TemplateFileError), + + #[error ("Hyper error")] + Hyper (#[from] hyper::Error), } diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 73bb72a..16c23b7 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -1,8 +1,23 @@ +#![warn (clippy::pedantic)] + +// I'm not sure if I like this one +#![allow (clippy::enum_glob_use)] + +// I don't see the point in documenting the errors outside of where the +// error type is defined. +#![allow (clippy::missing_errors_doc)] + +// I don't see the point of writing the type twice if I'm initializing a struct +// and the type is already in the struct definition. +#![allow (clippy::default_trait_access)] + +// False positive on futures::select! macro +#![allow (clippy::mut_mut)] + use std::{ borrow::Cow, - error::Error, - collections::*, - convert::{Infallible, TryFrom}, + collections::HashMap, + convert::TryFrom, iter::FromIterator, net::SocketAddr, path::{Path, PathBuf}, @@ -12,6 +27,11 @@ use std::{ time::Duration, }; +use chrono::{ + DateTime, + SecondsFormat, + Utc +}; use dashmap::DashMap; use futures::{ FutureExt, @@ -91,12 +111,6 @@ enum RequestRendezvous { type ResponseRendezvous = oneshot::Sender >; -use chrono::{ - DateTime, - SecondsFormat, - Utc -}; - #[derive (Clone)] pub struct ServerStatus { last_seen: DateTime , @@ -174,9 +188,11 @@ async fn handle_http_listen ( watcher_code: String, api_key: &[u8], ) --> Result , http::Error> +-> Result , RequestError> { - let trip_error = error_reply (StatusCode::UNAUTHORIZED, "Bad X-ApiKey"); + use RequestRendezvous::*; + + let trip_error = || Ok (error_reply (StatusCode::UNAUTHORIZED, "Bad X-ApiKey")?); let expected_tripcode = { let config = state.config.read ().await; @@ -184,16 +200,16 @@ async fn handle_http_listen ( match config.servers.get (&watcher_code) { None => { error! ("Denied http_listen for non-existent server name {}", watcher_code); - return trip_error; + return trip_error (); }, - Some (x) => (*x).tripcode.clone (), + Some (x) => (*x).tripcode, } }; let actual_tripcode = blake3::hash (api_key); if expected_tripcode != actual_tripcode { error! ("Denied http_listen for bad tripcode {}", base64::encode (actual_tripcode.as_bytes ())); - return trip_error; + return trip_error (); } // End of early returns @@ -206,8 +222,6 @@ async fn handle_http_listen ( status.last_seen = Utc::now (); } - use RequestRendezvous::*; - let (tx, rx) = oneshot::channel (); { @@ -220,7 +234,7 @@ async fn handle_http_listen ( // handle them immediately debug! ("Sending {} parked requests to server {}", v.len (), watcher_code); - return ok_reply (rmp_serde::to_vec (&v).unwrap ()); + return Ok (ok_reply (rmp_serde::to_vec (&v)?)?); } } @@ -234,14 +248,14 @@ async fn handle_http_listen ( x = rx.fuse () => match x { Ok (Ok (one_req)) => { debug! ("Unparking server {}", watcher_code); - ok_reply (rmp_serde::to_vec (&vec! [one_req]).unwrap ()) + Ok (ok_reply (rmp_serde::to_vec (&vec! [one_req])?)?) }, - Ok (Err (ShuttingDownError::ShuttingDown)) => error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon"), - Err (_) => error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error"), + Ok (Err (ShuttingDownError::ShuttingDown)) => Ok (error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon")?), + Err (_) => Ok (error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error")?), }, _ = delay_for (Duration::from_secs (30)).fuse () => { debug! ("Timed out http_listen for server {}", watcher_code); - return error_reply (StatusCode::NO_CONTENT, "No requests now, long-poll again") + return Ok (error_reply (StatusCode::NO_CONTENT, "No requests now, long-poll again")?) } } } @@ -253,26 +267,32 @@ async fn handle_http_response ( state: Arc , req_id: String, ) --> Result , http::Error> +-> Result , HandleHttpResponseError> { - let (parts, mut body) = req.into_parts (); - let resp_parts: http_serde::ResponseParts = rmp_serde::from_read_ref (&base64::decode (parts.headers.get (ptth_core::PTTH_MAGIC_HEADER).unwrap ()).unwrap ()).unwrap (); - - // Intercept the body packets here so we can check when the stream - // ends or errors out - #[derive (Debug)] enum BodyFinishedReason { StreamFinished, ClientDisconnected, } use BodyFinishedReason::*; + use HandleHttpResponseError::*; + + let (parts, mut body) = req.into_parts (); + + let magic_header = parts.headers.get (ptth_core::PTTH_MAGIC_HEADER).ok_or (MissingPtthMagicHeader)?; + + let magic_header = base64::decode (magic_header).map_err (PtthMagicHeaderNotBase64)?; + + let resp_parts: http_serde::ResponseParts = rmp_serde::from_read_ref (&magic_header).map_err (PtthMagicHeaderNotMsgPack)?; + + // Intercept the body packets here so we can check when the stream + // ends or errors out let (mut body_tx, body_rx) = mpsc::channel (2); let (body_finished_tx, body_finished_rx) = oneshot::channel (); let mut shutdown_watch_rx = state.shutdown_watch_rx.clone (); - spawn (async move { + let relay_task = spawn (async move { if shutdown_watch_rx.recv ().await == Some (false) { loop { let item = body.next ().await; @@ -285,7 +305,7 @@ async fn handle_http_response ( futures::select! { x = body_tx.send (item).fuse () => if let Err (_) = x { info! ("Body closed while relaying. (Client hung up?)"); - body_finished_tx.send (ClientDisconnected).unwrap (); + body_finished_tx.send (ClientDisconnected).map_err (|_| LostServer).unwrap (); break; }, _ = shutdown_watch_rx.recv ().fuse () => { @@ -296,7 +316,7 @@ async fn handle_http_response ( } else { debug! ("Finished relaying bytes"); - body_finished_tx.send (StreamFinished).unwrap (); + body_finished_tx.send (StreamFinished).map_err (|_| LostServer).unwrap (); break; } } @@ -313,7 +333,7 @@ async fn handle_http_response ( match response_rendezvous.remove (&req_id) { None => { error! ("Server tried to respond to non-existent request"); - return error_reply (StatusCode::BAD_REQUEST, "Request ID not found in response_rendezvous"); + return Ok (error_reply (StatusCode::BAD_REQUEST, "Request ID not found in response_rendezvous")?); }, Some ((_, x)) => x, } @@ -323,20 +343,22 @@ async fn handle_http_response ( if tx.send (Ok ((resp_parts, body))).is_err () { let msg = "Failed to connect to client"; error! (msg); - return error_reply (StatusCode::BAD_GATEWAY, msg); + return Ok (error_reply (StatusCode::BAD_GATEWAY, msg)?); } + relay_task.await?; + debug! ("Connected server to client for streaming."); match body_finished_rx.await { Ok (StreamFinished) => { - error_reply (StatusCode::OK, "StreamFinished") + Ok (error_reply (StatusCode::OK, "StreamFinished")?) }, Ok (ClientDisconnected) => { - error_reply (StatusCode::OK, "ClientDisconnected") + Ok (error_reply (StatusCode::OK, "ClientDisconnected")?) }, Err (e) => { debug! ("body_finished_rx {}", e); - error_reply (StatusCode::OK, "body_finished_rx Err") + Ok (error_reply (StatusCode::OK, "body_finished_rx Err")?) }, } } @@ -361,7 +383,7 @@ async fn handle_http_request ( let req = match http_serde::RequestParts::from_hyper (req.method, uri, req.headers) { Ok (x) => x, - _ => return error_reply (StatusCode::BAD_REQUEST, "Bad request"), + Err (_) => return error_reply (StatusCode::BAD_REQUEST, "Bad request"), }; let (tx, rx) = oneshot::channel (); @@ -375,6 +397,8 @@ async fn handle_http_request ( trace! ("Created request {}", req_id); { + use RequestRendezvous::*; + let mut request_rendezvous = state.request_rendezvous.lock ().await; let wrapped = http_serde::WrappedRequest { @@ -382,8 +406,6 @@ async fn handle_http_request ( req, }; - use RequestRendezvous::*; - let new_rendezvous = match request_rendezvous.remove (&watcher_code) { Some (ParkedClients (mut v)) => { debug! ("Parking request {} ({} already queued)", req_id, v.len ()); @@ -439,7 +461,7 @@ async fn handle_http_request ( let mut resp = Response::builder () .status (hyper::StatusCode::from (parts.status_code)); - for (k, v) in parts.headers.into_iter () { + for (k, v) in parts.headers { resp = resp.header (&k, v); } @@ -567,17 +589,17 @@ async fn handle_server_list_internal (state: &Arc ) async fn handle_server_list ( state: Arc -) -> Result , http::Error> +) -> Result , RequestError> { let page = handle_server_list_internal (&state).await; - let s = state.handlebars.render ("relay_server_list", &page).unwrap (); - ok_reply (s) + let s = state.handlebars.render ("relay_server_list", &page)?; + Ok (ok_reply (s)?) } #[instrument (level = "trace", skip (req, state))] async fn handle_all (req: Request , state: Arc ) --> Result , http::Error> +-> Result , RequestError> { let path = req.uri ().path (); //println! ("{}", path); @@ -592,44 +614,47 @@ async fn handle_all (req: Request , state: Arc ) return if let Some (request_code) = prefix_match ("/7ZSFUKGV/http_response/", path) { let request_code = request_code.into (); - handle_http_response (req, state, request_code).await + Ok (handle_http_response (req, state, request_code).await?) } else { - error_reply (StatusCode::BAD_REQUEST, "Can't POST this") + Ok (error_reply (StatusCode::BAD_REQUEST, "Can't POST this")?) }; } if let Some (listen_code) = prefix_match ("/7ZSFUKGV/http_listen/", path) { let api_key = match api_key { - None => return error_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key"), + None => return Ok (error_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key")?), Some (x) => x, }; handle_http_listen (state, listen_code.into (), api_key.as_bytes ()).await } else if let Some (rest) = prefix_match ("/frontend/servers/", path) { if rest == "" { - handle_server_list (state).await + Ok (handle_server_list (state).await?) } else if let Some (idx) = rest.find ('/') { let listen_code = String::from (&rest [0..idx]); let path = String::from (&rest [idx..]); let (parts, _) = req.into_parts (); - handle_http_request (parts, path, state, listen_code).await + Ok (handle_http_request (parts, path, state, listen_code).await?) } else { - error_reply (StatusCode::BAD_REQUEST, "Bad URI format") + Ok (error_reply (StatusCode::BAD_REQUEST, "Bad URI format")?) } } else if path == "/" { - let s = state.handlebars.render ("relay_root", &()).unwrap (); - ok_reply (s) + let s = state.handlebars.render ("relay_root", &())?; + Ok (ok_reply (s)?) } else if path == "/frontend/relay_up_check" { - error_reply (StatusCode::OK, "Relay is up") + Ok (error_reply (StatusCode::OK, "Relay is up")?) + } + else if path == "/frontend/test_mysterious_error" { + Err (RequestError::Mysterious) } else { - error_reply (StatusCode::OK, "Hi") + Ok (error_reply (StatusCode::OK, "Hi")?) } } @@ -641,10 +666,10 @@ pub fn load_templates (asset_root: &Path) let asset_root = asset_root.join ("handlebars/relay"); - for (k, v) in vec! [ + for (k, v) in &[ ("relay_server_list", "relay_server_list.html"), ("relay_root", "relay_root.html"), - ].into_iter () { + ] { handlebars.register_template_file (k, &asset_root.join (v))?; } @@ -670,7 +695,7 @@ pub async fn run_relay ( shutdown_oneshot: oneshot::Receiver <()>, config_reload_path: Option ) --> Result <(), Box > +-> Result <(), RelayError> { let addr = SocketAddr::from (( [0, 0, 0, 0], @@ -693,7 +718,7 @@ pub async fn run_relay ( let state = state.clone (); async { - Ok::<_, Infallible> (service_fn (move |req| { + Ok::<_, RequestError> (service_fn (move |req| { let state = state.clone (); handle_all (req, state) @@ -705,18 +730,18 @@ pub async fn run_relay ( .serve (make_svc); server.with_graceful_shutdown (async { + use ShuttingDownError::ShuttingDown; + shutdown_oneshot.await.ok (); state.shutdown_watch_tx.broadcast (true).unwrap (); - use ShuttingDownError::ShuttingDown; - let mut response_rendezvous = state.response_rendezvous.write ().await; let mut swapped = DashMap::default (); std::mem::swap (&mut swapped, &mut response_rendezvous); - for (_, sender) in swapped.into_iter () { + for (_, sender) in swapped { sender.send (Err (ShuttingDown)).ok (); } From aad7f8e7298123cc73299bcae981222677c4f8cb Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 18:39:51 +0000 Subject: [PATCH 066/208] :recycle: Remove all practically removable unwraps from ptth_relay --- crates/ptth_relay/src/lib.rs | 30 ++---------------------------- crates/ptth_relay/src/tests.rs | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 28 deletions(-) create mode 100644 crates/ptth_relay/src/tests.rs diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 16c23b7..d4d0a96 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -734,7 +734,7 @@ pub async fn run_relay ( shutdown_oneshot.await.ok (); - state.shutdown_watch_tx.broadcast (true).unwrap (); + state.shutdown_watch_tx.broadcast (true).expect ("Can't broadcast graceful shutdown"); let mut response_rendezvous = state.response_rendezvous.write ().await; let mut swapped = DashMap::default (); @@ -763,30 +763,4 @@ pub async fn run_relay ( } #[cfg (test)] -mod tests { - use super::*; - - #[test] - fn test_pretty_print_last_seen () { - use LastSeen::*; - - let last_seen = DateTime::parse_from_rfc3339 ("2019-05-29T00:00:00+00:00").unwrap ().with_timezone (&Utc); - - for (input, expected) in vec! [ - ("2019-05-28T23:59:59+00:00", Negative), - ("2019-05-29T00:00:00+00:00", Connected), - ("2019-05-29T00:00:59+00:00", Connected), - ("2019-05-29T00:01:30+00:00", Description ("1 m ago".into ())), - ("2019-05-29T00:59:30+00:00", Description ("59 m ago".into ())), - ("2019-05-29T01:00:30+00:00", Description ("1 h ago".into ())), - ("2019-05-29T10:00:00+00:00", Description ("10 h ago".into ())), - ("2019-05-30T00:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), - ("2019-05-30T10:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), - ("2019-05-31T00:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), - ].into_iter () { - let now = DateTime::parse_from_rfc3339 (input).unwrap ().with_timezone (&Utc); - let actual = pretty_print_last_seen (now, last_seen); - assert_eq! (actual, expected); - } - } -} +mod tests; diff --git a/crates/ptth_relay/src/tests.rs b/crates/ptth_relay/src/tests.rs new file mode 100644 index 0000000..c1e7b74 --- /dev/null +++ b/crates/ptth_relay/src/tests.rs @@ -0,0 +1,25 @@ +use super::*; + +#[test] +fn test_pretty_print_last_seen () { + use LastSeen::*; + + let last_seen = DateTime::parse_from_rfc3339 ("2019-05-29T00:00:00+00:00").unwrap ().with_timezone (&Utc); + + for (input, expected) in vec! [ + ("2019-05-28T23:59:59+00:00", Negative), + ("2019-05-29T00:00:00+00:00", Connected), + ("2019-05-29T00:00:59+00:00", Connected), + ("2019-05-29T00:01:30+00:00", Description ("1 m ago".into ())), + ("2019-05-29T00:59:30+00:00", Description ("59 m ago".into ())), + ("2019-05-29T01:00:30+00:00", Description ("1 h ago".into ())), + ("2019-05-29T10:00:00+00:00", Description ("10 h ago".into ())), + ("2019-05-30T00:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), + ("2019-05-30T10:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), + ("2019-05-31T00:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), + ].into_iter () { + let now = DateTime::parse_from_rfc3339 (input).unwrap ().with_timezone (&Utc); + let actual = pretty_print_last_seen (now, last_seen); + assert_eq! (actual, expected); + } +} From 47c59447f3c104fe52414de3b96eef08040c765e Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 18:50:51 +0000 Subject: [PATCH 067/208] :recycle: Splitting tests for file server into their own tests.rs --- crates/ptth_relay/src/errors.rs | 10 +- crates/ptth_server/Cargo.toml | 1 + crates/ptth_server/src/errors.rs | 6 + crates/ptth_server/src/file_server/errors.rs | 7 + .../{file_server.rs => file_server/mod.rs} | 221 +----------------- crates/ptth_server/src/file_server/tests.rs | 216 +++++++++++++++++ crates/ptth_server/src/lib.rs | 1 + 7 files changed, 239 insertions(+), 223 deletions(-) create mode 100644 crates/ptth_server/src/errors.rs create mode 100644 crates/ptth_server/src/file_server/errors.rs rename crates/ptth_server/src/{file_server.rs => file_server/mod.rs} (72%) create mode 100644 crates/ptth_server/src/file_server/tests.rs diff --git a/crates/ptth_relay/src/errors.rs b/crates/ptth_relay/src/errors.rs index b0d52dc..14af9e0 100644 --- a/crates/ptth_relay/src/errors.rs +++ b/crates/ptth_relay/src/errors.rs @@ -1,6 +1,6 @@ use thiserror::Error; -#[derive (Error, Debug)] +#[derive (Debug, Error)] pub enum ConfigError { #[error ("I/O error")] Io (#[from] std::io::Error), @@ -23,13 +23,13 @@ pub enum ConfigError { // I'm not sure how important this is, but it was already in the code -#[derive (Error, Debug)] +#[derive (Debug, Error)] pub enum ShuttingDownError { #[error ("Relay is shutting down")] ShuttingDown, } -#[derive (Error, Debug)] +#[derive (Debug, Error)] pub enum HandleHttpResponseError { #[error ("HTTP error")] Http (#[from] http::Error), @@ -50,7 +50,7 @@ pub enum HandleHttpResponseError { RelayingTaskPanicked (#[from] tokio::task::JoinError), } -#[derive (Error, Debug)] +#[derive (Debug, Error)] pub enum RequestError { #[error ("HTTP error")] Http (#[from] http::Error), @@ -68,7 +68,7 @@ pub enum RequestError { Mysterious, } -#[derive (Error, Debug)] +#[derive (Debug, Error)] pub enum RelayError { #[error ("Handlebars template file error")] TemplateFile (#[from] handlebars::TemplateFileError), diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index a0a61af..77004a9 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -22,6 +22,7 @@ reqwest = { version = "0.10.8", features = ["stream"] } rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} structopt = "0.3.20" +thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" tracing-futures = "0.2.4" diff --git a/crates/ptth_server/src/errors.rs b/crates/ptth_server/src/errors.rs new file mode 100644 index 0000000..25d8acc --- /dev/null +++ b/crates/ptth_server/src/errors.rs @@ -0,0 +1,6 @@ +use thiserror::Error; + +#[derive (Debug, Error)] +pub enum ServerError { + +} diff --git a/crates/ptth_server/src/file_server/errors.rs b/crates/ptth_server/src/file_server/errors.rs new file mode 100644 index 0000000..3cc5808 --- /dev/null +++ b/crates/ptth_server/src/file_server/errors.rs @@ -0,0 +1,7 @@ +use thiserror::Error; + +#[derive (Debug, Error)] +pub enum FileServerError { + +} + diff --git a/crates/ptth_server/src/file_server.rs b/crates/ptth_server/src/file_server/mod.rs similarity index 72% rename from crates/ptth_server/src/file_server.rs rename to crates/ptth_server/src/file_server/mod.rs index c111bbc..e5932a6 100644 --- a/crates/ptth_server/src/file_server.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -46,6 +46,8 @@ use ptth_core::{ prefix_match, }; +mod errors; + #[derive (Debug, Serialize)] pub struct ServerInfo { pub server_name: String, @@ -680,221 +682,4 @@ fn pretty_print_bytes (b: u64) -> String { } #[cfg (test)] -mod tests { - use std::{ - ffi::OsStr, - path::{ - Component, - Path, - }, - }; - - use maplit::*; - use tokio::runtime::Runtime; - - #[test] - fn icons () { - let video = "🎞️"; - let picture = "📷"; - let file = "📄"; - - for (input, expected) in vec! [ - ("copying_is_not_theft.mp4", video), - ("copying_is_not_theft.avi", video), - ("copying_is_not_theft.mkv", video), - ("copying_is_not_theft.webm", video), - ("lolcats.jpg", picture), - ("lolcats.jpeg", picture), - ("lolcats.png", picture), - ("lolcats.bmp", picture), - ("ptth.log", file), - ("README.md", file), - ("todo.txt", file), - ].into_iter () { - assert_eq! (super::get_icon (input), expected); - } - } - - #[test] - fn parse_range_header () { - for (input, expected) in vec! [ - ("", (None, None)), - ("bytes=0-", (Some (0), None)), - ("bytes=0-999", (Some (0), Some (1000))), - ("bytes=111-999", (Some (111), Some (1000))), - ].into_iter () { - let actual = super::parse_range_header (input); - assert_eq! (actual, expected); - } - - use super::ParsedRange::*; - - for (header, file_len, expected) in vec! [ - (None, 0, Ok (0..0)), - (None, 1024, Ok (0..1024)), - - (Some (""), 0, RangeNotSatisfiable (0)), - (Some (""), 1024, PartialContent (0..1024)), - - (Some ("bytes=0-"), 1024, PartialContent (0..1024)), - (Some ("bytes=0-999"), 1024, PartialContent (0..1000)), - (Some ("bytes=0-1023"), 1024, PartialContent (0..1024)), - (Some ("bytes=111-999"), 1024, PartialContent (111..1000)), - (Some ("bytes=111-1023"), 1024, PartialContent (111..1024)), - (Some ("bytes=200-100"), 1024, RangeNotSatisfiable (1024)), - - (Some ("bytes=0-"), 512, PartialContent (0..512)), - (Some ("bytes=0-1023"), 512, RangeNotSatisfiable (512)), - (Some ("bytes=1000-1023"), 512, RangeNotSatisfiable (512)), - ].into_iter () { - let actual = super::check_range (header, file_len); - assert_eq! (actual, expected); - } - } - - #[test] - fn pretty_print_bytes () { - for (input_after, expected_before, expected_after) in vec! [ - (1, "0 B", "1 B"), - (1024, "1023 B", "1 KiB"), - (1024 + 512, "1 KiB", "2 KiB"), - (1023 * 1024 + 512, "1023 KiB", "1 MiB"), - ((1024 + 512) * 1024, "1 MiB", "2 MiB"), - (1023 * 1024 * 1024 + 512 * 1024, "1023 MiB", "1 GiB"), - ((1024 + 512) * 1024 * 1024, "1 GiB", "2 GiB"), - - ].into_iter () { - let actual = super::pretty_print_bytes (input_after - 1); - assert_eq! (&actual, expected_before); - - let actual = super::pretty_print_bytes (input_after); - assert_eq! (&actual, expected_after); - } - } - - #[test] - fn i_hate_paths () { - let mut components = Path::new ("/home/user").components (); - - assert_eq! (components.next (), Some (Component::RootDir)); - assert_eq! (components.next (), Some (Component::Normal (OsStr::new ("home")))); - assert_eq! (components.next (), Some (Component::Normal (OsStr::new ("user")))); - assert_eq! (components.next (), None); - - let mut components = Path::new ("./home/user").components (); - - assert_eq! (components.next (), Some (Component::CurDir)); - assert_eq! (components.next (), Some (Component::Normal (OsStr::new ("home")))); - assert_eq! (components.next (), Some (Component::Normal (OsStr::new ("user")))); - assert_eq! (components.next (), None); - - let mut components = Path::new (".").components (); - - assert_eq! (components.next (), Some (Component::CurDir)); - assert_eq! (components.next (), None); - } - - #[test] - fn file_server () { - use ptth_core::{ - http_serde::Method, - }; - use super::*; - - tracing_subscriber::fmt ().try_init ().ok (); - let mut rt = Runtime::new ().unwrap (); - - rt.block_on (async { - let file_server_root = PathBuf::from ("./"); - let headers = Default::default (); - - { - use InternalResponse::*; - - let bad_passwords_path = "/files/src/bad_passwords.txt"; - - for (uri_path, expected) in vec! [ - ("/", Root), - ("/files", Redirect ("files/".to_string ())), - ("/files/?", InvalidQuery), - ("/files/src", Redirect ("src/".to_string ())), - ("/files/src/?", InvalidQuery), - (bad_passwords_path, ServeFile (ServeFileParams { - send_body: true, - range: 0..1_048_576, - range_requested: false, - file: AlwaysEqual::testing_blank (), - })), - ("/files/test/test.md", ServeFile (ServeFileParams { - send_body: true, - range: 0..144, - range_requested: false, - file: AlwaysEqual::testing_blank (), - })), - ("/ ", InvalidUri), - ].into_iter () { - let resp = internal_serve_all ( - &file_server_root, - Method::Get, - uri_path, - &headers, - None - ).await; - - assert_eq! (resp, expected); - } - - let resp = internal_serve_all ( - &file_server_root, - Method::Get, - bad_passwords_path, - &hashmap! { - "range".into () => b"bytes=0-2000000".to_vec (), - }, - None - ).await; - - assert_eq! (resp, RangeNotSatisfiable (1_048_576)); - - let resp = internal_serve_all ( - &file_server_root, - Method::Head, - bad_passwords_path, - &headers, - None - ).await; - - assert_eq! (resp, ServeFile (ServeFileParams { - send_body: false, - range: 0..1_048_576, - range_requested: false, - file: AlwaysEqual::testing_blank (), - })); - } - }); - } - - #[test] - fn parse_uri () { - use hyper::Uri; - - assert! (Uri::from_maybe_shared ("/").is_ok ()); - } - - #[test] - fn markdown () { - use super::*; - - for (input, expected) in vec! [ - ("", ""), - ( - "Hello world, this is a ~~complicated~~ *very simple* example.", - "

Hello world, this is a complicated very simple example.

\n" - ), - ].into_iter () { - let mut out = String::default (); - render_markdown (input.as_bytes (), &mut out).unwrap (); - assert_eq! (expected, &out); - } - } -} +mod tests; diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs new file mode 100644 index 0000000..f85361a --- /dev/null +++ b/crates/ptth_server/src/file_server/tests.rs @@ -0,0 +1,216 @@ +use std::{ + ffi::OsStr, + path::{ + Component, + Path, + }, +}; + +use maplit::*; +use tokio::runtime::Runtime; + +#[test] +fn icons () { + let video = "🎞️"; + let picture = "📷"; + let file = "📄"; + + for (input, expected) in vec! [ + ("copying_is_not_theft.mp4", video), + ("copying_is_not_theft.avi", video), + ("copying_is_not_theft.mkv", video), + ("copying_is_not_theft.webm", video), + ("lolcats.jpg", picture), + ("lolcats.jpeg", picture), + ("lolcats.png", picture), + ("lolcats.bmp", picture), + ("ptth.log", file), + ("README.md", file), + ("todo.txt", file), + ].into_iter () { + assert_eq! (super::get_icon (input), expected); + } +} + +#[test] +fn parse_range_header () { + for (input, expected) in vec! [ + ("", (None, None)), + ("bytes=0-", (Some (0), None)), + ("bytes=0-999", (Some (0), Some (1000))), + ("bytes=111-999", (Some (111), Some (1000))), + ].into_iter () { + let actual = super::parse_range_header (input); + assert_eq! (actual, expected); + } + + use super::ParsedRange::*; + + for (header, file_len, expected) in vec! [ + (None, 0, Ok (0..0)), + (None, 1024, Ok (0..1024)), + + (Some (""), 0, RangeNotSatisfiable (0)), + (Some (""), 1024, PartialContent (0..1024)), + + (Some ("bytes=0-"), 1024, PartialContent (0..1024)), + (Some ("bytes=0-999"), 1024, PartialContent (0..1000)), + (Some ("bytes=0-1023"), 1024, PartialContent (0..1024)), + (Some ("bytes=111-999"), 1024, PartialContent (111..1000)), + (Some ("bytes=111-1023"), 1024, PartialContent (111..1024)), + (Some ("bytes=200-100"), 1024, RangeNotSatisfiable (1024)), + + (Some ("bytes=0-"), 512, PartialContent (0..512)), + (Some ("bytes=0-1023"), 512, RangeNotSatisfiable (512)), + (Some ("bytes=1000-1023"), 512, RangeNotSatisfiable (512)), + ].into_iter () { + let actual = super::check_range (header, file_len); + assert_eq! (actual, expected); + } +} + +#[test] +fn pretty_print_bytes () { + for (input_after, expected_before, expected_after) in vec! [ + (1, "0 B", "1 B"), + (1024, "1023 B", "1 KiB"), + (1024 + 512, "1 KiB", "2 KiB"), + (1023 * 1024 + 512, "1023 KiB", "1 MiB"), + ((1024 + 512) * 1024, "1 MiB", "2 MiB"), + (1023 * 1024 * 1024 + 512 * 1024, "1023 MiB", "1 GiB"), + ((1024 + 512) * 1024 * 1024, "1 GiB", "2 GiB"), + + ].into_iter () { + let actual = super::pretty_print_bytes (input_after - 1); + assert_eq! (&actual, expected_before); + + let actual = super::pretty_print_bytes (input_after); + assert_eq! (&actual, expected_after); + } +} + +#[test] +fn i_hate_paths () { + let mut components = Path::new ("/home/user").components (); + + assert_eq! (components.next (), Some (Component::RootDir)); + assert_eq! (components.next (), Some (Component::Normal (OsStr::new ("home")))); + assert_eq! (components.next (), Some (Component::Normal (OsStr::new ("user")))); + assert_eq! (components.next (), None); + + let mut components = Path::new ("./home/user").components (); + + assert_eq! (components.next (), Some (Component::CurDir)); + assert_eq! (components.next (), Some (Component::Normal (OsStr::new ("home")))); + assert_eq! (components.next (), Some (Component::Normal (OsStr::new ("user")))); + assert_eq! (components.next (), None); + + let mut components = Path::new (".").components (); + + assert_eq! (components.next (), Some (Component::CurDir)); + assert_eq! (components.next (), None); +} + +#[test] +fn file_server () { + use ptth_core::{ + http_serde::Method, + }; + use super::*; + + tracing_subscriber::fmt ().try_init ().ok (); + let mut rt = Runtime::new ().unwrap (); + + rt.block_on (async { + let file_server_root = PathBuf::from ("./"); + let headers = Default::default (); + + { + use InternalResponse::*; + + let bad_passwords_path = "/files/src/bad_passwords.txt"; + + for (uri_path, expected) in vec! [ + ("/", Root), + ("/files", Redirect ("files/".to_string ())), + ("/files/?", InvalidQuery), + ("/files/src", Redirect ("src/".to_string ())), + ("/files/src/?", InvalidQuery), + (bad_passwords_path, ServeFile (ServeFileParams { + send_body: true, + range: 0..1_048_576, + range_requested: false, + file: AlwaysEqual::testing_blank (), + })), + ("/files/test/test.md", ServeFile (ServeFileParams { + send_body: true, + range: 0..144, + range_requested: false, + file: AlwaysEqual::testing_blank (), + })), + ("/ ", InvalidUri), + ].into_iter () { + let resp = internal_serve_all ( + &file_server_root, + Method::Get, + uri_path, + &headers, + None + ).await; + + assert_eq! (resp, expected); + } + + let resp = internal_serve_all ( + &file_server_root, + Method::Get, + bad_passwords_path, + &hashmap! { + "range".into () => b"bytes=0-2000000".to_vec (), + }, + None + ).await; + + assert_eq! (resp, RangeNotSatisfiable (1_048_576)); + + let resp = internal_serve_all ( + &file_server_root, + Method::Head, + bad_passwords_path, + &headers, + None + ).await; + + assert_eq! (resp, ServeFile (ServeFileParams { + send_body: false, + range: 0..1_048_576, + range_requested: false, + file: AlwaysEqual::testing_blank (), + })); + } + }); +} + +#[test] +fn parse_uri () { + use hyper::Uri; + + assert! (Uri::from_maybe_shared ("/").is_ok ()); +} + +#[test] +fn markdown () { + use super::*; + + for (input, expected) in vec! [ + ("", ""), + ( + "Hello world, this is a ~~complicated~~ *very simple* example.", + "

Hello world, this is a complicated very simple example.

\n" + ), + ].into_iter () { + let mut out = String::default (); + render_markdown (input.as_bytes (), &mut out).unwrap (); + assert_eq! (expected, &out); + } +} diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index a6b8d69..41dcaea 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -22,6 +22,7 @@ use ptth_core::{ prelude::*, }; +pub mod errors; pub mod file_server; pub mod load_toml; From c3ff3deb8eb6299d986c4025c39051aeaca9c64e Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 19:05:28 +0000 Subject: [PATCH 068/208] :recycle: Fix some clippy warnings --- crates/ptth_relay/src/lib.rs | 10 ++-- .../ptth_server/src/bin/ptth_file_server.rs | 18 +++---- crates/ptth_server/src/file_server/mod.rs | 52 +++++++++++-------- crates/ptth_server/src/lib.rs | 17 ++++-- crates/ptth_server/src/load_toml.rs | 2 +- 5 files changed, 58 insertions(+), 41 deletions(-) diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index d4d0a96..25247fd 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -1,5 +1,9 @@ #![warn (clippy::pedantic)] +// I don't see the point of writing the type twice if I'm initializing a struct +// and the type is already in the struct definition. +#![allow (clippy::default_trait_access)] + // I'm not sure if I like this one #![allow (clippy::enum_glob_use)] @@ -7,15 +11,11 @@ // error type is defined. #![allow (clippy::missing_errors_doc)] -// I don't see the point of writing the type twice if I'm initializing a struct -// and the type is already in the struct definition. -#![allow (clippy::default_trait_access)] - // False positive on futures::select! macro #![allow (clippy::mut_mut)] use std::{ - borrow::Cow, + borrow::Cow, collections::HashMap, convert::TryFrom, iter::FromIterator, diff --git a/crates/ptth_server/src/bin/ptth_file_server.rs b/crates/ptth_server/src/bin/ptth_file_server.rs index cd517ae..ae5eaf3 100644 --- a/crates/ptth_server/src/bin/ptth_file_server.rs +++ b/crates/ptth_server/src/bin/ptth_file_server.rs @@ -40,14 +40,16 @@ struct ServerState <'a> { } fn status_reply > (status: StatusCode, b: B) --> Response +-> Result , hyper::http::Error> { - Response::builder ().status (status).body (b.into ()).unwrap () + Response::builder ().status (status).body (b.into ()) } async fn handle_all (req: Request , state: Arc >) --> Result , String> +-> Result , hyper::http::Error> { + use std::str::FromStr; + debug! ("req.uri () = {:?}", req.uri ()); let path_and_query = req.uri ().path_and_query ().map (|x| x.as_str ()).unwrap_or_else (|| req.uri ().path ()); @@ -58,7 +60,7 @@ async fn handle_all (req: Request , state: Arc >) let ptth_req = match RequestParts::from_hyper (parts.method, path_and_query, parts.headers) { Ok (x) => x, - _ => return Ok (status_reply (StatusCode::BAD_REQUEST, "Bad request")), + _ => return Ok (status_reply (StatusCode::BAD_REQUEST, "Bad request")?), }; let default_root = PathBuf::from ("./"); @@ -79,10 +81,8 @@ async fn handle_all (req: Request , state: Arc >) let mut resp = Response::builder () .status (StatusCode::from (ptth_resp.parts.status_code)); - use std::str::FromStr; - for (k, v) in ptth_resp.parts.headers.into_iter () { - resp = resp.header (hyper::header::HeaderName::from_str (&k).unwrap (), v); + resp = resp.header (hyper::header::HeaderName::from_str (&k)?, v); } let body = ptth_resp.body @@ -90,9 +90,7 @@ async fn handle_all (req: Request , state: Arc >) .unwrap_or_else (Body::empty) ; - let resp = resp.body (body).unwrap (); - - Ok (resp) + resp.body (body) } #[derive (Deserialize)] diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index e5932a6..3715e2a 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -1,9 +1,12 @@ // Static file server that can plug into the PTTH reverse server +// I'm not sure if I like this one +#![allow (clippy::enum_glob_use)] + use std::{ borrow::Cow, cmp::min, - collections::*, + collections::HashMap, convert::{Infallible, TryInto}, error::Error, fmt::Debug, @@ -12,7 +15,9 @@ use std::{ }; use handlebars::Handlebars; -use percent_encoding::*; +use percent_encoding::{ + percent_decode, +}; use serde::Serialize; use tokio::{ fs::{ @@ -84,7 +89,7 @@ struct TemplateDirPage <'a> { } fn parse_range_header (range_str: &str) -> (Option , Option ) { - use lazy_static::*; + use lazy_static::lazy_static; lazy_static! { static ref RE: Regex = Regex::new (r"^bytes=(\d*)-(\d*)$").expect ("Couldn't compile regex for Range header"); @@ -94,17 +99,17 @@ fn parse_range_header (range_str: &str) -> (Option , Option ) { let caps = match RE.captures (range_str) { Some (x) => x, - _ => return (None, None), + None => return (None, None), }; let start = caps.get (1).map (|x| x.as_str ()); let end = caps.get (2).map (|x| x.as_str ()); - let start = start.map (|x| u64::from_str_radix (x, 10).ok ()).flatten (); + let start = start.and_then (|x| u64::from_str_radix (x, 10).ok ()); // HTTP specifies ranges as [start inclusive, end inclusive] // But that's dumb and [start inclusive, end exclusive) is better - let end = end.map (|x| u64::from_str_radix (x, 10).ok ().map (|x| x + 1)).flatten (); + let end = end.and_then (|x| u64::from_str_radix (x, 10).ok ().map (|x| x + 1)); (start, end) } @@ -151,9 +156,9 @@ fn check_range (range_str: Option <&str>, file_len: u64) fn get_icon (file_name: &str) -> &'static str { // Because my editor actually doesn't render these - let video = "🎞️"; - let picture = "📷"; - let file = "📄"; + let video = "\u{1f39e}\u{fe0f}"; + let picture = "\u{1f4f7}"; + let file = "\u{1f4c4}"; if file_name.ends_with (".mp4") || @@ -178,10 +183,15 @@ fn get_icon (file_name: &str) -> &'static str { async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry { + use percent_encoding::{ + CONTROLS, + utf8_percent_encode, + }; + let file_name = match entry.file_name ().into_string () { Ok (x) => x, Err (_) => return TemplateDirEntry { - icon: "⚠️", + icon: "\u{26a0}\u{fe0f}", trailing_slash: "", file_name: "File / directory name is not UTF-8".into (), encoded_file_name: "".into (), @@ -193,7 +203,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let metadata = match entry.metadata ().await { Ok (x) => x, Err (_) => return TemplateDirEntry { - icon: "⚠️", + icon: "\u{26a0}\u{fe0f}", trailing_slash: "", file_name: "Could not fetch metadata".into (), encoded_file_name: "".into (), @@ -204,7 +214,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let (trailing_slash, icon, size) = { let t = metadata.file_type (); - let icon_folder = "📁"; + let icon_folder = "\u{1f4c1}"; if t.is_dir () { ("/", icon_folder, "".into ()) @@ -214,8 +224,6 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry } }; - use percent_encoding::*; - let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string (); TemplateDirEntry { @@ -307,7 +315,7 @@ async fn serve_file ( let mut next_mark = mark_interval; loop { - let mut buffer = vec! [0u8; 65_536]; + let mut buffer = vec! [0_u8; 65_536]; let bytes_read: u64 = f.read (&mut buffer).await.unwrap ().try_into ().unwrap (); let bytes_read = min (bytes_left, bytes_read); @@ -353,11 +361,11 @@ async fn serve_file ( response.header (String::from ("content-length"), range.end.to_string ().into_bytes ()); } - if ! should_send_body { - response.status_code (StatusCode::NoContent); + if should_send_body { + response.content_length = Some (content_length); } else { - response.content_length = Some (content_length); + response.status_code (StatusCode::NoContent); } if let Some (body) = body { @@ -546,7 +554,7 @@ async fn internal_serve_all ( let file_len = file_md.len (); - let range_header = headers.get ("range").map (|v| std::str::from_utf8 (v).ok ()).flatten (); + let range_header = headers.get ("range").and_then (|v| std::str::from_utf8 (v).ok ()); match check_range (range_header, file_len) { ParsedRange::RangeNotSatisfiable (file_len) => RangeNotSatisfiable (file_len), @@ -557,7 +565,7 @@ async fn internal_serve_all ( MarkdownErr (MarkdownError::TooBig) } else { - let mut buffer = vec! [0u8; MAX_BUF_SIZE.try_into ().unwrap ()]; + let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().unwrap ()]; let bytes_read = file.read (&mut buffer).await.unwrap (); buffer.truncate (bytes_read); @@ -656,10 +664,10 @@ pub fn load_templates ( let asset_root = asset_root.join ("handlebars/server"); - for (k, v) in vec! [ + for (k, v) in &[ ("file_server_dir", "file_server_dir.html"), ("file_server_root", "file_server_root.html"), - ].into_iter () { + ] { handlebars.register_template_file (k, asset_root.join (v))?; } diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 41dcaea..588a61e 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -1,3 +1,12 @@ +#![warn (clippy::pedantic)] + +// I don't see the point in documenting the errors outside of where the +// error type is defined. +#![allow (clippy::missing_errors_doc)] + +// False positive on futures::select! macro +#![allow (clippy::mut_mut)] + use std::{ error::Error, path::PathBuf, @@ -30,6 +39,7 @@ pub mod load_toml; const BAD_PASSWORDS: &[u8] = include_bytes! ("bad_passwords.txt"); +#[must_use] pub fn password_is_bad (mut password: String) -> bool { password.make_ascii_lowercase (); @@ -66,7 +76,7 @@ async fn handle_req_resp <'a> ( debug! ("Unwrapped {} requests", wrapped_reqs.len ()); - for wrapped_req in wrapped_reqs.into_iter () { + for wrapped_req in wrapped_reqs { let state = state.clone (); tokio::spawn (async move { @@ -136,6 +146,7 @@ pub struct ConfigFile { } impl ConfigFile { + #[must_use] pub fn tripcode (&self) -> String { base64::encode (blake3::hash (self.api_key.as_bytes ()).as_bytes ()) } @@ -155,10 +166,10 @@ pub async fn run_server ( ) -> Result <(), Box > { - let asset_root = asset_root.unwrap_or_else (PathBuf::new); - use std::convert::TryInto; + let asset_root = asset_root.unwrap_or_else (PathBuf::new); + if password_is_bad (config_file.api_key.clone ()) { panic! ("API key is too weak, server can't use it"); } diff --git a/crates/ptth_server/src/load_toml.rs b/crates/ptth_server/src/load_toml.rs index 921ce1f..1922adb 100644 --- a/crates/ptth_server/src/load_toml.rs +++ b/crates/ptth_server/src/load_toml.rs @@ -14,7 +14,7 @@ fn load_inner < > ( mut f: File ) -> T { - let mut buffer = vec! [0u8; 4096]; + let mut buffer = vec! [0_u8; 4096]; let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read config")); buffer.truncate (bytes_read); From 720aae220142ff9bea1bde0f843d620d89a6cc70 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 19:19:59 +0000 Subject: [PATCH 069/208] :recycle: Working on errors for file server and server --- crates/ptth_server/src/bin/ptth_file_server.rs | 7 ++++--- crates/ptth_server/src/errors.rs | 10 ++++++++++ crates/ptth_server/src/file_server/errors.rs | 4 ++-- crates/ptth_server/src/file_server/mod.rs | 17 +++++++++-------- crates/ptth_server/src/lib.rs | 8 +++++--- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/crates/ptth_server/src/bin/ptth_file_server.rs b/crates/ptth_server/src/bin/ptth_file_server.rs index ae5eaf3..aefba3a 100644 --- a/crates/ptth_server/src/bin/ptth_file_server.rs +++ b/crates/ptth_server/src/bin/ptth_file_server.rs @@ -23,6 +23,7 @@ use ptth_core::{ prelude::*, }; use ptth_server::{ + errors::ServerError, file_server, load_toml, }; @@ -46,7 +47,7 @@ fn status_reply > (status: StatusCode, b: B) } async fn handle_all (req: Request , state: Arc >) --> Result , hyper::http::Error> +-> Result , ServerError> { use std::str::FromStr; @@ -76,7 +77,7 @@ async fn handle_all (req: Request , state: Arc >) &ptth_req.uri, &ptth_req.headers, state.hidden_path.as_deref () - ).await; + ).await?; let mut resp = Response::builder () .status (StatusCode::from (ptth_resp.parts.status_code)); @@ -90,7 +91,7 @@ async fn handle_all (req: Request , state: Arc >) .unwrap_or_else (Body::empty) ; - resp.body (body) + Ok (resp.body (body)?) } #[derive (Deserialize)] diff --git a/crates/ptth_server/src/errors.rs b/crates/ptth_server/src/errors.rs index 25d8acc..5708af7 100644 --- a/crates/ptth_server/src/errors.rs +++ b/crates/ptth_server/src/errors.rs @@ -2,5 +2,15 @@ use thiserror::Error; #[derive (Debug, Error)] pub enum ServerError { + #[error ("File server error")] + FileServer (#[from] super::file_server::errors::FileServerError), + #[error ("Hyper HTTP error")] + Http (#[from] hyper::http::Error), + + #[error ("Hyper invalid header name")] + InvalidHeaderName (#[from] hyper::header::InvalidHeaderName), + + #[error ("Can't parse wrapped requests")] + CantParseWrappedRequests (rmp_serde::decode::Error), } diff --git a/crates/ptth_server/src/file_server/errors.rs b/crates/ptth_server/src/file_server/errors.rs index 3cc5808..de32bf9 100644 --- a/crates/ptth_server/src/file_server/errors.rs +++ b/crates/ptth_server/src/file_server/errors.rs @@ -2,6 +2,6 @@ use thiserror::Error; #[derive (Debug, Error)] pub enum FileServerError { - + #[error ("Handlebars render error")] + Handlebars (#[from] handlebars::RenderError), } - diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 3715e2a..38dcfcf 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -51,7 +51,8 @@ use ptth_core::{ prefix_match, }; -mod errors; +pub mod errors; +use errors::FileServerError; #[derive (Debug, Serialize)] pub struct ServerInfo { @@ -239,11 +240,11 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry async fn serve_root ( handlebars: &Handlebars <'static>, server_info: &ServerInfo -) -> Response +) -> Result { - let s = handlebars.render ("file_server_root", &server_info).unwrap (); + let s = handlebars.render ("file_server_root", &server_info)?; - serve_html (s) + Ok (serve_html (s)) } fn serve_html (s: String) -> Response { @@ -615,11 +616,11 @@ pub async fn serve_all ( headers: &HashMap >, hidden_path: Option <&Path> ) - -> Response +-> Result { use InternalResponse::*; - match internal_serve_all (root, method, uri, headers, hidden_path).await { + Ok (match internal_serve_all (root, method, uri, headers, hidden_path).await { Favicon => serve_error (StatusCode::NotFound, ""), Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), InvalidUri => serve_error (StatusCode::BadRequest, "Invalid URI"), @@ -634,7 +635,7 @@ pub async fn serve_all ( }, Redirect (location) => serve_307 (location), - Root => serve_root (handlebars, server_info).await, + Root => serve_root (handlebars, server_info).await?, ServeDir (ServeDirParams { path, dir, @@ -651,7 +652,7 @@ pub async fn serve_all ( MarkdownError::NotUtf8 => serve_error (StatusCode::BadRequest, "File is not UTF-8"), }, MarkdownPreview (s) => serve_html (s), - } + }) } pub fn load_templates ( diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 588a61e..d764a75 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -35,6 +35,8 @@ pub mod errors; pub mod file_server; pub mod load_toml; +use errors::ServerError; + // Thanks to https://github.com/robsheldon/bad-passwords-index const BAD_PASSWORDS: &[u8] = include_bytes! ("bad_passwords.txt"); @@ -61,7 +63,7 @@ struct ServerState { async fn handle_req_resp <'a> ( state: &Arc , req_resp: reqwest::Response -) -> Result <(), ()> { +) -> Result <(), ServerError> { //println! ("Step 1"); let body = req_resp.bytes ().await.unwrap (); @@ -70,7 +72,7 @@ async fn handle_req_resp <'a> ( Ok (x) => x, Err (e) => { error! ("Can't parse wrapped requests: {:?}", e); - return Err (()); + return Err (ServerError::CantParseWrappedRequests (e)); }, }; @@ -97,7 +99,7 @@ async fn handle_req_resp <'a> ( &parts.uri, &parts.headers, state.hidden_path.as_deref () - ).await; + ).await.unwrap (); let mut resp_req = state.client .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) From f2129318428e007ebeb6679ca6024fed4f79f641 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 19:47:40 +0000 Subject: [PATCH 070/208] :recycle: Remove more unwraps --- crates/ptth_server/src/file_server/errors.rs | 27 ++++++++ crates/ptth_server/src/file_server/mod.rs | 66 +++++++++----------- crates/ptth_server/src/file_server/tests.rs | 6 +- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/crates/ptth_server/src/file_server/errors.rs b/crates/ptth_server/src/file_server/errors.rs index de32bf9..a4ae14b 100644 --- a/crates/ptth_server/src/file_server/errors.rs +++ b/crates/ptth_server/src/file_server/errors.rs @@ -1,7 +1,34 @@ use thiserror::Error; +#[derive (Debug, Error, PartialEq)] +pub enum MarkdownError { + #[error ("File is too big to process")] + TooBig, + + #[error ("File is not UTF-8")] + NotUtf8, +} + #[derive (Debug, Error)] pub enum FileServerError { #[error ("Handlebars render error")] Handlebars (#[from] handlebars::RenderError), + + #[error ("I/O error")] + Io (#[from] std::io::Error), + + #[error ("Request path is not UTF-8")] + PathNotUtf8 (std::str::Utf8Error), + + #[error ("Can't get file metadata")] + CantGetFileMetadata (std::io::Error), + + #[error ("No file name requested")] + NoFileNameRequested, + + #[error ("File path is not UTF-8")] + FilePathNotUtf8, + + #[error ("Markdown error")] + Markdown (#[from] MarkdownError), } diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 38dcfcf..74aec33 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -52,7 +52,10 @@ use ptth_core::{ }; pub mod errors; -use errors::FileServerError; +use errors::{ + FileServerError, + MarkdownError, +}; #[derive (Debug, Serialize)] pub struct ServerInfo { @@ -262,7 +265,7 @@ async fn serve_dir ( server_info: &ServerInfo, path: Cow <'_, str>, mut dir: ReadDir -) -> Response +) -> Result { let mut entries = vec! []; @@ -270,15 +273,15 @@ async fn serve_dir ( entries.push (read_dir_entry (entry).await); } - entries.sort_unstable_by (|a, b| a.file_name.partial_cmp (&b.file_name).unwrap ()); + entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name)); let s = handlebars.render ("file_server_dir", &TemplateDirPage { path, entries, server_info, - }).unwrap (); + })?; - serve_html (s) + Ok (serve_html (s)) } #[instrument (level = "debug", skip (f))] @@ -288,7 +291,7 @@ async fn serve_file ( range: Range , range_requested: bool ) - -> Response +-> Result { let (tx, rx) = channel (1); let body = if should_send_body { @@ -303,11 +306,10 @@ async fn serve_file ( let content_length = range.end - range.start; let seek = SeekFrom::Start (range.start); + f.seek (seek).await?; if should_send_body { tokio::spawn (async move { - f.seek (seek).await.unwrap (); - let mut tx = tx; let mut bytes_sent = 0; let mut bytes_left = content_length; @@ -373,7 +375,7 @@ async fn serve_file ( response.body (body); } - response + Ok (response) } fn serve_error ( @@ -443,13 +445,6 @@ struct ServeFileParams { file: AlwaysEqual , } -#[derive (Debug, PartialEq)] -enum MarkdownError { - TooBig, - // NotMarkdown, - NotUtf8, -} - #[derive (Debug, PartialEq)] enum InternalResponse { Favicon, @@ -475,7 +470,7 @@ async fn internal_serve_all ( headers: &HashMap >, hidden_path: Option <&Path> ) - -> InternalResponse +-> Result { use std::str::FromStr; use InternalResponse::*; @@ -483,7 +478,7 @@ async fn internal_serve_all ( info! ("Client requested {}", uri); let uri = match hyper::Uri::from_str (uri) { - Err (_) => return InvalidUri, + Err (_) => return Ok (InvalidUri), Ok (x) => x, }; @@ -492,28 +487,28 @@ async fn internal_serve_all ( Method::Head => false, m => { debug! ("Unsupported method {:?}", m); - return MethodNotAllowed; + return Ok (MethodNotAllowed); } }; if uri.path () == "/favicon.ico" { - return Favicon; + return Ok (Favicon); } let path = match prefix_match ("/files", uri.path ()) { Some (x) => x, - None => return Root, + None => return Ok (Root), }; if path == "" { - return Redirect ("files/".to_string ()); + return Ok (Redirect ("files/".to_string ())); } // TODO: There is totally a dir traversal attack in here somewhere let encoded_path = &path [1..]; - let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().unwrap (); + let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; let path = Path::new (&*path_s); let full_path = root.join (path); @@ -522,19 +517,20 @@ async fn internal_serve_all ( if let Some (hidden_path) = hidden_path { if full_path == hidden_path { - return Forbidden; + return Ok (Forbidden); } } let has_trailing_slash = path_s.is_empty () || path_s.ends_with ('/'); - if let Ok (dir) = read_dir (&full_path).await { + Ok (if let Ok (dir) = read_dir (&full_path).await { if ! has_trailing_slash { - return Redirect (format! ("{}/", path.file_name ().unwrap ().to_str ().unwrap ())); + let file_name = path.file_name ().ok_or (FileServerError::NoFileNameRequested)?; + return Ok (Redirect (format! ("{}/", file_name.to_str ().ok_or (FileServerError::FilePathNotUtf8)?))); } if uri.query ().is_some () { - return InvalidQuery; + return Ok (InvalidQuery); } let dir = dir.into (); @@ -547,10 +543,10 @@ async fn internal_serve_all ( else if let Ok (mut file) = File::open (&full_path).await { use std::os::unix::fs::PermissionsExt; - let file_md = file.metadata ().await.unwrap (); + let file_md = file.metadata ().await.map_err (FileServerError::CantGetFileMetadata)?; if file_md.permissions ().mode () == super::load_toml::CONFIG_PERMISSIONS_MODE { - return Forbidden; + return Ok (Forbidden); } let file_len = file_md.len (); @@ -567,10 +563,10 @@ async fn internal_serve_all ( } else { let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().unwrap ()]; - let bytes_read = file.read (&mut buffer).await.unwrap (); + let bytes_read = file.read (&mut buffer).await?; buffer.truncate (bytes_read); - MarkdownPreview (render_markdown_styled (&buffer).unwrap ()) + MarkdownPreview (render_markdown_styled (&buffer)?) } } else { @@ -603,7 +599,7 @@ async fn internal_serve_all ( } else { NotFound - } + }) } #[instrument (level = "debug", skip (handlebars, headers))] @@ -620,7 +616,7 @@ pub async fn serve_all ( { use InternalResponse::*; - Ok (match internal_serve_all (root, method, uri, headers, hidden_path).await { + Ok (match internal_serve_all (root, method, uri, headers, hidden_path).await? { Favicon => serve_error (StatusCode::NotFound, ""), Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), InvalidUri => serve_error (StatusCode::BadRequest, "Invalid URI"), @@ -639,13 +635,13 @@ pub async fn serve_all ( ServeDir (ServeDirParams { path, dir, - }) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await, + }) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?, ServeFile (ServeFileParams { file, send_body, range, range_requested, - }) => serve_file (file.into_inner (), send_body, range, range_requested).await, + }) => serve_file (file.into_inner (), send_body, range, range_requested).await?, MarkdownErr (e) => match e { MarkdownError::TooBig => serve_error (StatusCode::InternalServerError, "File is too big to preview as Markdown"), //MarkdownError::NotMarkdown => serve_error (StatusCode::BadRequest, "File is not Markdown"), diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index f85361a..d81407d 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -158,7 +158,7 @@ fn file_server () { None ).await; - assert_eq! (resp, expected); + assert_eq! (resp.unwrap (), expected); } let resp = internal_serve_all ( @@ -171,7 +171,7 @@ fn file_server () { None ).await; - assert_eq! (resp, RangeNotSatisfiable (1_048_576)); + assert_eq! (resp.unwrap (), RangeNotSatisfiable (1_048_576)); let resp = internal_serve_all ( &file_server_root, @@ -181,7 +181,7 @@ fn file_server () { None ).await; - assert_eq! (resp, ServeFile (ServeFileParams { + assert_eq! (resp.unwrap (), ServeFile (ServeFileParams { send_body: false, range: 0..1_048_576, range_requested: false, From eada65d94bb746d2938c2d4d88452605b91de692 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 19:59:47 +0000 Subject: [PATCH 071/208] :recycle: Remove unwraps from file server module --- crates/ptth_server/src/file_server/mod.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 74aec33..b27b95b 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -7,7 +7,7 @@ use std::{ borrow::Cow, cmp::min, collections::HashMap, - convert::{Infallible, TryInto}, + convert::{Infallible, TryFrom, TryInto}, error::Error, fmt::Debug, io::SeekFrom, @@ -319,28 +319,30 @@ async fn serve_file ( loop { let mut buffer = vec! [0_u8; 65_536]; - let bytes_read: u64 = f.read (&mut buffer).await.unwrap ().try_into ().unwrap (); - - let bytes_read = min (bytes_left, bytes_read); - - buffer.truncate (bytes_read.try_into ().unwrap ()); + let bytes_read = f.read (&mut buffer).await.expect ("Couldn't read from file"); if bytes_read == 0 { break; } + buffer.truncate (bytes_read); + + let bytes_read_64 = u64::try_from (bytes_read).expect ("Couldn't fit usize into u64"); + + let bytes_read_64 = min (bytes_left, bytes_read_64); + if tx.send (Ok::<_, Infallible> (buffer)).await.is_err () { warn! ("Cancelling file stream (Sent {} out of {} bytes)", bytes_sent, content_length); break; } - bytes_left -= bytes_read; + bytes_left -= bytes_read_64; if bytes_left == 0 { debug! ("Finished"); break; } - bytes_sent += bytes_read; + bytes_sent += bytes_read_64; while next_mark <= bytes_sent { trace! ("Sent {} bytes", next_mark); next_mark += mark_interval; @@ -558,11 +560,11 @@ async fn internal_serve_all ( ParsedRange::Ok (range) => { if uri.query () == Some ("as_markdown") { const MAX_BUF_SIZE: u32 = 1_000_000; - if file_len > MAX_BUF_SIZE.try_into ().unwrap () { + if file_len > MAX_BUF_SIZE.into () { MarkdownErr (MarkdownError::TooBig) } else { - let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().unwrap ()]; + let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().expect ("Couldn't fit u32 into usize")]; let bytes_read = file.read (&mut buffer).await?; buffer.truncate (bytes_read); From 7bd24506982aa15065d796e9296f273465401f9a Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 20:22:40 +0000 Subject: [PATCH 072/208] :recycle: Fix pedantic clippy warnings --- crates/ptth_core/Cargo.toml | 1 + crates/ptth_core/src/graceful_shutdown.rs | 13 ++++++++--- crates/ptth_core/src/http_serde.rs | 22 ++++++++++--------- crates/ptth_core/src/lib.rs | 3 +++ crates/ptth_relay/src/lib.rs | 8 ++++--- crates/ptth_relay/src/main.rs | 2 ++ crates/ptth_server/Cargo.toml | 1 + .../ptth_server/src/bin/ptth_file_server.rs | 22 +++++-------------- crates/ptth_server/src/bin/ptth_server.rs | 2 ++ crates/ptth_server/src/errors.rs | 3 +++ 10 files changed, 45 insertions(+), 32 deletions(-) diff --git a/crates/ptth_core/Cargo.toml b/crates/ptth_core/Cargo.toml index 7a584a0..937bee2 100644 --- a/crates/ptth_core/Cargo.toml +++ b/crates/ptth_core/Cargo.toml @@ -12,6 +12,7 @@ ctrlc = { version = "3.1.7", features = [ "termination" ] } futures = "0.3.7" hyper = "0.13.8" serde = {version = "1.0.117", features = ["derive"]} +thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" tracing-futures = "0.2.4" diff --git a/crates/ptth_core/src/graceful_shutdown.rs b/crates/ptth_core/src/graceful_shutdown.rs index 82bd541..38e1c36 100644 --- a/crates/ptth_core/src/graceful_shutdown.rs +++ b/crates/ptth_core/src/graceful_shutdown.rs @@ -1,3 +1,6 @@ +// False positive on futures::select! macro +#![allow (clippy::mut_mut)] + use std::{ cell::Cell, time::Duration, @@ -11,6 +14,7 @@ use tokio::{ use crate::prelude::*; +#[must_use] pub fn init () -> oneshot::Receiver <()> { let (tx, rx) = oneshot::channel::<()> (); @@ -44,10 +48,8 @@ use std::{ impl fmt::Display for ShutdownError { fn fmt (&self, f: &mut fmt::Formatter <'_>) -> fmt::Result { - use ShutdownError::*; - let desc = match self { - ForcedShutdown => "Shutdown was forced after a timeout", + ShutdownError::ForcedShutdown => "Shutdown was forced after a timeout", }; write! (f, "{}", desc) @@ -64,6 +66,10 @@ pub struct ForcedShutdown { } impl ForcedShutdown { + /// # Errors + /// + /// `ForcedShutdown` if the graceful shutdown doesn't complete in time + pub async fn wrap_server < T, F: Future @@ -91,6 +97,7 @@ impl ForcedShutdown { } } +#[must_use] pub fn init_with_force () -> (oneshot::Receiver <()>, ForcedShutdown) { let (tx, rx) = oneshot::channel (); diff --git a/crates/ptth_core/src/http_serde.rs b/crates/ptth_core/src/http_serde.rs index b2d4970..bcbe3ba 100644 --- a/crates/ptth_core/src/http_serde.rs +++ b/crates/ptth_core/src/http_serde.rs @@ -1,23 +1,19 @@ use std::{ - collections::*, + collections::HashMap, convert::{TryFrom, TryInto}, }; use serde::{Deserialize, Serialize}; +use thiserror::Error; use tokio::sync::mpsc; // Hyper doesn't seem to make it easy to de/ser requests // and responses and stuff like that, so I do it by hand here. +#[derive (Debug, Error)] pub enum Error { + #[error ("Unsupported method")] UnsupportedMethod, - InvalidHeaderName, -} - -impl From for Error { - fn from (_x: hyper::header::InvalidHeaderName) -> Self { - Self::InvalidHeaderName - } } #[derive (Debug, Deserialize, Serialize)] @@ -56,6 +52,10 @@ pub struct RequestParts { } impl RequestParts { + /// # Errors + /// + /// `UnsupportedMethod` if PTTH doesn't support the method + pub fn from_hyper ( method: hyper::Method, uri: String, @@ -67,8 +67,10 @@ impl RequestParts { let method = Method::try_from (method)?; let headers = HashMap::from_iter ( headers.into_iter () - .filter_map (|(k, v)| k.map (|k| (k, v))) - .map (|(k, v)| (String::from (k.as_str ()), v.as_bytes ().to_vec ())) + .filter_map (|(k, v)| { + let (k, v) = k.map (|k| (k, v))?; + Some ((String::from (k.as_str ()), v.as_bytes ().to_vec ())) + }) ); Ok (Self { diff --git a/crates/ptth_core/src/lib.rs b/crates/ptth_core/src/lib.rs index dd92c18..69bd7d6 100644 --- a/crates/ptth_core/src/lib.rs +++ b/crates/ptth_core/src/lib.rs @@ -1,3 +1,5 @@ +#![warn (clippy::pedantic)] + pub mod graceful_shutdown; pub mod http_serde; pub mod prelude; @@ -12,6 +14,7 @@ pub const PTTH_MAGIC_HEADER: &str = "X-PTTH-2LJYXWC4"; // The arguments are in order so they are in order overall: // e.g. prefix_match ("/prefix", "/prefix/middle/suffix") -> "/middle/suffix" +#[must_use] pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str> { if hay.starts_with (prefix) { diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 25247fd..1b94650 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -305,7 +305,7 @@ async fn handle_http_response ( futures::select! { x = body_tx.send (item).fuse () => if let Err (_) = x { info! ("Body closed while relaying. (Client hung up?)"); - body_finished_tx.send (ClientDisconnected).map_err (|_| LostServer).unwrap (); + body_finished_tx.send (ClientDisconnected).map_err (|_| LostServer)?; break; }, _ = shutdown_watch_rx.recv ().fuse () => { @@ -316,7 +316,7 @@ async fn handle_http_response ( } else { debug! ("Finished relaying bytes"); - body_finished_tx.send (StreamFinished).map_err (|_| LostServer).unwrap (); + body_finished_tx.send (StreamFinished).map_err (|_| LostServer)?; break; } } @@ -324,6 +324,8 @@ async fn handle_http_response ( else { debug! ("Can't relay bytes, relay is shutting down"); } + + Ok::<(), HandleHttpResponseError> (()) }); let body = Body::wrap_stream (body_rx); @@ -346,7 +348,7 @@ async fn handle_http_response ( return Ok (error_reply (StatusCode::BAD_GATEWAY, msg)?); } - relay_task.await?; + relay_task.await??; debug! ("Connected server to client for streaming."); match body_finished_rx.await { diff --git a/crates/ptth_relay/src/main.rs b/crates/ptth_relay/src/main.rs index 009d704..c8d1bce 100644 --- a/crates/ptth_relay/src/main.rs +++ b/crates/ptth_relay/src/main.rs @@ -1,3 +1,5 @@ +#![warn (clippy::pedantic)] + use std::{ convert::TryFrom, error::Error, diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index 77004a9..7fc1b84 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -13,6 +13,7 @@ base64 = "0.12.3" blake3 = "0.3.7" futures = "0.3.7" handlebars = "3.5.1" +http = "0.2.1" hyper = "0.13.8" lazy_static = "1.4.0" percent-encoding = "2.1.0" diff --git a/crates/ptth_server/src/bin/ptth_file_server.rs b/crates/ptth_server/src/bin/ptth_file_server.rs index aefba3a..1080286 100644 --- a/crates/ptth_server/src/bin/ptth_file_server.rs +++ b/crates/ptth_server/src/bin/ptth_file_server.rs @@ -1,3 +1,5 @@ +#![warn (clippy::pedantic)] + use std::{ error::Error, net::SocketAddr, @@ -40,12 +42,6 @@ struct ServerState <'a> { hidden_path: Option , } -fn status_reply > (status: StatusCode, b: B) --> Result , hyper::http::Error> -{ - Response::builder ().status (status).body (b.into ()) -} - async fn handle_all (req: Request , state: Arc >) -> Result , ServerError> { @@ -53,16 +49,13 @@ async fn handle_all (req: Request , state: Arc >) debug! ("req.uri () = {:?}", req.uri ()); - let path_and_query = req.uri ().path_and_query ().map (|x| x.as_str ()).unwrap_or_else (|| req.uri ().path ()); + let path_and_query = req.uri ().path_and_query ().map_or_else (|| req.uri ().path (), http::uri::PathAndQuery::as_str); let path_and_query = path_and_query.into (); let (parts, _) = req.into_parts (); - let ptth_req = match RequestParts::from_hyper (parts.method, path_and_query, parts.headers) { - Ok (x) => x, - _ => return Ok (status_reply (StatusCode::BAD_REQUEST, "Bad request")?), - }; + let ptth_req = RequestParts::from_hyper (parts.method, path_and_query, parts.headers)?; let default_root = PathBuf::from ("./"); let file_server_root: &std::path::Path = state.config.file_server_root @@ -82,14 +75,11 @@ async fn handle_all (req: Request , state: Arc >) let mut resp = Response::builder () .status (StatusCode::from (ptth_resp.parts.status_code)); - for (k, v) in ptth_resp.parts.headers.into_iter () { + for (k, v) in ptth_resp.parts.headers { resp = resp.header (hyper::header::HeaderName::from_str (&k)?, v); } - let body = ptth_resp.body - .map (Body::wrap_stream) - .unwrap_or_else (Body::empty) - ; + let body = ptth_resp.body.map_or_else (Body::empty, Body::wrap_stream); Ok (resp.body (body)?) } diff --git a/crates/ptth_server/src/bin/ptth_server.rs b/crates/ptth_server/src/bin/ptth_server.rs index 2718074..f08361e 100644 --- a/crates/ptth_server/src/bin/ptth_server.rs +++ b/crates/ptth_server/src/bin/ptth_server.rs @@ -1,3 +1,5 @@ +#![warn (clippy::pedantic)] + use std::{ error::Error, path::PathBuf, diff --git a/crates/ptth_server/src/errors.rs b/crates/ptth_server/src/errors.rs index 5708af7..c8c2d4b 100644 --- a/crates/ptth_server/src/errors.rs +++ b/crates/ptth_server/src/errors.rs @@ -13,4 +13,7 @@ pub enum ServerError { #[error ("Can't parse wrapped requests")] CantParseWrappedRequests (rmp_serde::decode::Error), + + #[error ("Can't convert Hyper request to PTTH request")] + CantConvertHyperToPtth (#[from] ptth_core::http_serde::Error), } From d6430e39a927b8bcd4f07ab21b956cecca7b1082 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 21:38:23 +0000 Subject: [PATCH 073/208] :recycle: Get rid of more unwraps and panics --- README.md | 7 +- crates/ptth_relay/src/tests.rs | 4 +- crates/ptth_server/Cargo.toml | 1 + .../ptth_server/src/bin/ptth_file_server.rs | 5 +- crates/ptth_server/src/bin/ptth_server.rs | 5 +- crates/ptth_server/src/errors.rs | 58 +++++- crates/ptth_server/src/file_server/errors.rs | 3 + crates/ptth_server/src/file_server/mod.rs | 195 ++++++++++-------- crates/ptth_server/src/file_server/tests.rs | 31 ++- crates/ptth_server/src/lib.rs | 28 +-- crates/ptth_server/src/load_toml.rs | 24 ++- src/tests.rs | 28 +-- 12 files changed, 251 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index 3b51c2f..c83c949 100644 --- a/README.md +++ b/README.md @@ -121,10 +121,11 @@ Client Relay Server O <----- O | P5 O <------ O - P6/H3 + P6/H3 | P7 + O -----> O ``` -We'll call these steps "P1" through "P6". +We'll call these steps "P1" through "P7". 1. The server makes a "listen" request to the relay, punching out through the server's firewall. @@ -138,6 +139,8 @@ to respond. 4. The server processes the request. (P4 == H2) 5. The server packages its response in another request to the relay. 6. The relay unwraps the request and forwards it to the client. (P6 == H3) +7. When the full response body has been streamed through the relay and to the +client, the relay will respond to the server. Every step of the normal HTTP process is inverted for the server: diff --git a/crates/ptth_relay/src/tests.rs b/crates/ptth_relay/src/tests.rs index c1e7b74..6b80c37 100644 --- a/crates/ptth_relay/src/tests.rs +++ b/crates/ptth_relay/src/tests.rs @@ -4,7 +4,7 @@ use super::*; fn test_pretty_print_last_seen () { use LastSeen::*; - let last_seen = DateTime::parse_from_rfc3339 ("2019-05-29T00:00:00+00:00").unwrap ().with_timezone (&Utc); + let last_seen = DateTime::parse_from_rfc3339 ("2019-05-29T00:00:00+00:00").expect ("Test case should be RFC3339").with_timezone (&Utc); for (input, expected) in vec! [ ("2019-05-28T23:59:59+00:00", Negative), @@ -18,7 +18,7 @@ fn test_pretty_print_last_seen () { ("2019-05-30T10:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), ("2019-05-31T00:00:00+00:00", Description ("2019-05-29T00:00:00Z".into ())), ].into_iter () { - let now = DateTime::parse_from_rfc3339 (input).unwrap ().with_timezone (&Utc); + let now = DateTime::parse_from_rfc3339 (input).expect ("Test case should be RFC3339").with_timezone (&Utc); let actual = pretty_print_last_seen (now, last_seen); assert_eq! (actual, expected); } diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index 7fc1b84..bb8ed1b 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -9,6 +9,7 @@ license = "AGPL-3.0" [dependencies] aho-corasick = "0.7.14" +anyhow = "1.0.34" base64 = "0.12.3" blake3 = "0.3.7" futures = "0.3.7" diff --git a/crates/ptth_server/src/bin/ptth_file_server.rs b/crates/ptth_server/src/bin/ptth_file_server.rs index 1080286..11482a0 100644 --- a/crates/ptth_server/src/bin/ptth_file_server.rs +++ b/crates/ptth_server/src/bin/ptth_file_server.rs @@ -1,7 +1,6 @@ #![warn (clippy::pedantic)] use std::{ - error::Error, net::SocketAddr, path::PathBuf, sync::Arc, @@ -91,11 +90,11 @@ pub struct ConfigFile { } #[tokio::main] -async fn main () -> Result <(), Box > { +async fn main () -> Result <(), anyhow::Error> { tracing_subscriber::fmt::init (); let path = PathBuf::from ("./config/ptth_server.toml"); - let config_file: ConfigFile = load_toml::load (&path); + let config_file: ConfigFile = load_toml::load (&path)?; info! ("file_server_root: {:?}", config_file.file_server_root); let addr = SocketAddr::from(([0, 0, 0, 0], 4000)); diff --git a/crates/ptth_server/src/bin/ptth_server.rs b/crates/ptth_server/src/bin/ptth_server.rs index f08361e..321d3dd 100644 --- a/crates/ptth_server/src/bin/ptth_server.rs +++ b/crates/ptth_server/src/bin/ptth_server.rs @@ -1,7 +1,6 @@ #![warn (clippy::pedantic)] use std::{ - error::Error, path::PathBuf, }; @@ -29,12 +28,12 @@ struct Opt { print_tripcode: bool, } -fn main () -> Result <(), Box > { +fn main () -> Result <(), anyhow::Error> { let opt = Opt::from_args (); tracing_subscriber::fmt::init (); let path = opt.config_path.clone ().unwrap_or_else (|| PathBuf::from ("./config/ptth_server.toml")); - let config_file: ConfigFile = load_toml::load (&path); + let config_file: ConfigFile = load_toml::load (&path)?; if opt.print_tripcode { println! (r#""{}" = "{}""#, config_file.name, config_file.tripcode ()); diff --git a/crates/ptth_server/src/errors.rs b/crates/ptth_server/src/errors.rs index c8c2d4b..a2bb0c9 100644 --- a/crates/ptth_server/src/errors.rs +++ b/crates/ptth_server/src/errors.rs @@ -1,19 +1,75 @@ use thiserror::Error; +#[derive (Debug, Error)] +pub enum LoadTomlError { + #[error ("Config file has bad permissions mode, it should be octal 0600")] + ConfigBadPermissions, + + #[error ("I/O")] + Io (#[from] std::io::Error), + + #[error ("UTF-8")] + Utf8 (#[from] std::string::FromUtf8Error), + + #[error ("TOML")] + Toml (#[from] toml::de::Error), +} + #[derive (Debug, Error)] pub enum ServerError { + #[error ("Loading TOML")] + LoadToml (#[from] LoadTomlError), + + #[error ("Loading Handlebars template file")] + LoadHandlebars (#[from] handlebars::TemplateFileError), + + #[error ("API key is too weak, server can't use it")] + WeakApiKey, + #[error ("File server error")] FileServer (#[from] super::file_server::errors::FileServerError), + // Hyper stuff + #[error ("Hyper HTTP error")] Http (#[from] hyper::http::Error), #[error ("Hyper invalid header name")] InvalidHeaderName (#[from] hyper::header::InvalidHeaderName), - #[error ("Can't parse wrapped requests")] + #[error ("API key invalid")] + ApiKeyInvalid (hyper::header::InvalidHeaderValue), + + // MessagePack stuff + + #[error ("Can't parse wrapped requests in Step 3")] CantParseWrappedRequests (rmp_serde::decode::Error), + #[error ("Can't encode PTTH response as MsgPack in Step 5")] + MessagePackEncodeResponse (rmp_serde::encode::Error), + #[error ("Can't convert Hyper request to PTTH request")] CantConvertHyperToPtth (#[from] ptth_core::http_serde::Error), + + // Reqwest stuff + + #[error ("Can't build HTTP client")] + CantBuildHttpClient (reqwest::Error), + + #[error ("Can't collect non-200 error response body in Step 3")] + Step3CollectBody (reqwest::Error), + + #[error ("Can't collect wrapped requests in Step 3")] + CantCollectWrappedRequests (reqwest::Error), + + #[error ("Error in Step 5, sending response to client through relay")] + Step5Responding (reqwest::Error), + + #[error ("Error in Step 7, getting response from relay after sending response to client")] + Step7AfterResponse (reqwest::Error), + + // UTF-8 + + #[error ("Step 3 relay response (non-200 OK) was not valid UTF-8")] + Step3ErrorResponseNotUtf8 (std::string::FromUtf8Error), } diff --git a/crates/ptth_server/src/file_server/errors.rs b/crates/ptth_server/src/file_server/errors.rs index a4ae14b..e5b7cc5 100644 --- a/crates/ptth_server/src/file_server/errors.rs +++ b/crates/ptth_server/src/file_server/errors.rs @@ -31,4 +31,7 @@ pub enum FileServerError { #[error ("Markdown error")] Markdown (#[from] MarkdownError), + + #[error ("Invalid URI")] + InvalidUri (#[from] hyper::http::uri::InvalidUri), } diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index b27b95b..7ef6b66 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -8,7 +8,6 @@ use std::{ cmp::min, collections::HashMap, convert::{Infallible, TryFrom, TryInto}, - error::Error, fmt::Debug, io::SeekFrom, path::{Path, PathBuf}, @@ -451,7 +450,6 @@ struct ServeFileParams { enum InternalResponse { Favicon, Forbidden, - InvalidUri, InvalidQuery, MethodNotAllowed, NotFound, @@ -465,6 +463,100 @@ enum InternalResponse { MarkdownPreview (String), } +fn internal_serve_dir ( + path_s: &str, + path: &Path, + dir: tokio::fs::ReadDir, + full_path: PathBuf, + uri: &hyper::Uri +) +-> Result +{ + let has_trailing_slash = path_s.is_empty () || path_s.ends_with ('/'); + + if ! has_trailing_slash { + let file_name = path.file_name ().ok_or (FileServerError::NoFileNameRequested)?; + let file_name = file_name.to_str ().ok_or (FileServerError::FilePathNotUtf8)?; + return Ok (InternalResponse::Redirect (format! ("{}/", file_name))); + } + + if uri.query ().is_some () { + return Ok (InternalResponse::InvalidQuery); + } + + let dir = dir.into (); + + Ok (InternalResponse::ServeDir (ServeDirParams { + dir, + path: full_path, + })) +} + +async fn internal_serve_file ( + mut file: tokio::fs::File, + uri: &hyper::Uri, + send_body: bool, + headers: &HashMap > +) +-> Result +{ + use std::os::unix::fs::PermissionsExt; + + let file_md = file.metadata ().await.map_err (FileServerError::CantGetFileMetadata)?; + if file_md.permissions ().mode () == super::load_toml::CONFIG_PERMISSIONS_MODE + { + return Ok (InternalResponse::Forbidden); + } + + let file_len = file_md.len (); + + let range_header = headers.get ("range").and_then (|v| std::str::from_utf8 (v).ok ()); + + Ok (match check_range (range_header, file_len) { + ParsedRange::RangeNotSatisfiable (file_len) => InternalResponse::RangeNotSatisfiable (file_len), + ParsedRange::Ok (range) => { + if uri.query () == Some ("as_markdown") { + const MAX_BUF_SIZE: u32 = 1_000_000; + if file_len > MAX_BUF_SIZE.into () { + InternalResponse::MarkdownErr (MarkdownError::TooBig) + } + else { + let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().expect ("Couldn't fit u32 into usize")]; + let bytes_read = file.read (&mut buffer).await?; + buffer.truncate (bytes_read); + + InternalResponse::MarkdownPreview (render_markdown_styled (&buffer)?) + } + } + else { + let file = file.into (); + + InternalResponse::ServeFile (ServeFileParams { + file, + send_body, + range, + range_requested: false, + }) + } + }, + ParsedRange::PartialContent (range) => { + if uri.query ().is_some () { + InternalResponse::InvalidQuery + } + else { + let file = file.into (); + + InternalResponse::ServeFile (ServeFileParams { + file, + send_body, + range, + range_requested: true, + }) + } + }, + }) +} + async fn internal_serve_all ( root: &Path, method: Method, @@ -479,10 +571,7 @@ async fn internal_serve_all ( info! ("Client requested {}", uri); - let uri = match hyper::Uri::from_str (uri) { - Err (_) => return Ok (InvalidUri), - Ok (x) => x, - }; + let uri = hyper::Uri::from_str (uri).map_err (FileServerError::InvalidUri)?; let send_body = match &method { Method::Get => true, @@ -523,85 +612,26 @@ async fn internal_serve_all ( } } - let has_trailing_slash = path_s.is_empty () || path_s.ends_with ('/'); - - Ok (if let Ok (dir) = read_dir (&full_path).await { - if ! has_trailing_slash { - let file_name = path.file_name ().ok_or (FileServerError::NoFileNameRequested)?; - return Ok (Redirect (format! ("{}/", file_name.to_str ().ok_or (FileServerError::FilePathNotUtf8)?))); - } - - if uri.query ().is_some () { - return Ok (InvalidQuery); - } - - let dir = dir.into (); - - ServeDir (ServeDirParams { + if let Ok (dir) = read_dir (&full_path).await { + internal_serve_dir ( + &path_s, + path, dir, - path: full_path, - }) + full_path, + &uri + ) } - else if let Ok (mut file) = File::open (&full_path).await { - use std::os::unix::fs::PermissionsExt; - - let file_md = file.metadata ().await.map_err (FileServerError::CantGetFileMetadata)?; - if file_md.permissions ().mode () == super::load_toml::CONFIG_PERMISSIONS_MODE - { - return Ok (Forbidden); - } - - let file_len = file_md.len (); - - let range_header = headers.get ("range").and_then (|v| std::str::from_utf8 (v).ok ()); - - match check_range (range_header, file_len) { - ParsedRange::RangeNotSatisfiable (file_len) => RangeNotSatisfiable (file_len), - ParsedRange::Ok (range) => { - if uri.query () == Some ("as_markdown") { - const MAX_BUF_SIZE: u32 = 1_000_000; - if file_len > MAX_BUF_SIZE.into () { - MarkdownErr (MarkdownError::TooBig) - } - else { - let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().expect ("Couldn't fit u32 into usize")]; - let bytes_read = file.read (&mut buffer).await?; - buffer.truncate (bytes_read); - - MarkdownPreview (render_markdown_styled (&buffer)?) - } - } - else { - let file = file.into (); - - ServeFile (ServeFileParams { - file, - send_body, - range, - range_requested: false, - }) - } - }, - ParsedRange::PartialContent (range) => { - if uri.query ().is_some () { - InvalidQuery - } - else { - let file = file.into (); - - ServeFile (ServeFileParams { - file, - send_body, - range, - range_requested: true, - }) - } - }, - } + else if let Ok (file) = File::open (&full_path).await { + internal_serve_file ( + file, + &uri, + send_body, + headers + ).await } else { - NotFound - }) + Ok (NotFound) + } } #[instrument (level = "debug", skip (handlebars, headers))] @@ -621,7 +651,6 @@ pub async fn serve_all ( Ok (match internal_serve_all (root, method, uri, headers, hidden_path).await? { Favicon => serve_error (StatusCode::NotFound, ""), Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), - InvalidUri => serve_error (StatusCode::BadRequest, "Invalid URI"), InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object"), MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method"), NotFound => serve_error (StatusCode::NotFound, "404 Not Found"), @@ -656,7 +685,7 @@ pub async fn serve_all ( pub fn load_templates ( asset_root: &Path ) --> Result , Box > +-> Result , handlebars::TemplateFileError> { let mut handlebars = Handlebars::new (); handlebars.set_strict_mode (true); diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index d81407d..cbb527f 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -119,7 +119,7 @@ fn file_server () { use super::*; tracing_subscriber::fmt ().try_init ().ok (); - let mut rt = Runtime::new ().unwrap (); + let mut rt = Runtime::new ().expect ("Can't create runtime"); rt.block_on (async { let file_server_root = PathBuf::from ("./"); @@ -127,6 +127,7 @@ fn file_server () { { use InternalResponse::*; + use crate::file_server::FileServerError; let bad_passwords_path = "/files/src/bad_passwords.txt"; @@ -148,8 +149,7 @@ fn file_server () { range_requested: false, file: AlwaysEqual::testing_blank (), })), - ("/ ", InvalidUri), - ].into_iter () { + ] { let resp = internal_serve_all ( &file_server_root, Method::Get, @@ -158,7 +158,24 @@ fn file_server () { None ).await; - assert_eq! (resp.unwrap (), expected); + assert_eq! (resp.expect ("This block only tests Ok (_) responses"), expected); + } + + for (uri_path, checker) in vec! [ + ("/ ", |e| match e { + FileServerError::InvalidUri (_) => (), + e => panic! ("Expected InvalidUri, got {:?}", e), + }), + ] { + let resp = internal_serve_all ( + &file_server_root, + Method::Get, + uri_path, + &headers, + None + ).await; + + checker (resp.unwrap_err ()); } let resp = internal_serve_all ( @@ -171,7 +188,7 @@ fn file_server () { None ).await; - assert_eq! (resp.unwrap (), RangeNotSatisfiable (1_048_576)); + assert_eq! (resp.expect ("Should be Ok (_)"), RangeNotSatisfiable (1_048_576)); let resp = internal_serve_all ( &file_server_root, @@ -181,7 +198,7 @@ fn file_server () { None ).await; - assert_eq! (resp.unwrap (), ServeFile (ServeFileParams { + assert_eq! (resp.expect ("Should be Ok (_)"), ServeFile (ServeFileParams { send_body: false, range: 0..1_048_576, range_requested: false, @@ -210,7 +227,7 @@ fn markdown () { ), ].into_iter () { let mut out = String::default (); - render_markdown (input.as_bytes (), &mut out).unwrap (); + render_markdown (input.as_bytes (), &mut out).expect ("Markdown sample failed"); assert_eq! (expected, &out); } } diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index d764a75..203af47 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -8,7 +8,6 @@ #![allow (clippy::mut_mut)] use std::{ - error::Error, path::PathBuf, sync::Arc, time::Duration, @@ -66,7 +65,7 @@ async fn handle_req_resp <'a> ( ) -> Result <(), ServerError> { //println! ("Step 1"); - let body = req_resp.bytes ().await.unwrap (); + let body = req_resp.bytes ().await.map_err (ServerError::CantCollectWrappedRequests)?; let wrapped_reqs: Vec = match rmp_serde::from_read_ref (&body) { Ok (x) => x, @@ -81,6 +80,8 @@ async fn handle_req_resp <'a> ( for wrapped_req in wrapped_reqs { let state = state.clone (); + // These have to detach, so we won't be able to catch the join errors. + tokio::spawn (async move { let (req_id, parts) = (wrapped_req.id, wrapped_req.req); @@ -99,11 +100,11 @@ async fn handle_req_resp <'a> ( &parts.uri, &parts.headers, state.hidden_path.as_deref () - ).await.unwrap (); + ).await?; let mut resp_req = state.client .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) - .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).unwrap ())); + .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?)); if let Some (length) = response.content_length { resp_req = resp_req.header ("Content-Length", length.to_string ()); @@ -112,7 +113,7 @@ async fn handle_req_resp <'a> ( resp_req = resp_req.body (reqwest::Body::wrap_stream (body)); } - let req = resp_req.build ().unwrap (); + let req = resp_req.build ().map_err (ServerError::Step5Responding)?; debug! ("{:?}", req.headers ()); @@ -120,7 +121,7 @@ async fn handle_req_resp <'a> ( match state.client.execute (req).await { Ok (r) => { let status = r.status (); - let text = r.text ().await.unwrap (); + let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?; debug! ("{:?} {:?}", status, text); }, Err (e) => { @@ -133,6 +134,7 @@ async fn handle_req_resp <'a> ( }, } + Ok::<(), ServerError> (()) }); } @@ -166,14 +168,14 @@ pub async fn run_server ( hidden_path: Option , asset_root: Option ) --> Result <(), Box > +-> Result <(), ServerError> { use std::convert::TryInto; let asset_root = asset_root.unwrap_or_else (PathBuf::new); if password_is_bad (config_file.api_key.clone ()) { - panic! ("API key is too weak, server can't use it"); + return Err (ServerError::WeakApiKey); } let server_info = file_server::ServerInfo { @@ -184,13 +186,13 @@ pub async fn run_server ( info! ("Tripcode is {}", config_file.tripcode ()); let mut headers = reqwest::header::HeaderMap::new (); - headers.insert ("X-ApiKey", config_file.api_key.try_into ().unwrap ()); + headers.insert ("X-ApiKey", config_file.api_key.try_into ().map_err (ServerError::ApiKeyInvalid)?); let client = Client::builder () .default_headers (headers) .timeout (Duration::from_secs (40)) - .build ().unwrap (); - let handlebars = file_server::load_templates (&asset_root).expect ("Can't load Handlebars templates"); + .build ().map_err (ServerError::CantBuildHttpClient)?; + let handlebars = file_server::load_templates (&asset_root)?; let state = Arc::new (ServerState { config: Config { @@ -260,8 +262,8 @@ pub async fn run_server ( } else if req_resp.status () != StatusCode::OK { error! ("{}", req_resp.status ()); - let body = req_resp.bytes ().await.unwrap (); - let body = String::from_utf8 (body.to_vec ()).unwrap (); + let body = req_resp.bytes ().await.map_err (ServerError::Step3CollectBody)?; + let body = String::from_utf8 (body.to_vec ()).map_err (ServerError::Step3ErrorResponseNotUtf8)?; error! ("{}", body); if backoff_delay != err_backoff_delay { error! ("Non-timeout issue, increasing backoff_delay"); diff --git a/crates/ptth_server/src/load_toml.rs b/crates/ptth_server/src/load_toml.rs index 1922adb..f92b73d 100644 --- a/crates/ptth_server/src/load_toml.rs +++ b/crates/ptth_server/src/load_toml.rs @@ -7,19 +7,21 @@ use std::{ use serde::de::DeserializeOwned; +use crate::errors::LoadTomlError; + pub const CONFIG_PERMISSIONS_MODE: u32 = 33152; fn load_inner < T: DeserializeOwned > ( mut f: File -) -> T { +) -> Result { let mut buffer = vec! [0_u8; 4096]; - let bytes_read = f.read (&mut buffer).unwrap_or_else (|_| panic! ("Can't read config")); + let bytes_read = f.read (&mut buffer)?; buffer.truncate (bytes_read); - let config_s = String::from_utf8 (buffer).unwrap_or_else (|_| panic! ("Can't parse config as UTF-8")); - toml::from_str (&config_s).unwrap_or_else (|e| panic! ("Can't parse config as TOML: {}", e)) + let config_s = String::from_utf8 (buffer)?; + Ok (toml::from_str (&config_s)?) } /// For files that contain public-viewable information @@ -29,8 +31,8 @@ pub fn load_public < P: AsRef + Debug > ( config_file_path: P -) -> T { - let f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); +) -> Result { + let f = File::open (&config_file_path)?; load_inner (f) } @@ -42,13 +44,15 @@ pub fn load < P: AsRef + Debug > ( config_file_path: P -) -> T { +) -> Result { use std::os::unix::fs::PermissionsExt; - let f = File::open (&config_file_path).unwrap_or_else (|_| panic! ("Can't open {:?}", config_file_path)); + let f = File::open (&config_file_path)?; - let mode = f.metadata ().unwrap ().permissions ().mode (); - assert_eq! (mode, CONFIG_PERMISSIONS_MODE, "Config file has bad permissions mode, it should be octal 0600"); + let mode = f.metadata ()?.permissions ().mode (); + if mode != CONFIG_PERMISSIONS_MODE { + return Err (LoadTomlError::ConfigBadPermissions); + } load_inner (f) } diff --git a/src/tests.rs b/src/tests.rs index a578242..6fef13c 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -23,7 +23,7 @@ fn end_to_end () { // and we don't care if another test already installed a subscriber. tracing_subscriber::fmt ().try_init ().ok (); - let mut rt = Runtime::new ().unwrap (); + let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); // Spawn the root task rt.block_on (async { @@ -41,14 +41,14 @@ fn end_to_end () { }, }; - let config = ptth_relay::config::Config::try_from (config_file).unwrap (); + let config = ptth_relay::config::Config::try_from (config_file).expect ("Can't load config"); - let relay_state = Arc::new (ptth_relay::RelayState::try_from (config).unwrap ()); + let relay_state = Arc::new (ptth_relay::RelayState::try_from (config).expect ("Can't create relay state")); let relay_state_2 = relay_state.clone (); let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); let task_relay = spawn (async move { - ptth_relay::run_relay (relay_state_2, stop_relay_rx, None).await.unwrap (); + ptth_relay::run_relay (relay_state_2, stop_relay_rx, None).await }); assert! (relay_state.list_servers ().await.is_empty ()); @@ -65,7 +65,7 @@ fn end_to_end () { let (stop_server_tx, stop_server_rx) = oneshot::channel (); let task_server = { spawn (async move { - ptth_server::run_server (config_file, stop_server_rx, None, None).await.unwrap (); + ptth_server::run_server (config_file, stop_server_rx, None, None).await }) }; @@ -77,15 +77,15 @@ fn end_to_end () { let client = Client::builder () .timeout (Duration::from_secs (2)) - .build ().unwrap (); + .build ().expect ("Couldn't build HTTP client"); let resp = client.get (&format! ("{}/frontend/relay_up_check", relay_url)) - .send ().await.unwrap ().bytes ().await.unwrap (); + .send ().await.expect ("Couldn't check if relay is up").bytes ().await.expect ("Couldn't check if relay is up"); assert_eq! (resp, "Relay is up\n"); let resp = client.get (&format! ("{}/frontend/servers/{}/files/COPYING", relay_url, server_name)) - .send ().await.unwrap ().bytes ().await.unwrap (); + .send ().await.expect ("Couldn't find license").bytes ().await.expect ("Couldn't find license"); if blake3::hash (&resp) != blake3::Hash::from ([ 0xca, 0x02, 0x92, 0x78, @@ -98,28 +98,28 @@ fn end_to_end () { 0x2c, 0x4a, 0xac, 0x1f, 0x1a, 0xbb, 0xa8, 0xef, ]) { - panic! ("{}", String::from_utf8 (resp.to_vec ()).unwrap ()); + panic! ("{}", String::from_utf8 (resp.to_vec ()).expect ("???")); } // Requesting a file from a server that isn't registered // will error out let resp = client.get (&format! ("{}/frontend/servers/obviously_this_server_does_not_exist/files/COPYING", relay_url)) - .send ().await.unwrap (); + .send ().await.expect ("Couldn't send request to bogus server"); assert_eq! (resp.status (), reqwest::StatusCode::NOT_FOUND); info! ("Shutting down end-to-end test"); - stop_server_tx.send (()).unwrap (); - stop_relay_tx.send (()).unwrap (); + stop_server_tx.send (()).expect ("Couldn't shut down server"); + stop_relay_tx.send (()).expect ("Couldn't shut down relay"); info! ("Sent stop messages"); - task_relay.await.unwrap (); + task_relay.await.expect ("Couldn't join relay").expect ("Relay error"); info! ("Relay stopped"); - task_server.await.unwrap (); + task_server.await.expect ("Couldn't join server").expect ("Server error"); info! ("Server stopped"); }); } From f4b0c64e01b20d8073ecd21183253a27ae40b53a Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 21:42:03 +0000 Subject: [PATCH 074/208] :recycle: Move ptth_file_server into a bin crate so ptth_server can shed some dependencies --- Cargo.toml | 1 + crates/ptth_file_server_bin/Cargo.toml | 41 +++++++++++++++++++ .../src/main.rs} | 0 3 files changed, 42 insertions(+) create mode 100644 crates/ptth_file_server_bin/Cargo.toml rename crates/{ptth_server/src/bin/ptth_file_server.rs => ptth_file_server_bin/src/main.rs} (100%) diff --git a/Cargo.toml b/Cargo.toml index 1c1eeca..2df9363 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ always_equal = { path = "crates/always_equal" } ptth_core = { path = "crates/ptth_core" } ptth_relay = { path = "crates/ptth_relay" } ptth_server = { path = "crates/ptth_server" } +ptth_file_server = { path = "crates/ptth_file_server_bin" } [workspace] diff --git a/crates/ptth_file_server_bin/Cargo.toml b/crates/ptth_file_server_bin/Cargo.toml new file mode 100644 index 0000000..4061612 --- /dev/null +++ b/crates/ptth_file_server_bin/Cargo.toml @@ -0,0 +1,41 @@ +[package] + +name = "ptth_file_server" +version = "0.1.0" +authors = ["Trish"] +edition = "2018" +license = "AGPL-3.0" + +[dependencies] + +aho-corasick = "0.7.14" +anyhow = "1.0.34" +base64 = "0.12.3" +blake3 = "0.3.7" +futures = "0.3.7" +handlebars = "3.5.1" +http = "0.2.1" +hyper = "0.13.8" +lazy_static = "1.4.0" +percent-encoding = "2.1.0" +pulldown-cmark = "0.8.0" +regex = "1.4.1" +reqwest = { version = "0.10.8", features = ["stream"] } +rmp-serde = "0.14.4" +serde = {version = "1.0.117", features = ["derive"]} +structopt = "0.3.20" +thiserror = "1.0.22" +tokio = { version = "0.2.22", features = ["full"] } +tracing = "0.1.21" +tracing-futures = "0.2.4" +tracing-subscriber = "0.2.15" +toml = "0.5.7" + +always_equal = { path = "../always_equal" } +ptth_core = { path = "../ptth_core" } +ptth_server = { path = "../ptth_server" } + +[dev-dependencies] + +maplit = "1.0.2" +rand = "0.6.5" diff --git a/crates/ptth_server/src/bin/ptth_file_server.rs b/crates/ptth_file_server_bin/src/main.rs similarity index 100% rename from crates/ptth_server/src/bin/ptth_file_server.rs rename to crates/ptth_file_server_bin/src/main.rs From 028970cdf0ee7fbfe2c80bd4b34994818285efb3 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 21:55:24 +0000 Subject: [PATCH 075/208] :recycle: Remove the hyper dep from ptth_server.exe --- crates/ptth_file_server_bin/Cargo.toml | 20 -------------------- crates/ptth_file_server_bin/src/main.rs | 3 +-- crates/ptth_server/Cargo.toml | 1 - crates/ptth_server/src/errors.rs | 8 ++++---- crates/ptth_server/src/file_server/errors.rs | 2 +- crates/ptth_server/src/file_server/mod.rs | 6 +++--- crates/ptth_server/src/file_server/tests.rs | 4 +--- crates/ptth_server/src/lib.rs | 2 +- 8 files changed, 11 insertions(+), 35 deletions(-) diff --git a/crates/ptth_file_server_bin/Cargo.toml b/crates/ptth_file_server_bin/Cargo.toml index 4061612..bf2fc2d 100644 --- a/crates/ptth_file_server_bin/Cargo.toml +++ b/crates/ptth_file_server_bin/Cargo.toml @@ -8,34 +8,14 @@ license = "AGPL-3.0" [dependencies] -aho-corasick = "0.7.14" anyhow = "1.0.34" -base64 = "0.12.3" -blake3 = "0.3.7" -futures = "0.3.7" handlebars = "3.5.1" http = "0.2.1" hyper = "0.13.8" -lazy_static = "1.4.0" -percent-encoding = "2.1.0" -pulldown-cmark = "0.8.0" -regex = "1.4.1" -reqwest = { version = "0.10.8", features = ["stream"] } -rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} structopt = "0.3.20" -thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } -tracing = "0.1.21" -tracing-futures = "0.2.4" tracing-subscriber = "0.2.15" -toml = "0.5.7" -always_equal = { path = "../always_equal" } ptth_core = { path = "../ptth_core" } ptth_server = { path = "../ptth_server" } - -[dev-dependencies] - -maplit = "1.0.2" -rand = "0.6.5" diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 11482a0..cec4bb2 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -24,7 +24,6 @@ use ptth_core::{ prelude::*, }; use ptth_server::{ - errors::ServerError, file_server, load_toml, }; @@ -42,7 +41,7 @@ struct ServerState <'a> { } async fn handle_all (req: Request , state: Arc >) --> Result , ServerError> +-> Result , anyhow::Error> { use std::str::FromStr; diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index bb8ed1b..0095a55 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -15,7 +15,6 @@ blake3 = "0.3.7" futures = "0.3.7" handlebars = "3.5.1" http = "0.2.1" -hyper = "0.13.8" lazy_static = "1.4.0" percent-encoding = "2.1.0" pulldown-cmark = "0.8.0" diff --git a/crates/ptth_server/src/errors.rs b/crates/ptth_server/src/errors.rs index a2bb0c9..4ebdbe1 100644 --- a/crates/ptth_server/src/errors.rs +++ b/crates/ptth_server/src/errors.rs @@ -32,13 +32,13 @@ pub enum ServerError { // Hyper stuff #[error ("Hyper HTTP error")] - Http (#[from] hyper::http::Error), + Http (#[from] http::Error), - #[error ("Hyper invalid header name")] - InvalidHeaderName (#[from] hyper::header::InvalidHeaderName), + //#[error ("Hyper invalid header name")] + //InvalidHeaderName (#[from] hyper::header::InvalidHeaderName), #[error ("API key invalid")] - ApiKeyInvalid (hyper::header::InvalidHeaderValue), + ApiKeyInvalid (http::header::InvalidHeaderValue), // MessagePack stuff diff --git a/crates/ptth_server/src/file_server/errors.rs b/crates/ptth_server/src/file_server/errors.rs index e5b7cc5..9845340 100644 --- a/crates/ptth_server/src/file_server/errors.rs +++ b/crates/ptth_server/src/file_server/errors.rs @@ -33,5 +33,5 @@ pub enum FileServerError { Markdown (#[from] MarkdownError), #[error ("Invalid URI")] - InvalidUri (#[from] hyper::http::uri::InvalidUri), + InvalidUri (#[from] http::uri::InvalidUri), } diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 7ef6b66..f205515 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -468,7 +468,7 @@ fn internal_serve_dir ( path: &Path, dir: tokio::fs::ReadDir, full_path: PathBuf, - uri: &hyper::Uri + uri: &http::Uri ) -> Result { @@ -494,7 +494,7 @@ fn internal_serve_dir ( async fn internal_serve_file ( mut file: tokio::fs::File, - uri: &hyper::Uri, + uri: &http::Uri, send_body: bool, headers: &HashMap > ) @@ -571,7 +571,7 @@ async fn internal_serve_all ( info! ("Client requested {}", uri); - let uri = hyper::Uri::from_str (uri).map_err (FileServerError::InvalidUri)?; + let uri = http::Uri::from_str (uri).map_err (FileServerError::InvalidUri)?; let send_body = match &method { Method::Get => true, diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index cbb527f..aa1a541 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -210,9 +210,7 @@ fn file_server () { #[test] fn parse_uri () { - use hyper::Uri; - - assert! (Uri::from_maybe_shared ("/").is_ok ()); + assert! (http::Uri::from_maybe_shared ("/").is_ok ()); } #[test] diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 203af47..ce63528 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -15,7 +15,7 @@ use std::{ use futures::FutureExt; use handlebars::Handlebars; -use hyper::{ +use http::status::{ StatusCode, }; use reqwest::Client; From b43a6c2e4b65c36c77eeb3fbca0ad12bce0172dd Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 22:12:25 +0000 Subject: [PATCH 076/208] :recycle: Move emoji icons into one place --- crates/ptth_server/src/file_server/mod.rs | 45 +++++++++++------------ 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index f205515..3e74509 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -51,11 +51,20 @@ use ptth_core::{ }; pub mod errors; + use errors::{ FileServerError, MarkdownError, }; +mod emoji { + pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; + pub const PICTURE: &str = "\u{1f4f7}"; + pub const FILE: &str = "\u{1f4c4}"; + pub const FOLDER: &str = "\u{1f4c1}"; + pub const ERROR: &str = "\u{26a0}\u{fe0f}"; +} + #[derive (Debug, Serialize)] pub struct ServerInfo { pub server_name: String, @@ -157,19 +166,13 @@ fn check_range (range_str: Option <&str>, file_len: u64) } fn get_icon (file_name: &str) -> &'static str { - // Because my editor actually doesn't render these - - let video = "\u{1f39e}\u{fe0f}"; - let picture = "\u{1f4f7}"; - let file = "\u{1f4c4}"; - if file_name.ends_with (".mp4") || file_name.ends_with (".avi") || file_name.ends_with (".mkv") || file_name.ends_with (".webm") { - video + emoji::VIDEO } else if file_name.ends_with (".jpg") || @@ -177,10 +180,10 @@ fn get_icon (file_name: &str) -> &'static str { file_name.ends_with (".png") || file_name.ends_with (".bmp") { - picture + emoji::PICTURE } else { - file + emoji::FILE } } @@ -194,7 +197,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let file_name = match entry.file_name ().into_string () { Ok (x) => x, Err (_) => return TemplateDirEntry { - icon: "\u{26a0}\u{fe0f}", + icon: emoji::ERROR, trailing_slash: "", file_name: "File / directory name is not UTF-8".into (), encoded_file_name: "".into (), @@ -206,7 +209,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let metadata = match entry.metadata ().await { Ok (x) => x, Err (_) => return TemplateDirEntry { - icon: "\u{26a0}\u{fe0f}", + icon: emoji::ERROR, trailing_slash: "", file_name: "Could not fetch metadata".into (), encoded_file_name: "".into (), @@ -217,10 +220,9 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let (trailing_slash, icon, size) = { let t = metadata.file_type (); - let icon_folder = "\u{1f4c1}"; if t.is_dir () { - ("/", icon_folder, "".into ()) + ("/", emoji::FOLDER, "".into ()) } else { ("", get_icon (&file_name), pretty_print_bytes (metadata.len ()).into ()) @@ -391,14 +393,6 @@ fn serve_error ( resp } -fn serve_307 (location: String) -> Response { - let mut resp = Response::default (); - resp.status_code (StatusCode::TemporaryRedirect); - resp.header ("location".to_string (), location.into_bytes ()); - resp.body_bytes (b"Redirecting...".to_vec ()); - resp -} - fn render_markdown (bytes: &[u8], out: &mut String) -> Result <(), MarkdownError> { use pulldown_cmark::{Parser, Options, html}; @@ -660,8 +654,13 @@ pub async fn serve_all ( .header ("content-range".to_string (), format! ("bytes */{}", file_len).into_bytes ()); resp }, - Redirect (location) => serve_307 (location), - + Redirect (location) => { + let mut resp = Response::default (); + resp.status_code (StatusCode::TemporaryRedirect); + resp.header ("location".to_string (), location.into_bytes ()); + resp.body_bytes (b"Redirecting...".to_vec ()); + resp + }, Root => serve_root (handlebars, server_info).await?, ServeDir (ServeDirParams { path, From b94a3a1e17ffcf8864a7d61c55d45fd915b41e5d Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 22:31:54 +0000 Subject: [PATCH 077/208] Move byte range code into its own file --- crates/ptth_server/src/file_server/mod.rs | 106 +++----------------- crates/ptth_server/src/file_server/range.rs | 75 ++++++++++++++ crates/ptth_server/src/file_server/tests.rs | 53 ++++++---- 3 files changed, 127 insertions(+), 107 deletions(-) create mode 100644 crates/ptth_server/src/file_server/range.rs diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 3e74509..be681ac 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -32,8 +32,6 @@ use tokio::{ }; use tracing::instrument; -use regex::Regex; - #[cfg (test)] use always_equal::test::AlwaysEqual; @@ -51,11 +49,17 @@ use ptth_core::{ }; pub mod errors; +mod range; use errors::{ FileServerError, MarkdownError, }; +use range::{ + check_range, + ParsedRange, + ValidParsedRange, +}; mod emoji { pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; @@ -100,71 +104,6 @@ struct TemplateDirPage <'a> { entries: Vec , } -fn parse_range_header (range_str: &str) -> (Option , Option ) { - use lazy_static::lazy_static; - - lazy_static! { - static ref RE: Regex = Regex::new (r"^bytes=(\d*)-(\d*)$").expect ("Couldn't compile regex for Range header"); - } - - debug! ("{}", range_str); - - let caps = match RE.captures (range_str) { - Some (x) => x, - None => return (None, None), - }; - let start = caps.get (1).map (|x| x.as_str ()); - let end = caps.get (2).map (|x| x.as_str ()); - - let start = start.and_then (|x| u64::from_str_radix (x, 10).ok ()); - - // HTTP specifies ranges as [start inclusive, end inclusive] - // But that's dumb and [start inclusive, end exclusive) is better - - let end = end.and_then (|x| u64::from_str_radix (x, 10).ok ().map (|x| x + 1)); - - (start, end) -} - -use std::ops::Range; - -#[derive (Debug, PartialEq)] -enum ParsedRange { - Ok (Range ), - PartialContent (Range ), - RangeNotSatisfiable (u64), -} - -fn check_range (range_str: Option <&str>, file_len: u64) - -> ParsedRange -{ - use ParsedRange::*; - - let not_satisfiable = RangeNotSatisfiable (file_len); - - let range_str = match range_str { - None => return Ok (0..file_len), - Some (x) => x, - }; - - let (start, end) = parse_range_header (range_str); - - let start = start.unwrap_or (0); - if start >= file_len { - return not_satisfiable; - } - - let end = end.unwrap_or (file_len); - if end > file_len { - return not_satisfiable; - } - if end < start { - return not_satisfiable; - } - - PartialContent (start..end) -} - fn get_icon (file_name: &str) -> &'static str { if file_name.ends_with (".mp4") || @@ -289,8 +228,7 @@ async fn serve_dir ( async fn serve_file ( mut f: File, should_send_body: bool, - range: Range , - range_requested: bool + range: ValidParsedRange ) -> Result { @@ -302,6 +240,8 @@ async fn serve_file ( None }; + let (range, range_requested) = (range.range, range.range_requested); + info! ("Serving range {}-{}", range.start, range.end); let content_length = range.end - range.start; @@ -435,8 +375,7 @@ struct ServeDirParams { #[derive (Debug, PartialEq)] struct ServeFileParams { send_body: bool, - range: Range , - range_requested: bool, + range: ValidParsedRange, file: AlwaysEqual , } @@ -508,8 +447,12 @@ async fn internal_serve_file ( Ok (match check_range (range_header, file_len) { ParsedRange::RangeNotSatisfiable (file_len) => InternalResponse::RangeNotSatisfiable (file_len), - ParsedRange::Ok (range) => { + ParsedRange::Valid (range) => { if uri.query () == Some ("as_markdown") { + if range.range_requested { + return Ok (InternalResponse::InvalidQuery); + } + const MAX_BUF_SIZE: u32 = 1_000_000; if file_len > MAX_BUF_SIZE.into () { InternalResponse::MarkdownErr (MarkdownError::TooBig) @@ -529,22 +472,6 @@ async fn internal_serve_file ( file, send_body, range, - range_requested: false, - }) - } - }, - ParsedRange::PartialContent (range) => { - if uri.query ().is_some () { - InternalResponse::InvalidQuery - } - else { - let file = file.into (); - - InternalResponse::ServeFile (ServeFileParams { - file, - send_body, - range, - range_requested: true, }) } }, @@ -670,8 +597,7 @@ pub async fn serve_all ( file, send_body, range, - range_requested, - }) => serve_file (file.into_inner (), send_body, range, range_requested).await?, + }) => serve_file (file.into_inner (), send_body, range).await?, MarkdownErr (e) => match e { MarkdownError::TooBig => serve_error (StatusCode::InternalServerError, "File is too big to preview as Markdown"), //MarkdownError::NotMarkdown => serve_error (StatusCode::BadRequest, "File is not Markdown"), diff --git a/crates/ptth_server/src/file_server/range.rs b/crates/ptth_server/src/file_server/range.rs new file mode 100644 index 0000000..2fe7fad --- /dev/null +++ b/crates/ptth_server/src/file_server/range.rs @@ -0,0 +1,75 @@ +use std::ops::Range; + +use regex::Regex; + +pub fn parse_range_header (range_str: &str) -> (Option , Option ) { + use lazy_static::lazy_static; + + lazy_static! { + static ref RE: Regex = Regex::new (r"^bytes=(\d*)-(\d*)$").expect ("Couldn't compile regex for Range header"); + } + + let caps = match RE.captures (range_str) { + Some (x) => x, + None => return (None, None), + }; + let start = caps.get (1).map (|x| x.as_str ()); + let end = caps.get (2).map (|x| x.as_str ()); + + let start = start.and_then (|x| u64::from_str_radix (x, 10).ok ()); + + // HTTP specifies ranges as [start inclusive, end inclusive] + // But that's dumb and [start inclusive, end exclusive) is better + + let end = end.and_then (|x| u64::from_str_radix (x, 10).ok ().map (|x| x + 1)); + + (start, end) +} + +#[derive (Debug, PartialEq)] +pub struct ValidParsedRange { + pub range: Range , + pub range_requested: bool, +} + +#[derive (Debug, PartialEq)] +pub enum ParsedRange { + Valid (ValidParsedRange), + RangeNotSatisfiable (u64), +} + +pub fn check_range (range_str: Option <&str>, file_len: u64) + -> ParsedRange +{ + use ParsedRange::*; + + let not_satisfiable = RangeNotSatisfiable (file_len); + + let range_str = match range_str { + None => return Valid (ValidParsedRange { + range: 0..file_len, + range_requested: false, + }), + Some (x) => x, + }; + + let (start, end) = parse_range_header (range_str); + + let start = start.unwrap_or (0); + if start >= file_len { + return not_satisfiable; + } + + let end = end.unwrap_or (file_len); + if end > file_len { + return not_satisfiable; + } + if end < start { + return not_satisfiable; + } + + Valid (ValidParsedRange { + range: start..end, + range_requested: true, + }) +} diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index aa1a541..3970c8b 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -34,33 +34,46 @@ fn icons () { #[test] fn parse_range_header () { + use super::range::{ + *, + ParsedRange::*, + }; + for (input, expected) in vec! [ ("", (None, None)), ("bytes=0-", (Some (0), None)), ("bytes=0-999", (Some (0), Some (1000))), ("bytes=111-999", (Some (111), Some (1000))), ].into_iter () { - let actual = super::parse_range_header (input); + let actual = parse_range_header (input); assert_eq! (actual, expected); } - use super::ParsedRange::*; + let ok_range = |range| Valid (ValidParsedRange { + range, + range_requested: false, + }); + + let partial_content = |range| Valid (ValidParsedRange { + range, + range_requested: true, + }); for (header, file_len, expected) in vec! [ - (None, 0, Ok (0..0)), - (None, 1024, Ok (0..1024)), + (None, 0, ok_range (0..0)), + (None, 1024, ok_range (0..1024)), (Some (""), 0, RangeNotSatisfiable (0)), - (Some (""), 1024, PartialContent (0..1024)), + (Some (""), 1024, partial_content (0..1024)), - (Some ("bytes=0-"), 1024, PartialContent (0..1024)), - (Some ("bytes=0-999"), 1024, PartialContent (0..1000)), - (Some ("bytes=0-1023"), 1024, PartialContent (0..1024)), - (Some ("bytes=111-999"), 1024, PartialContent (111..1000)), - (Some ("bytes=111-1023"), 1024, PartialContent (111..1024)), + (Some ("bytes=0-"), 1024, partial_content (0..1024)), + (Some ("bytes=0-999"), 1024, partial_content (0..1000)), + (Some ("bytes=0-1023"), 1024, partial_content (0..1024)), + (Some ("bytes=111-999"), 1024, partial_content (111..1000)), + (Some ("bytes=111-1023"), 1024, partial_content (111..1024)), (Some ("bytes=200-100"), 1024, RangeNotSatisfiable (1024)), - (Some ("bytes=0-"), 512, PartialContent (0..512)), + (Some ("bytes=0-"), 512, partial_content (0..512)), (Some ("bytes=0-1023"), 512, RangeNotSatisfiable (512)), (Some ("bytes=1000-1023"), 512, RangeNotSatisfiable (512)), ].into_iter () { @@ -139,14 +152,18 @@ fn file_server () { ("/files/src/?", InvalidQuery), (bad_passwords_path, ServeFile (ServeFileParams { send_body: true, - range: 0..1_048_576, - range_requested: false, + range: ValidParsedRange { + range: 0..1_048_576, + range_requested: false, + }, file: AlwaysEqual::testing_blank (), })), ("/files/test/test.md", ServeFile (ServeFileParams { send_body: true, - range: 0..144, - range_requested: false, + range: ValidParsedRange { + range: 0..144, + range_requested: false, + }, file: AlwaysEqual::testing_blank (), })), ] { @@ -200,8 +217,10 @@ fn file_server () { assert_eq! (resp.expect ("Should be Ok (_)"), ServeFile (ServeFileParams { send_body: false, - range: 0..1_048_576, - range_requested: false, + range: ValidParsedRange { + range: 0..1_048_576, + range_requested: false, + }, file: AlwaysEqual::testing_blank (), })); } From ab95485d921aa2e6bb015f293b243f92014a1f72 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 22:41:48 +0000 Subject: [PATCH 078/208] :recycle: Fix clippy issues --- crates/ptth_server/src/file_server/mod.rs | 18 ++--- crates/ptth_server/src/file_server/range.rs | 76 +++++++++++++++++---- crates/ptth_server/src/file_server/tests.rs | 56 +-------------- 3 files changed, 74 insertions(+), 76 deletions(-) diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index be681ac..fe2c497 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -55,11 +55,6 @@ use errors::{ FileServerError, MarkdownError, }; -use range::{ - check_range, - ParsedRange, - ValidParsedRange, -}; mod emoji { pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; @@ -228,7 +223,7 @@ async fn serve_dir ( async fn serve_file ( mut f: File, should_send_body: bool, - range: ValidParsedRange + range: range::ValidParsed ) -> Result { @@ -375,7 +370,7 @@ struct ServeDirParams { #[derive (Debug, PartialEq)] struct ServeFileParams { send_body: bool, - range: ValidParsedRange, + range: range::ValidParsed, file: AlwaysEqual , } @@ -445,15 +440,16 @@ async fn internal_serve_file ( let range_header = headers.get ("range").and_then (|v| std::str::from_utf8 (v).ok ()); - Ok (match check_range (range_header, file_len) { - ParsedRange::RangeNotSatisfiable (file_len) => InternalResponse::RangeNotSatisfiable (file_len), - ParsedRange::Valid (range) => { + Ok (match range::check (range_header, file_len) { + range::Parsed::NotSatisfiable (file_len) => InternalResponse::RangeNotSatisfiable (file_len), + range::Parsed::Valid (range) => { if uri.query () == Some ("as_markdown") { + const MAX_BUF_SIZE: u32 = 1_000_000; + if range.range_requested { return Ok (InternalResponse::InvalidQuery); } - const MAX_BUF_SIZE: u32 = 1_000_000; if file_len > MAX_BUF_SIZE.into () { InternalResponse::MarkdownErr (MarkdownError::TooBig) } diff --git a/crates/ptth_server/src/file_server/range.rs b/crates/ptth_server/src/file_server/range.rs index 2fe7fad..675ccd7 100644 --- a/crates/ptth_server/src/file_server/range.rs +++ b/crates/ptth_server/src/file_server/range.rs @@ -2,7 +2,8 @@ use std::ops::Range; use regex::Regex; -pub fn parse_range_header (range_str: &str) -> (Option , Option ) { +fn parse (range_str: &str) -> (Option , Option ) +{ use lazy_static::lazy_static; lazy_static! { @@ -27,33 +28,32 @@ pub fn parse_range_header (range_str: &str) -> (Option , Option ) { } #[derive (Debug, PartialEq)] -pub struct ValidParsedRange { +pub struct ValidParsed { pub range: Range , pub range_requested: bool, } #[derive (Debug, PartialEq)] -pub enum ParsedRange { - Valid (ValidParsedRange), - RangeNotSatisfiable (u64), +pub enum Parsed { + Valid (ValidParsed), + NotSatisfiable (u64), } -pub fn check_range (range_str: Option <&str>, file_len: u64) - -> ParsedRange +pub fn check (range_str: Option <&str>, file_len: u64) -> Parsed { - use ParsedRange::*; + use Parsed::*; - let not_satisfiable = RangeNotSatisfiable (file_len); + let not_satisfiable = NotSatisfiable (file_len); let range_str = match range_str { - None => return Valid (ValidParsedRange { + None => return Valid (ValidParsed { range: 0..file_len, range_requested: false, }), Some (x) => x, }; - let (start, end) = parse_range_header (range_str); + let (start, end) = parse (range_str); let start = start.unwrap_or (0); if start >= file_len { @@ -68,8 +68,60 @@ pub fn check_range (range_str: Option <&str>, file_len: u64) return not_satisfiable; } - Valid (ValidParsedRange { + Valid (ValidParsed { range: start..end, range_requested: true, }) } + +#[cfg (test)] +mod tests { + #[test] + fn test_byte_ranges () { + use super::*; + + for (input, expected) in vec! [ + ("", (None, None)), + ("bytes=0-", (Some (0), None)), + ("bytes=0-999", (Some (0), Some (1000))), + ("bytes=111-999", (Some (111), Some (1000))), + ].into_iter () { + let actual = parse (input); + assert_eq! (actual, expected); + } + + let ok_range = |range| Parsed::Valid (ValidParsed { + range, + range_requested: false, + }); + + let partial_content = |range| Parsed::Valid (ValidParsed { + range, + range_requested: true, + }); + + let not_satisfiable = |file_len| Parsed::NotSatisfiable (file_len); + + for (header, file_len, expected) in vec! [ + (None, 0, ok_range (0..0)), + (None, 1024, ok_range (0..1024)), + + (Some (""), 0, not_satisfiable (0)), + (Some (""), 1024, partial_content (0..1024)), + + (Some ("bytes=0-"), 1024, partial_content (0..1024)), + (Some ("bytes=0-999"), 1024, partial_content (0..1000)), + (Some ("bytes=0-1023"), 1024, partial_content (0..1024)), + (Some ("bytes=111-999"), 1024, partial_content (111..1000)), + (Some ("bytes=111-1023"), 1024, partial_content (111..1024)), + (Some ("bytes=200-100"), 1024, not_satisfiable (1024)), + + (Some ("bytes=0-"), 512, partial_content (0..512)), + (Some ("bytes=0-1023"), 512, not_satisfiable (512)), + (Some ("bytes=1000-1023"), 512, not_satisfiable (512)), + ].into_iter () { + let actual = check (header, file_len); + assert_eq! (actual, expected); + } + } +} diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index 3970c8b..0ada930 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -32,56 +32,6 @@ fn icons () { } } -#[test] -fn parse_range_header () { - use super::range::{ - *, - ParsedRange::*, - }; - - for (input, expected) in vec! [ - ("", (None, None)), - ("bytes=0-", (Some (0), None)), - ("bytes=0-999", (Some (0), Some (1000))), - ("bytes=111-999", (Some (111), Some (1000))), - ].into_iter () { - let actual = parse_range_header (input); - assert_eq! (actual, expected); - } - - let ok_range = |range| Valid (ValidParsedRange { - range, - range_requested: false, - }); - - let partial_content = |range| Valid (ValidParsedRange { - range, - range_requested: true, - }); - - for (header, file_len, expected) in vec! [ - (None, 0, ok_range (0..0)), - (None, 1024, ok_range (0..1024)), - - (Some (""), 0, RangeNotSatisfiable (0)), - (Some (""), 1024, partial_content (0..1024)), - - (Some ("bytes=0-"), 1024, partial_content (0..1024)), - (Some ("bytes=0-999"), 1024, partial_content (0..1000)), - (Some ("bytes=0-1023"), 1024, partial_content (0..1024)), - (Some ("bytes=111-999"), 1024, partial_content (111..1000)), - (Some ("bytes=111-1023"), 1024, partial_content (111..1024)), - (Some ("bytes=200-100"), 1024, RangeNotSatisfiable (1024)), - - (Some ("bytes=0-"), 512, partial_content (0..512)), - (Some ("bytes=0-1023"), 512, RangeNotSatisfiable (512)), - (Some ("bytes=1000-1023"), 512, RangeNotSatisfiable (512)), - ].into_iter () { - let actual = super::check_range (header, file_len); - assert_eq! (actual, expected); - } -} - #[test] fn pretty_print_bytes () { for (input_after, expected_before, expected_after) in vec! [ @@ -152,7 +102,7 @@ fn file_server () { ("/files/src/?", InvalidQuery), (bad_passwords_path, ServeFile (ServeFileParams { send_body: true, - range: ValidParsedRange { + range: range::ValidParsed { range: 0..1_048_576, range_requested: false, }, @@ -160,7 +110,7 @@ fn file_server () { })), ("/files/test/test.md", ServeFile (ServeFileParams { send_body: true, - range: ValidParsedRange { + range: range::ValidParsed { range: 0..144, range_requested: false, }, @@ -217,7 +167,7 @@ fn file_server () { assert_eq! (resp.expect ("Should be Ok (_)"), ServeFile (ServeFileParams { send_body: false, - range: ValidParsedRange { + range: range::ValidParsed { range: 0..1_048_576, range_requested: false, }, From 6e6e062c51506e82653d13f499b0adf35755deda Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 22:56:25 +0000 Subject: [PATCH 079/208] :recycle: Move Markdown previewing to its own file --- crates/ptth_server/src/file_server/errors.rs | 13 +-- .../ptth_server/src/file_server/markdown.rs | 56 +++++++++++++ crates/ptth_server/src/file_server/mod.rs | 81 +++++++------------ crates/ptth_server/src/file_server/tests.rs | 17 ---- 4 files changed, 88 insertions(+), 79 deletions(-) create mode 100644 crates/ptth_server/src/file_server/markdown.rs diff --git a/crates/ptth_server/src/file_server/errors.rs b/crates/ptth_server/src/file_server/errors.rs index 9845340..dbb9127 100644 --- a/crates/ptth_server/src/file_server/errors.rs +++ b/crates/ptth_server/src/file_server/errors.rs @@ -1,14 +1,5 @@ use thiserror::Error; -#[derive (Debug, Error, PartialEq)] -pub enum MarkdownError { - #[error ("File is too big to process")] - TooBig, - - #[error ("File is not UTF-8")] - NotUtf8, -} - #[derive (Debug, Error)] pub enum FileServerError { #[error ("Handlebars render error")] @@ -29,8 +20,8 @@ pub enum FileServerError { #[error ("File path is not UTF-8")] FilePathNotUtf8, - #[error ("Markdown error")] - Markdown (#[from] MarkdownError), + //#[error ("Markdown error")] + //Markdown (#[from] super::markdown::Error), #[error ("Invalid URI")] InvalidUri (#[from] http::uri::InvalidUri), diff --git a/crates/ptth_server/src/file_server/markdown.rs b/crates/ptth_server/src/file_server/markdown.rs new file mode 100644 index 0000000..2930481 --- /dev/null +++ b/crates/ptth_server/src/file_server/markdown.rs @@ -0,0 +1,56 @@ +#[derive (Debug, thiserror::Error, PartialEq)] +pub enum Error { + #[error ("File is too big to preview as Markdown")] + TooBig, + + #[error ("File is not UTF-8")] + NotUtf8, +} + +fn render (bytes: &[u8], out: &mut String) -> Result <(), Error> { + use pulldown_cmark::{Parser, Options, html}; + + let markdown_input = match std::str::from_utf8 (bytes) { + Err (_) => return Err (Error::NotUtf8), + Ok (x) => x, + }; + + let mut options = Options::empty (); + options.insert (Options::ENABLE_STRIKETHROUGH); + let parser = Parser::new_ext (markdown_input, options); + + html::push_html (out, parser); + + Ok (()) +} + +pub fn render_styled (bytes: &[u8]) -> Result { + // Write to String buffer. + let mut out = String::new (); + + out.push_str (""); + render (bytes, &mut out)?; + out.push_str (""); + + Ok (out) +} + +#[cfg (test)] +mod tests { + #[test] + fn markdown () { + use super::*; + + for (input, expected) in vec! [ + ("", ""), + ( + "Hello world, this is a ~~complicated~~ *very simple* example.", + "

Hello world, this is a complicated very simple example.

\n" + ), + ].into_iter () { + let mut out = String::default (); + render (input.as_bytes (), &mut out).expect ("Markdown sample failed"); + assert_eq! (expected, &out); + } + } +} diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index fe2c497..a34193f 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -49,12 +49,11 @@ use ptth_core::{ }; pub mod errors; +mod markdown; mod range; -use errors::{ - FileServerError, - MarkdownError, -}; +use errors::FileServerError; +use markdown::render_styled; mod emoji { pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; @@ -316,46 +315,6 @@ async fn serve_file ( Ok (response) } -fn serve_error ( - status_code: StatusCode, - msg: &str -) --> Response -{ - let mut resp = Response::default (); - resp.status_code (status_code); - resp.body_bytes (msg.as_bytes ().to_vec ()); - resp -} - -fn render_markdown (bytes: &[u8], out: &mut String) -> Result <(), MarkdownError> { - use pulldown_cmark::{Parser, Options, html}; - - let markdown_input = match std::str::from_utf8 (bytes) { - Err (_) => return Err (MarkdownError::NotUtf8), - Ok (x) => x, - }; - - let mut options = Options::empty (); - options.insert (Options::ENABLE_STRIKETHROUGH); - let parser = Parser::new_ext (markdown_input, options); - - html::push_html (out, parser); - - Ok (()) -} - -fn render_markdown_styled (bytes: &[u8]) -> Result { - // Write to String buffer. - let mut out = String::new (); - - out.push_str (""); - render_markdown (bytes, &mut out)?; - out.push_str (""); - - Ok (out) -} - // Sort of an internal API endpoint to make testing work better. // Eventually we could expose this as JSON or Msgpack or whatever. For now // it's just a Rust struct that we can test on without caring about @@ -387,7 +346,7 @@ enum InternalResponse { ServeDir (ServeDirParams), ServeFile (ServeFileParams), - MarkdownErr (MarkdownError), + MarkdownErr (markdown::Error), MarkdownPreview (String), } @@ -451,14 +410,17 @@ async fn internal_serve_file ( } if file_len > MAX_BUF_SIZE.into () { - InternalResponse::MarkdownErr (MarkdownError::TooBig) + InternalResponse::MarkdownErr (markdown::Error::TooBig) } else { let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().expect ("Couldn't fit u32 into usize")]; let bytes_read = file.read (&mut buffer).await?; buffer.truncate (bytes_read); - InternalResponse::MarkdownPreview (render_markdown_styled (&buffer)?) + match render_styled (&buffer) { + Ok (x) => InternalResponse::MarkdownPreview (x), + Err (x) => InternalResponse::MarkdownErr (x), + } } } else { @@ -565,6 +527,18 @@ pub async fn serve_all ( { use InternalResponse::*; + fn serve_error >> ( + status_code: StatusCode, + msg: S + ) + -> Response + { + let mut resp = Response::default (); + resp.status_code (status_code); + resp.body_bytes (msg.into ()); + resp + } + Ok (match internal_serve_all (root, method, uri, headers, hidden_path).await? { Favicon => serve_error (StatusCode::NotFound, ""), Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), @@ -594,10 +568,15 @@ pub async fn serve_all ( send_body, range, }) => serve_file (file.into_inner (), send_body, range).await?, - MarkdownErr (e) => match e { - MarkdownError::TooBig => serve_error (StatusCode::InternalServerError, "File is too big to preview as Markdown"), - //MarkdownError::NotMarkdown => serve_error (StatusCode::BadRequest, "File is not Markdown"), - MarkdownError::NotUtf8 => serve_error (StatusCode::BadRequest, "File is not UTF-8"), + MarkdownErr (e) => { + use markdown::Error::*; + let code = match &e { + TooBig => StatusCode::InternalServerError, + //NotMarkdown => serve_error (StatusCode::BadRequest, "File is not Markdown"), + NotUtf8 => StatusCode::BadRequest, + }; + + serve_error (code, e.to_string ()) }, MarkdownPreview (s) => serve_html (s), }) diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index 0ada930..a78e6f1 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -181,20 +181,3 @@ fn file_server () { fn parse_uri () { assert! (http::Uri::from_maybe_shared ("/").is_ok ()); } - -#[test] -fn markdown () { - use super::*; - - for (input, expected) in vec! [ - ("", ""), - ( - "Hello world, this is a ~~complicated~~ *very simple* example.", - "

Hello world, this is a complicated very simple example.

\n" - ), - ].into_iter () { - let mut out = String::default (); - render_markdown (input.as_bytes (), &mut out).expect ("Markdown sample failed"); - assert_eq! (expected, &out); - } -} From bb4c4e803a005992c6ec5886994cc0d9f27f3f7d Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 23:12:56 +0000 Subject: [PATCH 080/208] :recycle: Extract file server internal parts to their own file --- .../ptth_server/src/file_server/internal.rs | 236 ++++++++++++++++++ crates/ptth_server/src/file_server/mod.rs | 216 +--------------- crates/ptth_server/src/file_server/tests.rs | 5 + 3 files changed, 245 insertions(+), 212 deletions(-) create mode 100644 crates/ptth_server/src/file_server/internal.rs diff --git a/crates/ptth_server/src/file_server/internal.rs b/crates/ptth_server/src/file_server/internal.rs new file mode 100644 index 0000000..495fe84 --- /dev/null +++ b/crates/ptth_server/src/file_server/internal.rs @@ -0,0 +1,236 @@ +// Sort of an internal API endpoint to make testing easy. +// Eventually we could expose this as JSON or Msgpack or whatever. For now +// it's just a Rust struct that we can test on without caring about +// human-readable HTML + +use std::{ + collections::HashMap, + convert::TryInto, + path::{Path, PathBuf}, +}; + +use percent_encoding::percent_decode; +use tokio::{ + fs::{ + File, + read_dir, + ReadDir, + }, + io::AsyncReadExt, +}; + +#[cfg (test)] +use always_equal::test::AlwaysEqual; + +#[cfg (not (test))] +use always_equal::prod::AlwaysEqual; + +use ptth_core::{ + http_serde::Method, + prefix_match, + prelude::*, +}; + +use crate::{ + load_toml, +}; + +use super::{ + errors::FileServerError, + markdown, + markdown::render_styled, + range, +}; + +#[derive (Debug, PartialEq)] +pub struct ServeDirParams { + pub path: PathBuf, + pub dir: AlwaysEqual , +} + +#[derive (Debug, PartialEq)] +pub struct ServeFileParams { + pub send_body: bool, + pub range: range::ValidParsed, + pub file: AlwaysEqual , +} + +#[derive (Debug, PartialEq)] +pub enum InternalResponse { + Favicon, + Forbidden, + InvalidQuery, + MethodNotAllowed, + NotFound, + RangeNotSatisfiable (u64), + Redirect (String), + Root, + ServeDir (ServeDirParams), + ServeFile (ServeFileParams), + + MarkdownErr (markdown::Error), + MarkdownPreview (String), +} + +fn internal_serve_dir ( + path_s: &str, + path: &Path, + dir: tokio::fs::ReadDir, + full_path: PathBuf, + uri: &http::Uri +) +-> Result +{ + let has_trailing_slash = path_s.is_empty () || path_s.ends_with ('/'); + + if ! has_trailing_slash { + let file_name = path.file_name ().ok_or (FileServerError::NoFileNameRequested)?; + let file_name = file_name.to_str ().ok_or (FileServerError::FilePathNotUtf8)?; + return Ok (InternalResponse::Redirect (format! ("{}/", file_name))); + } + + if uri.query ().is_some () { + return Ok (InternalResponse::InvalidQuery); + } + + let dir = dir.into (); + + Ok (InternalResponse::ServeDir (ServeDirParams { + dir, + path: full_path, + })) +} + +async fn internal_serve_file ( + mut file: tokio::fs::File, + uri: &http::Uri, + send_body: bool, + headers: &HashMap > +) +-> Result +{ + use std::os::unix::fs::PermissionsExt; + + let file_md = file.metadata ().await.map_err (FileServerError::CantGetFileMetadata)?; + if file_md.permissions ().mode () == load_toml::CONFIG_PERMISSIONS_MODE + { + return Ok (InternalResponse::Forbidden); + } + + let file_len = file_md.len (); + + let range_header = headers.get ("range").and_then (|v| std::str::from_utf8 (v).ok ()); + + Ok (match range::check (range_header, file_len) { + range::Parsed::NotSatisfiable (file_len) => InternalResponse::RangeNotSatisfiable (file_len), + range::Parsed::Valid (range) => { + if uri.query () == Some ("as_markdown") { + const MAX_BUF_SIZE: u32 = 1_000_000; + + if range.range_requested { + return Ok (InternalResponse::InvalidQuery); + } + + if file_len > MAX_BUF_SIZE.into () { + InternalResponse::MarkdownErr (markdown::Error::TooBig) + } + else { + let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().expect ("Couldn't fit u32 into usize")]; + let bytes_read = file.read (&mut buffer).await?; + buffer.truncate (bytes_read); + + match render_styled (&buffer) { + Ok (x) => InternalResponse::MarkdownPreview (x), + Err (x) => InternalResponse::MarkdownErr (x), + } + } + } + else { + let file = file.into (); + + InternalResponse::ServeFile (ServeFileParams { + file, + send_body, + range, + }) + } + }, + }) +} + +pub async fn internal_serve_all ( + root: &Path, + method: Method, + uri: &str, + headers: &HashMap >, + hidden_path: Option <&Path> +) +-> Result +{ + use std::str::FromStr; + use InternalResponse::*; + + info! ("Client requested {}", uri); + + let uri = http::Uri::from_str (uri).map_err (FileServerError::InvalidUri)?; + + let send_body = match &method { + Method::Get => true, + Method::Head => false, + m => { + debug! ("Unsupported method {:?}", m); + return Ok (MethodNotAllowed); + } + }; + + if uri.path () == "/favicon.ico" { + return Ok (Favicon); + } + + let path = match prefix_match ("/files", uri.path ()) { + Some (x) => x, + None => return Ok (Root), + }; + + if path == "" { + return Ok (Redirect ("files/".to_string ())); + } + + // TODO: There is totally a dir traversal attack in here somewhere + + let encoded_path = &path [1..]; + + let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; + let path = Path::new (&*path_s); + + let full_path = root.join (path); + + debug! ("full_path = {:?}", full_path); + + if let Some (hidden_path) = hidden_path { + if full_path == hidden_path { + return Ok (Forbidden); + } + } + + if let Ok (dir) = read_dir (&full_path).await { + internal_serve_dir ( + &path_s, + path, + dir, + full_path, + &uri + ) + } + else if let Ok (file) = File::open (&full_path).await { + internal_serve_file ( + file, + &uri, + send_body, + headers + ).await + } + else { + Ok (NotFound) + } +} diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index a34193f..7d4ae13 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -7,22 +7,18 @@ use std::{ borrow::Cow, cmp::min, collections::HashMap, - convert::{Infallible, TryFrom, TryInto}, + convert::{Infallible, TryFrom}, fmt::Debug, io::SeekFrom, - path::{Path, PathBuf}, + path::Path, }; use handlebars::Handlebars; -use percent_encoding::{ - percent_decode, -}; use serde::Serialize; use tokio::{ fs::{ DirEntry, File, - read_dir, ReadDir, }, io::AsyncReadExt, @@ -32,12 +28,6 @@ use tokio::{ }; use tracing::instrument; -#[cfg (test)] -use always_equal::test::AlwaysEqual; - -#[cfg (not (test))] -use always_equal::prod::AlwaysEqual; - use ptth_core::{ http_serde::{ Method, @@ -45,15 +35,15 @@ use ptth_core::{ StatusCode, }, prelude::*, - prefix_match, }; pub mod errors; +mod internal; mod markdown; mod range; use errors::FileServerError; -use markdown::render_styled; +use internal::*; mod emoji { pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; @@ -315,204 +305,6 @@ async fn serve_file ( Ok (response) } -// Sort of an internal API endpoint to make testing work better. -// Eventually we could expose this as JSON or Msgpack or whatever. For now -// it's just a Rust struct that we can test on without caring about -// human-readable HTML - -#[derive (Debug, PartialEq)] -struct ServeDirParams { - path: PathBuf, - dir: AlwaysEqual , -} - -#[derive (Debug, PartialEq)] -struct ServeFileParams { - send_body: bool, - range: range::ValidParsed, - file: AlwaysEqual , -} - -#[derive (Debug, PartialEq)] -enum InternalResponse { - Favicon, - Forbidden, - InvalidQuery, - MethodNotAllowed, - NotFound, - RangeNotSatisfiable (u64), - Redirect (String), - Root, - ServeDir (ServeDirParams), - ServeFile (ServeFileParams), - - MarkdownErr (markdown::Error), - MarkdownPreview (String), -} - -fn internal_serve_dir ( - path_s: &str, - path: &Path, - dir: tokio::fs::ReadDir, - full_path: PathBuf, - uri: &http::Uri -) --> Result -{ - let has_trailing_slash = path_s.is_empty () || path_s.ends_with ('/'); - - if ! has_trailing_slash { - let file_name = path.file_name ().ok_or (FileServerError::NoFileNameRequested)?; - let file_name = file_name.to_str ().ok_or (FileServerError::FilePathNotUtf8)?; - return Ok (InternalResponse::Redirect (format! ("{}/", file_name))); - } - - if uri.query ().is_some () { - return Ok (InternalResponse::InvalidQuery); - } - - let dir = dir.into (); - - Ok (InternalResponse::ServeDir (ServeDirParams { - dir, - path: full_path, - })) -} - -async fn internal_serve_file ( - mut file: tokio::fs::File, - uri: &http::Uri, - send_body: bool, - headers: &HashMap > -) --> Result -{ - use std::os::unix::fs::PermissionsExt; - - let file_md = file.metadata ().await.map_err (FileServerError::CantGetFileMetadata)?; - if file_md.permissions ().mode () == super::load_toml::CONFIG_PERMISSIONS_MODE - { - return Ok (InternalResponse::Forbidden); - } - - let file_len = file_md.len (); - - let range_header = headers.get ("range").and_then (|v| std::str::from_utf8 (v).ok ()); - - Ok (match range::check (range_header, file_len) { - range::Parsed::NotSatisfiable (file_len) => InternalResponse::RangeNotSatisfiable (file_len), - range::Parsed::Valid (range) => { - if uri.query () == Some ("as_markdown") { - const MAX_BUF_SIZE: u32 = 1_000_000; - - if range.range_requested { - return Ok (InternalResponse::InvalidQuery); - } - - if file_len > MAX_BUF_SIZE.into () { - InternalResponse::MarkdownErr (markdown::Error::TooBig) - } - else { - let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().expect ("Couldn't fit u32 into usize")]; - let bytes_read = file.read (&mut buffer).await?; - buffer.truncate (bytes_read); - - match render_styled (&buffer) { - Ok (x) => InternalResponse::MarkdownPreview (x), - Err (x) => InternalResponse::MarkdownErr (x), - } - } - } - else { - let file = file.into (); - - InternalResponse::ServeFile (ServeFileParams { - file, - send_body, - range, - }) - } - }, - }) -} - -async fn internal_serve_all ( - root: &Path, - method: Method, - uri: &str, - headers: &HashMap >, - hidden_path: Option <&Path> -) --> Result -{ - use std::str::FromStr; - use InternalResponse::*; - - info! ("Client requested {}", uri); - - let uri = http::Uri::from_str (uri).map_err (FileServerError::InvalidUri)?; - - let send_body = match &method { - Method::Get => true, - Method::Head => false, - m => { - debug! ("Unsupported method {:?}", m); - return Ok (MethodNotAllowed); - } - }; - - if uri.path () == "/favicon.ico" { - return Ok (Favicon); - } - - let path = match prefix_match ("/files", uri.path ()) { - Some (x) => x, - None => return Ok (Root), - }; - - if path == "" { - return Ok (Redirect ("files/".to_string ())); - } - - // TODO: There is totally a dir traversal attack in here somewhere - - let encoded_path = &path [1..]; - - let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; - let path = Path::new (&*path_s); - - let full_path = root.join (path); - - debug! ("full_path = {:?}", full_path); - - if let Some (hidden_path) = hidden_path { - if full_path == hidden_path { - return Ok (Forbidden); - } - } - - if let Ok (dir) = read_dir (&full_path).await { - internal_serve_dir ( - &path_s, - path, - dir, - full_path, - &uri - ) - } - else if let Ok (file) = File::open (&full_path).await { - internal_serve_file ( - file, - &uri, - send_body, - headers - ).await - } - else { - Ok (NotFound) - } -} - #[instrument (level = "debug", skip (handlebars, headers))] pub async fn serve_all ( handlebars: &Handlebars <'static>, diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index a78e6f1..c11bdc7 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -76,6 +76,11 @@ fn i_hate_paths () { #[test] fn file_server () { + use std::path::PathBuf; + + #[cfg (test)] + use always_equal::test::AlwaysEqual; + use ptth_core::{ http_serde::Method, }; From 8079b3f7780d7a60abea2a70ecccb9335dd2261e Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 23:15:45 +0000 Subject: [PATCH 081/208] :recycle: Remove "internal" from names in the internal module --- .../ptth_server/src/file_server/internal.rs | 40 +++++++++---------- crates/ptth_server/src/file_server/mod.rs | 9 ++--- crates/ptth_server/src/file_server/tests.rs | 16 ++++---- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/crates/ptth_server/src/file_server/internal.rs b/crates/ptth_server/src/file_server/internal.rs index 495fe84..07ddb81 100644 --- a/crates/ptth_server/src/file_server/internal.rs +++ b/crates/ptth_server/src/file_server/internal.rs @@ -56,7 +56,7 @@ pub struct ServeFileParams { } #[derive (Debug, PartialEq)] -pub enum InternalResponse { +pub enum Response { Favicon, Forbidden, InvalidQuery, @@ -72,49 +72,49 @@ pub enum InternalResponse { MarkdownPreview (String), } -fn internal_serve_dir ( +fn serve_dir ( path_s: &str, path: &Path, dir: tokio::fs::ReadDir, full_path: PathBuf, uri: &http::Uri ) --> Result +-> Result { let has_trailing_slash = path_s.is_empty () || path_s.ends_with ('/'); if ! has_trailing_slash { let file_name = path.file_name ().ok_or (FileServerError::NoFileNameRequested)?; let file_name = file_name.to_str ().ok_or (FileServerError::FilePathNotUtf8)?; - return Ok (InternalResponse::Redirect (format! ("{}/", file_name))); + return Ok (Response::Redirect (format! ("{}/", file_name))); } if uri.query ().is_some () { - return Ok (InternalResponse::InvalidQuery); + return Ok (Response::InvalidQuery); } let dir = dir.into (); - Ok (InternalResponse::ServeDir (ServeDirParams { + Ok (Response::ServeDir (ServeDirParams { dir, path: full_path, })) } -async fn internal_serve_file ( +async fn serve_file ( mut file: tokio::fs::File, uri: &http::Uri, send_body: bool, headers: &HashMap > ) --> Result +-> Result { use std::os::unix::fs::PermissionsExt; let file_md = file.metadata ().await.map_err (FileServerError::CantGetFileMetadata)?; if file_md.permissions ().mode () == load_toml::CONFIG_PERMISSIONS_MODE { - return Ok (InternalResponse::Forbidden); + return Ok (Response::Forbidden); } let file_len = file_md.len (); @@ -122,17 +122,17 @@ async fn internal_serve_file ( let range_header = headers.get ("range").and_then (|v| std::str::from_utf8 (v).ok ()); Ok (match range::check (range_header, file_len) { - range::Parsed::NotSatisfiable (file_len) => InternalResponse::RangeNotSatisfiable (file_len), + range::Parsed::NotSatisfiable (file_len) => Response::RangeNotSatisfiable (file_len), range::Parsed::Valid (range) => { if uri.query () == Some ("as_markdown") { const MAX_BUF_SIZE: u32 = 1_000_000; if range.range_requested { - return Ok (InternalResponse::InvalidQuery); + return Ok (Response::InvalidQuery); } if file_len > MAX_BUF_SIZE.into () { - InternalResponse::MarkdownErr (markdown::Error::TooBig) + Response::MarkdownErr (markdown::Error::TooBig) } else { let mut buffer = vec! [0_u8; MAX_BUF_SIZE.try_into ().expect ("Couldn't fit u32 into usize")]; @@ -140,15 +140,15 @@ async fn internal_serve_file ( buffer.truncate (bytes_read); match render_styled (&buffer) { - Ok (x) => InternalResponse::MarkdownPreview (x), - Err (x) => InternalResponse::MarkdownErr (x), + Ok (x) => Response::MarkdownPreview (x), + Err (x) => Response::MarkdownErr (x), } } } else { let file = file.into (); - InternalResponse::ServeFile (ServeFileParams { + Response::ServeFile (ServeFileParams { file, send_body, range, @@ -158,17 +158,17 @@ async fn internal_serve_file ( }) } -pub async fn internal_serve_all ( +pub async fn serve_all ( root: &Path, method: Method, uri: &str, headers: &HashMap >, hidden_path: Option <&Path> ) --> Result +-> Result { use std::str::FromStr; - use InternalResponse::*; + use Response::*; info! ("Client requested {}", uri); @@ -214,7 +214,7 @@ pub async fn internal_serve_all ( } if let Ok (dir) = read_dir (&full_path).await { - internal_serve_dir ( + serve_dir ( &path_s, path, dir, @@ -223,7 +223,7 @@ pub async fn internal_serve_all ( ) } else if let Ok (file) = File::open (&full_path).await { - internal_serve_file ( + serve_file ( file, &uri, send_body, diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 7d4ae13..4ff2678 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -43,7 +43,6 @@ mod markdown; mod range; use errors::FileServerError; -use internal::*; mod emoji { pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; @@ -317,7 +316,7 @@ pub async fn serve_all ( ) -> Result { - use InternalResponse::*; + use internal::Response::*; fn serve_error >> ( status_code: StatusCode, @@ -331,7 +330,7 @@ pub async fn serve_all ( resp } - Ok (match internal_serve_all (root, method, uri, headers, hidden_path).await? { + Ok (match internal::serve_all (root, method, uri, headers, hidden_path).await? { Favicon => serve_error (StatusCode::NotFound, ""), Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object"), @@ -351,11 +350,11 @@ pub async fn serve_all ( resp }, Root => serve_root (handlebars, server_info).await?, - ServeDir (ServeDirParams { + ServeDir (internal::ServeDirParams { path, dir, }) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?, - ServeFile (ServeFileParams { + ServeFile (internal::ServeFileParams { file, send_body, range, diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index c11bdc7..0b78385 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -94,7 +94,7 @@ fn file_server () { let headers = Default::default (); { - use InternalResponse::*; + use internal::Response::*; use crate::file_server::FileServerError; let bad_passwords_path = "/files/src/bad_passwords.txt"; @@ -105,7 +105,7 @@ fn file_server () { ("/files/?", InvalidQuery), ("/files/src", Redirect ("src/".to_string ())), ("/files/src/?", InvalidQuery), - (bad_passwords_path, ServeFile (ServeFileParams { + (bad_passwords_path, ServeFile (internal::ServeFileParams { send_body: true, range: range::ValidParsed { range: 0..1_048_576, @@ -113,7 +113,7 @@ fn file_server () { }, file: AlwaysEqual::testing_blank (), })), - ("/files/test/test.md", ServeFile (ServeFileParams { + ("/files/test/test.md", ServeFile (internal::ServeFileParams { send_body: true, range: range::ValidParsed { range: 0..144, @@ -122,7 +122,7 @@ fn file_server () { file: AlwaysEqual::testing_blank (), })), ] { - let resp = internal_serve_all ( + let resp = internal::serve_all ( &file_server_root, Method::Get, uri_path, @@ -139,7 +139,7 @@ fn file_server () { e => panic! ("Expected InvalidUri, got {:?}", e), }), ] { - let resp = internal_serve_all ( + let resp = internal::serve_all ( &file_server_root, Method::Get, uri_path, @@ -150,7 +150,7 @@ fn file_server () { checker (resp.unwrap_err ()); } - let resp = internal_serve_all ( + let resp = internal::serve_all ( &file_server_root, Method::Get, bad_passwords_path, @@ -162,7 +162,7 @@ fn file_server () { assert_eq! (resp.expect ("Should be Ok (_)"), RangeNotSatisfiable (1_048_576)); - let resp = internal_serve_all ( + let resp = internal::serve_all ( &file_server_root, Method::Head, bad_passwords_path, @@ -170,7 +170,7 @@ fn file_server () { None ).await; - assert_eq! (resp.expect ("Should be Ok (_)"), ServeFile (ServeFileParams { + assert_eq! (resp.expect ("Should be Ok (_)"), ServeFile (internal::ServeFileParams { send_body: false, range: range::ValidParsed { range: 0..1_048_576, From 7925d9be95df600c84efd084ec77c81c0da3e651 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 23:24:25 +0000 Subject: [PATCH 082/208] :recycle: Move server endpoints to their own file --- crates/ptth_relay/src/lib.rs | 201 +------------------- crates/ptth_relay/src/server_endpoint.rs | 227 +++++++++++++++++++++++ 2 files changed, 231 insertions(+), 197 deletions(-) create mode 100644 crates/ptth_relay/src/server_endpoint.rs diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 1b94650..c3ed79f 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -21,9 +21,7 @@ use std::{ iter::FromIterator, net::SocketAddr, path::{Path, PathBuf}, - sync::{ - Arc, - }, + sync::Arc, time::Duration, }; @@ -33,10 +31,6 @@ use chrono::{ Utc }; use dashmap::DashMap; -use futures::{ - FutureExt, - stream::StreamExt, -}; use handlebars::Handlebars; use hyper::{ Body, @@ -51,15 +45,12 @@ use serde::{ Serialize, }; use tokio::{ - spawn, sync::{ Mutex, - mpsc, oneshot, RwLock, watch, }, - time::delay_for, }; use ptth_core::{ @@ -71,6 +62,7 @@ use ptth_core::{ pub mod config; pub mod errors; pub mod git_version; +mod server_endpoint; pub use config::Config; pub use errors::*; @@ -180,191 +172,6 @@ fn error_reply (status: StatusCode, b: &str) .body (format! ("{}\n", b).into ()) } -// Servers will come here and either handle queued requests from parked clients, -// or park themselves until a request comes in. - -async fn handle_http_listen ( - state: Arc , - watcher_code: String, - api_key: &[u8], -) --> Result , RequestError> -{ - use RequestRendezvous::*; - - let trip_error = || Ok (error_reply (StatusCode::UNAUTHORIZED, "Bad X-ApiKey")?); - - let expected_tripcode = { - let config = state.config.read ().await; - - match config.servers.get (&watcher_code) { - None => { - error! ("Denied http_listen for non-existent server name {}", watcher_code); - return trip_error (); - }, - Some (x) => (*x).tripcode, - } - }; - let actual_tripcode = blake3::hash (api_key); - - if expected_tripcode != actual_tripcode { - error! ("Denied http_listen for bad tripcode {}", base64::encode (actual_tripcode.as_bytes ())); - return trip_error (); - } - - // End of early returns - - { - let mut server_status = state.server_status.lock ().await; - - let mut status = server_status.entry (watcher_code.clone ()).or_insert_with (Default::default); - - status.last_seen = Utc::now (); - } - - let (tx, rx) = oneshot::channel (); - - { - let mut request_rendezvous = state.request_rendezvous.lock ().await; - - if let Some (ParkedClients (v)) = request_rendezvous.remove (&watcher_code) - { - if ! v.is_empty () { - // 1 or more clients were parked - Make the server - // handle them immediately - - debug! ("Sending {} parked requests to server {}", v.len (), watcher_code); - return Ok (ok_reply (rmp_serde::to_vec (&v)?)?); - } - } - - debug! ("Parking server {}", watcher_code); - request_rendezvous.insert (watcher_code.clone (), ParkedServer (tx)); - } - - // No clients were parked - make the server long-poll - - futures::select! { - x = rx.fuse () => match x { - Ok (Ok (one_req)) => { - debug! ("Unparking server {}", watcher_code); - Ok (ok_reply (rmp_serde::to_vec (&vec! [one_req])?)?) - }, - Ok (Err (ShuttingDownError::ShuttingDown)) => Ok (error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon")?), - Err (_) => Ok (error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error")?), - }, - _ = delay_for (Duration::from_secs (30)).fuse () => { - debug! ("Timed out http_listen for server {}", watcher_code); - return Ok (error_reply (StatusCode::NO_CONTENT, "No requests now, long-poll again")?) - } - } -} - -// Servers will come here to stream responses to clients - -async fn handle_http_response ( - req: Request , - state: Arc , - req_id: String, -) --> Result , HandleHttpResponseError> -{ - #[derive (Debug)] - enum BodyFinishedReason { - StreamFinished, - ClientDisconnected, - } - use BodyFinishedReason::*; - use HandleHttpResponseError::*; - - let (parts, mut body) = req.into_parts (); - - let magic_header = parts.headers.get (ptth_core::PTTH_MAGIC_HEADER).ok_or (MissingPtthMagicHeader)?; - - let magic_header = base64::decode (magic_header).map_err (PtthMagicHeaderNotBase64)?; - - let resp_parts: http_serde::ResponseParts = rmp_serde::from_read_ref (&magic_header).map_err (PtthMagicHeaderNotMsgPack)?; - - // Intercept the body packets here so we can check when the stream - // ends or errors out - - let (mut body_tx, body_rx) = mpsc::channel (2); - let (body_finished_tx, body_finished_rx) = oneshot::channel (); - let mut shutdown_watch_rx = state.shutdown_watch_rx.clone (); - - let relay_task = spawn (async move { - if shutdown_watch_rx.recv ().await == Some (false) { - loop { - let item = body.next ().await; - - if let Some (item) = item { - if let Ok (bytes) = &item { - trace! ("Relaying {} bytes", bytes.len ()); - } - - futures::select! { - x = body_tx.send (item).fuse () => if let Err (_) = x { - info! ("Body closed while relaying. (Client hung up?)"); - body_finished_tx.send (ClientDisconnected).map_err (|_| LostServer)?; - break; - }, - _ = shutdown_watch_rx.recv ().fuse () => { - debug! ("Closing stream: relay is shutting down"); - break; - }, - } - } - else { - debug! ("Finished relaying bytes"); - body_finished_tx.send (StreamFinished).map_err (|_| LostServer)?; - break; - } - } - } - else { - debug! ("Can't relay bytes, relay is shutting down"); - } - - Ok::<(), HandleHttpResponseError> (()) - }); - - let body = Body::wrap_stream (body_rx); - - let tx = { - let response_rendezvous = state.response_rendezvous.read ().await; - match response_rendezvous.remove (&req_id) { - None => { - error! ("Server tried to respond to non-existent request"); - return Ok (error_reply (StatusCode::BAD_REQUEST, "Request ID not found in response_rendezvous")?); - }, - Some ((_, x)) => x, - } - }; - - // UKAUFFY4 (Send half) - if tx.send (Ok ((resp_parts, body))).is_err () { - let msg = "Failed to connect to client"; - error! (msg); - return Ok (error_reply (StatusCode::BAD_GATEWAY, msg)?); - } - - relay_task.await??; - - debug! ("Connected server to client for streaming."); - match body_finished_rx.await { - Ok (StreamFinished) => { - Ok (error_reply (StatusCode::OK, "StreamFinished")?) - }, - Ok (ClientDisconnected) => { - Ok (error_reply (StatusCode::OK, "ClientDisconnected")?) - }, - Err (e) => { - debug! ("body_finished_rx {}", e); - Ok (error_reply (StatusCode::OK, "body_finished_rx Err")?) - }, - } -} - // Clients will come here to start requests, and always park for at least // a short amount of time. @@ -616,7 +423,7 @@ async fn handle_all (req: Request , state: Arc ) return if let Some (request_code) = prefix_match ("/7ZSFUKGV/http_response/", path) { let request_code = request_code.into (); - Ok (handle_http_response (req, state, request_code).await?) + Ok (server_endpoint::handle_response (req, state, request_code).await?) } else { Ok (error_reply (StatusCode::BAD_REQUEST, "Can't POST this")?) @@ -628,7 +435,7 @@ async fn handle_all (req: Request , state: Arc ) None => return Ok (error_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key")?), Some (x) => x, }; - handle_http_listen (state, listen_code.into (), api_key.as_bytes ()).await + server_endpoint::handle_listen (state, listen_code.into (), api_key.as_bytes ()).await } else if let Some (rest) = prefix_match ("/frontend/servers/", path) { if rest == "" { diff --git a/crates/ptth_relay/src/server_endpoint.rs b/crates/ptth_relay/src/server_endpoint.rs new file mode 100644 index 0000000..dc4083a --- /dev/null +++ b/crates/ptth_relay/src/server_endpoint.rs @@ -0,0 +1,227 @@ +use std::{ + sync::Arc, + time::Duration, +}; + +use chrono::Utc; +use futures::{ + FutureExt, + stream::StreamExt, +}; +use hyper::{ + Body, + Response, + Request, + StatusCode, +}; +use tokio::{ + spawn, + sync::{ + mpsc, + oneshot, + }, + time::delay_for, +}; + +use ptth_core::{ + http_serde, + prelude::*, +}; + +use super::{ + error_reply, + errors::{ + RequestError, + ShuttingDownError, + }, + HandleHttpResponseError, + ok_reply, + RelayState, +}; + +// Servers will come here and either handle queued requests from parked clients, +// or park themselves until a request comes in. +// Step 1 + +pub async fn handle_listen ( + state: Arc , + watcher_code: String, + api_key: &[u8], +) +-> Result , RequestError> +{ + use super::RequestRendezvous::*; + + let trip_error = || Ok (error_reply (StatusCode::UNAUTHORIZED, "Bad X-ApiKey")?); + + let expected_tripcode = { + let config = state.config.read ().await; + + match config.servers.get (&watcher_code) { + None => { + error! ("Denied http_listen for non-existent server name {}", watcher_code); + return trip_error (); + }, + Some (x) => (*x).tripcode, + } + }; + let actual_tripcode = blake3::hash (api_key); + + if expected_tripcode != actual_tripcode { + error! ("Denied http_listen for bad tripcode {}", base64::encode (actual_tripcode.as_bytes ())); + return trip_error (); + } + + // End of early returns + + { + let mut server_status = state.server_status.lock ().await; + + let mut status = server_status.entry (watcher_code.clone ()).or_insert_with (Default::default); + + status.last_seen = Utc::now (); + } + + let (tx, rx) = oneshot::channel (); + + { + let mut request_rendezvous = state.request_rendezvous.lock ().await; + + if let Some (ParkedClients (v)) = request_rendezvous.remove (&watcher_code) + { + if ! v.is_empty () { + // 1 or more clients were parked - Make the server + // handle them immediately + + debug! ("Sending {} parked requests to server {}", v.len (), watcher_code); + return Ok (ok_reply (rmp_serde::to_vec (&v)?)?); + } + } + + debug! ("Parking server {}", watcher_code); + request_rendezvous.insert (watcher_code.clone (), ParkedServer (tx)); + } + + // No clients were parked - make the server long-poll + + futures::select! { + x = rx.fuse () => match x { + Ok (Ok (one_req)) => { + debug! ("Unparking server {}", watcher_code); + Ok (ok_reply (rmp_serde::to_vec (&vec! [one_req])?)?) + }, + Ok (Err (ShuttingDownError::ShuttingDown)) => Ok (error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon")?), + Err (_) => Ok (error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error")?), + }, + _ = delay_for (Duration::from_secs (30)).fuse () => { + debug! ("Timed out http_listen for server {}", watcher_code); + return Ok (error_reply (StatusCode::NO_CONTENT, "No requests now, long-poll again")?) + } + } +} + +// Servers will come here to stream responses to clients +// Step 5 + +pub async fn handle_response ( + req: Request , + state: Arc , + req_id: String, +) +-> Result , HandleHttpResponseError> +{ + #[derive (Debug)] + enum BodyFinishedReason { + StreamFinished, + ClientDisconnected, + } + use BodyFinishedReason::*; + use HandleHttpResponseError::*; + + let (parts, mut body) = req.into_parts (); + + let magic_header = parts.headers.get (ptth_core::PTTH_MAGIC_HEADER).ok_or (MissingPtthMagicHeader)?; + + let magic_header = base64::decode (magic_header).map_err (PtthMagicHeaderNotBase64)?; + + let resp_parts: http_serde::ResponseParts = rmp_serde::from_read_ref (&magic_header).map_err (PtthMagicHeaderNotMsgPack)?; + + // Intercept the body packets here so we can check when the stream + // ends or errors out + + let (mut body_tx, body_rx) = mpsc::channel (2); + let (body_finished_tx, body_finished_rx) = oneshot::channel (); + let mut shutdown_watch_rx = state.shutdown_watch_rx.clone (); + + let relay_task = spawn (async move { + if shutdown_watch_rx.recv ().await == Some (false) { + loop { + let item = body.next ().await; + + if let Some (item) = item { + if let Ok (bytes) = &item { + trace! ("Relaying {} bytes", bytes.len ()); + } + + futures::select! { + x = body_tx.send (item).fuse () => if let Err (_) = x { + info! ("Body closed while relaying. (Client hung up?)"); + body_finished_tx.send (ClientDisconnected).map_err (|_| LostServer)?; + break; + }, + _ = shutdown_watch_rx.recv ().fuse () => { + debug! ("Closing stream: relay is shutting down"); + break; + }, + } + } + else { + debug! ("Finished relaying bytes"); + body_finished_tx.send (StreamFinished).map_err (|_| LostServer)?; + break; + } + } + } + else { + debug! ("Can't relay bytes, relay is shutting down"); + } + + Ok::<(), HandleHttpResponseError> (()) + }); + + let body = Body::wrap_stream (body_rx); + + let tx = { + let response_rendezvous = state.response_rendezvous.read ().await; + match response_rendezvous.remove (&req_id) { + None => { + error! ("Server tried to respond to non-existent request"); + return Ok (error_reply (StatusCode::BAD_REQUEST, "Request ID not found in response_rendezvous")?); + }, + Some ((_, x)) => x, + } + }; + + // UKAUFFY4 (Send half) + if tx.send (Ok ((resp_parts, body))).is_err () { + let msg = "Failed to connect to client"; + error! (msg); + return Ok (error_reply (StatusCode::BAD_GATEWAY, msg)?); + } + + relay_task.await??; + + debug! ("Connected server to client for streaming."); + match body_finished_rx.await { + Ok (StreamFinished) => { + Ok (error_reply (StatusCode::OK, "StreamFinished")?) + }, + Ok (ClientDisconnected) => { + Ok (error_reply (StatusCode::OK, "ClientDisconnected")?) + }, + Err (e) => { + debug! ("body_finished_rx {}", e); + Ok (error_reply (StatusCode::OK, "body_finished_rx Err")?) + }, + } +} From 14df0bdf92faf0a3f360b207a87c248ce56b3288 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 29 Nov 2020 23:47:02 +0000 Subject: [PATCH 083/208] :whale: Fix Docker --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6406606..61778c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update \ # Make sure the dependencies are all cached so we won't hammer crates.io ADD old-git.tar.gz . -RUN git checkout bbb88c01e8ad28d256c8faf316f46d7214184ec2 \ +RUN git checkout 7925d9be95df600c84efd084ec77c81c0da3e651 \ && git reset --hard \ && cargo check -p ptth_relay @@ -21,7 +21,7 @@ ARG gitcommithash=HEAD RUN git checkout "$gitcommithash" \ && git reset --hard \ -&& echo "pub const GIT_VERSION: Option <&str> = Some (\"$(git rev-parse HEAD)\");" > src/git_version.rs \ +&& echo "pub const GIT_VERSION: Option <&str> = Some (\"$(git rev-parse HEAD)\");" > crates/ptth_relay/src/git_version.rs \ && cargo test --release --all \ && cargo build --release -p ptth_relay @@ -32,7 +32,7 @@ RUN apt-get update \ && apt-get upgrade -y COPY --from=build /usr/src/target/release/ptth_relay /root -COPY --from=build /usr/src/src/git_version.rs /root/git_version.rs +COPY --from=build /usr/src/crates/ptth-relay/src/git_version.rs /root/git_version.rs COPY --from=build /usr/src/handlebars /root/handlebars WORKDIR /root From e59bb5b7bc4f7e35583d19e486b9c98004462fb6 Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 30 Nov 2020 15:52:15 +0000 Subject: [PATCH 084/208] :boom: Break relay config again --- crates/ptth_relay/src/config.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index 9ea5d83..e6d1a46 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -14,11 +14,11 @@ use crate::errors::ConfigError; // set up the HTTP server pub mod file { - use std::collections::HashMap; use serde::Deserialize; #[derive (Deserialize)] pub struct Server { + pub name: String, pub tripcode: String, pub display_name: Option , } @@ -26,7 +26,7 @@ pub mod file { #[derive (Deserialize)] pub struct Config { pub port: Option , - pub servers: HashMap , + pub servers: Vec , } } @@ -62,7 +62,7 @@ impl TryFrom for Config { fn try_from (f: file::Config) -> Result { let servers = f.servers.into_iter () - .map (|(k, v)| Ok::<_, ConfigError> ((k, v.try_into ()?))); + .map (|server| Ok::<_, ConfigError> ((server.name.clone (), server.try_into ()?))); let servers = itertools::process_results (servers, |i| HashMap::from_iter (i))?; From b40eda4a6968df1b6c3b753c7e5ccf0c86a0d6a4 Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 30 Nov 2020 15:55:14 +0000 Subject: [PATCH 085/208] :bug: Fix tests --- src/tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests.rs b/src/tests.rs index 6fef13c..3b8a6cf 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -15,7 +15,6 @@ use tokio::{ #[test] fn end_to_end () { - use maplit::*; use reqwest::Client; use tracing::{debug, info}; @@ -33,12 +32,13 @@ fn end_to_end () { debug! ("Relay is expecting tripcode {}", tripcode); let config_file = ptth_relay::config::file::Config { port: None, - servers: hashmap! { - server_name.into () => ptth_relay::config::file::Server { + servers: vec! [ + ptth_relay::config::file::Server { + name: server_name.to_string (), tripcode, display_name: None, }, - }, + ], }; let config = ptth_relay::config::Config::try_from (config_file).expect ("Can't load config"); From 00c29a64649e6762fa65393ba28740866517086a Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 30 Nov 2020 15:57:14 +0000 Subject: [PATCH 086/208] :docker: Fix Dockerfile again --- Dockerfile | 2 +- build-and-minimize.bash | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 61778c8..c19e011 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ RUN apt-get update \ && apt-get upgrade -y COPY --from=build /usr/src/target/release/ptth_relay /root -COPY --from=build /usr/src/crates/ptth-relay/src/git_version.rs /root/git_version.rs +COPY --from=build /usr/src/crates/ptth_relay/src/git_version.rs /root/git_version.rs COPY --from=build /usr/src/handlebars /root/handlebars WORKDIR /root diff --git a/build-and-minimize.bash b/build-and-minimize.bash index 155ec58..4c96733 100755 --- a/build-and-minimize.bash +++ b/build-and-minimize.bash @@ -5,6 +5,8 @@ # the base Debian layer, and repacks it so I can upload to servers a little # faster. +set -euo pipefail + TEMP_GIBBERISH="ptth_build_L6KLMVS6" TEMP_TAR="$TEMP_GIBBERISH/ptth.tar" UPLOADABLE_TAR="$PWD/ptth_latest.tar.gz" From bbeb4060328c0972d006081a8e1c9cdccbd01adc Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 30 Nov 2020 16:15:22 +0000 Subject: [PATCH 087/208] Removed unused dep --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2df9363..11b237f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,6 @@ license = "AGPL-3.0" base64 = "0.12.3" blake3 = "0.3.7" -maplit = "1.0.2" reqwest = { version = "0.10.8", features = ["stream"] } tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" From e8c020fbc6e354366fdc05c47c70e6e7d6036af4 Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 30 Nov 2020 16:15:27 +0000 Subject: [PATCH 088/208] Update --print-tripcode option --- crates/ptth_server/src/bin/ptth_server.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/ptth_server/src/bin/ptth_server.rs b/crates/ptth_server/src/bin/ptth_server.rs index 321d3dd..6fb0fe4 100644 --- a/crates/ptth_server/src/bin/ptth_server.rs +++ b/crates/ptth_server/src/bin/ptth_server.rs @@ -36,7 +36,8 @@ fn main () -> Result <(), anyhow::Error> { let config_file: ConfigFile = load_toml::load (&path)?; if opt.print_tripcode { - println! (r#""{}" = "{}""#, config_file.name, config_file.tripcode ()); + println! (r#"name = "{}""#, config_file.name); + println! (r#"tripcode = "{}""#, config_file.tripcode ()); return Ok (()); } From 4cbb4b72ca70745826f6c7240835202e9f79bd95 Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 1 Dec 2020 14:47:55 +0000 Subject: [PATCH 089/208] :construction: :whale: Experimenting with a new Dockerfile --- Cargo.toml | 3 --- new-Dockerfile | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 new-Dockerfile diff --git a/Cargo.toml b/Cargo.toml index 11b237f..896475b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,11 +17,8 @@ tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" tracing-subscriber = "0.2.15" -always_equal = { path = "crates/always_equal" } -ptth_core = { path = "crates/ptth_core" } ptth_relay = { path = "crates/ptth_relay" } ptth_server = { path = "crates/ptth_server" } -ptth_file_server = { path = "crates/ptth_file_server_bin" } [workspace] diff --git a/new-Dockerfile b/new-Dockerfile new file mode 100644 index 0000000..fdf668e --- /dev/null +++ b/new-Dockerfile @@ -0,0 +1,68 @@ +# Mix of https://whitfin.io/speeding-up-rust-docker-builds/ +# and a "minimal Rust Docker image!" GH repo which was just +# "lol use Alpine duh" + +FROM rust:1.47-slim-buster as build + +#RUN apk add libseccomp-dev + +WORKDIR / +ENV USER root + +# create empty shell projects +RUN cargo new --bin ptth + +WORKDIR /ptth + +RUN \ +cargo new --lib crates/always_equal && \ +cargo new --lib crates/ptth_core && \ +cargo new --bin crates/ptth_file_server_bin && \ +cargo new --bin crates/ptth_relay && \ +cargo new --bin crates/ptth_server + +# copy over your manifests +COPY ./Cargo.lock ./Cargo.lock +COPY ./Cargo.toml ./Cargo.toml +COPY ./crates/always_equal/Cargo.toml ./crates/always_equal/Cargo.toml +COPY ./crates/ptth_core/Cargo.toml ./crates/ptth_core/Cargo.toml +COPY ./crates/ptth_relay/Cargo.toml ./crates/ptth_relay/Cargo.toml +COPY ./crates/ptth_file_server_bin/Cargo.toml ./crates/ptth_file_server_bin/Cargo.toml +COPY ./crates/ptth_server/Cargo.toml ./crates/ptth_server/Cargo.toml + +# this build step will cache your dependencies +RUN cargo build --release -p ptth_relay + +RUN \ +rm \ +src/*.rs \ +crates/always_equal/src/*.rs \ +crates/ptth_core/src/*.rs \ +crates/ptth_file_server_bin/src/*.rs \ +crates/ptth_relay/src/*.rs \ +crates/ptth_server/src/*.rs + +# copy source tree +COPY ./src ./src +COPY ./crates ./crates +COPY ./handlebars ./handlebars + +# Bug in cargo's incremental build logic, triggered by +# Docker doing something funny with mtimes? Maybe? +RUN touch crates/ptth_core/src/lib.rs + +# build for release +RUN cargo build --release -p ptth_relay + +FROM debian:buster-slim + +RUN apt-get update \ +&& apt-get install -y libssl1.1 ca-certificates \ +&& apt-get upgrade -y + +COPY --from=build /ptth/target/release/ptth_relay /root/ptth_relay +COPY --from=build /ptth/crates/ptth_relay/src/git_version.rs /root/git_version.rs +COPY --from=build /ptth/handlebars /root/handlebars + +WORKDIR /root +ENTRYPOINT ["./ptth_relay"] From 865bd1f01f4d8b7be5f82d883b97f9e38218d2af Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 10 Dec 2020 05:49:23 +0000 Subject: [PATCH 090/208] :construction: Can't remember what I was working on here --- new-Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/new-Dockerfile b/new-Dockerfile index fdf668e..0481715 100644 --- a/new-Dockerfile +++ b/new-Dockerfile @@ -1,6 +1,5 @@ -# Mix of https://whitfin.io/speeding-up-rust-docker-builds/ -# and a "minimal Rust Docker image!" GH repo which was just -# "lol use Alpine duh" +# https://whitfin.io/speeding-up-rust-docker-builds/ +# TODO: https://stackoverflow.com/questions/57389547/how-to-define-the-context-for-a-docker-build-as-a-specific-commit-on-one-of-the FROM rust:1.47-slim-buster as build From c4b12eb806139133771f2008f2814b5ff730a70c Mon Sep 17 00:00:00 2001 From: _ <> Date: Thu, 10 Dec 2020 06:24:56 +0000 Subject: [PATCH 091/208] :checkered_flag: Builds on Windows. --- crates/ptth_server/src/file_server/internal.rs | 11 +++++++---- crates/ptth_server/src/load_toml.rs | 13 +++++++++++++ src/tests.rs | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/ptth_server/src/file_server/internal.rs b/crates/ptth_server/src/file_server/internal.rs index 07ddb81..695c0b3 100644 --- a/crates/ptth_server/src/file_server/internal.rs +++ b/crates/ptth_server/src/file_server/internal.rs @@ -109,12 +109,15 @@ async fn serve_file ( ) -> Result { - use std::os::unix::fs::PermissionsExt; - let file_md = file.metadata ().await.map_err (FileServerError::CantGetFileMetadata)?; - if file_md.permissions ().mode () == load_toml::CONFIG_PERMISSIONS_MODE + + #[cfg (unix)] { - return Ok (Response::Forbidden); + use std::os::unix::fs::PermissionsExt; + if file_md.permissions ().mode () == load_toml::CONFIG_PERMISSIONS_MODE + { + return Ok (Response::Forbidden); + } } let file_len = file_md.len (); diff --git a/crates/ptth_server/src/load_toml.rs b/crates/ptth_server/src/load_toml.rs index f92b73d..6851467 100644 --- a/crates/ptth_server/src/load_toml.rs +++ b/crates/ptth_server/src/load_toml.rs @@ -39,6 +39,7 @@ pub fn load_public < /// For files that may contain secrets and should have permissions or other /// safeties checked +#[cfg (unix)] pub fn load < T: DeserializeOwned, P: AsRef + Debug @@ -56,3 +57,15 @@ pub fn load < load_inner (f) } + +// The permission check doesn't work on Windows + +#[cfg (not (unix))] +pub fn load < + T: DeserializeOwned, + P: AsRef + Debug +> ( + config_file_path: P +) -> Result { + load_public (config_file_path) +} diff --git a/src/tests.rs b/src/tests.rs index 3b8a6cf..a0eb534 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -69,7 +69,7 @@ fn end_to_end () { }) }; - delay_for (Duration::from_millis (500)).await; + delay_for (Duration::from_millis (1000)).await; assert_eq! (relay_state.list_servers ().await, vec! [ server_name.to_string (), From 4014290f98cf3d0755aaf587f3ec881b676e9eb5 Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 11 Dec 2020 21:04:59 +0000 Subject: [PATCH 092/208] :pencil: docs (YNQAQKJS) add plan for 3rd auth route --- issues/2020-12Dec/auth-route-YNQAQKJS.md | 98 ++++++++++++++++++++++++ todo.md | 19 +++-- 2 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 issues/2020-12Dec/auth-route-YNQAQKJS.md diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md new file mode 100644 index 0000000..2561d30 --- /dev/null +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -0,0 +1,98 @@ +# Auth route for scrapers + +(Find this issue with `git grep YNQAQKJS`) + +## Problem statement + +PTTH has 2 auth routes: + +- A fixed API key for servers +- Whatever the end user puts in front of the HTML client + +"Whatever" is hard for scrapers to deal with. This barrier to scraping +is blocking these issues: + +- EOTPXGR3 Remote `tail -f` +- UPAQ3ZPT Audit logging of the relay itself +- YMFMSV2R Add Prometheus metrics + +## Proposal + +Add a 3rd auth route meeting these criteria: + +- Enabled by a feature flag, disabled by default +- Bootstrapped by the user-friendly HTML frontend +- Suitable for headless automated scrapers + +It will probably involve an API key like the servers use. Public-key +crypto is stronger, but involves more work. I think we should plan to +start with something weak, and also plan to deprecate it once something +stronger is ready. + +## Proposed impl plan + +- Add feature flags to ptth_relay.toml for dev mode and scrapers +- Make sure Docker release CAN build +- Add failing test to block releases +- Make sure `cargo test` fails and Docker release can NOT build +- Add hard-coded hash of 1 API key, with 1 week expiration +- (POC) Test with curl +- Manually create SQLite DB for API keys, add 1 hash +- Impl DB reads +- Remove hard-coded API key +- Make sure `cargo test` passes and Docker CAN build +- (MVP) Test with curl +- Impl and test DB init / migration +- Impl DB writes (Add / revoke keys) as CLI commands +- Implement API (Behind X-Email auth) for that, test with curl +- Set up mitmproxy or something to add X-Email header in dev env +- Implement web UI (Behind X-Email) + +POC is the proof-of-concept - At this point we will know that in theory the +feature can work. + +MVP is the first deployable version - I could put it in prod, manually fudge +the SQLite DB to add a 1-month key, and let people start building scrapers. + +Details: + +Dev mode will allow anonymous users to generate scraper keys. In prod mode, +(the default) clients will need to have the X-Email header set or use a +scraper key to do anything. + +Design the DB so that the servers can share it one day. + +Design the API so that new types of auth / keys can be added one day, and +the old ones deprecated. + +## Open questions + +**Who generates the API key? The scraper client, or the PTTH relay server?** + +The precedent from big cloud vendors seems to be that the server generates +tokens. This is probably to avoid a situation where clients with vulnerable +crypto code or just bad code generate low-entropy keys. By putting that +responsibility on the server, the server can enforce high-entropy keys. + +**Should the key rotate? If so, how?** + +The key should _at least_ expire. If it expires every 30 or 90 days, then a +human is slightly inconvenienced to service their scraper regularly. + +When adding other features, we must consider the use cases: + +1. A really dumb Bash script that shells out to curl +2. A Python script +3. A sophisticated desktop app in C#, Rust, or C++ +4. Eventually replacing the fixed API keys used in ptth_server + +For the Bash script, rotation will probably be difficult, and I'm okay if +our support for that is merely "It'll work for 30 days at a time, then you +need to rotate keys manually." + +For the Python script, rotation could be automated, but cryptography is +still probably difficult. I think some AWS services require actual crypto +keys, and not just high-entropy password keys. + +For the sophisticated desktop app, cryptography is on the table, but this +is the least likely use case to ever happen, too. diff --git a/todo.md b/todo.md index 02488b8..da42c81 100644 --- a/todo.md +++ b/todo.md @@ -1,15 +1,20 @@ -- Estimate bandwidth per server? +Interesting issues will get a unique ID with +`dd if=/dev/urandom bs=5 count=1 | base32` + +- Report server version in HTML +- [YNQAQKJS](issues/2020-12Dec/auth-route-YNQAQKJS.md) Open new auth route for spiders / scrapers +- Track / Estimate bandwidth per server? +- EOTPXGR3 Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) - "Preview as" feature for Markdown (It's not threaded through the relay yet) -- Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) - Make a debug client to replicate the issue Firefox is having with turtling -- Add Prometheus metrics +- YMFMSV2R Add Prometheus metrics - Not working great behind reverse proxies - Impl multi-range / multi-part byte serving - Deny unused HTTP methods for endpoints - ETag cache based on mtime - Server-side hash? -- Log / audit log? +- UPAQ3ZPT Log / audit log? - Prevent directory traversal attacks in file_server.rs - Error handling @@ -75,7 +80,7 @@ what happens. I might have to build a client that imitates this behavior, since it's hard to control. -## Server won't work on Windows +## Server can't protect its API key on Windows -This is because I use Unix-specific file permissions to protect the server -config. +This is because I use a dumb hack with Unix permissions to protect the config +file on Linux. From f6486b2c1a34363a33b0de52f6dbab855d667253 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 01:26:58 +0000 Subject: [PATCH 093/208] :wrench: config (ptth_relay): add feature flags - dev mode - scraper auth These will gate features I'm adding soon. --- crates/ptth_relay/src/config.rs | 14 +++++++++++ crates/ptth_relay/src/lib.rs | 6 +++++ issues/2020-12Dec/auth-route-YNQAQKJS.md | 32 ++++++++++++------------ src/tests.rs | 1 + 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index e6d1a46..f8ad9bd 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -23,10 +23,22 @@ pub mod file { pub display_name: Option , } + // Stuff that's identical between the file and the runtime structures + + #[derive (Default, Deserialize)] + pub struct Isomorphic { + #[serde (default)] + pub enable_dev_mode: bool, + #[serde (default)] + pub enable_scraper_auth: bool, + } + #[derive (Deserialize)] pub struct Config { pub port: Option , pub servers: Vec , + #[serde (flatten)] + pub iso: Isomorphic, } } @@ -39,6 +51,7 @@ pub struct Server { pub struct Config { pub servers: HashMap , + pub iso: file::Isomorphic, } impl TryFrom for Server { @@ -68,6 +81,7 @@ impl TryFrom for Config { Ok (Self { servers, + iso: f.iso, }) } } diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index c3ed79f..ee9161b 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -495,6 +495,12 @@ async fn reload_config ( (*config) = new_config; debug! ("Loaded {} server configs", config.servers.len ()); + debug! ("enable_dev_mode: {}", config.iso.enable_dev_mode); + debug! ("enable_scraper_auth: {}", config.iso.enable_scraper_auth); + + if config.iso.enable_dev_mode { + error! ("Dev mode is enabled! This might turn off some security features. If you see this in production, escalate it to someone!"); + } Ok (()) } diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 2561d30..fec8b62 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -31,22 +31,22 @@ stronger is ready. ## Proposed impl plan -- Add feature flags to ptth_relay.toml for dev mode and scrapers -- Make sure Docker release CAN build -- Add failing test to block releases -- Make sure `cargo test` fails and Docker release can NOT build -- Add hard-coded hash of 1 API key, with 1 week expiration -- (POC) Test with curl -- Manually create SQLite DB for API keys, add 1 hash -- Impl DB reads -- Remove hard-coded API key -- Make sure `cargo test` passes and Docker CAN build -- (MVP) Test with curl -- Impl and test DB init / migration -- Impl DB writes (Add / revoke keys) as CLI commands -- Implement API (Behind X-Email auth) for that, test with curl -- Set up mitmproxy or something to add X-Email header in dev env -- Implement web UI (Behind X-Email) +- (X) Add feature flags to ptth_relay.toml for dev mode and scrapers +- ( ) Make sure Docker release CAN build +- ( ) Add failing test to block releases +- ( ) Make sure `cargo test` fails and Docker release can NOT build +- ( ) Add hard-coded hash of 1 API key, with 1 week expiration +- ( ) (POC) Test with curl +- ( ) Manually create SQLite DB for API keys, add 1 hash +- ( ) Impl DB reads +- ( ) Remove hard-coded API key +- ( ) Make sure `cargo test` passes and Docker CAN build +- ( ) (MVP) Test with curl +- ( ) Impl and test DB init / migration +- ( ) Impl DB writes (Add / revoke keys) as CLI commands +- ( ) Implement API (Behind X-Email auth) for that, test with curl +- ( ) Set up mitmproxy or something to add X-Email header in dev env +- ( ) Implement web UI (Behind X-Email) POC is the proof-of-concept - At this point we will know that in theory the feature can work. diff --git a/src/tests.rs b/src/tests.rs index a0eb534..993faef 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -39,6 +39,7 @@ fn end_to_end () { display_name: None, }, ], + iso: Default::default (), }; let config = ptth_relay::config::Config::try_from (config_file).expect ("Can't load config"); From 951fe27b5f0b79bbac3d8ddff77d1610cf56b687 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 01:35:47 +0000 Subject: [PATCH 094/208] :heavy_minus_sign: update (build scripts): remove old build.bash It was one line and wasn't used --- build.bash | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 build.bash diff --git a/build.bash b/build.bash deleted file mode 100755 index 68d5058..0000000 --- a/build.bash +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -sudo docker build -t ptth . From 0c5a37b441c9893b184199281002b2b009dafa7b Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 01:53:20 +0000 Subject: [PATCH 095/208] :whale: build (ptth_relay): clean up Docker build process The new method is much nicer and doesn't require the manual make-old-git step. The top-level command is actually build_and_minimize.bash, which uses `git archive` to unpack the last Git commit and build with _that_ Dockerfile and Docker context. This is better for determinism. It's similar to our build process for that one big project at work. --- .gitignore | 1 - Cargo.lock | 2235 +++++++++++++++++ Dockerfile | 85 +- ...d-minimize.bash => build_and_minimize.bash | 4 +- crates/ptth_relay/src/git_version.rs | 2 +- crates/ptth_relay/src/git_version.txt | 1 + issues/2020-12Dec/auth-route-YNQAQKJS.md | 2 +- make-old-git-tar.bash | 1 - new-Dockerfile | 67 - run_docker_image.bash | 3 + 10 files changed, 2305 insertions(+), 96 deletions(-) create mode 100644 Cargo.lock rename build-and-minimize.bash => build_and_minimize.bash (80%) create mode 100644 crates/ptth_relay/src/git_version.txt delete mode 100755 make-old-git-tar.bash delete mode 100644 new-Dockerfile create mode 100755 run_docker_image.bash diff --git a/.gitignore b/.gitignore index bb1ca12..a688c68 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -/Cargo.lock /config /*.tar.gz /ptth_server.toml diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..03f4fff --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2235 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +dependencies = [ + "const-random", +] + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "always_equal" +version = "0.1.0" + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "anyhow" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "blake3" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9ff35b701f3914bdb8fad3368d822c766ef2858b2583198e41639b936f09d3f" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 0.1.10", + "constant_time_eq", + "crypto-mac", + "digest 0.9.0", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.3", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "cc" +version = "1.0.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "js-sys", + "libc", + "num-integer", + "num-traits", + "time", + "wasm-bindgen", + "winapi 0.3.9", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term 0.11.0", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "const-random" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02dc82c12dc2ee6e1ded861cf7d582b46f66f796d1b6c93fa28b911ead95da02" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc757bbb9544aa296c2ae00c679e81f886b37e28e59097defe0cf524306f6685" +dependencies = [ + "getrandom 0.2.0", + "proc-macro-hack", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array 0.14.4", + "subtle", +] + +[[package]] +name = "ctrlc" +version = "3.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b57a92e9749e10f25a171adcebfafe72991d45e7ec2dcb853e8f83d9dafaeb08" +dependencies = [ + "nix", + "winapi 0.3.9", +] + +[[package]] +name = "dashmap" +version = "3.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5" +dependencies = [ + "ahash", + "cfg-if 0.1.10", + "num_cpus", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95314d38584ffbfda215621d723e0a3906f032e03ae5551e650058dac83d4797" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46" + +[[package]] +name = "futures-executor" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f8e0c9258abaea85e78ebdda17ef9666d390e987f006be6080dfe354b708cb" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b" + +[[package]] +name = "futures-macro" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11" + +[[package]] +name = "futures-task" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project 1.0.1", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "generator" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc" +dependencies = [ + "cc", + "libc", + "log", + "rustc_version", + "winapi 0.3.9", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8025cf36f917e6a52cce185b7c7177689b838b7ec138364e50cc2277a56cf4" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "handlebars" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2764f9796c0ddca4b82c07f25dd2cb3db30b9a8f47940e78e1c883d9e95c3db9" +dependencies = [ + "log", + "pest", + "pest_derive", + "quick-error", + "serde", + "serde_json", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "hyper" +version = "0.13.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.1", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-tls", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "js-sys" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "loom" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed" +dependencies = [ + "cfg-if 0.1.10", + "generator", + "scoped-tls", + "serde", + "serde_json", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.1", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio", + "miow 0.3.5", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "native-tls" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a1cda389c26d6b88f3d2dc38aa1b750fe87d298cc5d795ec9e975f402f00372" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nix" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83450fe6a6142ddd95fb064b746083fc4ef1705fe81f64a64e1d4b39f54a1055" +dependencies = [ + "bitflags", + "cc", + "cfg-if 0.1.10", + "libc", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "openssl" +version = "0.10.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" +dependencies = [ + "bitflags", + "cfg-if 0.1.10", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +dependencies = [ + "pin-project-internal 1.0.1", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "ptth" +version = "0.1.0" +dependencies = [ + "base64", + "blake3", + "ptth_relay", + "ptth_server", + "reqwest", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "ptth_core" +version = "0.1.0" +dependencies = [ + "ctrlc", + "futures", + "hyper", + "serde", + "thiserror", + "tokio", + "tracing", + "tracing-futures", +] + +[[package]] +name = "ptth_file_server" +version = "0.1.0" +dependencies = [ + "anyhow", + "handlebars", + "http", + "hyper", + "ptth_core", + "ptth_server", + "serde", + "structopt", + "tokio", + "tracing-subscriber", +] + +[[package]] +name = "ptth_relay" +version = "0.1.0" +dependencies = [ + "base64", + "blake3", + "chrono", + "dashmap", + "futures", + "handlebars", + "http", + "hyper", + "itertools", + "ptth_core", + "rmp-serde", + "serde", + "thiserror", + "tokio", + "toml", + "tracing", + "tracing-futures", + "tracing-subscriber", + "ulid", +] + +[[package]] +name = "ptth_server" +version = "0.1.0" +dependencies = [ + "aho-corasick", + "always_equal", + "anyhow", + "base64", + "blake3", + "futures", + "handlebars", + "http", + "lazy_static", + "maplit", + "percent-encoding", + "ptth_core", + "pulldown-cmark", + "rand 0.6.5", + "regex", + "reqwest", + "rmp-serde", + "serde", + "structopt", + "thiserror", + "tokio", + "toml", + "tracing", + "tracing-futures", + "tracing-subscriber", +] + +[[package]] +name = "pulldown-cmark" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" +dependencies = [ + "bitflags", + "getopts", + "memchr", + "unicase", +] + +[[package]] +name = "quick-error" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ac73b1112776fc109b2e61909bc46c7e1bf0d7f690ffb1676553acce16d5cda" + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.15", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.15", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.9", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "wasm-bindgen", + "winapi 0.3.9", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "regex" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_urlencoded", + "tokio", + "tokio-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rmp" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f10b46df14cf1ee1ac7baa4d2fbc2c52c0622a4b82fa8740e37bc452ac0184f" +dependencies = [ + "byteorder", + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce7d70c926fe472aed493b902010bccc17fa9f7284145cb8772fd22fdb052d8" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "scoped-tls" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + +[[package]] +name = "security-framework" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer", + "digest 0.8.1", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "sharded-slab" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127" +dependencies = [ + "lazy_static", + "loom", +] + +[[package]] +name = "signal-hook-registry" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce32ea0c6c56d5eacaeb814fbed9960547021d3edd010ded1425f180536b20ab" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" + +[[package]] +name = "socket2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "redox_syscall", + "winapi 0.3.9", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126d630294ec449fae0b16f964e35bf3c74f940da9dca17ee9b905f7b3112eb8" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65e51c492f9e23a220534971ff5afc14037289de430e3c83f9daf6a1b6ae91e8" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subtle" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" + +[[package]] +name = "syn" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "tinyvec" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" + +[[package]] +name = "tokio" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-named-pipes", + "mio-uds", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cf45bb0bef80604d001caaec0d09da99611b3c0fd39d3080468875cdb65645" +dependencies = [ + "serde", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" +dependencies = [ + "cfg-if 0.1.10", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401" +dependencies = [ + "ansi_term 0.12.1", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "ulid" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e95a59b292ca0cf9b45be2e52294d1ca6cb24eb11b08ef4376f73f1a00c549" +dependencies = [ + "chrono", + "lazy_static", + "rand 0.6.5", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "url" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" +dependencies = [ + "cfg-if 0.1.10", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" +dependencies = [ + "cfg-if 0.1.10", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" + +[[package]] +name = "web-sys" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] diff --git a/Dockerfile b/Dockerfile index c19e011..a2749e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,40 +1,77 @@ -FROM rust:1.47-slim-buster as build +# https://whitfin.io/speeding-up-rust-docker-builds/ +# TODO: https://stackoverflow.com/questions/57389547/how-to-define-the-context-for-a-docker-build-as-a-specific-commit-on-one-of-the -WORKDIR /usr/src +# 1.47 slim-buster +FROM rust@sha256:2a902de987345f126fe59daca200afae1fccb6f68e14e9a27c0fd9cf39f9743f as build -RUN apt-get update \ -&& apt-get install -y git pkg-config libssl-dev +#RUN apk add libseccomp-dev -# Make sure the dependencies are all cached so we won't hammer crates.io +WORKDIR / +ENV USER root -ADD old-git.tar.gz . -RUN git checkout 7925d9be95df600c84efd084ec77c81c0da3e651 \ -&& git reset --hard \ -&& cargo check -p ptth_relay +# create empty shell projects +RUN cargo new --bin ptth -RUN cargo test --release --all \ -&& cargo build --release -p ptth_relay +WORKDIR /ptth -COPY .git .git +RUN \ +cargo new --lib crates/always_equal && \ +cargo new --lib crates/ptth_core && \ +cargo new --bin crates/ptth_file_server_bin && \ +cargo new --bin crates/ptth_relay && \ +cargo new --bin crates/ptth_server -ARG gitcommithash=HEAD +# copy over your manifests +COPY ./Cargo.lock ./ +COPY ./Cargo.toml ./ +COPY ./crates/always_equal/Cargo.toml ./crates/always_equal/ +COPY ./crates/ptth_core/Cargo.toml ./crates/ptth_core/ +COPY ./crates/ptth_relay/Cargo.toml ./crates/ptth_relay/ +COPY ./crates/ptth_file_server_bin/Cargo.toml ./crates/ptth_file_server_bin/ +COPY ./crates/ptth_server/Cargo.toml ./crates/ptth_server/ -RUN git checkout "$gitcommithash" \ -&& git reset --hard \ -&& echo "pub const GIT_VERSION: Option <&str> = Some (\"$(git rev-parse HEAD)\");" > crates/ptth_relay/src/git_version.rs \ -&& cargo test --release --all \ -&& cargo build --release -p ptth_relay +# this build step will cache your dependencies +RUN cargo build --release -p ptth_relay -FROM debian:buster-slim as deploy +RUN \ +rm \ +src/*.rs \ +crates/always_equal/src/*.rs \ +crates/ptth_core/src/*.rs \ +crates/ptth_file_server_bin/src/*.rs \ +crates/ptth_relay/src/*.rs \ +crates/ptth_server/src/*.rs + +# Copy source tree +# Yes, I tried a few variations on the syntax. Dockerfiles are just rough. + +COPY ./src/ ./src +COPY ./crates/ ./crates +COPY ./handlebars/ ./handlebars + +# Bug in cargo's incremental build logic, triggered by +# Docker doing something funny with mtimes? Maybe? +RUN touch crates/ptth_core/src/lib.rs + +ARG git_version +RUN echo -n "$git_version" > crates/ptth_relay/src/git_version.txt + +# build for release +# gate only on ptth_relay tests for now +RUN \ +cargo build --release -p ptth_relay && \ +cargo test --release -p ptth_relay + +# buster-slim +FROM debian@sha256:062bbd9a1a58c9c5b8fc9d83a206371127ef268cfcc65f1a01227c6faebdb212 RUN apt-get update \ && apt-get install -y libssl1.1 ca-certificates \ && apt-get upgrade -y -COPY --from=build /usr/src/target/release/ptth_relay /root -COPY --from=build /usr/src/crates/ptth_relay/src/git_version.rs /root/git_version.rs -COPY --from=build /usr/src/handlebars /root/handlebars +COPY --from=build /ptth/target/release/ptth_relay /root/ +COPY --from=build /ptth/crates/ptth_relay/src/git_version.txt /root/ +COPY --from=build /ptth/handlebars /root/handlebars WORKDIR /root - -CMD ["./ptth_relay"] +ENTRYPOINT ["./ptth_relay"] diff --git a/build-and-minimize.bash b/build_and_minimize.bash similarity index 80% rename from build-and-minimize.bash rename to build_and_minimize.bash index 4c96733..365758f 100755 --- a/build-and-minimize.bash +++ b/build_and_minimize.bash @@ -10,15 +10,17 @@ set -euo pipefail TEMP_GIBBERISH="ptth_build_L6KLMVS6" TEMP_TAR="$TEMP_GIBBERISH/ptth.tar" UPLOADABLE_TAR="$PWD/ptth_latest.tar.gz" +GIT_COMMITISH=$(git rev-parse main) # This is magic and will need to be updated whenever we update the # Debian layer. BOTTOM_LAYER="cec906613726ec32de92af0ec1cd6692c34df78782227f4415cd12c47a264dd4" +rm -rf "$TEMP_GIBBERISH" mkdir -p "$TEMP_GIBBERISH/ptth" -sudo docker build -t ptth:latest . +git archive --format=tar "$GIT_COMMITISH" | sudo docker build -t ptth:latest --build-arg "git_version=$GIT_COMMITISH" - sudo docker image save ptth:latest | pv > "$TEMP_TAR" tar -C "$TEMP_GIBBERISH/ptth" -xf "$TEMP_TAR" diff --git a/crates/ptth_relay/src/git_version.rs b/crates/ptth_relay/src/git_version.rs index ff014a3..12a4fd8 100644 --- a/crates/ptth_relay/src/git_version.rs +++ b/crates/ptth_relay/src/git_version.rs @@ -1 +1 @@ -pub const GIT_VERSION: Option <&str> = None; +pub const GIT_VERSION: &str = include_str! ("git_version.txt"); diff --git a/crates/ptth_relay/src/git_version.txt b/crates/ptth_relay/src/git_version.txt new file mode 100644 index 0000000..d5d3349 --- /dev/null +++ b/crates/ptth_relay/src/git_version.txt @@ -0,0 +1 @@ +(Unknown) \ No newline at end of file diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index fec8b62..4fef75c 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -32,7 +32,7 @@ stronger is ready. ## Proposed impl plan - (X) Add feature flags to ptth_relay.toml for dev mode and scrapers -- ( ) Make sure Docker release CAN build +- (X) Make sure Docker release CAN build - ( ) Add failing test to block releases - ( ) Make sure `cargo test` fails and Docker release can NOT build - ( ) Add hard-coded hash of 1 API key, with 1 week expiration diff --git a/make-old-git-tar.bash b/make-old-git-tar.bash deleted file mode 100755 index 6d9cb84..0000000 --- a/make-old-git-tar.bash +++ /dev/null @@ -1 +0,0 @@ -tar -czf old-git.tar.gz .git diff --git a/new-Dockerfile b/new-Dockerfile deleted file mode 100644 index 0481715..0000000 --- a/new-Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -# https://whitfin.io/speeding-up-rust-docker-builds/ -# TODO: https://stackoverflow.com/questions/57389547/how-to-define-the-context-for-a-docker-build-as-a-specific-commit-on-one-of-the - -FROM rust:1.47-slim-buster as build - -#RUN apk add libseccomp-dev - -WORKDIR / -ENV USER root - -# create empty shell projects -RUN cargo new --bin ptth - -WORKDIR /ptth - -RUN \ -cargo new --lib crates/always_equal && \ -cargo new --lib crates/ptth_core && \ -cargo new --bin crates/ptth_file_server_bin && \ -cargo new --bin crates/ptth_relay && \ -cargo new --bin crates/ptth_server - -# copy over your manifests -COPY ./Cargo.lock ./Cargo.lock -COPY ./Cargo.toml ./Cargo.toml -COPY ./crates/always_equal/Cargo.toml ./crates/always_equal/Cargo.toml -COPY ./crates/ptth_core/Cargo.toml ./crates/ptth_core/Cargo.toml -COPY ./crates/ptth_relay/Cargo.toml ./crates/ptth_relay/Cargo.toml -COPY ./crates/ptth_file_server_bin/Cargo.toml ./crates/ptth_file_server_bin/Cargo.toml -COPY ./crates/ptth_server/Cargo.toml ./crates/ptth_server/Cargo.toml - -# this build step will cache your dependencies -RUN cargo build --release -p ptth_relay - -RUN \ -rm \ -src/*.rs \ -crates/always_equal/src/*.rs \ -crates/ptth_core/src/*.rs \ -crates/ptth_file_server_bin/src/*.rs \ -crates/ptth_relay/src/*.rs \ -crates/ptth_server/src/*.rs - -# copy source tree -COPY ./src ./src -COPY ./crates ./crates -COPY ./handlebars ./handlebars - -# Bug in cargo's incremental build logic, triggered by -# Docker doing something funny with mtimes? Maybe? -RUN touch crates/ptth_core/src/lib.rs - -# build for release -RUN cargo build --release -p ptth_relay - -FROM debian:buster-slim - -RUN apt-get update \ -&& apt-get install -y libssl1.1 ca-certificates \ -&& apt-get upgrade -y - -COPY --from=build /ptth/target/release/ptth_relay /root/ptth_relay -COPY --from=build /ptth/crates/ptth_relay/src/git_version.rs /root/git_version.rs -COPY --from=build /ptth/handlebars /root/handlebars - -WORKDIR /root -ENTRYPOINT ["./ptth_relay"] diff --git a/run_docker_image.bash b/run_docker_image.bash new file mode 100755 index 0000000..7857c93 --- /dev/null +++ b/run_docker_image.bash @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +sudo docker run -it -v $PWD/config:/root/config -e RUST_LOG=ptth=trace ptth:latest From 8e171fbf081a637d9707d3b7d5eba866cd99de36 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 05:18:29 +0000 Subject: [PATCH 096/208] :arrow_up: deps (ptth_relay): update Docker build to Rust 1.48 and latest Buster --- Dockerfile | 8 ++++---- rust-toolchain | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index a2749e9..0e13bd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # https://whitfin.io/speeding-up-rust-docker-builds/ # TODO: https://stackoverflow.com/questions/57389547/how-to-define-the-context-for-a-docker-build-as-a-specific-commit-on-one-of-the -# 1.47 slim-buster -FROM rust@sha256:2a902de987345f126fe59daca200afae1fccb6f68e14e9a27c0fd9cf39f9743f as build +# rust:1.48-slim-buster +FROM rust@sha256:cb6b98346ef41a2062d4d8f099127d880f2ef7c1515d00215fc9ea713b99167b as build #RUN apk add libseccomp-dev @@ -62,8 +62,8 @@ RUN \ cargo build --release -p ptth_relay && \ cargo test --release -p ptth_relay -# buster-slim -FROM debian@sha256:062bbd9a1a58c9c5b8fc9d83a206371127ef268cfcc65f1a01227c6faebdb212 +# debian:buster-slim +FROM debian@sha256:240f770008bdc538fecc8d3fa7a32a533eac55c14cbc56a9a8a6f7d741b47e33 RUN apt-get update \ && apt-get install -y libssl1.1 ca-certificates \ diff --git a/rust-toolchain b/rust-toolchain index 21998d3..9db5ea1 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.47.0 +1.48.0 From 1c798afdf0d829e6d005e9909f359c976575c852 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 05:23:23 +0000 Subject: [PATCH 097/208] :package: build (ptth_relay): update ID of bottom layer that we minimize out --- build_and_minimize.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_and_minimize.bash b/build_and_minimize.bash index 365758f..97b812a 100755 --- a/build_and_minimize.bash +++ b/build_and_minimize.bash @@ -15,7 +15,7 @@ GIT_COMMITISH=$(git rev-parse main) # This is magic and will need to be updated whenever we update the # Debian layer. -BOTTOM_LAYER="cec906613726ec32de92af0ec1cd6692c34df78782227f4415cd12c47a264dd4" +BOTTOM_LAYER="7900806df64cc5adea32b37d8484569598376cac1c66f0810c01b7ad2458e9fd" rm -rf "$TEMP_GIBBERISH" mkdir -p "$TEMP_GIBBERISH/ptth" From 31626844ca5a214a38a84b2bad85dd32d8bd9fe8 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 05:31:29 +0000 Subject: [PATCH 098/208] :arrow_up: deps: cargo update --- Cargo.lock | 319 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 193 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03f4fff..ad87ae5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,9 +42,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" +checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4" [[package]] name = "arrayref" @@ -87,6 +87,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.2.1" @@ -155,9 +161,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.62" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cfg-if" @@ -211,10 +217,20 @@ dependencies = [ ] [[package]] -name = "const-random" -version = "0.1.11" +name = "console_error_panic_hook" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02dc82c12dc2ee6e1ded861cf7d582b46f66f796d1b6c93fa28b911ead95da02" +checksum = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211" +dependencies = [ + "cfg-if 0.1.10", + "wasm-bindgen", +] + +[[package]] +name = "const-random" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f590d95d011aa80b063ffe3253422ed5aa462af4e9867d43ce8337562bac77c4" dependencies = [ "const-random-macro", "proc-macro-hack", @@ -222,12 +238,14 @@ dependencies = [ [[package]] name = "const-random-macro" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc757bbb9544aa296c2ae00c679e81f886b37e28e59097defe0cf524306f6685" +checksum = "615f6e27d000a2bffbc7f2f6a8669179378fa27ee4d0a509e985dfc0a7defb40" dependencies = [ "getrandom 0.2.0", + "lazy_static", "proc-macro-hack", + "tiny-keccak", ] [[package]] @@ -252,6 +270,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-mac" version = "0.8.0" @@ -301,12 +325,6 @@ dependencies = [ "generic-array 0.14.4", ] -[[package]] -name = "dtoa" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" - [[package]] name = "either" version = "1.6.1" @@ -383,9 +401,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95314d38584ffbfda215621d723e0a3906f032e03ae5551e650058dac83d4797" +checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" dependencies = [ "futures-channel", "futures-core", @@ -398,9 +416,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151" +checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" dependencies = [ "futures-core", "futures-sink", @@ -408,15 +426,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46" +checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" [[package]] name = "futures-executor" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5f8e0c9258abaea85e78ebdda17ef9666d390e987f006be6080dfe354b708cb" +checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" dependencies = [ "futures-core", "futures-task", @@ -425,15 +443,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b" +checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" [[package]] name = "futures-macro" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe" +checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -443,24 +461,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11" +checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" [[package]] name = "futures-task" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c" +checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34" +checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" dependencies = [ "futures-channel", "futures-core", @@ -469,7 +487,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.1", + "pin-project 1.0.2", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -646,7 +664,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.1", + "pin-project 1.0.2", "socket2", "tokio", "tower-service", @@ -720,9 +738,9 @@ checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" dependencies = [ "wasm-bindgen", ] @@ -745,9 +763,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" [[package]] name = "log" @@ -816,9 +834,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.6.22" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" dependencies = [ "cfg-if 0.1.10", "fuchsia-zircon", @@ -827,7 +845,7 @@ dependencies = [ "kernel32-sys", "libc", "log", - "miow 0.2.1", + "miow 0.2.2", "net2", "slab", "winapi 0.2.8", @@ -841,7 +859,7 @@ checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" dependencies = [ "log", "mio", - "miow 0.3.5", + "miow 0.3.6", "winapi 0.3.9", ] @@ -858,9 +876,9 @@ dependencies = [ [[package]] name = "miow" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" dependencies = [ "kernel32-sys", "net2", @@ -870,9 +888,9 @@ dependencies = [ [[package]] name = "miow" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" dependencies = [ "socket2", "winapi 0.3.9", @@ -880,9 +898,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a1cda389c26d6b88f3d2dc38aa1b750fe87d298cc5d795ec9e975f402f00372" +checksum = "6fcc7939b5edc4e4f86b1b4a04bb1498afaaf871b1a6691838ed06fcb48d3a3f" dependencies = [ "lazy_static", "libc", @@ -898,9 +916,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" dependencies = [ "cfg-if 0.1.10", "libc", @@ -950,9 +968,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" [[package]] name = "opaque-debug" @@ -962,12 +980,12 @@ checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" [[package]] name = "openssl" -version = "0.10.30" +version = "0.10.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" +checksum = "8d008f51b1acffa0d3450a68606e6a51c123012edaacb0f4e1426bd978869187" dependencies = [ "bitflags", - "cfg-if 0.1.10", + "cfg-if 1.0.0", "foreign-types", "lazy_static", "libc", @@ -982,9 +1000,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.58" +version = "0.9.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +checksum = "de52d8eabd217311538a39bba130d7dea1f1e118010fee7a033d966845e7d5fe" dependencies = [ "autocfg 1.0.1", "cc", @@ -1053,11 +1071,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" dependencies = [ - "pin-project-internal 1.0.1", + "pin-project-internal 1.0.2", ] [[package]] @@ -1073,9 +1091,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" dependencies = [ "proc-macro2", "quote", @@ -1088,6 +1106,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" +[[package]] +name = "pin-project-lite" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" + [[package]] name = "pin-utils" version = "0.1.0" @@ -1155,7 +1179,7 @@ dependencies = [ name = "ptth" version = "0.1.0" dependencies = [ - "base64", + "base64 0.12.3", "blake3", "ptth_relay", "ptth_server", @@ -1199,7 +1223,7 @@ dependencies = [ name = "ptth_relay" version = "0.1.0" dependencies = [ - "base64", + "base64 0.12.3", "blake3", "chrono", "dashmap", @@ -1227,7 +1251,7 @@ dependencies = [ "aho-corasick", "always_equal", "anyhow", - "base64", + "base64 0.12.3", "blake3", "futures", "handlebars", @@ -1480,11 +1504,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" +checksum = "fb15d6255c792356a0f578d8a645c677904dc02e862bebe2ecc18e0c01b9a0ce" dependencies = [ - "base64", + "base64 0.13.0", "bytes", "encoding_rs", "futures-core", @@ -1501,7 +1525,7 @@ dependencies = [ "mime_guess", "native-tls", "percent-encoding", - "pin-project-lite", + "pin-project-lite 0.2.0", "serde", "serde_urlencoded", "tokio", @@ -1509,6 +1533,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-bindgen-test", "web-sys", "winreg", ] @@ -1605,18 +1630,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" dependencies = [ "proc-macro2", "quote", @@ -1625,9 +1650,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" +checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" dependencies = [ "itoa", "ryu", @@ -1636,14 +1661,14 @@ dependencies = [ [[package]] name = "serde_urlencoded" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" dependencies = [ - "dtoa", + "form_urlencoded", "itoa", + "ryu", "serde", - "url", ] [[package]] @@ -1685,17 +1710,17 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.4.2" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" +checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" [[package]] name = "socket2" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" +checksum = "2c29947abdee2a218277abeca306f25789c938e500ea5a9d4b12a5a504466902" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "redox_syscall", "winapi 0.3.9", @@ -1709,9 +1734,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "structopt" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126d630294ec449fae0b16f964e35bf3c74f940da9dca17ee9b905f7b3112eb8" +checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" dependencies = [ "clap", "lazy_static", @@ -1720,9 +1745,9 @@ dependencies = [ [[package]] name = "structopt-derive" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e51c492f9e23a220534971ff5afc14037289de430e3c83f9daf6a1b6ae91e8" +checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", @@ -1739,9 +1764,9 @@ checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" [[package]] name = "syn" -version = "1.0.48" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" +checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44" dependencies = [ "proc-macro2", "quote", @@ -1812,16 +1837,34 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "0.3.4" +name = "tiny-keccak" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinyvec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.22" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" dependencies = [ "bytes", "fnv", @@ -1834,7 +1877,7 @@ dependencies = [ "mio-named-pipes", "mio-uds", "num_cpus", - "pin-project-lite", + "pin-project-lite 0.1.11", "signal-hook-registry", "slab", "tokio-macros", @@ -1843,9 +1886,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ "proc-macro2", "quote", @@ -1872,7 +1915,7 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite", + "pin-project-lite 0.1.11", "tokio", ] @@ -1893,13 +1936,13 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" [[package]] name = "tracing" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" +checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "log", - "pin-project-lite", + "pin-project-lite 0.2.0", "tracing-attributes", "tracing-core", ] @@ -2026,18 +2069,18 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.13" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" [[package]] name = "unicode-width" @@ -2065,9 +2108,9 @@ dependencies = [ [[package]] name = "vcpkg" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" [[package]] name = "vec_map" @@ -2105,11 +2148,11 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "serde", "serde_json", "wasm-bindgen-macro", @@ -2117,9 +2160,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" dependencies = [ "bumpalo", "lazy_static", @@ -2132,11 +2175,11 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -2144,9 +2187,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2154,9 +2197,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" dependencies = [ "proc-macro2", "quote", @@ -2167,15 +2210,39 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0355fa0c1f9b792a09b6dcb6a8be24d51e71e6d74972f9eb4a44c4c004d24a25" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27e07b46b98024c2ba2f9e83a10c2ef0515f057f2da299c1762a2017de80438b" +dependencies = [ + "proc-macro2", + "quote", +] [[package]] name = "web-sys" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" dependencies = [ "js-sys", "wasm-bindgen", From 6861560274dd15d03f63fb362e8bf1184ed56e82 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 15:07:45 +0000 Subject: [PATCH 099/208] :pencil: docs: update todo.md --- todo.md | 1 + 1 file changed, 1 insertion(+) diff --git a/todo.md b/todo.md index da42c81..5efa75f 100644 --- a/todo.md +++ b/todo.md @@ -1,6 +1,7 @@ Interesting issues will get a unique ID with `dd if=/dev/urandom bs=5 count=1 | base32` +- Diagram dependencies - Report server version in HTML - [YNQAQKJS](issues/2020-12Dec/auth-route-YNQAQKJS.md) Open new auth route for spiders / scrapers - Track / Estimate bandwidth per server? From cc96af6110dd08a2cb89f34fee320126f987e90f Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 15:10:14 +0000 Subject: [PATCH 100/208] :pencil: docs: improve plan for scraper keys --- issues/2020-12Dec/auth-route-YNQAQKJS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 4fef75c..8307ccf 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -35,11 +35,11 @@ stronger is ready. - (X) Make sure Docker release CAN build - ( ) Add failing test to block releases - ( ) Make sure `cargo test` fails and Docker release can NOT build -- ( ) Add hard-coded hash of 1 API key, with 1 week expiration +- ( ) Add hash of 1 scraper key to ptth_relay.toml, with 1 week expiration - ( ) (POC) Test with curl -- ( ) Manually create SQLite DB for API keys, add 1 hash +- ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads -- ( ) Remove hard-coded API key +- ( ) Remove scraper key from config file - ( ) Make sure `cargo test` passes and Docker CAN build - ( ) (MVP) Test with curl - ( ) Impl and test DB init / migration From 9bc4b57058be2106d47a90888952d87549ff186f Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 15:35:33 +0000 Subject: [PATCH 101/208] :star: new (ptth_relay): add serde deserializer for blake3 hashes --- crates/ptth_relay/src/config.rs | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index f8ad9bd..4e8a707 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -4,18 +4,68 @@ use std::{ collections::HashMap, convert::{TryFrom, TryInto}, + fmt, iter::FromIterator, + ops::Deref, path::Path, }; +use serde::{ + Deserialize, + Deserializer, + de::{ + self, + Visitor, + }, +}; + use crate::errors::ConfigError; +pub struct BlakeHashWrapper (blake3::Hash); + +impl Deref for BlakeHashWrapper { + type Target = blake3::Hash; + + fn deref (&self) -> &::Target { + &self.0 + } +} + +struct BlakeHashVisitor; + +impl <'de> Visitor <'de> for BlakeHashVisitor { + type Value = blake3::Hash; + + fn expecting (&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str ("a 32-byte blake3 hash, encoded as base64") + } + + fn visit_str (self, value: &str) + -> Result + { + let bytes: Vec = base64::decode (value).map_err (|_| E::custom (format! ("str is not base64: {}", value)))?; + let bytes: [u8; 32] = (&bytes [..]).try_into ().map_err (|_| E::custom (format! ("decode base64 is not 32 bytes long: {}", value)))?; + + let tripcode = blake3::Hash::from (bytes); + + Ok (tripcode) + } +} + +impl <'de> Deserialize <'de> for BlakeHashWrapper { + fn deserialize > (deserializer: D) -> Result { + Ok (BlakeHashWrapper (deserializer.deserialize_str (BlakeHashVisitor)?)) + } +} + // Stuff we need to load from the config file and use to // set up the HTTP server pub mod file { use serde::Deserialize; + use super::*; + #[derive (Deserialize)] pub struct Server { pub name: String, @@ -31,6 +81,8 @@ pub mod file { pub enable_dev_mode: bool, #[serde (default)] pub enable_scraper_auth: bool, + + pub dev_scraper_key: Option , } #[derive (Deserialize)] From b43106393a8244ad660d1efe1521aea8d70beca9 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 15:50:38 +0000 Subject: [PATCH 102/208] :shirt: refactor (ptth_relay): use the new blake3 deserializer --- crates/ptth_relay/src/config.rs | 19 ++++++++++++------- src/tests.rs | 6 ++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index 4e8a707..fac50b2 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -23,6 +23,16 @@ use crate::errors::ConfigError; pub struct BlakeHashWrapper (blake3::Hash); +impl BlakeHashWrapper { + pub fn from_key (bytes: &[u8]) -> Self { + Self (blake3::hash (bytes)) + } + + pub fn encode_base64 (&self) -> String { + base64::encode (self.as_bytes ()) + } +} + impl Deref for BlakeHashWrapper { type Target = blake3::Hash; @@ -69,7 +79,7 @@ pub mod file { #[derive (Deserialize)] pub struct Server { pub name: String, - pub tripcode: String, + pub tripcode: BlakeHashWrapper, pub display_name: Option , } @@ -110,13 +120,8 @@ impl TryFrom for Server { type Error = ConfigError; fn try_from (f: file::Server) -> Result { - let bytes: Vec = base64::decode (f.tripcode)?; - let bytes: [u8; 32] = (&bytes [..]).try_into ().map_err (|_| ConfigError::TripcodeBadLength)?; - - let tripcode = blake3::Hash::from (bytes); - Ok (Self { - tripcode, + tripcode: *f.tripcode, display_name: f.display_name, }) } diff --git a/src/tests.rs b/src/tests.rs index 993faef..65d94e5 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -18,6 +18,8 @@ fn end_to_end () { use reqwest::Client; use tracing::{debug, info}; + use ptth_relay::config::BlakeHashWrapper; + // Prefer this form for tests, since all tests share one process // and we don't care if another test already installed a subscriber. @@ -28,8 +30,8 @@ fn end_to_end () { rt.block_on (async { let server_name = "aliens_wildland"; let api_key = "AnacondaHardcoverGrannyUnlatchLankinessMutate"; - let tripcode = base64::encode (blake3::hash (api_key.as_bytes ()).as_bytes ()); - debug! ("Relay is expecting tripcode {}", tripcode); + let tripcode = BlakeHashWrapper::from_key (api_key.as_bytes ()); + debug! ("Relay is expecting tripcode {}", tripcode.encode_base64 ()); let config_file = ptth_relay::config::file::Config { port: None, servers: vec! [ From bf8e483d16d97bb8280d5a7a74ee2fa3c5e9c03f Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 15:57:22 +0000 Subject: [PATCH 103/208] :shirt: refactor: merge the servers' config file and runtime representations --- crates/ptth_relay/src/config.rs | 21 +++------------------ crates/ptth_relay/src/server_endpoint.rs | 2 +- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index fac50b2..cfcde8a 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -78,6 +78,7 @@ pub mod file { #[derive (Deserialize)] pub struct Server { + // This is duplicated in the hashmap, but it's not a problem pub name: String, pub tripcode: BlakeHashWrapper, pub display_name: Option , @@ -106,33 +107,17 @@ pub mod file { // Stuff we actually need at runtime -pub struct Server { - pub tripcode: blake3::Hash, - pub display_name: Option , -} - pub struct Config { - pub servers: HashMap , + pub servers: HashMap , pub iso: file::Isomorphic, } -impl TryFrom for Server { - type Error = ConfigError; - - fn try_from (f: file::Server) -> Result { - Ok (Self { - tripcode: *f.tripcode, - display_name: f.display_name, - }) - } -} - impl TryFrom for Config { type Error = ConfigError; fn try_from (f: file::Config) -> Result { let servers = f.servers.into_iter () - .map (|server| Ok::<_, ConfigError> ((server.name.clone (), server.try_into ()?))); + .map (|server| Ok::<_, ConfigError> ((server.name.clone (), server))); let servers = itertools::process_results (servers, |i| HashMap::from_iter (i))?; diff --git a/crates/ptth_relay/src/server_endpoint.rs b/crates/ptth_relay/src/server_endpoint.rs index dc4083a..c6e9795 100644 --- a/crates/ptth_relay/src/server_endpoint.rs +++ b/crates/ptth_relay/src/server_endpoint.rs @@ -62,7 +62,7 @@ pub async fn handle_listen ( error! ("Denied http_listen for non-existent server name {}", watcher_code); return trip_error (); }, - Some (x) => (*x).tripcode, + Some (x) => *(*x).tripcode, } }; let actual_tripcode = blake3::hash (api_key); From 0eb1e7e38f8eff8f16f2bc8257d2edc9c23a26d3 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 17:11:22 +0000 Subject: [PATCH 104/208] :star: new: add code for scraper keys to expire and have limited durations --- Cargo.lock | 1 + crates/ptth_relay/Cargo.toml | 2 +- crates/ptth_relay/src/config.rs | 73 +------- crates/ptth_relay/src/key_validity.rs | 222 +++++++++++++++++++++++ crates/ptth_relay/src/lib.rs | 5 +- issues/2020-12Dec/auth-route-YNQAQKJS.md | 2 - src/tests.rs | 2 +- 7 files changed, 238 insertions(+), 69 deletions(-) create mode 100644 crates/ptth_relay/src/key_validity.rs diff --git a/Cargo.lock b/Cargo.lock index ad87ae5..dce88da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,6 +187,7 @@ dependencies = [ "libc", "num-integer", "num-traits", + "serde", "time", "wasm-bindgen", "winapi 0.3.9", diff --git a/crates/ptth_relay/Cargo.toml b/crates/ptth_relay/Cargo.toml index 938364f..58d03e3 100644 --- a/crates/ptth_relay/Cargo.toml +++ b/crates/ptth_relay/Cargo.toml @@ -10,7 +10,7 @@ license = "AGPL-3.0" base64 = "0.12.3" blake3 = "0.3.7" -chrono = "0.4.19" +chrono = {version = "0.4.19", features = ["serde"]} dashmap = "3.11.10" futures = "0.3.7" handlebars = "3.5.1" diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index cfcde8a..b444810 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -3,78 +3,20 @@ use std::{ collections::HashMap, - convert::{TryFrom, TryInto}, - fmt, + convert::{TryFrom}, iter::FromIterator, - ops::Deref, path::Path, }; -use serde::{ - Deserialize, - Deserializer, - de::{ - self, - Visitor, - }, -}; - use crate::errors::ConfigError; -pub struct BlakeHashWrapper (blake3::Hash); - -impl BlakeHashWrapper { - pub fn from_key (bytes: &[u8]) -> Self { - Self (blake3::hash (bytes)) - } - - pub fn encode_base64 (&self) -> String { - base64::encode (self.as_bytes ()) - } -} - -impl Deref for BlakeHashWrapper { - type Target = blake3::Hash; - - fn deref (&self) -> &::Target { - &self.0 - } -} - -struct BlakeHashVisitor; - -impl <'de> Visitor <'de> for BlakeHashVisitor { - type Value = blake3::Hash; - - fn expecting (&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str ("a 32-byte blake3 hash, encoded as base64") - } - - fn visit_str (self, value: &str) - -> Result - { - let bytes: Vec = base64::decode (value).map_err (|_| E::custom (format! ("str is not base64: {}", value)))?; - let bytes: [u8; 32] = (&bytes [..]).try_into ().map_err (|_| E::custom (format! ("decode base64 is not 32 bytes long: {}", value)))?; - - let tripcode = blake3::Hash::from (bytes); - - Ok (tripcode) - } -} - -impl <'de> Deserialize <'de> for BlakeHashWrapper { - fn deserialize > (deserializer: D) -> Result { - Ok (BlakeHashWrapper (deserializer.deserialize_str (BlakeHashVisitor)?)) - } -} - // Stuff we need to load from the config file and use to // set up the HTTP server pub mod file { use serde::Deserialize; - use super::*; + use crate::key_validity::*; #[derive (Deserialize)] pub struct Server { @@ -84,16 +26,21 @@ pub mod file { pub display_name: Option , } + #[derive (Deserialize)] + pub struct DevMode { + pub scraper_key: Option >, + } + // Stuff that's identical between the file and the runtime structures #[derive (Default, Deserialize)] pub struct Isomorphic { - #[serde (default)] - pub enable_dev_mode: bool, #[serde (default)] pub enable_scraper_auth: bool, - pub dev_scraper_key: Option , + // If any of these fields are used, we are in dev mode and have to + // show extra warnings, since some auth may be weakened + pub dev_mode: Option , } #[derive (Deserialize)] diff --git a/crates/ptth_relay/src/key_validity.rs b/crates/ptth_relay/src/key_validity.rs new file mode 100644 index 0000000..30a314b --- /dev/null +++ b/crates/ptth_relay/src/key_validity.rs @@ -0,0 +1,222 @@ +use std::{ + convert::TryInto, + fmt, + ops::Deref, +}; + +use chrono::{DateTime, Duration, Utc}; +use serde::{ + de::{ + self, + Visitor, + }, + Deserialize, + Deserializer, +}; + +pub struct BlakeHashWrapper (blake3::Hash); + +impl BlakeHashWrapper { + pub fn from_key (bytes: &[u8]) -> Self { + Self (blake3::hash (bytes)) + } + + pub fn encode_base64 (&self) -> String { + base64::encode (self.as_bytes ()) + } +} + +impl Deref for BlakeHashWrapper { + type Target = blake3::Hash; + + fn deref (&self) -> &::Target { + &self.0 + } +} + +struct BlakeHashVisitor; + +impl <'de> Visitor <'de> for BlakeHashVisitor { + type Value = blake3::Hash; + + fn expecting (&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str ("a 32-byte blake3 hash, encoded as base64") + } + + fn visit_str (self, value: &str) + -> Result + { + let bytes: Vec = base64::decode (value).map_err (|_| E::custom (format! ("str is not base64: {}", value)))?; + let bytes: [u8; 32] = (&bytes [..]).try_into ().map_err (|_| E::custom (format! ("decode base64 is not 32 bytes long: {}", value)))?; + + let tripcode = blake3::Hash::from (bytes); + + Ok (tripcode) + } +} + +impl <'de> Deserialize <'de> for BlakeHashWrapper { + fn deserialize > (deserializer: D) -> Result { + Ok (BlakeHashWrapper (deserializer.deserialize_str (BlakeHashVisitor)?)) + } +} + +pub struct Valid7Days; +//pub struct Valid30Days; +//pub struct Valid90Days; + +pub trait MaxValidDuration { + fn dur () -> Duration; +} + +impl MaxValidDuration for Valid7Days { + fn dur () -> Duration { + Duration::days (7) + } +} + +#[derive (Deserialize)] +pub struct ScraperKey { + pub not_before: DateTime , + pub not_after: DateTime , + pub hash: BlakeHashWrapper, + _phantom: std::marker::PhantomData , +} + +#[derive (Copy, Clone, Debug, PartialEq)] +pub enum KeyValidity { + Valid, + + WrongKey, + ClockIsBehind, + Expired, + DurationTooLong (Duration), + DurationNegative, +} + +impl ScraperKey { + pub fn is_valid (&self, now: DateTime , input: &[u8]) -> KeyValidity { + use KeyValidity::*; + + // I put this first because I think the constant-time check should run + // before anything else. But I'm not a crypto expert, so it's just + // guesswork. + if blake3::hash (input) != *self.hash { + return WrongKey; + } + + if self.not_after < self.not_before { + return DurationNegative; + } + + let max_dur = V::dur (); + let actual_dur = self.not_after - self.not_before; + + if actual_dur > max_dur { + return DurationTooLong (max_dur); + } + + if now >= self.not_after { + return Expired; + } + + if now < self.not_before { + return ClockIsBehind; + } + + return Valid; + } +} + +#[cfg (test)] +mod tests { + use chrono::{Utc}; + use super::*; + use KeyValidity::*; + + #[test] + fn duration_negative () { + let zero_time = Utc::now (); + + let key = ScraperKey:: { + not_before: zero_time + Duration::days (1 + 2), + not_after: zero_time + Duration::days (1), + hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()), + _phantom: Default::default (), + }; + + let err = DurationNegative; + + for (input, expected) in &[ + (zero_time + Duration::days (0), err), + (zero_time + Duration::days (2), err), + (zero_time + Duration::days (100), err), + ] { + assert_eq! (key.is_valid (*input, "bad_password".as_bytes ()), *expected); + } + } + + #[test] + fn key_valid_too_long () { + let zero_time = Utc::now (); + + let key = ScraperKey:: { + not_before: zero_time + Duration::days (1), + not_after: zero_time + Duration::days (1 + 8), + hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()), + _phantom: Default::default (), + }; + + let err = DurationTooLong (Duration::days (7)); + + for (input, expected) in &[ + (zero_time + Duration::days (0), err), + (zero_time + Duration::days (2), err), + (zero_time + Duration::days (100), err), + ] { + assert_eq! (key.is_valid (*input, "bad_password".as_bytes ()), *expected); + } + } + + #[test] + fn normal_key () { + let zero_time = Utc::now (); + + let key = ScraperKey:: { + not_before: zero_time + Duration::days (1), + not_after: zero_time + Duration::days (1 + 7), + hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()), + _phantom: Default::default (), + }; + + for (input, expected) in &[ + (zero_time + Duration::days (0), ClockIsBehind), + (zero_time + Duration::days (2), Valid), + (zero_time + Duration::days (1 + 7), Expired), + (zero_time + Duration::days (100), Expired), + ] { + assert_eq! (key.is_valid (*input, "bad_password".as_bytes ()), *expected); + } + } + + #[test] + fn wrong_key () { + let zero_time = Utc::now (); + + let key = ScraperKey:: { + not_before: zero_time + Duration::days (1), + not_after: zero_time + Duration::days (1 + 7), + hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()), + _phantom: Default::default (), + }; + + for (input, expected) in &[ + (zero_time + Duration::days (0), WrongKey), + (zero_time + Duration::days (2), WrongKey), + (zero_time + Duration::days (1 + 7), WrongKey), + (zero_time + Duration::days (100), WrongKey), + ] { + assert_eq! (key.is_valid (*input, "badder_password".as_bytes ()), *expected); + } + } +} diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index ee9161b..be4f63f 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -62,6 +62,8 @@ use ptth_core::{ pub mod config; pub mod errors; pub mod git_version; +pub mod key_validity; + mod server_endpoint; pub use config::Config; @@ -495,10 +497,9 @@ async fn reload_config ( (*config) = new_config; debug! ("Loaded {} server configs", config.servers.len ()); - debug! ("enable_dev_mode: {}", config.iso.enable_dev_mode); debug! ("enable_scraper_auth: {}", config.iso.enable_scraper_auth); - if config.iso.enable_dev_mode { + if config.iso.dev_mode.is_some () { error! ("Dev mode is enabled! This might turn off some security features. If you see this in production, escalate it to someone!"); } diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 8307ccf..80bae96 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -33,8 +33,6 @@ stronger is ready. - (X) Add feature flags to ptth_relay.toml for dev mode and scrapers - (X) Make sure Docker release CAN build -- ( ) Add failing test to block releases -- ( ) Make sure `cargo test` fails and Docker release can NOT build - ( ) Add hash of 1 scraper key to ptth_relay.toml, with 1 week expiration - ( ) (POC) Test with curl - ( ) Manually create SQLite DB for scraper keys, add 1 hash diff --git a/src/tests.rs b/src/tests.rs index 65d94e5..5de6e56 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -18,7 +18,7 @@ fn end_to_end () { use reqwest::Client; use tracing::{debug, info}; - use ptth_relay::config::BlakeHashWrapper; + use ptth_relay::key_validity::BlakeHashWrapper; // Prefer this form for tests, since all tests share one process // and we don't care if another test already installed a subscriber. From 004b98229a147000554754010a27558ba47b2bd8 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 17:12:38 +0000 Subject: [PATCH 105/208] :bug: bug: fix serde expecting phantom data in the config file --- crates/ptth_relay/src/key_validity.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/ptth_relay/src/key_validity.rs b/crates/ptth_relay/src/key_validity.rs index 30a314b..8488409 100644 --- a/crates/ptth_relay/src/key_validity.rs +++ b/crates/ptth_relay/src/key_validity.rs @@ -80,6 +80,8 @@ pub struct ScraperKey { pub not_before: DateTime , pub not_after: DateTime , pub hash: BlakeHashWrapper, + + #[serde (default)] _phantom: std::marker::PhantomData , } From 6961fde7dcb6da1facf6e05ffe6acb27a4b69a0b Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 17:14:10 +0000 Subject: [PATCH 106/208] :pencil: docs: update plan --- issues/2020-12Dec/auth-route-YNQAQKJS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 80bae96..14d0ea4 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -33,7 +33,8 @@ stronger is ready. - (X) Add feature flags to ptth_relay.toml for dev mode and scrapers - (X) Make sure Docker release CAN build -- ( ) Add hash of 1 scraper key to ptth_relay.toml, with 1 week expiration +- (X) Add hash of 1 scraper key to ptth_relay.toml, with 1 week expiration +- ( ) Accept scraper key for some testing endpoint - ( ) (POC) Test with curl - ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads From 6d68a77364491a258b7d3f596f5a9a433ee97848 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 17:50:40 +0000 Subject: [PATCH 107/208] :star: new (ptth_relay): add test endpoint for scrapers Scrapers can auth using a shared (but hashed) API key. The hash of the key is specified in ptth_relay.toml, and forces dev mode on. --- crates/ptth_relay/src/key_validity.rs | 33 +++++++++++++------ crates/ptth_relay/src/lib.rs | 42 +++++++++++++++++++++++- issues/2020-12Dec/auth-route-YNQAQKJS.md | 5 +-- 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/crates/ptth_relay/src/key_validity.rs b/crates/ptth_relay/src/key_validity.rs index 8488409..2bc2593 100644 --- a/crates/ptth_relay/src/key_validity.rs +++ b/crates/ptth_relay/src/key_validity.rs @@ -1,6 +1,6 @@ use std::{ convert::TryInto, - fmt, + fmt::{self, Debug, Formatter}, ops::Deref, }; @@ -14,8 +14,15 @@ use serde::{ Deserializer, }; +#[derive (Copy, Clone, PartialEq, Eq)] pub struct BlakeHashWrapper (blake3::Hash); +impl Debug for BlakeHashWrapper { + fn fmt (&self, f: &mut Formatter <'_>) -> Result <(), fmt::Error> { + write! (f, "{}", self.encode_base64 ()) + } +} + impl BlakeHashWrapper { pub fn from_key (bytes: &[u8]) -> Self { Self (blake3::hash (bytes)) @@ -89,7 +96,7 @@ pub struct ScraperKey { pub enum KeyValidity { Valid, - WrongKey, + WrongKey (BlakeHashWrapper), ClockIsBehind, Expired, DurationTooLong (Duration), @@ -103,8 +110,9 @@ impl ScraperKey { // I put this first because I think the constant-time check should run // before anything else. But I'm not a crypto expert, so it's just // guesswork. - if blake3::hash (input) != *self.hash { - return WrongKey; + let input_hash = BlakeHashWrapper::from_key (input); + if input_hash != self.hash { + return WrongKey (input_hash); } if self.not_after < self.not_before { @@ -212,13 +220,18 @@ mod tests { _phantom: Default::default (), }; - for (input, expected) in &[ - (zero_time + Duration::days (0), WrongKey), - (zero_time + Duration::days (2), WrongKey), - (zero_time + Duration::days (1 + 7), WrongKey), - (zero_time + Duration::days (100), WrongKey), + for input in &[ + zero_time + Duration::days (0), + zero_time + Duration::days (2), + zero_time + Duration::days (1 + 7), + zero_time + Duration::days (100), ] { - assert_eq! (key.is_valid (*input, "badder_password".as_bytes ()), *expected); + let validity = key.is_valid (*input, "badder_password".as_bytes ()); + + match validity { + WrongKey (_) => (), + _ => panic! ("Expected WrongKey here"), + } } } } diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index be4f63f..1ea4d2b 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -434,7 +434,7 @@ async fn handle_all (req: Request , state: Arc ) if let Some (listen_code) = prefix_match ("/7ZSFUKGV/http_listen/", path) { let api_key = match api_key { - None => return Ok (error_reply (StatusCode::UNAUTHORIZED, "Can't register as server without an API key")?), + None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run server without an API key")?), Some (x) => x, }; server_endpoint::handle_listen (state, listen_code.into (), api_key.as_bytes ()).await @@ -464,6 +464,46 @@ async fn handle_all (req: Request , state: Arc ) else if path == "/frontend/test_mysterious_error" { Err (RequestError::Mysterious) } + else if path == "/scraper/v1/test" || path == "/scraper/api/test" { + use key_validity::KeyValidity; + + let api_key = match api_key { + None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run scraper without an API key")?), + Some (x) => x, + }; + + let bad_key = || error_reply (StatusCode::FORBIDDEN, "403 Forbidden"); + + { + let config = state.config.read ().await; + + let dev_mode = match &config.iso.dev_mode { + None => return Ok (bad_key ()?), + Some (x) => x, + }; + + let expected_key = match &dev_mode.scraper_key { + None => return Ok (bad_key ()?), + Some (x) => x, + }; + + let now = chrono::Utc::now (); + + match expected_key.is_valid (now, api_key.as_bytes ()) { + KeyValidity::Valid => (), + KeyValidity::WrongKey (bad_hash) => { + error! ("Bad scraper key with hash {:?}", bad_hash); + return Ok (bad_key ()?); + } + err => { + error! ("Bad scraper key {:?}", err); + return Ok (bad_key ()?); + }, + } + } + + Ok (error_reply (StatusCode::OK, "You're valid!")?) + } else { Ok (error_reply (StatusCode::OK, "Hi")?) } diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 14d0ea4..12f2e38 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -34,8 +34,9 @@ stronger is ready. - (X) Add feature flags to ptth_relay.toml for dev mode and scrapers - (X) Make sure Docker release CAN build - (X) Add hash of 1 scraper key to ptth_relay.toml, with 1 week expiration -- ( ) Accept scraper key for some testing endpoint -- ( ) (POC) Test with curl +- (X) Accept scraper key for some testing endpoint +- (X) (POC) Test with curl +- ( ) Clean up scraper endpoint - ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads - ( ) Remove scraper key from config file From ca6f28135112891cd18284592d426439148400c2 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 18:06:49 +0000 Subject: [PATCH 108/208] :pencil: docs: add glossary section --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index c83c949..2279a7d 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,28 @@ Inside the tunnel The server can run behind a firewall, because it is actually a special HTTP client. +## Glossary + +- "Tunnel" - The reverse HTTP tunnel between ptth_relay and ptth_server. +ptth_server connects out to ptth_relay, then ptth_relay forwards incoming +connections to ptth_server through the tunnel. +- "Relay" or "Relay server" - The ptth_relay app. This must run on a server +that can accept incoming HTTP connections. +- "Server" or "destination server" - The ptth_server app. This should run behind +a firewall. It will connect out to the relay and accept incoming connections +through the PTTH tunnel. +- "Client" - Any client that connects to the relay in order to reach a +destination server. Admins must terminate TLS between +ptth_relay and all clients. +- "Frontend" - The human-friendly HTTP+HTML interface that ptth_relay either +serves directly or relays from ptth_server. This interface has no auth by +default. Admins must provide their own auth in front of ptth_relay. +OAuth2 is recommended. +- "Backend API" - The HTTP API that ptth_server uses to establish the tunnel. +Noted in the code with the cookie "7ZSFUKGV". +- "Scraper API" - An optional HTTP API for scraper clients to access ptth_relay and +the destination servers using machine-friendly auth. + ## How to configure The server must be configured first so that its tripcode can be registered From 622895f77db867ada65cea1af0c448da2a4124d9 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sat, 12 Dec 2020 18:19:51 +0000 Subject: [PATCH 109/208] :pencil: docs: update readme, which was out of date --- README.md | 73 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 2279a7d..ebfeec2 100644 --- a/README.md +++ b/README.md @@ -19,44 +19,46 @@ client. ## Glossary -- "Tunnel" - The reverse HTTP tunnel between ptth_relay and ptth_server. -ptth_server connects out to ptth_relay, then ptth_relay forwards incoming -connections to ptth_server through the tunnel. -- "Relay" or "Relay server" - The ptth_relay app. This must run on a server -that can accept incoming HTTP connections. -- "Server" or "destination server" - The ptth_server app. This should run behind -a firewall. It will connect out to the relay and accept incoming connections -through the PTTH tunnel. -- "Client" - Any client that connects to the relay in order to reach a +(sorted alphabetically) + +- **Backend API** - The HTTP API that ptth_server uses to establish the tunnel. +Noted in the code with the cookie "7ZSFUKGV". +- **Client** - Any client that connects to ptth_relay in order to reach a destination server. Admins must terminate TLS between ptth_relay and all clients. -- "Frontend" - The human-friendly HTTP+HTML interface that ptth_relay either -serves directly or relays from ptth_server. This interface has no auth by -default. Admins must provide their own auth in front of ptth_relay. -OAuth2 is recommended. -- "Backend API" - The HTTP API that ptth_server uses to establish the tunnel. -Noted in the code with the cookie "7ZSFUKGV". -- "Scraper API" - An optional HTTP API for scraper clients to access ptth_relay and +- **Frontend** - The human-friendly, browser-friendly HTTP+HTML interface +that ptth_relay serves directly or relays from ptth_server. +This interface has no auth by default. Admins must provide their own auth +in front of ptth_relay. OAuth2 is recommended. +- **ptth_file_server** - A standalone file server. It uses the same code +as ptth_server, so production environments don't need it. +- **ptth_relay** or **Relay server** - The ptth_relay app. This must run on a server +that can accept incoming HTTP connections. +- **ptth_server** or **Destination server** - The ptth_server app. This should run behind +a firewall. It will connect out to the relay and accept incoming connections +through the PTTH tunnel. +- **Scraper API** - An optional HTTP API for scraper clients to access ptth_relay and the destination servers using machine-friendly auth. +- **Tripcode** - The base64 hash of a server's private API key. When adding +a new server, the tripcode must be copied to ptth_relay.toml on the relay +server. +- **Tunnel** - The reverse HTTP tunnel between ptth_relay and ptth_server. +ptth_server connects out to ptth_relay, then ptth_relay forwards incoming +connections to ptth_server through the tunnel. -## How to configure +## Configuration -The server must be configured first so that its tripcode can be registered -with the relay. - -Configuring a server: +ptth_server: - Copy ptth_server or ptth_server.exe onto the server - Create ptth_server.toml in the server's working dir -- Add a human-readable name and a secret API key generated by diceware method -- Run the server and use Ctrl+C to close it. It will print a tripcode to -the terminal. Copy that to the relay. +- Add a human-readable name and a secret API key generated by diceware +- Run `ptth_server --print-tripcode` and copy the output into ptth_relay.toml -Configuring the relay: +ptth_relay first-time config: - Copy ptth_relay onto the server (A Dockerfile is provided with no guarantees) - Create ptth_relay.toml in the relay's working dir -- Add the tripcodes of all servers to the server_tripcodes table Example server config: (Won't run because the key is too weak) @@ -64,20 +66,22 @@ Example server config: (Won't run because the key is too weak) name = "my_server" api_key = "secretpassword" relay_url = "http://127.0.0.1:4000" +file_server_root = "./data" ``` -Example relay config: +ptth_relay.toml: ``` -[server_tripcodes] +[[servers]] +name = "my_server" +tripcode = "czpCob1t1T7IU9zIlYyoNRomyeN7pqKSg1R0EUPz6Pw=" -my_server = "czpCob1t1T7IU9zIlYyoNRomyeN7pqKSg1R0EUPz6Pw=" -some_other_server = "su2wWbTyf5xih4yiCTfAzqDlASatV+0dI+UVKFBIsEI=" +[[servers] +name = "some_other_server" +tripcode = "su2wWbTyf5xih4yiCTfAzqDlASatV+0dI+UVKFBIsEI=" ``` -## How to run - -The relay should run first so that the server(s) can connect to it without error. +## Use 1. Start the relay 2. Start a server @@ -95,9 +99,6 @@ If you only have pre-built binaries: - `./ptth_server` - `firefox http://127.0.0.1:4000/servers/my_server/files/` -"ptth_file_server" is not needed for production. It's a standalone build of -the file server module, for testing. - To run the relay behind Nginx, these directives improve time-to-first-byte when streaming: From 1e81421444538e7232c8b9d9cefe97284204dd56 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 01:12:56 +0000 Subject: [PATCH 110/208] :shirt: refactor: Extract functions for scraper API endpoints --- crates/ptth_relay/src/lib.rs | 127 ++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 1ea4d2b..49b1a3c 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -408,22 +408,94 @@ async fn handle_server_list ( Ok (ok_reply (s)?) } +#[instrument (level = "trace", skip (req, state))] +async fn handle_scraper_api_v1 ( + req: Request , + state: Arc , + path_rest: &str +) +-> Result , RequestError> +{ + use key_validity::KeyValidity; + + let api_key = req.headers ().get ("X-ApiKey"); + + let api_key = match api_key { + None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run scraper without an API key")?), + Some (x) => x, + }; + + let bad_key = || error_reply (StatusCode::FORBIDDEN, "403 Forbidden"); + + { + let config = state.config.read ().await; + + let dev_mode = match &config.iso.dev_mode { + None => return Ok (bad_key ()?), + Some (x) => x, + }; + + let expected_key = match &dev_mode.scraper_key { + None => return Ok (bad_key ()?), + Some (x) => x, + }; + + let now = chrono::Utc::now (); + + match expected_key.is_valid (now, api_key.as_bytes ()) { + KeyValidity::Valid => (), + KeyValidity::WrongKey (bad_hash) => { + error! ("Bad scraper key with hash {:?}", bad_hash); + return Ok (bad_key ()?); + } + err => { + error! ("Bad scraper key {:?}", err); + return Ok (bad_key ()?); + }, + } + } + + if path_rest == "test" { + Ok (error_reply (StatusCode::OK, "You're valid!")?) + } + else { + Ok (error_reply (StatusCode::NOT_FOUND, "Unknown API endpoint")?) + } +} + +#[instrument (level = "trace", skip (req, state))] +async fn handle_scraper_api ( + req: Request , + state: Arc , + path_rest: &str +) +-> Result , RequestError> +{ + if let Some (rest) = prefix_match ("v1/", path_rest) { + handle_scraper_api_v1 (req, state, rest).await + } + else if let Some (rest) = prefix_match ("api/", path_rest) { + handle_scraper_api_v1 (req, state, rest).await + } + else { + Ok (error_reply (StatusCode::NOT_FOUND, "Unknown scraper API version")?) + } +} + #[instrument (level = "trace", skip (req, state))] async fn handle_all (req: Request , state: Arc ) -> Result , RequestError> { - let path = req.uri ().path (); + let path = req.uri ().path ().to_string (); //println! ("{}", path); debug! ("Request path: {}", path); - let api_key = req.headers ().get ("X-ApiKey"); - if req.method () == Method::POST { // This is stuff the server can use. Clients can't // POST right now - return if let Some (request_code) = prefix_match ("/7ZSFUKGV/http_response/", path) { + return if let Some (request_code) = prefix_match ("/7ZSFUKGV/http_response/", &path) { let request_code = request_code.into (); Ok (server_endpoint::handle_response (req, state, request_code).await?) } @@ -432,14 +504,16 @@ async fn handle_all (req: Request , state: Arc ) }; } - if let Some (listen_code) = prefix_match ("/7ZSFUKGV/http_listen/", path) { + if let Some (listen_code) = prefix_match ("/7ZSFUKGV/http_listen/", &path) { + let api_key = req.headers ().get ("X-ApiKey"); + let api_key = match api_key { None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run server without an API key")?), Some (x) => x, }; server_endpoint::handle_listen (state, listen_code.into (), api_key.as_bytes ()).await } - else if let Some (rest) = prefix_match ("/frontend/servers/", path) { + else if let Some (rest) = prefix_match ("/frontend/servers/", &path) { if rest == "" { Ok (handle_server_list (state).await?) } @@ -464,45 +538,8 @@ async fn handle_all (req: Request , state: Arc ) else if path == "/frontend/test_mysterious_error" { Err (RequestError::Mysterious) } - else if path == "/scraper/v1/test" || path == "/scraper/api/test" { - use key_validity::KeyValidity; - - let api_key = match api_key { - None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run scraper without an API key")?), - Some (x) => x, - }; - - let bad_key = || error_reply (StatusCode::FORBIDDEN, "403 Forbidden"); - - { - let config = state.config.read ().await; - - let dev_mode = match &config.iso.dev_mode { - None => return Ok (bad_key ()?), - Some (x) => x, - }; - - let expected_key = match &dev_mode.scraper_key { - None => return Ok (bad_key ()?), - Some (x) => x, - }; - - let now = chrono::Utc::now (); - - match expected_key.is_valid (now, api_key.as_bytes ()) { - KeyValidity::Valid => (), - KeyValidity::WrongKey (bad_hash) => { - error! ("Bad scraper key with hash {:?}", bad_hash); - return Ok (bad_key ()?); - } - err => { - error! ("Bad scraper key {:?}", err); - return Ok (bad_key ()?); - }, - } - } - - Ok (error_reply (StatusCode::OK, "You're valid!")?) + else if let Some (rest) = prefix_match ("/scraper/", &path) { + handle_scraper_api (req, state, rest).await } else { Ok (error_reply (StatusCode::OK, "Hi")?) From 670ce30667909387af0eaeb1ebc5970ebfdd0e11 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 01:54:54 +0000 Subject: [PATCH 111/208] :white_check_mark: test: add end-to-end test for scraper API --- Cargo.lock | 1 + Cargo.toml | 1 + crates/ptth_relay/src/config.rs | 2 + crates/ptth_relay/src/key_validity.rs | 19 ++++++-- crates/ptth_relay/src/lib.rs | 16 +++++-- issues/2020-12Dec/auth-route-YNQAQKJS.md | 3 +- src/tests.rs | 59 ++++++++++++++++++++++-- 7 files changed, 87 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dce88da..edeec04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1182,6 +1182,7 @@ version = "0.1.0" dependencies = [ "base64 0.12.3", "blake3", + "chrono", "ptth_relay", "ptth_server", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 896475b..7532e78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ license = "AGPL-3.0" base64 = "0.12.3" blake3 = "0.3.7" +chrono = {version = "0.4.19", features = ["serde"]} reqwest = { version = "0.10.8", features = ["stream"] } tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index b444810..d152293 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -55,6 +55,7 @@ pub mod file { // Stuff we actually need at runtime pub struct Config { + pub port: Option , pub servers: HashMap , pub iso: file::Isomorphic, } @@ -69,6 +70,7 @@ impl TryFrom for Config { let servers = itertools::process_results (servers, |i| HashMap::from_iter (i))?; Ok (Self { + port: f.port, servers, iso: f.iso, }) diff --git a/crates/ptth_relay/src/key_validity.rs b/crates/ptth_relay/src/key_validity.rs index 2bc2593..c75603e 100644 --- a/crates/ptth_relay/src/key_validity.rs +++ b/crates/ptth_relay/src/key_validity.rs @@ -84,9 +84,9 @@ impl MaxValidDuration for Valid7Days { #[derive (Deserialize)] pub struct ScraperKey { - pub not_before: DateTime , - pub not_after: DateTime , - pub hash: BlakeHashWrapper, + not_before: DateTime , + not_after: DateTime , + hash: BlakeHashWrapper, #[serde (default)] _phantom: std::marker::PhantomData , @@ -103,6 +103,19 @@ pub enum KeyValidity { DurationNegative, } +impl ScraperKey { + pub fn new (input: &[u8]) -> Self { + let now = Utc::now (); + + Self { + not_before: now, + not_after: now + Duration::days (7), + hash: BlakeHashWrapper::from_key (input), + _phantom: Default::default (), + } + } +} + impl ScraperKey { pub fn is_valid (&self, now: DateTime , input: &[u8]) -> KeyValidity { use KeyValidity::*; diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 49b1a3c..cf45021 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -471,6 +471,12 @@ async fn handle_scraper_api ( ) -> Result , RequestError> { + { + if ! state.config.read ().await.iso.enable_scraper_auth { + return Ok (error_reply (StatusCode::FORBIDDEN, "Scraper API disabled")?); + } + } + if let Some (rest) = prefix_match ("v1/", path_rest) { handle_scraper_api_v1 (req, state, rest).await } @@ -590,11 +596,6 @@ pub async fn run_relay ( ) -> Result <(), RelayError> { - let addr = SocketAddr::from (( - [0, 0, 0, 0], - 4000, - )); - if let Some (config_reload_path) = config_reload_path { let state_2 = state.clone (); tokio::spawn (async move { @@ -619,6 +620,11 @@ pub async fn run_relay ( } }); + let addr = SocketAddr::from (( + [0, 0, 0, 0], + state.config.read ().await.port.unwrap_or (4000), + )); + let server = Server::bind (&addr) .serve (make_svc); diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 12f2e38..05e9464 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -36,7 +36,8 @@ stronger is ready. - (X) Add hash of 1 scraper key to ptth_relay.toml, with 1 week expiration - (X) Accept scraper key for some testing endpoint - (X) (POC) Test with curl -- ( ) Clean up scraper endpoint +- (X) Clean up scraper endpoint +- ( ) Add end-to-end tests for scraper endpoint - ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads - ( ) Remove scraper key from config file diff --git a/src/tests.rs b/src/tests.rs index 5de6e56..55856bb 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,5 +1,5 @@ use std::{ - convert::TryFrom, + convert::{TryFrom, TryInto}, sync::{ Arc, }, @@ -13,17 +13,17 @@ use tokio::{ time::delay_for, }; +use reqwest::Client; +use tracing::{debug, info}; + #[test] fn end_to_end () { - use reqwest::Client; - use tracing::{debug, info}; - use ptth_relay::key_validity::BlakeHashWrapper; // Prefer this form for tests, since all tests share one process // and we don't care if another test already installed a subscriber. - tracing_subscriber::fmt ().try_init ().ok (); + //tracing_subscriber::fmt ().try_init ().ok (); let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); // Spawn the root task @@ -126,3 +126,52 @@ fn end_to_end () { info! ("Server stopped"); }); } + +#[test] +fn scraper_endpoints () { + let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); + + rt.block_on (async { + use ptth_relay::*; + + let config_file = config::file::Config { + port: Some (4001), + servers: vec! [ + + ], + iso: config::file::Isomorphic { + enable_scraper_auth: true, + dev_mode: Some (config::file::DevMode { + scraper_key: Some (key_validity::ScraperKey::new (b"bogus")), + }), + }, + }; + + let config = config::Config::try_from (config_file).expect ("Can't load config"); + + let relay_state = Arc::new (RelayState::try_from (config).expect ("Can't create relay state")); + let relay_state_2 = relay_state.clone (); + let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); + let task_relay = spawn (async move { + run_relay (relay_state_2, stop_relay_rx, None).await + }); + + let relay_url = "http://127.0.0.1:4001"; + + let mut headers = reqwest::header::HeaderMap::new (); + headers.insert ("X-ApiKey", "bogus".try_into ().unwrap ()); + + let client = Client::builder () + .default_headers (headers) + .timeout (Duration::from_secs (2)) + .build ().expect ("Couldn't build HTTP client"); + + let resp = client.get (&format! ("{}/scraper/api/test", relay_url)) + .send ().await.expect ("Couldn't check if relay is up").bytes ().await.expect ("Couldn't check if relay is up"); + + assert_eq! (resp, "You're valid!\n"); + + stop_relay_tx.send (()).expect ("Couldn't shut down relay"); + task_relay.await.expect ("Couldn't join relay").expect ("Relay error"); + }); +} From dc2958ad7ac1fe1557c7fe98c52123d0f9e7e048 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 01:51:42 +0000 Subject: [PATCH 112/208] :pencil: docs: add todo --- todo.md | 1 + 1 file changed, 1 insertion(+) diff --git a/todo.md b/todo.md index 5efa75f..4d1598c 100644 --- a/todo.md +++ b/todo.md @@ -3,6 +3,7 @@ Interesting issues will get a unique ID with - Diagram dependencies - Report server version in HTML +- Apply https://github.com/hexops/dockerfile to Dockerfile - [YNQAQKJS](issues/2020-12Dec/auth-route-YNQAQKJS.md) Open new auth route for spiders / scrapers - Track / Estimate bandwidth per server? - EOTPXGR3 Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) From 0e4839e146cd929854e72b636a36bd25b8d09ab4 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 02:05:22 +0000 Subject: [PATCH 113/208] :whale: build: add Tini to Docker image, simplify Git version injection Building the Git version into the code meant that Cargo had to recompile ptth_relay every time I built. This is annoying and it doesn't add anything. I changed it to read the Git version from a text file which is absent by default, and present in the Docker image. --- Dockerfile | 14 +++++++------- crates/ptth_relay/src/git_version.rs | 15 ++++++++++++++- crates/ptth_relay/src/git_version.txt | 1 - crates/ptth_relay/src/main.rs | 7 +++++-- todo.md | 2 +- 5 files changed, 27 insertions(+), 12 deletions(-) delete mode 100644 crates/ptth_relay/src/git_version.txt diff --git a/Dockerfile b/Dockerfile index 0e13bd0..48b1753 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,9 +53,6 @@ COPY ./handlebars/ ./handlebars # Docker doing something funny with mtimes? Maybe? RUN touch crates/ptth_core/src/lib.rs -ARG git_version -RUN echo -n "$git_version" > crates/ptth_relay/src/git_version.txt - # build for release # gate only on ptth_relay tests for now RUN \ @@ -66,12 +63,15 @@ cargo test --release -p ptth_relay FROM debian@sha256:240f770008bdc538fecc8d3fa7a32a533eac55c14cbc56a9a8a6f7d741b47e33 RUN apt-get update \ -&& apt-get install -y libssl1.1 ca-certificates \ -&& apt-get upgrade -y +&& apt-get upgrade -y \ +&& apt-get install -y libssl1.1 ca-certificates tini \ +&& mkdir -p /root COPY --from=build /ptth/target/release/ptth_relay /root/ -COPY --from=build /ptth/crates/ptth_relay/src/git_version.txt /root/ COPY --from=build /ptth/handlebars /root/handlebars +ARG git_version +RUN echo -n "$git_version" > /root/git_version.txt + WORKDIR /root -ENTRYPOINT ["./ptth_relay"] +ENTRYPOINT ["/usr/bin/tini", "--", "./ptth_relay"] diff --git a/crates/ptth_relay/src/git_version.rs b/crates/ptth_relay/src/git_version.rs index 12a4fd8..7a1dc70 100644 --- a/crates/ptth_relay/src/git_version.rs +++ b/crates/ptth_relay/src/git_version.rs @@ -1 +1,14 @@ -pub const GIT_VERSION: &str = include_str! ("git_version.txt"); +pub fn read_git_version () -> Option { + use std::{ + io::Read, + fs::File, + }; + + let mut buf = vec! [0u8; 512]; + + let mut f = File::open ("git_version.txt").ok ()?; + let bytes_read = f.read (&mut buf).ok ()?; + buf.truncate (bytes_read); + + Some (String::from_utf8 (buf).ok ()?) +} diff --git a/crates/ptth_relay/src/git_version.txt b/crates/ptth_relay/src/git_version.txt deleted file mode 100644 index d5d3349..0000000 --- a/crates/ptth_relay/src/git_version.txt +++ /dev/null @@ -1 +0,0 @@ -(Unknown) \ No newline at end of file diff --git a/crates/ptth_relay/src/main.rs b/crates/ptth_relay/src/main.rs index c8d1bce..bd3c9c6 100644 --- a/crates/ptth_relay/src/main.rs +++ b/crates/ptth_relay/src/main.rs @@ -16,7 +16,7 @@ use tracing_subscriber::{ use ptth_relay::{ Config, - git_version::GIT_VERSION, + git_version::read_git_version, RelayState, run_relay, }; @@ -32,7 +32,10 @@ async fn main () -> Result <(), Box > { let config_path = PathBuf::from ("config/ptth_relay.toml"); let config = Config::from_file (&config_path).await?; - info! ("ptth_relay Git version: {:?}", GIT_VERSION); + match read_git_version () { + Some (x) => info! ("ptth_relay Git version: {:?}", x), + None => info! ("ptth_relay not built from Git"), + } let (shutdown_rx, forced_shutdown) = ptth_core::graceful_shutdown::init_with_force (); diff --git a/todo.md b/todo.md index 4d1598c..f96bb92 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,7 @@ Interesting issues will get a unique ID with `dd if=/dev/urandom bs=5 count=1 | base32` -- Diagram dependencies +- Move Git version out of source code and into a plain file in the Docker image - Report server version in HTML - Apply https://github.com/hexops/dockerfile to Dockerfile - [YNQAQKJS](issues/2020-12Dec/auth-route-YNQAQKJS.md) Open new auth route for spiders / scrapers From 532f99f7702549ab44b7706f42b90bd6006c0329 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 02:25:18 +0000 Subject: [PATCH 114/208] :lipstick: update: add dev mode and server Git version to server list --- Dockerfile | 15 +++++++++------ crates/ptth_relay/src/git_version.rs | 10 +++++----- crates/ptth_relay/src/lib.rs | 6 ++++++ crates/ptth_relay/src/main.rs | 2 +- handlebars/relay/relay_server_list.html | 16 +++++++++++++++- run_docker_image.bash | 6 +++++- todo.md | 2 -- 7 files changed, 41 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 48b1753..c597284 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,14 +64,17 @@ FROM debian@sha256:240f770008bdc538fecc8d3fa7a32a533eac55c14cbc56a9a8a6f7d741b47 RUN apt-get update \ && apt-get upgrade -y \ -&& apt-get install -y libssl1.1 ca-certificates tini \ -&& mkdir -p /root +&& apt-get install -y libssl1.1 ca-certificates tini -COPY --from=build /ptth/target/release/ptth_relay /root/ -COPY --from=build /ptth/handlebars /root/handlebars +RUN addgroup --gid 10001 nonroot && adduser --system --uid 10000 --gid 10001 nonroot + +USER nonroot +WORKDIR /home/nonroot + +COPY --from=build /ptth/target/release/ptth_relay ./ +COPY --from=build /ptth/handlebars ./handlebars ARG git_version -RUN echo -n "$git_version" > /root/git_version.txt +RUN echo -n "$git_version" > ./git_version.txt -WORKDIR /root ENTRYPOINT ["/usr/bin/tini", "--", "./ptth_relay"] diff --git a/crates/ptth_relay/src/git_version.rs b/crates/ptth_relay/src/git_version.rs index 7a1dc70..a44088d 100644 --- a/crates/ptth_relay/src/git_version.rs +++ b/crates/ptth_relay/src/git_version.rs @@ -1,13 +1,13 @@ -pub fn read_git_version () -> Option { - use std::{ - io::Read, +pub async fn read_git_version () -> Option { + use tokio::{ fs::File, + io::AsyncReadExt, }; let mut buf = vec! [0u8; 512]; - let mut f = File::open ("git_version.txt").ok ()?; - let bytes_read = f.read (&mut buf).ok ()?; + let mut f = File::open ("git_version.txt").await.ok ()?; + let bytes_read = f.read (&mut buf).await.ok ()?; buf.truncate (bytes_read); Some (String::from_utf8 (buf).ok ()?) diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index cf45021..999e5d9 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -336,15 +336,19 @@ struct ServerEntry <'a> { #[derive (Serialize)] struct ServerListPage <'a> { + dev_mode: bool, + git_version: Option , servers: Vec >, } async fn handle_server_list_internal (state: &Arc ) -> ServerListPage <'static> { + let dev_mode; let display_names: HashMap = { let guard = state.config.read ().await; + dev_mode = guard.iso.dev_mode.is_some (); let servers = (*guard).servers.iter () .map (|(k, v)| { let display_name = v.display_name @@ -394,6 +398,8 @@ async fn handle_server_list_internal (state: &Arc ) servers.sort_by (|a, b| a.display_name.cmp (&b.display_name)); ServerListPage { + dev_mode, + git_version: git_version::read_git_version ().await, servers, } } diff --git a/crates/ptth_relay/src/main.rs b/crates/ptth_relay/src/main.rs index bd3c9c6..d00332d 100644 --- a/crates/ptth_relay/src/main.rs +++ b/crates/ptth_relay/src/main.rs @@ -32,7 +32,7 @@ async fn main () -> Result <(), Box > { let config_path = PathBuf::from ("config/ptth_relay.toml"); let config = Config::from_file (&config_path).await?; - match read_git_version () { + match read_git_version ().await { Some (x) => info! ("ptth_relay Git version: {:?}", x), None => info! ("ptth_relay not built from Git"), } diff --git a/handlebars/relay/relay_server_list.html b/handlebars/relay/relay_server_list.html index 3fd8a8a..2075d61 100644 --- a/handlebars/relay/relay_server_list.html +++ b/handlebars/relay/relay_server_list.html @@ -34,6 +34,11 @@

Server list

+{{#if dev_mode}} +
Relay is in dev mode. This should never be seen in production!
+{{/if}} + +
{{#if servers}} @@ -54,8 +59,17 @@
{{else}} - (No servers have reported since this relay started) + (No servers are registered in the config file) {{/if}} +
+ +
+{{#if git_version}} +Git version: {{git_version}} +{{else}} +Not built from Git +{{/if}} +
diff --git a/run_docker_image.bash b/run_docker_image.bash index 7857c93..090c4c4 100755 --- a/run_docker_image.bash +++ b/run_docker_image.bash @@ -1,3 +1,7 @@ #!/usr/bin/env bash -sudo docker run -it -v $PWD/config:/root/config -e RUST_LOG=ptth=trace ptth:latest +sudo docker run -it \ +-v $PWD/config:/home/nonroot/config \ +-e RUST_LOG=ptth=trace \ +-p 4000:4000 \ +ptth:latest diff --git a/todo.md b/todo.md index f96bb92..da42c81 100644 --- a/todo.md +++ b/todo.md @@ -1,9 +1,7 @@ Interesting issues will get a unique ID with `dd if=/dev/urandom bs=5 count=1 | base32` -- Move Git version out of source code and into a plain file in the Docker image - Report server version in HTML -- Apply https://github.com/hexops/dockerfile to Dockerfile - [YNQAQKJS](issues/2020-12Dec/auth-route-YNQAQKJS.md) Open new auth route for spiders / scrapers - Track / Estimate bandwidth per server? - EOTPXGR3 Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) From 5f947ed73cafd320433a3f8cd2f7f69873e3e004 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 03:29:54 +0000 Subject: [PATCH 115/208] :shirt: refactor: Extract relay_state module --- crates/ptth_relay/src/lib.rs | 96 +------------------ crates/ptth_relay/src/relay_state.rs | 116 +++++++++++++++++++++++ crates/ptth_relay/src/server_endpoint.rs | 2 + crates/ptth_relay/src/tests.rs | 11 +++ todo.md | 1 - 5 files changed, 132 insertions(+), 94 deletions(-) create mode 100644 crates/ptth_relay/src/relay_state.rs diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 999e5d9..1c1c7fa 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -17,7 +17,6 @@ use std::{ borrow::Cow, collections::HashMap, - convert::TryFrom, iter::FromIterator, net::SocketAddr, path::{Path, PathBuf}, @@ -46,10 +45,7 @@ use serde::{ }; use tokio::{ sync::{ - Mutex, oneshot, - RwLock, - watch, }, }; @@ -63,101 +59,15 @@ pub mod config; pub mod errors; pub mod git_version; pub mod key_validity; +mod relay_state; mod server_endpoint; pub use config::Config; pub use errors::*; +pub use relay_state::RelayState; -/* - -Here's what we need to handle: - -When a request comes in: - -- Park the client in response_rendezvous -- Look up the server ID in request_rendezvous -- If a server is parked, unpark it and send the request -- Otherwise, queue the request - -When a server comes to listen: - -- Look up the server ID in request_rendezvous -- Either return all pending requests, or park the server - -When a server comes to respond: - -- Look up the parked client in response_rendezvous -- Unpark the client and begin streaming - -So we need these lookups to be fast: - -- Server IDs, where (1 server) or (0 or many clients) -can be parked -- Request IDs, where 1 client is parked - -*/ - -enum RequestRendezvous { - ParkedClients (Vec ), - ParkedServer (oneshot::Sender >), -} - -type ResponseRendezvous = oneshot::Sender >; - -#[derive (Clone)] -pub struct ServerStatus { - last_seen: DateTime , -} - -impl Default for ServerStatus { - fn default () -> Self { - Self { - last_seen: Utc::now (), - } - } -} - -pub struct RelayState { - config: RwLock , - handlebars: Arc >, - - // Key: Server ID - request_rendezvous: Mutex >, - server_status: Mutex >, - - // Key: Request ID - response_rendezvous: RwLock >, - - shutdown_watch_tx: watch::Sender , - shutdown_watch_rx: watch::Receiver , -} - -impl TryFrom for RelayState { - type Error = RelayError; - - fn try_from (config: Config) -> Result { - let (shutdown_watch_tx, shutdown_watch_rx) = watch::channel (false); - - Ok (Self { - config: config.into (), - handlebars: Arc::new (load_templates (&PathBuf::new ())?), - request_rendezvous: Default::default (), - server_status: Default::default (), - response_rendezvous: Default::default (), - shutdown_watch_tx, - shutdown_watch_rx, - }) - } -} - -impl RelayState { - pub async fn list_servers (&self) -> Vec { - self.request_rendezvous.lock ().await.iter () - .map (|(k, _)| (*k).clone ()) - .collect () - } -} +use relay_state::*; fn ok_reply > (b: B) -> Result , http::Error> diff --git a/crates/ptth_relay/src/relay_state.rs b/crates/ptth_relay/src/relay_state.rs new file mode 100644 index 0000000..6b7f0c1 --- /dev/null +++ b/crates/ptth_relay/src/relay_state.rs @@ -0,0 +1,116 @@ +use std::{ + collections::HashMap, + convert::TryFrom, + path::{PathBuf}, + sync::Arc, +}; + +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use handlebars::Handlebars; +use tokio::sync::{ + Mutex, + RwLock, + oneshot, + watch, +}; + +use crate::{ + Body, + Config, + RelayError, + ShuttingDownError, + load_templates, +}; + +use ptth_core::http_serde; + +/* + +Here's what we need to handle: + +When a request comes in: + +- Park the client in response_rendezvous +- Look up the server ID in request_rendezvous +- If a server is parked, unpark it and send the request +- Otherwise, queue the request + +When a server comes to listen: + +- Look up the server ID in request_rendezvous +- Either return all pending requests, or park the server + +When a server comes to respond: + +- Look up the parked client in response_rendezvous +- Unpark the client and begin streaming + +So we need these lookups to be fast: + +- Server IDs, where (1 server) or (0 or many clients) +can be parked +- Request IDs, where 1 client is parked + +*/ + +pub enum RequestRendezvous { + ParkedClients (Vec ), + ParkedServer (oneshot::Sender >), +} + +type ResponseRendezvous = oneshot::Sender >; + +#[derive (Clone)] +pub struct ServerStatus { + pub last_seen: DateTime , +} + +impl Default for ServerStatus { + fn default () -> Self { + Self { + last_seen: Utc::now (), + } + } +} + +pub struct RelayState { + pub config: RwLock , + pub handlebars: Arc >, + + // Key: Server ID + pub request_rendezvous: Mutex >, + pub server_status: Mutex >, + + // Key: Request ID + pub response_rendezvous: RwLock >, + + pub shutdown_watch_tx: watch::Sender , + pub shutdown_watch_rx: watch::Receiver , +} + +impl TryFrom for RelayState { + type Error = RelayError; + + fn try_from (config: Config) -> Result { + let (shutdown_watch_tx, shutdown_watch_rx) = watch::channel (false); + + Ok (Self { + config: config.into (), + handlebars: Arc::new (load_templates (&PathBuf::new ())?), + request_rendezvous: Default::default (), + server_status: Default::default (), + response_rendezvous: Default::default (), + shutdown_watch_tx, + shutdown_watch_rx, + }) + } +} + +impl RelayState { + pub async fn list_servers (&self) -> Vec { + self.request_rendezvous.lock ().await.iter () + .map (|(k, _)| (*k).clone ()) + .collect () + } +} diff --git a/crates/ptth_relay/src/server_endpoint.rs b/crates/ptth_relay/src/server_endpoint.rs index c6e9795..f907552 100644 --- a/crates/ptth_relay/src/server_endpoint.rs +++ b/crates/ptth_relay/src/server_endpoint.rs @@ -75,6 +75,8 @@ pub async fn handle_listen ( // End of early returns { + // TODO: Move into relay_state.rs + let mut server_status = state.server_status.lock ().await; let mut status = server_status.entry (watcher_code.clone ()).or_insert_with (Default::default); diff --git a/crates/ptth_relay/src/tests.rs b/crates/ptth_relay/src/tests.rs index 6b80c37..b40cc1a 100644 --- a/crates/ptth_relay/src/tests.rs +++ b/crates/ptth_relay/src/tests.rs @@ -1,3 +1,5 @@ +use tokio::runtime::Runtime; + use super::*; #[test] @@ -23,3 +25,12 @@ fn test_pretty_print_last_seen () { assert_eq! (actual, expected); } } + +#[test] +fn scraper_endpoints () { + let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); + + rt.block_on (async { + + }); +} diff --git a/todo.md b/todo.md index da42c81..7f9fcf4 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,6 @@ Interesting issues will get a unique ID with `dd if=/dev/urandom bs=5 count=1 | base32` -- Report server version in HTML - [YNQAQKJS](issues/2020-12Dec/auth-route-YNQAQKJS.md) Open new auth route for spiders / scrapers - Track / Estimate bandwidth per server? - EOTPXGR3 Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) From 32e48697d58d7257f6f09c5138104e59d9f46979 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 03:42:00 +0000 Subject: [PATCH 116/208] :shirt: refactor: extract scraper_api module --- crates/ptth_relay/src/lib.rs | 82 +--------------------- crates/ptth_relay/src/scraper_api.rs | 100 +++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 80 deletions(-) create mode 100644 crates/ptth_relay/src/scraper_api.rs diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 1c1c7fa..9b5ce9d 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -61,6 +61,7 @@ pub mod git_version; pub mod key_validity; mod relay_state; +mod scraper_api; mod server_endpoint; pub use config::Config; @@ -68,6 +69,7 @@ pub use errors::*; pub use relay_state::RelayState; use relay_state::*; +use scraper_api::*; fn ok_reply > (b: B) -> Result , http::Error> @@ -324,86 +326,6 @@ async fn handle_server_list ( Ok (ok_reply (s)?) } -#[instrument (level = "trace", skip (req, state))] -async fn handle_scraper_api_v1 ( - req: Request , - state: Arc , - path_rest: &str -) --> Result , RequestError> -{ - use key_validity::KeyValidity; - - let api_key = req.headers ().get ("X-ApiKey"); - - let api_key = match api_key { - None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run scraper without an API key")?), - Some (x) => x, - }; - - let bad_key = || error_reply (StatusCode::FORBIDDEN, "403 Forbidden"); - - { - let config = state.config.read ().await; - - let dev_mode = match &config.iso.dev_mode { - None => return Ok (bad_key ()?), - Some (x) => x, - }; - - let expected_key = match &dev_mode.scraper_key { - None => return Ok (bad_key ()?), - Some (x) => x, - }; - - let now = chrono::Utc::now (); - - match expected_key.is_valid (now, api_key.as_bytes ()) { - KeyValidity::Valid => (), - KeyValidity::WrongKey (bad_hash) => { - error! ("Bad scraper key with hash {:?}", bad_hash); - return Ok (bad_key ()?); - } - err => { - error! ("Bad scraper key {:?}", err); - return Ok (bad_key ()?); - }, - } - } - - if path_rest == "test" { - Ok (error_reply (StatusCode::OK, "You're valid!")?) - } - else { - Ok (error_reply (StatusCode::NOT_FOUND, "Unknown API endpoint")?) - } -} - -#[instrument (level = "trace", skip (req, state))] -async fn handle_scraper_api ( - req: Request , - state: Arc , - path_rest: &str -) --> Result , RequestError> -{ - { - if ! state.config.read ().await.iso.enable_scraper_auth { - return Ok (error_reply (StatusCode::FORBIDDEN, "Scraper API disabled")?); - } - } - - if let Some (rest) = prefix_match ("v1/", path_rest) { - handle_scraper_api_v1 (req, state, rest).await - } - else if let Some (rest) = prefix_match ("api/", path_rest) { - handle_scraper_api_v1 (req, state, rest).await - } - else { - Ok (error_reply (StatusCode::NOT_FOUND, "Unknown scraper API version")?) - } -} - #[instrument (level = "trace", skip (req, state))] async fn handle_all (req: Request , state: Arc ) -> Result , RequestError> diff --git a/crates/ptth_relay/src/scraper_api.rs b/crates/ptth_relay/src/scraper_api.rs new file mode 100644 index 0000000..b63bbf7 --- /dev/null +++ b/crates/ptth_relay/src/scraper_api.rs @@ -0,0 +1,100 @@ +use std::{ + sync::Arc, +}; + +use hyper::{ + Body, + Request, + Response, + StatusCode, +}; +use tracing::{ + error, + instrument, +}; + +use crate::{ + RequestError, + error_reply, + key_validity::KeyValidity, + prefix_match, + relay_state::RelayState, +}; + +#[instrument (level = "trace", skip (req, state))] +pub async fn handle_scraper_api_v1 ( + req: Request , + state: Arc , + path_rest: &str +) +-> Result , RequestError> +{ + let api_key = req.headers ().get ("X-ApiKey"); + + let api_key = match api_key { + None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run scraper without an API key")?), + Some (x) => x, + }; + + let bad_key = || error_reply (StatusCode::FORBIDDEN, "403 Forbidden"); + + { + let config = state.config.read ().await; + + let dev_mode = match &config.iso.dev_mode { + None => return Ok (bad_key ()?), + Some (x) => x, + }; + + let expected_key = match &dev_mode.scraper_key { + None => return Ok (bad_key ()?), + Some (x) => x, + }; + + let now = chrono::Utc::now (); + + match expected_key.is_valid (now, api_key.as_bytes ()) { + KeyValidity::Valid => (), + KeyValidity::WrongKey (bad_hash) => { + error! ("Bad scraper key with hash {:?}", bad_hash); + return Ok (bad_key ()?); + } + err => { + error! ("Bad scraper key {:?}", err); + return Ok (bad_key ()?); + }, + } + } + + if path_rest == "test" { + Ok (error_reply (StatusCode::OK, "You're valid!")?) + } + else { + Ok (error_reply (StatusCode::NOT_FOUND, "Unknown API endpoint")?) + } +} + +#[instrument (level = "trace", skip (req, state))] +pub async fn handle_scraper_api ( + req: Request , + state: Arc , + path_rest: &str +) +-> Result , RequestError> +{ + { + if ! state.config.read ().await.iso.enable_scraper_auth { + return Ok (error_reply (StatusCode::FORBIDDEN, "Scraper API disabled")?); + } + } + + if let Some (rest) = prefix_match ("v1/", path_rest) { + handle_scraper_api_v1 (req, state, rest).await + } + else if let Some (rest) = prefix_match ("api/", path_rest) { + handle_scraper_api_v1 (req, state, rest).await + } + else { + Ok (error_reply (StatusCode::NOT_FOUND, "Unknown scraper API version")?) + } +} From 0737edd8f88ba8640b3034ca0c18ada48a47ba33 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 04:03:30 +0000 Subject: [PATCH 117/208] :shirt: refactor: move handlebars out of RelayState This will make it simpler to separate logic and presentation. --- crates/ptth_relay/src/lib.rs | 20 ++++++++---- crates/ptth_relay/src/main.rs | 1 + crates/ptth_relay/src/relay_state.rs | 2 -- crates/ptth_relay/src/scraper_api.rs | 46 ++++++++++++++++++++++++++++ src/tests.rs | 17 ++++++++-- 5 files changed, 76 insertions(+), 10 deletions(-) diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 9b5ce9d..1ab9c1f 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -317,17 +317,22 @@ async fn handle_server_list_internal (state: &Arc ) } async fn handle_server_list ( - state: Arc + state: Arc , + handlebars: Arc > ) -> Result , RequestError> { let page = handle_server_list_internal (&state).await; - let s = state.handlebars.render ("relay_server_list", &page)?; + let s = handlebars.render ("relay_server_list", &page)?; Ok (ok_reply (s)?) } #[instrument (level = "trace", skip (req, state))] -async fn handle_all (req: Request , state: Arc ) +async fn handle_all ( + req: Request , + state: Arc , + handlebars: Arc > +) -> Result , RequestError> { let path = req.uri ().path ().to_string (); @@ -359,7 +364,7 @@ async fn handle_all (req: Request , state: Arc ) } else if let Some (rest) = prefix_match ("/frontend/servers/", &path) { if rest == "" { - Ok (handle_server_list (state).await?) + Ok (handle_server_list (state, handlebars).await?) } else if let Some (idx) = rest.find ('/') { let listen_code = String::from (&rest [0..idx]); @@ -373,7 +378,7 @@ async fn handle_all (req: Request , state: Arc ) } } else if path == "/" { - let s = state.handlebars.render ("relay_root", &())?; + let s = handlebars.render ("relay_root", &())?; Ok (ok_reply (s)?) } else if path == "/frontend/relay_up_check" { @@ -429,6 +434,7 @@ async fn reload_config ( pub async fn run_relay ( state: Arc , + handlebars: Arc >, shutdown_oneshot: oneshot::Receiver <()>, config_reload_path: Option ) @@ -448,12 +454,14 @@ pub async fn run_relay ( let make_svc = make_service_fn (|_conn| { let state = state.clone (); + let handlebars = handlebars.clone (); async { Ok::<_, RequestError> (service_fn (move |req| { let state = state.clone (); + let handlebars = handlebars.clone (); - handle_all (req, state) + handle_all (req, state, handlebars) })) } }); diff --git a/crates/ptth_relay/src/main.rs b/crates/ptth_relay/src/main.rs index d00332d..abe446d 100644 --- a/crates/ptth_relay/src/main.rs +++ b/crates/ptth_relay/src/main.rs @@ -42,6 +42,7 @@ async fn main () -> Result <(), Box > { forced_shutdown.wrap_server ( run_relay ( Arc::new (RelayState::try_from (config)?), + Arc::new (ptth_relay::load_templates (&PathBuf::new ())?), shutdown_rx, Some (config_path) ) diff --git a/crates/ptth_relay/src/relay_state.rs b/crates/ptth_relay/src/relay_state.rs index 6b7f0c1..f91d76a 100644 --- a/crates/ptth_relay/src/relay_state.rs +++ b/crates/ptth_relay/src/relay_state.rs @@ -76,7 +76,6 @@ impl Default for ServerStatus { pub struct RelayState { pub config: RwLock , - pub handlebars: Arc >, // Key: Server ID pub request_rendezvous: Mutex >, @@ -97,7 +96,6 @@ impl TryFrom for RelayState { Ok (Self { config: config.into (), - handlebars: Arc::new (load_templates (&PathBuf::new ())?), request_rendezvous: Default::default (), server_status: Default::default (), response_rendezvous: Default::default (), diff --git a/crates/ptth_relay/src/scraper_api.rs b/crates/ptth_relay/src/scraper_api.rs index b63bbf7..ff58a32 100644 --- a/crates/ptth_relay/src/scraper_api.rs +++ b/crates/ptth_relay/src/scraper_api.rs @@ -98,3 +98,49 @@ pub async fn handle_scraper_api ( Ok (error_reply (StatusCode::NOT_FOUND, "Unknown scraper API version")?) } } + +#[cfg (test)] +mod tests { + use std::{ + convert::TryFrom, + }; + + use tokio::runtime::Runtime; + use crate::{ + config, + key_validity, + }; + use super::*; + + #[test] + fn auth () { + let input = Request::builder () + .method ("GET") + .uri ("http://127.0.0.1:4000/scraper/v1/test") + .header ("X-ApiKey", "bogus") + .body (Body::empty ()) + .unwrap (); + + let config_file = config::file::Config { + port: Some (4000), + servers: vec! [], + iso: config::file::Isomorphic { + enable_scraper_auth: true, + dev_mode: Some (config::file::DevMode { + scraper_key: Some (key_validity::ScraperKey::new (b"bogus")), + }), + }, + }; + + let config = config::Config::try_from (config_file).expect ("Can't load config"); + + let relay_state = Arc::new (RelayState::try_from (config).expect ("Can't create relay state")); + + let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); + + rt.block_on (async move { + let actual = handle_scraper_api (input, relay_state, "").await; + let actual = actual.expect ("Relay didn't respond"); + }); + } +} diff --git a/src/tests.rs b/src/tests.rs index 55856bb..6a4c28f 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,5 +1,6 @@ use std::{ convert::{TryFrom, TryInto}, + path::PathBuf, sync::{ Arc, }, @@ -16,6 +17,8 @@ use tokio::{ use reqwest::Client; use tracing::{debug, info}; +use ptth_relay::load_templates; + #[test] fn end_to_end () { use ptth_relay::key_validity::BlakeHashWrapper; @@ -51,7 +54,12 @@ fn end_to_end () { let relay_state_2 = relay_state.clone (); let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); let task_relay = spawn (async move { - ptth_relay::run_relay (relay_state_2, stop_relay_rx, None).await + ptth_relay::run_relay ( + relay_state_2, + Arc::new (load_templates (&PathBuf::new ())?), + stop_relay_rx, + None + ).await }); assert! (relay_state.list_servers ().await.is_empty ()); @@ -153,7 +161,12 @@ fn scraper_endpoints () { let relay_state_2 = relay_state.clone (); let (stop_relay_tx, stop_relay_rx) = oneshot::channel (); let task_relay = spawn (async move { - run_relay (relay_state_2, stop_relay_rx, None).await + run_relay ( + relay_state_2, + Arc::new (load_templates (&PathBuf::new ())?), + stop_relay_rx, + None + ).await }); let relay_url = "http://127.0.0.1:4001"; From 9c7b2b7a861f4b51f77c49efec83bfd921c67e23 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 04:44:37 +0000 Subject: [PATCH 118/208] :white_check_mark: test: add tests for scraper API test endpoint --- crates/ptth_relay/src/scraper_api.rs | 155 ++++++++++++++++++++++----- src/tests.rs | 22 +++- 2 files changed, 150 insertions(+), 27 deletions(-) diff --git a/crates/ptth_relay/src/scraper_api.rs b/crates/ptth_relay/src/scraper_api.rs index ff58a32..d642cbb 100644 --- a/crates/ptth_relay/src/scraper_api.rs +++ b/crates/ptth_relay/src/scraper_api.rs @@ -102,7 +102,7 @@ pub async fn handle_scraper_api ( #[cfg (test)] mod tests { use std::{ - convert::TryFrom, + convert::{TryFrom, TryInto}, }; use tokio::runtime::Runtime; @@ -112,35 +112,140 @@ mod tests { }; use super::*; + #[derive (Clone)] + struct TestCase { + // Inputs + path_rest: &'static str, + valid_key: Option <&'static str>, + input_key: Option <&'static str>, + + // Expected + expected_status: StatusCode, + expected_headers: Vec <(&'static str, &'static str)>, + expected_body: &'static str, + } + + impl TestCase { + fn path_rest (&self, v: &'static str) -> Self { + let mut x = self.clone (); + x.path_rest = v; + x + } + + fn valid_key (&self, v: Option <&'static str>) -> Self { + let mut x = self.clone (); + x.valid_key = v; + x + } + + fn input_key (&self, v: Option <&'static str>) -> Self { + let mut x = self.clone (); + x.input_key = v; + x + } + + fn expected_status (&self, v: StatusCode) -> Self { + let mut x = self.clone (); + x.expected_status = v; + x + } + + fn expected_headers (&self, v: Vec <(&'static str, &'static str)>) -> Self { + let mut x = self.clone (); + x.expected_headers = v; + x + } + + fn expected_body (&self, v: &'static str) -> Self { + let mut x = self.clone (); + x.expected_body = v; + x + } + + fn expected (&self, sc: StatusCode, body: &'static str) -> Self { + self + .expected_status (sc) + .expected_body (body) + } + + async fn test (&self) { + let mut input = Request::builder () + .method ("GET") + .uri (format! ("http://127.0.0.1:4000/scraper/{}", self.path_rest)); + + if let Some (input_key) = self.input_key { + input = input.header ("X-ApiKey", input_key); + } + let input = input.body (Body::empty ()).unwrap (); + + let config_file = config::file::Config { + port: Some (4000), + servers: vec! [], + iso: config::file::Isomorphic { + enable_scraper_auth: true, + dev_mode: self.valid_key.map (|key| config::file::DevMode { + scraper_key: Some (key_validity::ScraperKey::new (key.as_bytes ())), + }), + }, + }; + + let config = config::Config::try_from (config_file).expect ("Can't load config"); + + let relay_state = Arc::new (RelayState::try_from (config).expect ("Can't create relay state")); + + let actual = handle_scraper_api (input, relay_state, self.path_rest).await; + let actual = actual.expect ("Relay didn't respond"); + let (actual_head, actual_body) = actual.into_parts (); + + let mut expected_headers = hyper::header::HeaderMap::new (); + + for (key, value) in &self.expected_headers { + expected_headers.insert (*key, (*value).try_into ().expect ("Couldn't convert header value")); + } + + assert_eq! (actual_head.status, self.expected_status); + assert_eq! (actual_head.headers, expected_headers); + + let actual_body = hyper::body::to_bytes (actual_body).await; + let actual_body = actual_body.expect ("Body should be convertible to bytes"); + let actual_body = actual_body.to_vec (); + let actual_body = String::from_utf8 (actual_body).expect ("Body should be UTF-8"); + + assert_eq! (&actual_body, self.expected_body); + } + } + #[test] fn auth () { - let input = Request::builder () - .method ("GET") - .uri ("http://127.0.0.1:4000/scraper/v1/test") - .header ("X-ApiKey", "bogus") - .body (Body::empty ()) - .unwrap (); - - let config_file = config::file::Config { - port: Some (4000), - servers: vec! [], - iso: config::file::Isomorphic { - enable_scraper_auth: true, - dev_mode: Some (config::file::DevMode { - scraper_key: Some (key_validity::ScraperKey::new (b"bogus")), - }), - }, - }; - - let config = config::Config::try_from (config_file).expect ("Can't load config"); - - let relay_state = Arc::new (RelayState::try_from (config).expect ("Can't create relay state")); - let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); rt.block_on (async move { - let actual = handle_scraper_api (input, relay_state, "").await; - let actual = actual.expect ("Relay didn't respond"); + let base_case = TestCase { + path_rest: "v1/test", + valid_key: Some ("bogus"), + input_key: Some ("bogus"), + expected_status: StatusCode::OK, + expected_headers: vec! [ + ("content-type", "text/plain"), + ], + expected_body: "You're valid!\n", + }; + + for case in &[ + base_case.clone (), + base_case.path_rest ("v9999/test") + .expected (StatusCode::NOT_FOUND, "Unknown scraper API version\n"), + base_case.valid_key (None) + .expected (StatusCode::FORBIDDEN, "403 Forbidden\n"), + base_case.input_key (Some ("borgus")) + .expected (StatusCode::FORBIDDEN, "403 Forbidden\n"), + base_case.path_rest ("v1/toast") + .expected (StatusCode::NOT_FOUND, "Unknown API endpoint\n"), + base_case.input_key (None) + .expected (StatusCode::FORBIDDEN, "Can't run scraper without an API key\n"), + ] { + case.test ().await; + } }); } } diff --git a/src/tests.rs b/src/tests.rs index 6a4c28f..ad5a9be 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -179,8 +179,26 @@ fn scraper_endpoints () { .timeout (Duration::from_secs (2)) .build ().expect ("Couldn't build HTTP client"); - let resp = client.get (&format! ("{}/scraper/api/test", relay_url)) - .send ().await.expect ("Couldn't check if relay is up").bytes ().await.expect ("Couldn't check if relay is up"); + let mut resp = None; + for _ in 0usize..5 { + let x = client.get (&format! ("{}/scraper/api/test", relay_url)) + .send ().await; + match x { + Err (_) => { + // Probably a reqwest error cause the port is in + // use or something. Try again. + () + }, + Ok (x) => { + resp = Some (x); + break; + }, + }; + + delay_for (Duration::from_millis (200)).await; + } + let resp = resp.expect ("Reqwest repeatedly failed to connect to the relay"); + let resp = resp.bytes ().await.expect ("Couldn't check if relay is up"); assert_eq! (resp, "You're valid!\n"); From 5d1b68dc90c75300cb2c9f5dfcbc4554417d20c1 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 04:47:47 +0000 Subject: [PATCH 119/208] :scroll: logging: don't log handlebars object --- crates/ptth_relay/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 1ab9c1f..5cf5798 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -327,7 +327,7 @@ async fn handle_server_list ( Ok (ok_reply (s)?) } -#[instrument (level = "trace", skip (req, state))] +#[instrument (level = "trace", skip (req, state, handlebars))] async fn handle_all ( req: Request , state: Arc , From 4c52d88be00aa9fe00f1d892851b4d387e5bac98 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 04:56:43 +0000 Subject: [PATCH 120/208] :pencil: docs: check off todo for scraper API --- issues/2020-12Dec/auth-route-YNQAQKJS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 05e9464..9954d91 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -37,7 +37,7 @@ stronger is ready. - (X) Accept scraper key for some testing endpoint - (X) (POC) Test with curl - (X) Clean up scraper endpoint -- ( ) Add end-to-end tests for scraper endpoint +- (X) Add (almost) end-to-end tests for scraper endpoint - ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads - ( ) Remove scraper key from config file From 78bffc74c325d8241f8c9115e7e68bc35a789b9d Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 13 Dec 2020 05:04:04 +0000 Subject: [PATCH 121/208] :pencil: docs: plan remaining tasks on scraper API --- README.md | 70 ++++++++++++------------ issues/2020-12Dec/auth-route-YNQAQKJS.md | 11 ++++ 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index ebfeec2..be77f24 100644 --- a/README.md +++ b/README.md @@ -4,48 +4,19 @@ An HTTP server that can run behind a firewall by connecting out to a relay. ``` Outside the tunnel -+--------+ +------------+ +-------------+ -| Client | ------> | PTTH relay | <----- | PTTH server | -+--------+ +------------+ +-------------+ ++--------+ +------------+ +-------------+ +| Client | >>> | PTTH relay | <<< | PTTH server | ++--------+ +------------+ +-------------+ Inside the tunnel -+--------+ -------------- +-------------+ -| Client | ----------------------------> | Server | -+--------+ -------------- +-------------+ ++--------+ -------------- +-------------+ +| Client | >>> >>> >>> | Server | ++--------+ -------------- +-------------+ ``` The server can run behind a firewall, because it is actually a special HTTP client. -## Glossary - -(sorted alphabetically) - -- **Backend API** - The HTTP API that ptth_server uses to establish the tunnel. -Noted in the code with the cookie "7ZSFUKGV". -- **Client** - Any client that connects to ptth_relay in order to reach a -destination server. Admins must terminate TLS between -ptth_relay and all clients. -- **Frontend** - The human-friendly, browser-friendly HTTP+HTML interface -that ptth_relay serves directly or relays from ptth_server. -This interface has no auth by default. Admins must provide their own auth -in front of ptth_relay. OAuth2 is recommended. -- **ptth_file_server** - A standalone file server. It uses the same code -as ptth_server, so production environments don't need it. -- **ptth_relay** or **Relay server** - The ptth_relay app. This must run on a server -that can accept incoming HTTP connections. -- **ptth_server** or **Destination server** - The ptth_server app. This should run behind -a firewall. It will connect out to the relay and accept incoming connections -through the PTTH tunnel. -- **Scraper API** - An optional HTTP API for scraper clients to access ptth_relay and -the destination servers using machine-friendly auth. -- **Tripcode** - The base64 hash of a server's private API key. When adding -a new server, the tripcode must be copied to ptth_relay.toml on the relay -server. -- **Tunnel** - The reverse HTTP tunnel between ptth_relay and ptth_server. -ptth_server connects out to ptth_relay, then ptth_relay forwards incoming -connections to ptth_server through the tunnel. - ## Configuration ptth_server: @@ -109,6 +80,35 @@ proxy_request_buffering off; proxy_buffering off; ``` +## Glossary + +(sorted alphabetically) + +- **Backend API** - The HTTP API that ptth_server uses to establish the tunnel. +Noted in the code with the cookie "7ZSFUKGV". +- **Client** - Any client that connects to ptth_relay in order to reach a +destination server. Admins must terminate TLS between +ptth_relay and all clients. +- **Frontend** - The human-friendly, browser-friendly HTTP+HTML interface +that ptth_relay serves directly or relays from ptth_server. +This interface has no auth by default. Admins must provide their own auth +in front of ptth_relay. OAuth2 is recommended. +- **ptth_file_server** - A standalone file server. It uses the same code +as ptth_server, so production environments don't need it. +- **ptth_relay** or **Relay server** - The ptth_relay app. This must run on a server +that can accept incoming HTTP connections. +- **ptth_server** or **Destination server** - The ptth_server app. This should run behind +a firewall. It will connect out to the relay and accept incoming connections +through the PTTH tunnel. +- **Scraper API** - An optional HTTP API for scraper clients to access ptth_relay and +the destination servers using machine-friendly auth. +- **Tripcode** - The base64 hash of a server's private API key. When adding +a new server, the tripcode must be copied to ptth_relay.toml on the relay +server. +- **Tunnel** - The reverse HTTP tunnel between ptth_relay and ptth_server. +ptth_server connects out to ptth_relay, then ptth_relay forwards incoming +connections to ptth_server through the tunnel. + ## Comparison with normal HTTP Normal HTTP: diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 9954d91..e6d4319 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -38,6 +38,7 @@ stronger is ready. - (X) (POC) Test with curl - (X) Clean up scraper endpoint - (X) Add (almost) end-to-end tests for scraper endpoint +- ( ) Add real scraper endpoints - ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads - ( ) Remove scraper key from config file @@ -66,6 +67,16 @@ Design the DB so that the servers can share it one day. Design the API so that new types of auth / keys can be added one day, and the old ones deprecated. +Endpoints needed: + +- Query server list +- Query directory in server +- GET file with byte range (identical to frontend file API) + +These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it. +For compatibility with wget spidering, I _might_ do XML or HTML that's +machine-readable. We'll see. + ## Open questions **Who generates the API key? The scraper client, or the PTTH relay server?** From e865ac56c7c1b561a994defb72636e44ea1bfed0 Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Sun, 13 Dec 2020 20:05:52 -0600 Subject: [PATCH 122/208] :rotating_light: refactor: fix some clippy / cargo check warnings --- crates/ptth_relay/src/relay_state.rs | 4 ---- issues/2020-12Dec/auth-route-YNQAQKJS.md | 6 +++--- src/tests.rs | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/crates/ptth_relay/src/relay_state.rs b/crates/ptth_relay/src/relay_state.rs index f91d76a..d7afcbf 100644 --- a/crates/ptth_relay/src/relay_state.rs +++ b/crates/ptth_relay/src/relay_state.rs @@ -1,13 +1,10 @@ use std::{ collections::HashMap, convert::TryFrom, - path::{PathBuf}, - sync::Arc, }; use chrono::{DateTime, Utc}; use dashmap::DashMap; -use handlebars::Handlebars; use tokio::sync::{ Mutex, RwLock, @@ -20,7 +17,6 @@ use crate::{ Config, RelayError, ShuttingDownError, - load_templates, }; use ptth_core::http_serde; diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index e6d4319..bfff13d 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -69,9 +69,9 @@ the old ones deprecated. Endpoints needed: -- Query server list -- Query directory in server -- GET file with byte range (identical to frontend file API) +- ( ) Query server list +- ( ) Query directory in server +- ( ) GET file with byte range (identical to frontend file API) These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it. For compatibility with wget spidering, I _might_ do XML or HTML that's diff --git a/src/tests.rs b/src/tests.rs index ad5a9be..b3cf8a8 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -187,7 +187,6 @@ fn scraper_endpoints () { Err (_) => { // Probably a reqwest error cause the port is in // use or something. Try again. - () }, Ok (x) => { resp = Some (x); From fa5aa8b05a756512d6e278f6b473f93e402021be Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Mon, 14 Dec 2020 01:07:13 -0600 Subject: [PATCH 123/208] :construction: wip: add server list API endpoint --- Cargo.lock | 1 + crates/ptth_relay/Cargo.toml | 1 + crates/ptth_relay/src/config.rs | 2 +- crates/ptth_relay/src/lib.rs | 69 ++++++------------- crates/ptth_relay/src/scraper_api.rs | 91 +++++++++++++++++++++++-- handlebars/relay/relay_server_list.html | 2 +- 6 files changed, 112 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edeec04..36c91c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1237,6 +1237,7 @@ dependencies = [ "ptth_core", "rmp-serde", "serde", + "serde_json", "thiserror", "tokio", "toml", diff --git a/crates/ptth_relay/Cargo.toml b/crates/ptth_relay/Cargo.toml index 58d03e3..d41d751 100644 --- a/crates/ptth_relay/Cargo.toml +++ b/crates/ptth_relay/Cargo.toml @@ -19,6 +19,7 @@ hyper = "0.13.8" itertools = "0.9.0" rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} +serde_json = "1.0.60" thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } toml = "0.5.7" diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index d152293..3877f47 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -36,7 +36,7 @@ pub mod file { #[derive (Default, Deserialize)] pub struct Isomorphic { #[serde (default)] - pub enable_scraper_auth: bool, + pub enable_scraper_api: bool, // If any of these fields are used, we are in dev mode and have to // show extra warnings, since some auth may be weakened diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index 5cf5798..e74799a 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -15,9 +15,7 @@ #![allow (clippy::mut_mut)] use std::{ - borrow::Cow, - collections::HashMap, - iter::FromIterator, + borrow::Cow, net::SocketAddr, path::{Path, PathBuf}, sync::Arc, @@ -69,7 +67,6 @@ pub use errors::*; pub use relay_state::RelayState; use relay_state::*; -use scraper_api::*; fn ok_reply > (b: B) -> Result , http::Error> @@ -241,7 +238,7 @@ fn pretty_print_last_seen ( #[derive (Serialize)] struct ServerEntry <'a> { - id: String, + name: String, display_name: String, last_seen: Cow <'a, str>, } @@ -256,62 +253,40 @@ struct ServerListPage <'a> { async fn handle_server_list_internal (state: &Arc ) -> ServerListPage <'static> { - let dev_mode; - let display_names: HashMap = { - let guard = state.config.read ().await; - - dev_mode = guard.iso.dev_mode.is_some (); - let servers = (*guard).servers.iter () - .map (|(k, v)| { - let display_name = v.display_name - .clone () - .unwrap_or_else (|| k.clone ()); - - (k.clone (), display_name) - }); - - HashMap::from_iter (servers) - }; + use LastSeen::*; - let server_statuses = { - let guard = state.server_status.lock ().await; - (*guard).clone () + let dev_mode = { + let guard = state.config.read ().await; + guard.iso.dev_mode.is_some () }; + let git_version = git_version::read_git_version ().await; + + let server_list = scraper_api::v1_server_list (&state).await; let now = Utc::now (); - let mut servers: Vec <_> = display_names.into_iter () - .map (|(id, display_name)| { - use LastSeen::*; - - let status = match server_statuses.get (&id) { - None => return ServerEntry { - display_name, - id, - last_seen: "Never".into (), + let servers = server_list.servers.into_iter () + .map (|x| { + let last_seen = match x.last_seen { + None => "Never".into (), + Some (x) => match pretty_print_last_seen (now, x) { + Negative => "Error (negative time)".into (), + Connected => "Connected".into (), + Description (s) => s.into (), }, - Some (x) => x, - }; - - let last_seen = match pretty_print_last_seen (now, status.last_seen) { - Negative => "Error (negative time)".into (), - Connected => "Connected".into (), - Description (s) => s.into (), }; ServerEntry { - display_name, - id, + name: x.name, + display_name: x.display_name, last_seen, } }) .collect (); - servers.sort_by (|a, b| a.display_name.cmp (&b.display_name)); - ServerListPage { dev_mode, - git_version: git_version::read_git_version ().await, + git_version, servers, } } @@ -388,7 +363,7 @@ async fn handle_all ( Err (RequestError::Mysterious) } else if let Some (rest) = prefix_match ("/scraper/", &path) { - handle_scraper_api (req, state, rest).await + scraper_api::handle (req, state, rest).await } else { Ok (error_reply (StatusCode::OK, "Hi")?) @@ -423,7 +398,7 @@ async fn reload_config ( (*config) = new_config; debug! ("Loaded {} server configs", config.servers.len ()); - debug! ("enable_scraper_auth: {}", config.iso.enable_scraper_auth); + debug! ("enable_scraper_api: {}", config.iso.enable_scraper_api); if config.iso.dev_mode.is_some () { error! ("Dev mode is enabled! This might turn off some security features. If you see this in production, escalate it to someone!"); diff --git a/crates/ptth_relay/src/scraper_api.rs b/crates/ptth_relay/src/scraper_api.rs index d642cbb..81de53e 100644 --- a/crates/ptth_relay/src/scraper_api.rs +++ b/crates/ptth_relay/src/scraper_api.rs @@ -1,13 +1,20 @@ use std::{ + collections::HashMap, + iter::FromIterator, sync::Arc, }; +use chrono::{DateTime, Utc}; use hyper::{ Body, Request, Response, StatusCode, }; +use serde::{ + Serialize, + Serializer, +}; use tracing::{ error, instrument, @@ -21,8 +28,78 @@ use crate::{ relay_state::RelayState, }; +// JSON is probably Good Enough For Now, so I'll just make everything +// a struct and lazily serialize it right before leaving the +// top-level handle () fn. + +fn serialize_last_seen (x: &Option >, s: S) +-> Result +{ + match x { + None => s.serialize_none (), + Some (x) => s.serialize_str (&x.to_rfc3339 ()), + } +} + +#[derive (Serialize)] +pub struct Server { + pub name: String, + pub display_name: String, + #[serde (serialize_with = "serialize_last_seen")] + pub last_seen: Option >, +} + +#[derive (Serialize)] +pub struct ServerList { + pub servers: Vec , +} + +pub async fn v1_server_list (state: &Arc ) +-> ServerList +{ + // name --> display_name + let display_names: HashMap = { + let guard = state.config.read ().await; + + let servers = (*guard).servers.iter () + .map (|(k, v)| { + let display_name = v.display_name + .clone () + .unwrap_or_else (|| k.clone ()); + + (k.clone (), display_name) + }); + + HashMap::from_iter (servers) + }; + + // name --> status + let server_statuses = { + let guard = state.server_status.lock ().await; + (*guard).clone () + }; + + let mut servers: Vec <_> = display_names.into_iter () + .map (|(name, display_name)| { + let last_seen = server_statuses.get (&name).map (|x| x.last_seen); + + Server { + display_name, + name, + last_seen, + } + }) + .collect (); + + servers.sort_by (|a, b| a.display_name.cmp (&b.display_name)); + + ServerList { + servers, + } +} + #[instrument (level = "trace", skip (req, state))] -pub async fn handle_scraper_api_v1 ( +async fn api_v1 ( req: Request , state: Arc , path_rest: &str @@ -69,13 +146,17 @@ pub async fn handle_scraper_api_v1 ( if path_rest == "test" { Ok (error_reply (StatusCode::OK, "You're valid!")?) } + else if path_rest == "server_list" { + let x = v1_server_list (&state).await; + Ok (error_reply (StatusCode::OK, &serde_json::to_string (&x).unwrap ())?) + } else { Ok (error_reply (StatusCode::NOT_FOUND, "Unknown API endpoint")?) } } #[instrument (level = "trace", skip (req, state))] -pub async fn handle_scraper_api ( +pub async fn handle ( req: Request , state: Arc , path_rest: &str @@ -83,16 +164,16 @@ pub async fn handle_scraper_api ( -> Result , RequestError> { { - if ! state.config.read ().await.iso.enable_scraper_auth { + if ! state.config.read ().await.iso.enable_scraper_api { return Ok (error_reply (StatusCode::FORBIDDEN, "Scraper API disabled")?); } } if let Some (rest) = prefix_match ("v1/", path_rest) { - handle_scraper_api_v1 (req, state, rest).await + api_v1 (req, state, rest).await } else if let Some (rest) = prefix_match ("api/", path_rest) { - handle_scraper_api_v1 (req, state, rest).await + api_v1 (req, state, rest).await } else { Ok (error_reply (StatusCode::NOT_FOUND, "Unknown scraper API version")?) diff --git a/handlebars/relay/relay_server_list.html b/handlebars/relay/relay_server_list.html index 2075d61..5e8a621 100644 --- a/handlebars/relay/relay_server_list.html +++ b/handlebars/relay/relay_server_list.html @@ -51,7 +51,7 @@ {{#each servers}} - {{this.display_name}} + {{this.display_name}} {{this.last_seen}} {{/each}} From cdc890ad895ab0dc4067ab4e16521010d2f49a4e Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Mon, 14 Dec 2020 01:08:00 -0600 Subject: [PATCH 124/208] :pencil: docs: update scraper auth todo --- issues/2020-12Dec/auth-route-YNQAQKJS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index bfff13d..18b855b 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -38,6 +38,8 @@ stronger is ready. - (X) (POC) Test with curl - (X) Clean up scraper endpoint - (X) Add (almost) end-to-end tests for scraper endpoint +- ( ) Add tests for scraper endpoints +- ( ) Factor v1 API into v1 module - ( ) Add real scraper endpoints - ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads @@ -69,7 +71,7 @@ the old ones deprecated. Endpoints needed: -- ( ) Query server list +- (X) Query server list - ( ) Query directory in server - ( ) GET file with byte range (identical to frontend file API) From 11f4b0e65b58d0cc715fca9aa4108f18e0a137c2 Mon Sep 17 00:00:00 2001 From: _ <> Date: Mon, 14 Dec 2020 14:17:52 +0000 Subject: [PATCH 125/208] :white_check_mark: test: Fix tests broken by recent commits --- crates/ptth_relay/src/scraper_api.rs | 4 ++-- src/tests.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/ptth_relay/src/scraper_api.rs b/crates/ptth_relay/src/scraper_api.rs index 81de53e..32ba64e 100644 --- a/crates/ptth_relay/src/scraper_api.rs +++ b/crates/ptth_relay/src/scraper_api.rs @@ -263,7 +263,7 @@ mod tests { port: Some (4000), servers: vec! [], iso: config::file::Isomorphic { - enable_scraper_auth: true, + enable_scraper_api: true, dev_mode: self.valid_key.map (|key| config::file::DevMode { scraper_key: Some (key_validity::ScraperKey::new (key.as_bytes ())), }), @@ -274,7 +274,7 @@ mod tests { let relay_state = Arc::new (RelayState::try_from (config).expect ("Can't create relay state")); - let actual = handle_scraper_api (input, relay_state, self.path_rest).await; + let actual = super::handle (input, relay_state, self.path_rest).await; let actual = actual.expect ("Relay didn't respond"); let (actual_head, actual_body) = actual.into_parts (); diff --git a/src/tests.rs b/src/tests.rs index b3cf8a8..dabe3d5 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -148,7 +148,7 @@ fn scraper_endpoints () { ], iso: config::file::Isomorphic { - enable_scraper_auth: true, + enable_scraper_api: true, dev_mode: Some (config::file::DevMode { scraper_key: Some (key_validity::ScraperKey::new (b"bogus")), }), From cda627fa4bc1ff493aaec115675aaaa73b93c799 Mon Sep 17 00:00:00 2001 From: _ <> Date: Tue, 15 Dec 2020 05:15:17 +0000 Subject: [PATCH 126/208] :star: new: add JSON API in server for dir listings --- Cargo.lock | 1 + crates/ptth_core/src/lib.rs | 17 +++ crates/ptth_file_server_bin/src/main.rs | 3 +- crates/ptth_server/Cargo.toml | 1 + .../ptth_server/src/file_server/internal.rs | 88 +++++++++++-- crates/ptth_server/src/file_server/mod.rs | 99 +++++++++++--- crates/ptth_server/src/file_server/tests.rs | 2 +- crates/ptth_server/src/lib.rs | 122 ++++++++++-------- issues/2020-12Dec/auth-route-YNQAQKJS.md | 13 +- 9 files changed, 249 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 36c91c3..0d49e91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1269,6 +1269,7 @@ dependencies = [ "reqwest", "rmp-serde", "serde", + "serde_json", "structopt", "thiserror", "tokio", diff --git a/crates/ptth_core/src/lib.rs b/crates/ptth_core/src/lib.rs index 69bd7d6..4d2440a 100644 --- a/crates/ptth_core/src/lib.rs +++ b/crates/ptth_core/src/lib.rs @@ -24,3 +24,20 @@ pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str> None } } + +#[cfg (test)] +mod tests { + use super::*; + #[test] + fn prefix () { + for (p, h, expected) in &[ + ("/files/", "/files/a", Some ("a")), + ("/files/", "/files/abc/def", Some ("abc/def")), + ("/files/", "/files", None), + ("/files/", "/not_files", None), + ("/files/", "/files/", Some ("")), + ] { + assert_eq! (prefix_match (*p, *h), *expected); + } + } +} diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index cec4bb2..33acbd7 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -44,6 +44,7 @@ async fn handle_all (req: Request , state: Arc >) -> Result , anyhow::Error> { use std::str::FromStr; + use hyper::header::HeaderName; debug! ("req.uri () = {:?}", req.uri ()); @@ -74,7 +75,7 @@ async fn handle_all (req: Request , state: Arc >) .status (StatusCode::from (ptth_resp.parts.status_code)); for (k, v) in ptth_resp.parts.headers { - resp = resp.header (hyper::header::HeaderName::from_str (&k)?, v); + resp = resp.header (HeaderName::from_str (&k)?, v); } let body = ptth_resp.body.map_or_else (Body::empty, Body::wrap_stream); diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index 0095a55..81345a8 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -22,6 +22,7 @@ regex = "1.4.1" reqwest = { version = "0.10.8", features = ["stream"] } rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} +serde_json = "1.0.60" structopt = "0.3.20" thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } diff --git a/crates/ptth_server/src/file_server/internal.rs b/crates/ptth_server/src/file_server/internal.rs index 695c0b3..9b5080a 100644 --- a/crates/ptth_server/src/file_server/internal.rs +++ b/crates/ptth_server/src/file_server/internal.rs @@ -42,10 +42,17 @@ use super::{ range, }; +#[derive (Debug, PartialEq)] +pub enum OutputFormat { + Json, + Html, +} + #[derive (Debug, PartialEq)] pub struct ServeDirParams { pub path: PathBuf, pub dir: AlwaysEqual , + pub format: OutputFormat, } #[derive (Debug, PartialEq)] @@ -59,11 +66,12 @@ pub struct ServeFileParams { pub enum Response { Favicon, Forbidden, - InvalidQuery, MethodNotAllowed, NotFound, RangeNotSatisfiable (u64), Redirect (String), + InvalidQuery, + Root, ServeDir (ServeDirParams), ServeFile (ServeFileParams), @@ -77,7 +85,8 @@ fn serve_dir ( path: &Path, dir: tokio::fs::ReadDir, full_path: PathBuf, - uri: &http::Uri + uri: &http::Uri, + format: OutputFormat ) -> Result { @@ -98,6 +107,7 @@ fn serve_dir ( Ok (Response::ServeDir (ServeDirParams { dir, path: full_path, + format, })) } @@ -161,6 +171,53 @@ async fn serve_file ( }) } +async fn serve_api ( + root: &Path, + uri: &http::Uri, + hidden_path: Option <&Path>, + path: &str +) +-> Result +{ + use Response::*; + + match prefix_match ("/v1/dir/", path) { + None => (), + Some (path) => { + let encoded_path = &path [0..]; + + let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; + let path = Path::new (&*path_s); + + let full_path = root.join (path); + + debug! ("full_path = {:?}", full_path); + + if let Some (hidden_path) = hidden_path { + if full_path == hidden_path { + return Ok (Forbidden); + } + } + + return if let Ok (dir) = read_dir (&full_path).await { + serve_dir ( + &path_s, + path, + dir, + full_path, + &uri, + OutputFormat::Json + ) + } + else { + Ok (NotFound) + }; + }, + }; + + Ok (NotFound) +} + pub async fn serve_all ( root: &Path, method: Method, @@ -186,22 +243,28 @@ pub async fn serve_all ( } }; - if uri.path () == "/favicon.ico" { + let path = uri.path (); + + if path == "/favicon.ico" { return Ok (Favicon); } - let path = match prefix_match ("/files", uri.path ()) { - Some (x) => x, - None => return Ok (Root), - }; - - if path == "" { - return Ok (Redirect ("files/".to_string ())); + if path == "/" { + return Ok (Root); } + if let Some (path) = prefix_match ("/api", path) { + return serve_api (root, &uri, hidden_path, path).await; + } + + let path = match prefix_match ("/files/", path) { + Some (x) => x, + None => return Ok (NotFound), + }; + // TODO: There is totally a dir traversal attack in here somewhere - let encoded_path = &path [1..]; + let encoded_path = &path [0..]; let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; let path = Path::new (&*path_s); @@ -222,7 +285,8 @@ pub async fn serve_all ( path, dir, full_path, - &uri + &uri, + OutputFormat::Html ) } else if let Ok (file) = File::open (&full_path).await { diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 4ff2678..40d9d84 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -58,7 +58,19 @@ pub struct ServerInfo { } #[derive (Serialize)] -struct TemplateDirEntry { +struct DirEntryJson { + name: String, + size: u64, + is_dir: bool, +} + +#[derive (Serialize)] +struct DirJson { + entries: Vec , +} + +#[derive (Serialize)] +struct DirEntryHtml { icon: &'static str, trailing_slash: &'static str, @@ -79,12 +91,12 @@ struct TemplateDirEntry { } #[derive (Serialize)] -struct TemplateDirPage <'a> { +struct DirHtml <'a> { #[serde (flatten)] server_info: &'a ServerInfo, path: Cow <'a, str>, - entries: Vec , + entries: Vec , } fn get_icon (file_name: &str) -> &'static str { @@ -109,7 +121,7 @@ fn get_icon (file_name: &str) -> &'static str { } } -async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry +async fn read_dir_entry_html (entry: DirEntry) -> DirEntryHtml { use percent_encoding::{ CONTROLS, @@ -118,7 +130,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let file_name = match entry.file_name ().into_string () { Ok (x) => x, - Err (_) => return TemplateDirEntry { + Err (_) => return DirEntryHtml { icon: emoji::ERROR, trailing_slash: "", file_name: "File / directory name is not UTF-8".into (), @@ -130,7 +142,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let metadata = match entry.metadata ().await { Ok (x) => x, - Err (_) => return TemplateDirEntry { + Err (_) => return DirEntryHtml { icon: emoji::ERROR, trailing_slash: "", file_name: "Could not fetch metadata".into (), @@ -153,7 +165,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string (); - TemplateDirEntry { + DirEntryHtml { icon, trailing_slash: &trailing_slash, file_name, @@ -163,6 +175,20 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry } } +async fn read_dir_entry_json (entry: DirEntry) -> Option +{ + let name = entry.file_name ().into_string ().ok ()?; + let metadata = entry.metadata ().await.ok ()?; + let is_dir = metadata.is_dir (); + let size = metadata.len (); + + Some (DirEntryJson { + name, + size, + is_dir, + }) +} + async fn serve_root ( handlebars: &Handlebars <'static>, server_info: &ServerInfo @@ -182,8 +208,33 @@ fn serve_html (s: String) -> Response { resp } +async fn serve_dir_json ( + mut dir: ReadDir +) -> Result +{ + let mut entries = vec! []; + + while let Ok (Some (entry)) = dir.next_entry ().await { + if let Some (entry) = read_dir_entry_json (entry).await { + entries.push (entry); + } + } + + entries.sort_unstable_by (|a, b| a.name.cmp (&b.name)); + + let dir = DirJson { + entries, + }; + + let mut response = Response::default (); + response.header ("content-type".to_string (), "application/json; charset=UTF-8".to_string ().into_bytes ()); + response.body_bytes (serde_json::to_string (&dir).unwrap ().into_bytes ()); + + Ok (response) +} + #[instrument (level = "debug", skip (handlebars, dir))] -async fn serve_dir ( +async fn serve_dir_html ( handlebars: &Handlebars <'static>, server_info: &ServerInfo, path: Cow <'_, str>, @@ -193,12 +244,12 @@ async fn serve_dir ( let mut entries = vec! []; while let Ok (Some (entry)) = dir.next_entry ().await { - entries.push (read_dir_entry (entry).await); + entries.push (read_dir_entry_html (entry).await); } entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name)); - let s = handlebars.render ("file_server_dir", &TemplateDirPage { + let s = handlebars.render ("file_server_dir", &DirHtml { path, entries, server_info, @@ -316,7 +367,10 @@ pub async fn serve_all ( ) -> Result { - use internal::Response::*; + use internal::{ + OutputFormat, + Response::*, + }; fn serve_error >> ( status_code: StatusCode, @@ -331,11 +385,10 @@ pub async fn serve_all ( } Ok (match internal::serve_all (root, method, uri, headers, hidden_path).await? { - Favicon => serve_error (StatusCode::NotFound, ""), - Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), - InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object"), - MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method"), - NotFound => serve_error (StatusCode::NotFound, "404 Not Found"), + Favicon => serve_error (StatusCode::NotFound, "Not found\n"), + Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden\n"), + MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method\n"), + NotFound => serve_error (StatusCode::NotFound, "404 Not Found\nAre you missing a trailing slash?\n"), RangeNotSatisfiable (file_len) => { let mut resp = Response::default (); resp.status_code (StatusCode::RangeNotSatisfiable) @@ -344,16 +397,22 @@ pub async fn serve_all ( }, Redirect (location) => { let mut resp = Response::default (); - resp.status_code (StatusCode::TemporaryRedirect); - resp.header ("location".to_string (), location.into_bytes ()); - resp.body_bytes (b"Redirecting...".to_vec ()); + resp.status_code (StatusCode::TemporaryRedirect) + .header ("location".to_string (), location.into_bytes ()); + resp.body_bytes (b"Redirecting...\n".to_vec ()); resp }, + InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"), + Root => serve_root (handlebars, server_info).await?, ServeDir (internal::ServeDirParams { path, dir, - }) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?, + format + }) => match format { + OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?, + OutputFormat::Html => serve_dir_html (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?, + }, ServeFile (internal::ServeFileParams { file, send_body, diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index 0b78385..9f096be 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -101,7 +101,7 @@ fn file_server () { for (uri_path, expected) in vec! [ ("/", Root), - ("/files", Redirect ("files/".to_string ())), + ("/files", NotFound), ("/files/?", InvalidQuery), ("/files/src", Redirect ("src/".to_string ())), ("/files/src/?", InvalidQuery), diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index ce63528..bccc824 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -15,9 +15,6 @@ use std::{ use futures::FutureExt; use handlebars::Handlebars; -use http::status::{ - StatusCode, -}; use reqwest::Client; use serde::Deserialize; use tokio::{ @@ -59,10 +56,70 @@ struct ServerState { hidden_path: Option , } -async fn handle_req_resp <'a> ( +async fn handle_one_req ( + state: &Arc , + wrapped_req: http_serde::WrappedRequest +) -> Result <(), ServerError> +{ + let (req_id, parts) = (wrapped_req.id, wrapped_req.req); + + debug! ("Handling request {}", req_id); + + let default_root = PathBuf::from ("./"); + let file_server_root: &std::path::Path = state.config.file_server_root + .as_ref () + .unwrap_or (&default_root); + + let response = file_server::serve_all ( + &state.handlebars, + &state.server_info, + file_server_root, + parts.method, + &parts.uri, + &parts.headers, + state.hidden_path.as_deref () + ).await?; + + let mut resp_req = state.client + .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) + .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?)); + + if let Some (length) = response.content_length { + resp_req = resp_req.header ("Content-Length", length.to_string ()); + } + if let Some (body) = response.body { + resp_req = resp_req.body (reqwest::Body::wrap_stream (body)); + } + + let req = resp_req.build ().map_err (ServerError::Step5Responding)?; + + debug! ("{:?}", req.headers ()); + + //println! ("Step 6"); + match state.client.execute (req).await { + Ok (r) => { + let status = r.status (); + let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?; + debug! ("{:?} {:?}", status, text); + }, + Err (e) => { + if e.is_request () { + warn! ("Error while POSTing response. Client probably hung up."); + } + else { + error! ("Err: {:?}", e); + } + }, + } + + Ok::<(), ServerError> (()) +} + +async fn handle_req_resp ( state: &Arc , req_resp: reqwest::Response -) -> Result <(), ServerError> { +) -> Result <(), ServerError> +{ //println! ("Step 1"); let body = req_resp.bytes ().await.map_err (ServerError::CantCollectWrappedRequests)?; @@ -83,58 +140,7 @@ async fn handle_req_resp <'a> ( // These have to detach, so we won't be able to catch the join errors. tokio::spawn (async move { - let (req_id, parts) = (wrapped_req.id, wrapped_req.req); - - debug! ("Handling request {}", req_id); - - let default_root = PathBuf::from ("./"); - let file_server_root: &std::path::Path = state.config.file_server_root - .as_ref () - .unwrap_or (&default_root); - - let response = file_server::serve_all ( - &state.handlebars, - &state.server_info, - file_server_root, - parts.method, - &parts.uri, - &parts.headers, - state.hidden_path.as_deref () - ).await?; - - let mut resp_req = state.client - .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) - .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?)); - - if let Some (length) = response.content_length { - resp_req = resp_req.header ("Content-Length", length.to_string ()); - } - if let Some (body) = response.body { - resp_req = resp_req.body (reqwest::Body::wrap_stream (body)); - } - - let req = resp_req.build ().map_err (ServerError::Step5Responding)?; - - debug! ("{:?}", req.headers ()); - - //println! ("Step 6"); - match state.client.execute (req).await { - Ok (r) => { - let status = r.status (); - let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?; - debug! ("{:?} {:?}", status, text); - }, - Err (e) => { - if e.is_request () { - warn! ("Error while POSTing response. Client probably hung up."); - } - else { - error! ("Err: {:?}", e); - } - }, - } - - Ok::<(), ServerError> (()) + handle_one_req (&state, wrapped_req).await }); } @@ -172,6 +178,8 @@ pub async fn run_server ( { use std::convert::TryInto; + use http::status::StatusCode; + let asset_root = asset_root.unwrap_or_else (PathBuf::new); if password_is_bad (config_file.api_key.clone ()) { diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 18b855b..e161f08 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -37,10 +37,11 @@ stronger is ready. - (X) Accept scraper key for some testing endpoint - (X) (POC) Test with curl - (X) Clean up scraper endpoint -- (X) Add (almost) end-to-end tests for scraper endpoint -- ( ) Add tests for scraper endpoints -- ( ) Factor v1 API into v1 module -- ( ) Add real scraper endpoints +- (X) Add (almost) end-to-end tests for test scraper endpoint +- ( ) Thread server endpoints through relay scraper auth +- ( ) Add tests for other scraper endpoints +- (don't care) Factor v1 API into v1 module +- (X) Add real scraper endpoints - ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads - ( ) Remove scraper key from config file @@ -72,8 +73,8 @@ the old ones deprecated. Endpoints needed: - (X) Query server list -- ( ) Query directory in server -- ( ) GET file with byte range (identical to frontend file API) +- (X) Query directory in server +- (not needed) GET file with byte range (identical to frontend file API) These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it. For compatibility with wget spidering, I _might_ do XML or HTML that's From 9ac44cfeb7a938ad6a8df5a002a6d4db53f87b36 Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 16 Dec 2020 14:46:03 +0000 Subject: [PATCH 127/208] :star: new: finish MVP for scraper auth. Adding a SQLite DB to properly track the keys is going to take a while. For now I'll just keep them in the config file and give them 30-day expirations. --- crates/ptth_relay/src/config.rs | 29 +++++++-- crates/ptth_relay/src/key_validity.rs | 41 +++++++----- crates/ptth_relay/src/lib.rs | 2 + crates/ptth_relay/src/scraper_api.rs | 80 ++++++++++++++++-------- issues/2020-12Dec/auth-route-YNQAQKJS.md | 50 ++++++++++++++- src/tests.rs | 16 ++--- 6 files changed, 161 insertions(+), 57 deletions(-) diff --git a/crates/ptth_relay/src/config.rs b/crates/ptth_relay/src/config.rs index 3877f47..d444081 100644 --- a/crates/ptth_relay/src/config.rs +++ b/crates/ptth_relay/src/config.rs @@ -8,7 +8,10 @@ use std::{ path::Path, }; -use crate::errors::ConfigError; +use crate::{ + errors::ConfigError, + key_validity::*, +}; // Stuff we need to load from the config file and use to // set up the HTTP server @@ -28,7 +31,7 @@ pub mod file { #[derive (Deserialize)] pub struct DevMode { - pub scraper_key: Option >, + } // Stuff that's identical between the file and the runtime structures @@ -45,19 +48,25 @@ pub mod file { #[derive (Deserialize)] pub struct Config { - pub port: Option , - pub servers: Vec , #[serde (flatten)] pub iso: Isomorphic, + + pub port: Option , + pub servers: Vec , + + // Adding a DB will take a while, so I'm moving these out of dev mode. + pub scraper_keys: Vec >, } } // Stuff we actually need at runtime pub struct Config { + pub iso: file::Isomorphic, + pub port: Option , pub servers: HashMap , - pub iso: file::Isomorphic, + pub scraper_keys: HashMap >, } impl TryFrom for Config { @@ -69,10 +78,18 @@ impl TryFrom for Config { let servers = itertools::process_results (servers, |i| HashMap::from_iter (i))?; + let scraper_keys = if f.iso.enable_scraper_api { + HashMap::from_iter (f.scraper_keys.into_iter ().map (|key| (key.hash.encode_base64 (), key))) + } + else { + Default::default () + }; + Ok (Self { + iso: f.iso, port: f.port, servers, - iso: f.iso, + scraper_keys, }) } } diff --git a/crates/ptth_relay/src/key_validity.rs b/crates/ptth_relay/src/key_validity.rs index c75603e..dbb3704 100644 --- a/crates/ptth_relay/src/key_validity.rs +++ b/crates/ptth_relay/src/key_validity.rs @@ -69,7 +69,7 @@ impl <'de> Deserialize <'de> for BlakeHashWrapper { } pub struct Valid7Days; -//pub struct Valid30Days; +pub struct Valid30Days; //pub struct Valid90Days; pub trait MaxValidDuration { @@ -82,11 +82,19 @@ impl MaxValidDuration for Valid7Days { } } +impl MaxValidDuration for Valid30Days { + fn dur () -> Duration { + Duration::days (30) + } +} + #[derive (Deserialize)] pub struct ScraperKey { + name: String, + not_before: DateTime , not_after: DateTime , - hash: BlakeHashWrapper, + pub hash: BlakeHashWrapper, #[serde (default)] _phantom: std::marker::PhantomData , @@ -103,11 +111,12 @@ pub enum KeyValidity { DurationNegative, } -impl ScraperKey { - pub fn new (input: &[u8]) -> Self { +impl ScraperKey { + pub fn new_30_day > (name: S, input: &[u8]) -> Self { let now = Utc::now (); Self { + name: name.into (), not_before: now, not_after: now + Duration::days (7), hash: BlakeHashWrapper::from_key (input), @@ -161,7 +170,8 @@ mod tests { fn duration_negative () { let zero_time = Utc::now (); - let key = ScraperKey:: { + let key = ScraperKey:: { + name: "automated testing".to_string (), not_before: zero_time + Duration::days (1 + 2), not_after: zero_time + Duration::days (1), hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()), @@ -183,14 +193,15 @@ mod tests { fn key_valid_too_long () { let zero_time = Utc::now (); - let key = ScraperKey:: { + let key = ScraperKey:: { + name: "automated testing".to_string (), not_before: zero_time + Duration::days (1), - not_after: zero_time + Duration::days (1 + 8), + not_after: zero_time + Duration::days (1 + 31), hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()), _phantom: Default::default (), }; - let err = DurationTooLong (Duration::days (7)); + let err = DurationTooLong (Duration::days (30)); for (input, expected) in &[ (zero_time + Duration::days (0), err), @@ -205,9 +216,10 @@ mod tests { fn normal_key () { let zero_time = Utc::now (); - let key = ScraperKey:: { + let key = ScraperKey:: { + name: "automated testing".to_string (), not_before: zero_time + Duration::days (1), - not_after: zero_time + Duration::days (1 + 7), + not_after: zero_time + Duration::days (1 + 30), hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()), _phantom: Default::default (), }; @@ -215,7 +227,7 @@ mod tests { for (input, expected) in &[ (zero_time + Duration::days (0), ClockIsBehind), (zero_time + Duration::days (2), Valid), - (zero_time + Duration::days (1 + 7), Expired), + (zero_time + Duration::days (1 + 30), Expired), (zero_time + Duration::days (100), Expired), ] { assert_eq! (key.is_valid (*input, "bad_password".as_bytes ()), *expected); @@ -226,9 +238,10 @@ mod tests { fn wrong_key () { let zero_time = Utc::now (); - let key = ScraperKey:: { + let key = ScraperKey:: { + name: "automated testing".to_string (), not_before: zero_time + Duration::days (1), - not_after: zero_time + Duration::days (1 + 7), + not_after: zero_time + Duration::days (1 + 30), hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()), _phantom: Default::default (), }; @@ -236,7 +249,7 @@ mod tests { for input in &[ zero_time + Duration::days (0), zero_time + Duration::days (2), - zero_time + Duration::days (1 + 7), + zero_time + Duration::days (1 + 30), zero_time + Duration::days (100), ] { let validity = key.is_valid (*input, "badder_password".as_bytes ()); diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index e74799a..f3c0057 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -338,6 +338,8 @@ async fn handle_all ( server_endpoint::handle_listen (state, listen_code.into (), api_key.as_bytes ()).await } else if let Some (rest) = prefix_match ("/frontend/servers/", &path) { + // DRY T4H76LB3 + if rest == "" { Ok (handle_server_list (state, handlebars).await?) } diff --git a/crates/ptth_relay/src/scraper_api.rs b/crates/ptth_relay/src/scraper_api.rs index 32ba64e..ab391fc 100644 --- a/crates/ptth_relay/src/scraper_api.rs +++ b/crates/ptth_relay/src/scraper_api.rs @@ -23,11 +23,21 @@ use tracing::{ use crate::{ RequestError, error_reply, - key_validity::KeyValidity, + key_validity::*, prefix_match, relay_state::RelayState, }; +// Not sure if this is the best way to do a hard-coded string table, but +// it'll keep the tests up to date + +mod strings { + pub const FORBIDDEN: &str = "403 Forbidden"; + pub const NO_API_KEY: &str = "Can't auth as scraper without API key"; + pub const UNKNOWN_API_VERSION: &str = "Unknown scraper API version"; + pub const UNKNOWN_API_ENDPOINT: &str = "Unknown scraper API endpoint"; +} + // JSON is probably Good Enough For Now, so I'll just make everything // a struct and lazily serialize it right before leaving the // top-level handle () fn. @@ -109,27 +119,25 @@ async fn api_v1 ( let api_key = req.headers ().get ("X-ApiKey"); let api_key = match api_key { - None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run scraper without an API key")?), + None => return Ok (error_reply (StatusCode::FORBIDDEN, strings::NO_API_KEY)?), Some (x) => x, }; - let bad_key = || error_reply (StatusCode::FORBIDDEN, "403 Forbidden"); + let actual_hash = BlakeHashWrapper::from_key (api_key.as_bytes ()); + + let bad_key = || error_reply (StatusCode::FORBIDDEN, strings::FORBIDDEN); { let config = state.config.read ().await; - let dev_mode = match &config.iso.dev_mode { - None => return Ok (bad_key ()?), + let expected_key = match config.scraper_keys.get (&actual_hash.encode_base64 ()) { Some (x) => x, - }; - - let expected_key = match &dev_mode.scraper_key { None => return Ok (bad_key ()?), - Some (x) => x, }; let now = chrono::Utc::now (); + // The hash in is_valid is redundant match expected_key.is_valid (now, api_key.as_bytes ()) { KeyValidity::Valid => (), KeyValidity::WrongKey (bad_hash) => { @@ -150,8 +158,25 @@ async fn api_v1 ( let x = v1_server_list (&state).await; Ok (error_reply (StatusCode::OK, &serde_json::to_string (&x).unwrap ())?) } + else if let Some (rest) = prefix_match ("server/", path_rest) { + // DRY T4H76LB3 + + if let Some (idx) = rest.find ('/') { + let listen_code = String::from (&rest [0..idx]); + let path = String::from (&rest [idx..]); + let (parts, _) = req.into_parts (); + + // This is ugly. I don't like having scraper_api know about the + // crate root. + + Ok (crate::handle_http_request (parts, path, state, listen_code).await?) + } + else { + Ok (error_reply (StatusCode::BAD_REQUEST, "Bad URI format")?) + } + } else { - Ok (error_reply (StatusCode::NOT_FOUND, "Unknown API endpoint")?) + Ok (error_reply (StatusCode::NOT_FOUND, strings::UNKNOWN_API_ENDPOINT)?) } } @@ -176,7 +201,7 @@ pub async fn handle ( api_v1 (req, state, rest).await } else { - Ok (error_reply (StatusCode::NOT_FOUND, "Unknown scraper API version")?) + Ok (error_reply (StatusCode::NOT_FOUND, strings::UNKNOWN_API_VERSION)?) } } @@ -203,7 +228,7 @@ mod tests { // Expected expected_status: StatusCode, expected_headers: Vec <(&'static str, &'static str)>, - expected_body: &'static str, + expected_body: String, } impl TestCase { @@ -237,16 +262,16 @@ mod tests { x } - fn expected_body (&self, v: &'static str) -> Self { + fn expected_body (&self, v: String) -> Self { let mut x = self.clone (); x.expected_body = v; x } - fn expected (&self, sc: StatusCode, body: &'static str) -> Self { + fn expected (&self, sc: StatusCode, body: &str) -> Self { self .expected_status (sc) - .expected_body (body) + .expected_body (format! ("{}\n", body)) } async fn test (&self) { @@ -260,14 +285,15 @@ mod tests { let input = input.body (Body::empty ()).unwrap (); let config_file = config::file::Config { - port: Some (4000), - servers: vec! [], iso: config::file::Isomorphic { enable_scraper_api: true, - dev_mode: self.valid_key.map (|key| config::file::DevMode { - scraper_key: Some (key_validity::ScraperKey::new (key.as_bytes ())), - }), + dev_mode: Default::default (), }, + port: Some (4000), + servers: vec! [], + scraper_keys: vec! [ + self.valid_key.map (|x| key_validity::ScraperKey::new_30_day ("automated test", x.as_bytes ())), + ].into_iter ().filter_map (|x| x).collect (), }; let config = config::Config::try_from (config_file).expect ("Can't load config"); @@ -292,7 +318,7 @@ mod tests { let actual_body = actual_body.to_vec (); let actual_body = String::from_utf8 (actual_body).expect ("Body should be UTF-8"); - assert_eq! (&actual_body, self.expected_body); + assert_eq! (actual_body, self.expected_body); } } @@ -309,21 +335,21 @@ mod tests { expected_headers: vec! [ ("content-type", "text/plain"), ], - expected_body: "You're valid!\n", + expected_body: "You're valid!\n".to_string (), }; for case in &[ base_case.clone (), base_case.path_rest ("v9999/test") - .expected (StatusCode::NOT_FOUND, "Unknown scraper API version\n"), + .expected (StatusCode::NOT_FOUND, strings::UNKNOWN_API_VERSION), base_case.valid_key (None) - .expected (StatusCode::FORBIDDEN, "403 Forbidden\n"), + .expected (StatusCode::FORBIDDEN, strings::FORBIDDEN), base_case.input_key (Some ("borgus")) - .expected (StatusCode::FORBIDDEN, "403 Forbidden\n"), + .expected (StatusCode::FORBIDDEN, strings::FORBIDDEN), base_case.path_rest ("v1/toast") - .expected (StatusCode::NOT_FOUND, "Unknown API endpoint\n"), + .expected (StatusCode::NOT_FOUND, strings::UNKNOWN_API_ENDPOINT), base_case.input_key (None) - .expected (StatusCode::FORBIDDEN, "Can't run scraper without an API key\n"), + .expected (StatusCode::FORBIDDEN, strings::NO_API_KEY), ] { case.test ().await; } diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index e161f08..9c114f7 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -2,6 +2,52 @@ (Find this issue with `git grep YNQAQKJS`) +## Test curl commands + +(In production the API key should be loaded from a file. Putting it in the +Bash command is bad, because it will be saved to Bash's history file. Putting +it in environment variables is slightly better) + +`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/api/test` + +Should return "You're valid!" + +`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server_list` + +Should return a JSON object listing all the servers. + +`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/api/v1/dir/` + +Proxies into the "aliens_wildland" server and retrieves a JSON object listing +the file server root. (The server must be running a new version of ptth_server +which can serve the JSON API) + +`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/api/v1/dir/src/` + +Same, but retrieves the listing for "/src". + +`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs` + +There is no special API for retrieving files yet - But the existing server +API will be is proxied through the new scraper API on the relay. + +`curl --head -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs` + +PTTH supports HEAD requests. This request will yield a "204 No Content", with +the "content-length" header. + +`curl -H "range: bytes=100-199" -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs` + +PTTH supports byte range requests. This request will skip 100 bytes into the +file, and read 100 bytes. + +To avoid fence-post errors, most programming languages use half-open ranges. +e.g. `0..3` means "0, 1, 2". However, HTTP byte ranges are closed ranges. +e.g. `0..3` means "0, 1, 2, 3". So 100-199 means 199 is the last byte retrieved. + +By polling with HEAD and byte range requests, a scraper client can approximate +`tail -f` behavior of a server-side file. + ## Problem statement PTTH has 2 auth routes: @@ -38,7 +84,7 @@ stronger is ready. - (X) (POC) Test with curl - (X) Clean up scraper endpoint - (X) Add (almost) end-to-end tests for test scraper endpoint -- ( ) Thread server endpoints through relay scraper auth +- (X) Thread server endpoints through relay scraper auth - ( ) Add tests for other scraper endpoints - (don't care) Factor v1 API into v1 module - (X) Add real scraper endpoints @@ -80,7 +126,7 @@ These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it. For compatibility with wget spidering, I _might_ do XML or HTML that's machine-readable. We'll see. -## Open questions +## Decision journal **Who generates the API key? The scraper client, or the PTTH relay server?** diff --git a/src/tests.rs b/src/tests.rs index dabe3d5..45e2f91 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -36,6 +36,7 @@ fn end_to_end () { let tripcode = BlakeHashWrapper::from_key (api_key.as_bytes ()); debug! ("Relay is expecting tripcode {}", tripcode.encode_base64 ()); let config_file = ptth_relay::config::file::Config { + iso: Default::default (), port: None, servers: vec! [ ptth_relay::config::file::Server { @@ -44,7 +45,7 @@ fn end_to_end () { display_name: None, }, ], - iso: Default::default (), + scraper_keys: vec! [], }; let config = ptth_relay::config::Config::try_from (config_file).expect ("Can't load config"); @@ -143,16 +144,15 @@ fn scraper_endpoints () { use ptth_relay::*; let config_file = config::file::Config { - port: Some (4001), - servers: vec! [ - - ], iso: config::file::Isomorphic { enable_scraper_api: true, - dev_mode: Some (config::file::DevMode { - scraper_key: Some (key_validity::ScraperKey::new (b"bogus")), - }), + dev_mode: Default::default (), }, + port: Some (4001), + servers: vec! [], + scraper_keys: vec! [ + key_validity::ScraperKey::new_30_day ("automated testing", b"bogus") + ], }; let config = config::Config::try_from (config_file).expect ("Can't load config"); From 1e160ec55bf87af29c145d7bcca2b5a6aa6de329 Mon Sep 17 00:00:00 2001 From: _ <> Date: Wed, 16 Dec 2020 14:57:47 +0000 Subject: [PATCH 128/208] :star: new: add subcommand in ptth_relay to hash API keys from the terminal --- Cargo.lock | 1 + crates/ptth_relay/Cargo.toml | 1 + crates/ptth_relay/src/main.rs | 28 +++++++++++++++++++++++++--- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d49e91..c6fc347 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1228,6 +1228,7 @@ dependencies = [ "base64 0.12.3", "blake3", "chrono", + "clap", "dashmap", "futures", "handlebars", diff --git a/crates/ptth_relay/Cargo.toml b/crates/ptth_relay/Cargo.toml index d41d751..fe3fb13 100644 --- a/crates/ptth_relay/Cargo.toml +++ b/crates/ptth_relay/Cargo.toml @@ -11,6 +11,7 @@ license = "AGPL-3.0" base64 = "0.12.3" blake3 = "0.3.7" chrono = {version = "0.4.19", features = ["serde"]} +clap = "2.33.3" dashmap = "3.11.10" futures = "0.3.7" handlebars = "3.5.1" diff --git a/crates/ptth_relay/src/main.rs b/crates/ptth_relay/src/main.rs index abe446d..0d676da 100644 --- a/crates/ptth_relay/src/main.rs +++ b/crates/ptth_relay/src/main.rs @@ -7,6 +7,7 @@ use std::{ sync::Arc, }; +use clap::{App, SubCommand}; use tracing::{info}; use tracing_subscriber::{ fmt, @@ -29,12 +30,33 @@ async fn main () -> Result <(), Box > { .init () ; + let matches = App::new ("ptth_relay") + .author ("Trish") + .about ("Relay server for the PTTH backwards HTTP server") + .subcommand (SubCommand::with_name ("hash-api-key")) + .get_matches (); + + if matches.subcommand_matches ("hash-api-key").is_some () { + use std::io; + use ptth_relay::key_validity::BlakeHashWrapper; + + println! ("Enter key (it will be visible in the terminal)"); + + let mut key = String::new (); + io::stdin ().read_line (&mut key)?; + + println! ("{}", BlakeHashWrapper::from_key (key.trim_end ().as_bytes ()).encode_base64 ()); + return Ok (()); + } + let config_path = PathBuf::from ("config/ptth_relay.toml"); let config = Config::from_file (&config_path).await?; - match read_git_version ().await { - Some (x) => info! ("ptth_relay Git version: {:?}", x), - None => info! ("ptth_relay not built from Git"), + if let Some (x) = read_git_version ().await { + info! ("ptth_relay Git version: {:?}", x); + } + else { + info! ("ptth_relay not built from Git"); } let (shutdown_rx, forced_shutdown) = ptth_core::graceful_shutdown::init_with_force (); From 3bc8323cc893b3903bed7a35827b1e931bf948c2 Mon Sep 17 00:00:00 2001 From: Trisha Date: Wed, 16 Dec 2020 10:33:03 -0600 Subject: [PATCH 129/208] :pencil: docs: update example curl commands --- issues/2020-12Dec/auth-route-YNQAQKJS.md | 33 +++++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 9c114f7..2fdd142 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -4,39 +4,52 @@ ## Test curl commands -(In production the API key should be loaded from a file. Putting it in the -Bash command is bad, because it will be saved to Bash's history file. Putting -it in environment variables is slightly better) +Put your API key into a header file, like this: -`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/api/test` +``` +X-ApiKey: bad_password +``` + +Export the scraper API's URL prefix to an environment variable: + +`export API=http://127.0.0.1:4000/scraper` + +Call it "scraper-secret.txt" or something else obviously secret. +Don't check it into Git. The key will expire every 30 days and need +to be rotated manually. (for now) + +New versions of Curl can load headers from a text file. All commands +will use this feature to load the API key. + +`curl --header @scraper-secret.txt $API/api/test` Should return "You're valid!" -`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server_list` +`curl --header @scraper-secret.txt $API/v1/server_list` Should return a JSON object listing all the servers. -`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/api/v1/dir/` +`curl --header @scraper-secret.txt $API/v1/server/aliens_wildland/api/v1/dir/` Proxies into the "aliens_wildland" server and retrieves a JSON object listing the file server root. (The server must be running a new version of ptth_server which can serve the JSON API) -`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/api/v1/dir/src/` +`curl --header @scraper-secret.txt $API/v1/server/aliens_wildland/api/v1/dir/src/` Same, but retrieves the listing for "/src". -`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs` +`curl --header @scraper-secret.txt $API/v1/server/aliens_wildland/files/src/tests.rs` There is no special API for retrieving files yet - But the existing server API will be is proxied through the new scraper API on the relay. -`curl --head -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs` +`curl --header @scraper-secret.txt $API/v1/server/aliens_wildland/files/src/tests.rs` PTTH supports HEAD requests. This request will yield a "204 No Content", with the "content-length" header. -`curl -H "range: bytes=100-199" -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs` +`curl --header @scraper-secret.txt -H "range: bytes=100-199" $API/v1/server/aliens_wildland/files/src/tests.rs` PTTH supports byte range requests. This request will skip 100 bytes into the file, and read 100 bytes. From 626946b7b786659c985acf3b76ad98c8547acb5e Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Wed, 16 Dec 2020 19:06:15 -0600 Subject: [PATCH 130/208] :sound: update: demote some boring logs from debug to trace --- crates/ptth_relay/src/lib.rs | 15 ++++++++++----- crates/ptth_relay/src/server_endpoint.rs | 10 +++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/ptth_relay/src/lib.rs b/crates/ptth_relay/src/lib.rs index f3c0057..d3fcaf9 100644 --- a/crates/ptth_relay/src/lib.rs +++ b/crates/ptth_relay/src/lib.rs @@ -141,7 +141,7 @@ async fn handle_http_request ( // disconnects right as we're sending this. // Then what? - debug! ( + trace! ( "Sending request {} directly to server {}", req_id, watcher_code, @@ -395,12 +395,17 @@ async fn reload_config ( config_reload_path: &Path ) -> Result <(), ConfigError> { let new_config = Config::from_file (config_reload_path).await?; - let mut config = state.config.write ().await; - (*config) = new_config; - debug! ("Loaded {} server configs", config.servers.len ()); - debug! ("enable_scraper_api: {}", config.iso.enable_scraper_api); + trace! ("Reloading config"); + if config.servers.len () != new_config.servers.len () { + debug! ("Loaded {} server configs", config.servers.len ()); + } + if config.iso.enable_scraper_api != new_config.iso.enable_scraper_api { + debug! ("enable_scraper_api: {}", config.iso.enable_scraper_api); + } + + (*config) = new_config; if config.iso.dev_mode.is_some () { error! ("Dev mode is enabled! This might turn off some security features. If you see this in production, escalate it to someone!"); diff --git a/crates/ptth_relay/src/server_endpoint.rs b/crates/ptth_relay/src/server_endpoint.rs index f907552..a5df0f3 100644 --- a/crates/ptth_relay/src/server_endpoint.rs +++ b/crates/ptth_relay/src/server_endpoint.rs @@ -100,7 +100,7 @@ pub async fn handle_listen ( } } - debug! ("Parking server {}", watcher_code); + trace! ("Parking server {}", watcher_code); request_rendezvous.insert (watcher_code.clone (), ParkedServer (tx)); } @@ -109,14 +109,14 @@ pub async fn handle_listen ( futures::select! { x = rx.fuse () => match x { Ok (Ok (one_req)) => { - debug! ("Unparking server {}", watcher_code); + trace! ("Unparking server {}", watcher_code); Ok (ok_reply (rmp_serde::to_vec (&vec! [one_req])?)?) }, Ok (Err (ShuttingDownError::ShuttingDown)) => Ok (error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon")?), Err (_) => Ok (error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error")?), }, _ = delay_for (Duration::from_secs (30)).fuse () => { - debug! ("Timed out http_listen for server {}", watcher_code); + trace! ("Timed out http_listen for server {}", watcher_code); return Ok (error_reply (StatusCode::NO_CONTENT, "No requests now, long-poll again")?) } } @@ -178,7 +178,7 @@ pub async fn handle_response ( } } else { - debug! ("Finished relaying bytes"); + trace! ("Finished relaying bytes"); body_finished_tx.send (StreamFinished).map_err (|_| LostServer)?; break; } @@ -213,7 +213,7 @@ pub async fn handle_response ( relay_task.await??; - debug! ("Connected server to client for streaming."); + trace! ("Connected server to client for streaming."); match body_finished_rx.await { Ok (StreamFinished) => { Ok (error_reply (StatusCode::OK, "StreamFinished")?) From c70f44f4e40afd02879590c8952ad69c02041438 Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 18 Dec 2020 16:58:41 +0000 Subject: [PATCH 131/208] :pencil: docs: update todo --- issues/2020-12Dec/metrics-K5NPHQHP.md | 36 +++++++++++++++++++++++++++ todo.md | 15 ++++++++--- 2 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 issues/2020-12Dec/metrics-K5NPHQHP.md diff --git a/issues/2020-12Dec/metrics-K5NPHQHP.md b/issues/2020-12Dec/metrics-K5NPHQHP.md new file mode 100644 index 0000000..fb37bf5 --- /dev/null +++ b/issues/2020-12Dec/metrics-K5NPHQHP.md @@ -0,0 +1,36 @@ +# Metrics + instance data + log + +(K5NPHQHP) + +## ptth_server metrics + +These are just like Prometheus. Either counters, or gauges, scalar or histograms. + +- CPU usage counter +- RAM usage gauge +- Requests received counter +- Responses sent counter +- Tokio gauges? +- Outbound byte counter +- Uptime gauge +- Last seen UTC time +- Free disk space gauge + +## ptth_server instance data + +These are captured during process start and then don't change. They're kinda +like metrics, but static. + +- Machine ID +- Git version (if possible) +- Server ID +- Random instance ID +- UTC time at startup + +## ptth_server logs + +Basically what the tracing crate puts out. See if we can route that over the +network? + +- Metrics, captured every 5 minutes or whatever +- Probably "warn" level logs from `tracing` diff --git a/todo.md b/todo.md index 7f9fcf4..ccbc6f8 100644 --- a/todo.md +++ b/todo.md @@ -1,12 +1,21 @@ Interesting issues will get a unique ID with `dd if=/dev/urandom bs=5 count=1 | base32` -- [YNQAQKJS](issues/2020-12Dec/auth-route-YNQAQKJS.md) Open new auth route for spiders / scrapers +- Dark mode? +- [K5NPHQHP](issues/2020-12Dec/metrics-K5NPHQHP.md) API for metrics + instance data + recent logs on ptth_server +- API for remote mtime +- Scraper `tail -f` example +- API for remote inotify (or similar) +- Scraper `rsync -ru` example +- Make TS text browser, in ptth_relay, using ptth_server APIs, and tail -f +- Make TS read-only SQLite browser (this is a long shot) +- [YNQAQKJS](issues/2020-12Dec/auth-route-YNQAQKJS.md) Add database for +scraper keys - Track / Estimate bandwidth per server? - EOTPXGR3 Remote `tail -f` (_Complicated_) (Maybe use chunked encoding or something?) - "Preview as" feature for Markdown (It's not threaded through the relay yet) - Make a debug client to replicate the issue Firefox is having with turtling -- YMFMSV2R Add Prometheus metrics +- YMFMSV2R / K5NPHQHP Add Prometheus metrics - Not working great behind reverse proxies - Impl multi-range / multi-part byte serving @@ -37,7 +46,7 @@ Sometimes I get the turtle icon in Firefox's network debugger. But this happens even with Caddy running a static file server, so I can't prove that it's on my side. The VPS is cheap, and the datacenter is far away. -## Embedded asssets +## Embedded assets The bad_passwords file is huge. Since it's static, it should only be in physical RAM when the server first launches, and then the kernel will let it be paged From 1cf3ff5313465c659094527bfa66efeb397b1db4 Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 18 Dec 2020 17:01:26 +0000 Subject: [PATCH 132/208] :lipstick: update: add logo and favicon --- README.md | 2 ++ assets/favicon.png | Bin 0 -> 643 bytes assets/logo-128-pixel.png | Bin 0 -> 2324 bytes 3 files changed, 2 insertions(+) create mode 100644 assets/favicon.png create mode 100644 assets/logo-128-pixel.png diff --git a/README.md b/README.md index be77f24..b4c252b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![The PTTH logo, a green box sitting on a black conveyor belt. The box has an arrow pointing left, and the text "PTTH", in white. The conveyor belt has an arrow pointing right, in white.](assets/logo-128-pixel.png) + # PTTH An HTTP server that can run behind a firewall by connecting out to a relay. diff --git a/assets/favicon.png b/assets/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..392baa847ea7cef68987fda6e08820c6a2d5b64d GIT binary patch literal 643 zcmV-}0(||6P)EX>4Tx04R}tkv&MmKp2MKrivmh4t5Z6$WUFh;2+|sRVYG*P%E_RU~=gnG-*gu zTpR`0f`dPcRR6lU)_NUJ*i=<~T-VW*Kvml!Wj2x<`QTcQKyjU-#z-t2v7S0g-r?8KzCVK|H-_ z8=UuvBdjQ^#OK7LCS8#Dk?V@bZ=4G*3p_Jorc?985n{2>#!4HrqNx#25l2-`r+gvf zvC4UivsSLM<~{ifLpgnAnd`I$k-#FBAVGwJDoQBBMvQiy6bmUjkNfxsT)#vvgpt)9ZtvT_HO>D00IuC~zb{Jtc>n+a32;bRa{vG?BLDy{BLR4&KXw2B00(qQ zO+^Rf3=$A5G}HKaACCV-Jp($rwh7G}5d{}#A=iS*vtRs{p%3UK dsLe|NOD9XqC1)t8!$SZ7002ovPDHLkV1n_>4Nd?6 literal 0 HcmV?d00001 diff --git a/assets/logo-128-pixel.png b/assets/logo-128-pixel.png new file mode 100644 index 0000000000000000000000000000000000000000..55e09d1c70de50150279f78d62f4ff687ff7682d GIT binary patch literal 2324 zcmY*a2{hDQ8~+bxXvjK|r3{ksg|Z}M9qU+Pl0r;FS?0CO*q1C5Mz-wPLe?n~S+mbf z)+9AinrxLMl`Xs`L%#8y@BPmC&biOMzw^7#x%YSPbMNyc+g&g}43&Ta0B{&(VRDf@ z5)X_A!tVR@-Ch6yh=emXwnG^k!wG?SPn@p@03foWv-B+b438$xkyB)q`9lQ*hV5Ki zK1zRn=zu_y>`t|O#|AD*Yv3#{x=4X1S@c#F;@#;-77kF&aXr$75keMx)iS%`DIS!ao3P64G$*cxsg^EW z+g{{(q@4-^WrgR7lWn?l--Ttx5(_iBv|3BGRS)%YuY zBl@S-9T7N_&t*wDwwix5DjVa2qsx_rNw&B9<3^4XIyg5C#G4emE@tLszT#Z<+O%;h zTNVB0w6XI}cW8+-lZ=ta6XT7s?S_mW`l~Dct5cM-FaPWT^)rE#)0TS??6@2vSU3d% z0Oykf11aq>OK_9fb7vpE=t%RfNg;m3`J`7-fGyHVjA* zL~_Z`$LLq#iU!G6G_2z3&V-bs;8PnAAZ+u_Uz_ z+GiFE-Nw`_5e?p7d-lT4gB@S0%$9c=dxt*;C1XU~qNrLT-Gx_rBUV;dA{G|D49pw{ zI}WJKZr`K~oywqOuX$O%%Y!prGk+V*fU#n>AN}E7;T>6iYL7-DC{l)3*hg+yI3# zLXFag(@Gz5wD7llE{uPL{ql}S4)I-)8ZnjZC7_qu!0PpcnLY;PCdKWHZ@qV$0mTmA zUt4@eKQnfI?0UHG(Q|s|gwhj9N3hSgA>@%eGIs~=YV~Yul*WGf4W0~P%^>!wHtXjW z^fObNo~h;D8$sVtLKZ6iVFlW2TbqqfC%ROJ7DQhk-kX+uc<*^xW3p=Z=71G5a4P@( z_Ah;b$hFJ(rs7`Ygyv_M9|rC zJ-2T8B2g=N|Lx(Yb9x&=Ul#=shT1C#-maMv=gN+!5pxxpeghg)?qs2_E2I- z2`B+Z-6=i$DI{Y*vRFvCnx{M80Kc1@!>J%HBLr?&o-vzw8q!^Dm5ZVJLd?0>(-)4p z-0!Yu4PG1F0y;(E_G+$DC?YRgVp|Q(%mx`t(rPOV;-Ke*LG$gj2q^Zj zp(AI%t%&cj=(pUX|9o_(b{<$(dAUr%%dcI~2fSh`RQE=#x3@)-WdH z$Udu>A0w}Ftv9W$O|#|c(*}#vM2;#TY-Reu%U4rUir?JwXU+;&!SSVy$Q~X@e5QwZ z3`LsVQYzqrJk%dsNHlD?i4T?SseirI8SUs;kn4}Jvf_=5j4Z3I*)kBk+$tsrC4F2E z_e}H@i8h#PN=`mxBAdmUr7RTl5_C$u{rpUWg4Cz|w`;j0dok1$Phv_6lq;TAxE|T_ zDX@d^eL}INrRC$u-BU|p09)>EFhN7&ZRE|Ht$tH&I7qzAay>UzsJ$PCe>$bAU)sC> zA)P~s1C{-n!ANGG%B4FT9m_pzPo1_W*48@X@eOT3ix*ycYmk>4tNV`_o!E*5NkK)W spoRyJ`dM~~1>LaV0J5u7QaV$hs&v(Wulh Date: Fri, 18 Dec 2020 18:08:17 +0000 Subject: [PATCH 133/208] :lipstick: update: add base64 favicons and WIP dark mode --- assets/favicon.png | Bin 643 -> 595 bytes handlebars/relay/relay_server_list.html | 6 +++ handlebars/server/file_server_dir.html | 66 +++++++++++++++++++++++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/assets/favicon.png b/assets/favicon.png index 392baa847ea7cef68987fda6e08820c6a2d5b64d..b8eb845866cec4aafb2aab59131f763247857060 100644 GIT binary patch delta 89 zcmZo>z09(Ki7C*=)5S5w;&gHXL(~5hCN>F%;z<%7hnx>EIJG@8Wk_Fe@hq#eiP#Ch t&Q+^c{ntF`BF(_$BI&?!;U4RfB!-y9Z#iVs&rf3j0#8>zmvv4FO#qS9AL0N2 delta 138 zcmV;50CoS<1cL>z0s?<*NklQQ3# + {{path}} {{server_name}} + +

{{server_name}}

{{path}}

@@ -61,5 +120,8 @@ + + + From 7a47c705d7f75ea0babfc6214f69694699cdad54 Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 18 Dec 2020 18:10:00 +0000 Subject: [PATCH 134/208] :heavy_plus_sign: update: add a few other assets from the logo --- assets/favicon.ico | Bin 0 -> 318 bytes assets/logo.svg | 119 +++++++++++++++++++++++++++++++++ assets/media preview.svg | 138 +++++++++++++++++++++++++++++++++++++++ todo.md | 2 +- 4 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 assets/favicon.ico create mode 100644 assets/logo.svg create mode 100644 assets/media preview.svg diff --git a/assets/favicon.ico b/assets/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..569fdf0ef39586c42fa0bf042a25e9332f1d8110 GIT binary patch literal 318 zcma)2K?=Yi42!S_)E#!#4#Bfe+DrCxm%3gC6HJM*F;x% + + + + + + + image/svg+xml + + + + + + + + + + + PTTH + + + + diff --git a/assets/media preview.svg b/assets/media preview.svg new file mode 100644 index 0000000..f48b3b0 --- /dev/null +++ b/assets/media preview.svg @@ -0,0 +1,138 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + PTTH + + + + + + + diff --git a/todo.md b/todo.md index ccbc6f8..b9e0bc1 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,7 @@ Interesting issues will get a unique ID with `dd if=/dev/urandom bs=5 count=1 | base32` -- Dark mode? +- (WIP) Dark mode? - [K5NPHQHP](issues/2020-12Dec/metrics-K5NPHQHP.md) API for metrics + instance data + recent logs on ptth_server - API for remote mtime - Scraper `tail -f` example From d03c1a54762b32d39659553d70d83034760a18b9 Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 18 Dec 2020 20:43:34 +0000 Subject: [PATCH 135/208] :heavy_plus_sign: update: add InstanceMetrics and replace ServerInfo --- Cargo.lock | 3 + crates/ptth_file_server_bin/Cargo.toml | 1 + crates/ptth_file_server_bin/src/main.rs | 20 +++-- crates/ptth_server/Cargo.toml | 2 + .../ptth_server/src/file_server/internal.rs | 3 + crates/ptth_server/src/file_server/metrics.rs | 76 +++++++++++++++++++ crates/ptth_server/src/file_server/mod.rs | 33 ++++---- crates/ptth_server/src/lib.rs | 17 +++-- 8 files changed, 125 insertions(+), 30 deletions(-) create mode 100644 crates/ptth_server/src/file_server/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index c6fc347..3bb3d07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1218,6 +1218,7 @@ dependencies = [ "serde", "structopt", "tokio", + "tracing", "tracing-subscriber", ] @@ -1257,6 +1258,7 @@ dependencies = [ "anyhow", "base64 0.12.3", "blake3", + "chrono", "futures", "handlebars", "http", @@ -1278,6 +1280,7 @@ dependencies = [ "tracing", "tracing-futures", "tracing-subscriber", + "ulid", ] [[package]] diff --git a/crates/ptth_file_server_bin/Cargo.toml b/crates/ptth_file_server_bin/Cargo.toml index bf2fc2d..04b67b4 100644 --- a/crates/ptth_file_server_bin/Cargo.toml +++ b/crates/ptth_file_server_bin/Cargo.toml @@ -15,6 +15,7 @@ hyper = "0.13.8" serde = {version = "1.0.117", features = ["derive"]} structopt = "0.3.20" tokio = { version = "0.2.22", features = ["full"] } +tracing = "0.1.21" tracing-subscriber = "0.2.15" ptth_core = { path = "../ptth_core" } diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 33acbd7..affc7c1 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -18,13 +18,17 @@ use hyper::{ StatusCode, }; use serde::Deserialize; +use tracing::debug; use ptth_core::{ http_serde::RequestParts, prelude::*, }; use ptth_server::{ - file_server, + file_server::{ + self, + metrics::InstanceMetrics, + }, load_toml, }; @@ -36,7 +40,7 @@ pub struct Config { struct ServerState <'a> { config: Config, handlebars: handlebars::Handlebars <'a>, - server_info: file_server::ServerInfo, + instance_metrics: InstanceMetrics, hidden_path: Option , } @@ -63,7 +67,7 @@ async fn handle_all (req: Request , state: Arc >) let ptth_resp = file_server::serve_all ( &state.handlebars, - &state.server_info, + &state.instance_metrics, file_server_root, ptth_req.method, &ptth_req.uri, @@ -101,14 +105,18 @@ async fn main () -> Result <(), anyhow::Error> { let handlebars = file_server::load_templates (&PathBuf::new ())?; + let instance_metrics = InstanceMetrics::new ( + config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) + ); + + debug! ("{:?}", instance_metrics); + let state = Arc::new (ServerState { config: Config { file_server_root: config_file.file_server_root, }, handlebars, - server_info: crate::file_server::ServerInfo { - server_name: config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()), - }, + instance_metrics, hidden_path: Some (path), }); diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index 81345a8..4f5b160 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -12,6 +12,7 @@ aho-corasick = "0.7.14" anyhow = "1.0.34" base64 = "0.12.3" blake3 = "0.3.7" +chrono = {version = "0.4.19", features = ["serde"]} futures = "0.3.7" handlebars = "3.5.1" http = "0.2.1" @@ -30,6 +31,7 @@ tracing = "0.1.21" tracing-futures = "0.2.4" tracing-subscriber = "0.2.15" toml = "0.5.7" +ulid = "0.4.1" always_equal = { path = "../always_equal" } ptth_core = { path = "../ptth_core" } diff --git a/crates/ptth_server/src/file_server/internal.rs b/crates/ptth_server/src/file_server/internal.rs index 9b5080a..de6335a 100644 --- a/crates/ptth_server/src/file_server/internal.rs +++ b/crates/ptth_server/src/file_server/internal.rs @@ -218,6 +218,9 @@ async fn serve_api ( Ok (NotFound) } +// Handle the requests internally without knowing anything about PTTH or +// HTML / handlebars + pub async fn serve_all ( root: &Path, method: Method, diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs new file mode 100644 index 0000000..19f697c --- /dev/null +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -0,0 +1,76 @@ +use chrono::{DateTime, Utc}; +use ulid::Ulid; + +fn serialize_ulid (t: &Ulid, s: S) +-> Result +{ + let t = t.to_string (); + s.serialize_str (&t) +} + +// Instance metrics are captured when the ptth_server process starts. +// They don't change after that. + +#[derive (Debug, serde::Serialize)] +pub struct InstanceMetrics { + // D-Bus machine ID, if we're on Linux + pub machine_id: Option , + + // Git version that ptth_server was built from (unimplemented) + pub _git_version: Option , + + // User-assigned and human-readable name for this server. + // Must be unique within a relay. + pub server_name: String, + + // Random base64 instance ID. ptth_server generates this at process start. + // It's a fallback for detecting outages without relying on any clocks. + #[serde (serialize_with = "serialize_ulid")] + pub instance_id: Ulid, + + pub startup_utc: DateTime , +} + +fn get_machine_id () -> Option { + use std::{ + fs::File, + io::Read, + }; + + let mut buf = vec! [0u8; 1024]; + let mut f = File::open ("/etc/machine-id").ok ()?; + let bytes_read = f.read (&mut buf).ok ()?; + + buf.truncate (bytes_read); + + let s = std::str::from_utf8 (&buf).ok ()?; + let s = s.trim_end ().to_string (); + + Some (s) +} + +impl InstanceMetrics { + pub fn new (server_name: String) -> Self + { + Self { + machine_id: get_machine_id (), + _git_version: None, + server_name, + instance_id: ulid::Ulid::new (), + startup_utc: Utc::now (), + } + } +} + +#[cfg (test)] +mod tests { + use super::*; + + #[test] + fn ulid_null () { + let a = InstanceMetrics::new ("bogus".to_string ()); + let b = InstanceMetrics::new ("bogus".to_string ()); + + assert_ne! (a.instance_id, b.instance_id); + } +} diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 40d9d84..019fc98 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -8,7 +8,6 @@ use std::{ cmp::min, collections::HashMap, convert::{Infallible, TryFrom}, - fmt::Debug, io::SeekFrom, path::Path, }; @@ -38,11 +37,14 @@ use ptth_core::{ }; pub mod errors; +pub mod metrics; + mod internal; mod markdown; mod range; use errors::FileServerError; +use metrics::InstanceMetrics; mod emoji { pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; @@ -52,11 +54,6 @@ mod emoji { pub const ERROR: &str = "\u{26a0}\u{fe0f}"; } -#[derive (Debug, Serialize)] -pub struct ServerInfo { - pub server_name: String, -} - #[derive (Serialize)] struct DirEntryJson { name: String, @@ -93,7 +90,7 @@ struct DirEntryHtml { #[derive (Serialize)] struct DirHtml <'a> { #[serde (flatten)] - server_info: &'a ServerInfo, + instance_metrics: &'a InstanceMetrics, path: Cow <'a, str>, entries: Vec , @@ -191,10 +188,10 @@ async fn read_dir_entry_json (entry: DirEntry) -> Option async fn serve_root ( handlebars: &Handlebars <'static>, - server_info: &ServerInfo + instance_metrics: &InstanceMetrics ) -> Result { - let s = handlebars.render ("file_server_root", &server_info)?; + let s = handlebars.render ("file_server_root", &instance_metrics)?; Ok (serve_html (s)) } @@ -233,10 +230,10 @@ async fn serve_dir_json ( Ok (response) } -#[instrument (level = "debug", skip (handlebars, dir))] +#[instrument (level = "debug", skip (handlebars, instance_metrics, dir))] async fn serve_dir_html ( handlebars: &Handlebars <'static>, - server_info: &ServerInfo, + instance_metrics: &InstanceMetrics, path: Cow <'_, str>, mut dir: ReadDir ) -> Result @@ -252,7 +249,7 @@ async fn serve_dir_html ( let s = handlebars.render ("file_server_dir", &DirHtml { path, entries, - server_info, + instance_metrics, })?; Ok (serve_html (s)) @@ -355,10 +352,14 @@ async fn serve_file ( Ok (response) } -#[instrument (level = "debug", skip (handlebars, headers))] +// Pass a request to the internal decision-making logic. +// When it returns, prettify it as HTML or JSON based on what the client +// asked for. + +#[instrument (level = "debug", skip (handlebars, headers, instance_metrics))] pub async fn serve_all ( handlebars: &Handlebars <'static>, - server_info: &ServerInfo, + instance_metrics: &InstanceMetrics, root: &Path, method: Method, uri: &str, @@ -404,14 +405,14 @@ pub async fn serve_all ( }, InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"), - Root => serve_root (handlebars, server_info).await?, + Root => serve_root (handlebars, instance_metrics).await?, ServeDir (internal::ServeDirParams { path, dir, format }) => match format { OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?, - OutputFormat::Html => serve_dir_html (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?, + OutputFormat::Html => serve_dir_html (handlebars, instance_metrics, path.to_string_lossy (), dir.into_inner ()).await?, }, ServeFile (internal::ServeFileParams { file, diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index bccc824..3fed368 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -51,11 +51,14 @@ pub fn password_is_bad (mut password: String) -> bool { struct ServerState { config: Config, handlebars: Handlebars <'static>, - server_info: file_server::ServerInfo, + instance_metrics: file_server::metrics::InstanceMetrics, client: Client, hidden_path: Option , } +// Unwrap a request from PTTH format and pass it into file_server. +// When file_server responds, wrap it back up and stream it to the relay. + async fn handle_one_req ( state: &Arc , wrapped_req: http_serde::WrappedRequest @@ -72,7 +75,7 @@ async fn handle_one_req ( let response = file_server::serve_all ( &state.handlebars, - &state.server_info, + &state.instance_metrics, file_server_root, parts.method, &parts.uri, @@ -186,10 +189,6 @@ pub async fn run_server ( return Err (ServerError::WeakApiKey); } - let server_info = file_server::ServerInfo { - server_name: config_file.name.clone (), - }; - info! ("Server name is {}", config_file.name); info! ("Tripcode is {}", config_file.tripcode ()); @@ -202,13 +201,15 @@ pub async fn run_server ( .build ().map_err (ServerError::CantBuildHttpClient)?; let handlebars = file_server::load_templates (&asset_root)?; + let instance_metrics = file_server::metrics::InstanceMetrics::new (config_file.name); + let state = Arc::new (ServerState { config: Config { relay_url: config_file.relay_url, file_server_root: config_file.file_server_root, }, handlebars, - server_info, + instance_metrics, client, hidden_path, }); @@ -233,7 +234,7 @@ pub async fn run_server ( debug! ("http_listen"); - let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, config_file.name)).send (); + let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, state.instance_metrics.server_name)).send (); let err_backoff_delay = std::cmp::min (30_000, backoff_delay * 2 + 500); From 72b3b33206556ddf47c1122fd87a743aefa1ecac Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 18 Dec 2020 23:41:52 +0000 Subject: [PATCH 136/208] :rotating_light: refactor: fix some Clippy lints --- .../ptth_server/src/file_server/internal.rs | 64 +++++++++---------- crates/ptth_server/src/file_server/metrics.rs | 7 +- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/crates/ptth_server/src/file_server/internal.rs b/crates/ptth_server/src/file_server/internal.rs index de6335a..109ccd3 100644 --- a/crates/ptth_server/src/file_server/internal.rs +++ b/crates/ptth_server/src/file_server/internal.rs @@ -181,39 +181,39 @@ async fn serve_api ( { use Response::*; - match prefix_match ("/v1/dir/", path) { - None => (), - Some (path) => { - let encoded_path = &path [0..]; - - let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; - let path = Path::new (&*path_s); - - let full_path = root.join (path); - - debug! ("full_path = {:?}", full_path); - - if let Some (hidden_path) = hidden_path { - if full_path == hidden_path { - return Ok (Forbidden); - } + // API versioning will be major-only, so I'll keep adding stuff to v1 + // until I need to deprecate or break something. + + if let Some (path) = prefix_match ("/v1/dir/", path) { + let encoded_path = &path [0..]; + + let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; + let path = Path::new (&*path_s); + + let full_path = root.join (path); + + debug! ("full_path = {:?}", full_path); + + if let Some (hidden_path) = hidden_path { + if full_path == hidden_path { + return Ok (Forbidden); } - - return if let Ok (dir) = read_dir (&full_path).await { - serve_dir ( - &path_s, - path, - dir, - full_path, - &uri, - OutputFormat::Json - ) - } - else { - Ok (NotFound) - }; - }, - }; + } + + return if let Ok (dir) = read_dir (&full_path).await { + serve_dir ( + &path_s, + path, + dir, + full_path, + &uri, + OutputFormat::Json + ) + } + else { + Ok (NotFound) + }; + } Ok (NotFound) } diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs index 19f697c..6c74d3f 100644 --- a/crates/ptth_server/src/file_server/metrics.rs +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -17,7 +17,7 @@ pub struct InstanceMetrics { pub machine_id: Option , // Git version that ptth_server was built from (unimplemented) - pub _git_version: Option , + pub git_version: Option , // User-assigned and human-readable name for this server. // Must be unique within a relay. @@ -37,7 +37,7 @@ fn get_machine_id () -> Option { io::Read, }; - let mut buf = vec! [0u8; 1024]; + let mut buf = vec! [0; 1024]; let mut f = File::open ("/etc/machine-id").ok ()?; let bytes_read = f.read (&mut buf).ok ()?; @@ -50,11 +50,12 @@ fn get_machine_id () -> Option { } impl InstanceMetrics { + #[must_use] pub fn new (server_name: String) -> Self { Self { machine_id: get_machine_id (), - _git_version: None, + git_version: None, server_name, instance_id: ulid::Ulid::new (), startup_utc: Utc::now (), From b54be58abc74b432f69bb0968f8d1b58630c0b5b Mon Sep 17 00:00:00 2001 From: _ <> Date: Fri, 18 Dec 2020 23:45:30 +0000 Subject: [PATCH 137/208] :rotating_light: refactor: rename InstanceMetrics to PerInstance --- crates/ptth_file_server_bin/src/main.rs | 6 +++--- crates/ptth_server/src/file_server/metrics.rs | 8 ++++---- crates/ptth_server/src/file_server/mod.rs | 9 ++++----- crates/ptth_server/src/lib.rs | 4 ++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index affc7c1..5921497 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -27,7 +27,7 @@ use ptth_core::{ use ptth_server::{ file_server::{ self, - metrics::InstanceMetrics, + metrics, }, load_toml, }; @@ -40,7 +40,7 @@ pub struct Config { struct ServerState <'a> { config: Config, handlebars: handlebars::Handlebars <'a>, - instance_metrics: InstanceMetrics, + instance_metrics: metrics::PerInstance, hidden_path: Option , } @@ -105,7 +105,7 @@ async fn main () -> Result <(), anyhow::Error> { let handlebars = file_server::load_templates (&PathBuf::new ())?; - let instance_metrics = InstanceMetrics::new ( + let instance_metrics = metrics::PerInstance::new ( config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) ); diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs index 6c74d3f..e760843 100644 --- a/crates/ptth_server/src/file_server/metrics.rs +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -12,7 +12,7 @@ fn serialize_ulid (t: &Ulid, s: S) // They don't change after that. #[derive (Debug, serde::Serialize)] -pub struct InstanceMetrics { +pub struct PerInstance { // D-Bus machine ID, if we're on Linux pub machine_id: Option , @@ -49,7 +49,7 @@ fn get_machine_id () -> Option { Some (s) } -impl InstanceMetrics { +impl PerInstance { #[must_use] pub fn new (server_name: String) -> Self { @@ -69,8 +69,8 @@ mod tests { #[test] fn ulid_null () { - let a = InstanceMetrics::new ("bogus".to_string ()); - let b = InstanceMetrics::new ("bogus".to_string ()); + let a = PerInstance::new ("bogus".to_string ()); + let b = PerInstance::new ("bogus".to_string ()); assert_ne! (a.instance_id, b.instance_id); } diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 019fc98..37ee6a0 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -44,7 +44,6 @@ mod markdown; mod range; use errors::FileServerError; -use metrics::InstanceMetrics; mod emoji { pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; @@ -90,7 +89,7 @@ struct DirEntryHtml { #[derive (Serialize)] struct DirHtml <'a> { #[serde (flatten)] - instance_metrics: &'a InstanceMetrics, + instance_metrics: &'a metrics::PerInstance, path: Cow <'a, str>, entries: Vec , @@ -188,7 +187,7 @@ async fn read_dir_entry_json (entry: DirEntry) -> Option async fn serve_root ( handlebars: &Handlebars <'static>, - instance_metrics: &InstanceMetrics + instance_metrics: &metrics::PerInstance ) -> Result { let s = handlebars.render ("file_server_root", &instance_metrics)?; @@ -233,7 +232,7 @@ async fn serve_dir_json ( #[instrument (level = "debug", skip (handlebars, instance_metrics, dir))] async fn serve_dir_html ( handlebars: &Handlebars <'static>, - instance_metrics: &InstanceMetrics, + instance_metrics: &metrics::PerInstance, path: Cow <'_, str>, mut dir: ReadDir ) -> Result @@ -359,7 +358,7 @@ async fn serve_file ( #[instrument (level = "debug", skip (handlebars, headers, instance_metrics))] pub async fn serve_all ( handlebars: &Handlebars <'static>, - instance_metrics: &InstanceMetrics, + instance_metrics: &metrics::PerInstance, root: &Path, method: Method, uri: &str, diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 3fed368..a5a8173 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -51,7 +51,7 @@ pub fn password_is_bad (mut password: String) -> bool { struct ServerState { config: Config, handlebars: Handlebars <'static>, - instance_metrics: file_server::metrics::InstanceMetrics, + instance_metrics: file_server::metrics::PerInstance, client: Client, hidden_path: Option , } @@ -201,7 +201,7 @@ pub async fn run_server ( .build ().map_err (ServerError::CantBuildHttpClient)?; let handlebars = file_server::load_templates (&asset_root)?; - let instance_metrics = file_server::metrics::InstanceMetrics::new (config_file.name); + let instance_metrics = file_server::metrics::PerInstance::new (config_file.name); let state = Arc::new (ServerState { config: Config { From d052f42507b44299b40fe254f5d13cc1b5638d83 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 17:41:00 +0000 Subject: [PATCH 138/208] :construction: wip: add placeholder for gauges --- crates/ptth_server/src/file_server/metrics.rs | 5 +++++ crates/ptth_server/src/lib.rs | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs index e760843..9db8d51 100644 --- a/crates/ptth_server/src/file_server/metrics.rs +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -31,6 +31,11 @@ pub struct PerInstance { pub startup_utc: DateTime , } +#[derive (Debug, serde::Serialize)] +pub struct Gauges { + pub rss_mib: u64, +} + fn get_machine_id () -> Option { use std::{ fs::File, diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index a5a8173..0bf214e 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -18,7 +18,10 @@ use handlebars::Handlebars; use reqwest::Client; use serde::Deserialize; use tokio::{ - sync::oneshot, + sync::{ + oneshot, + RwLock, + }, time::delay_for, }; @@ -52,6 +55,7 @@ struct ServerState { config: Config, handlebars: Handlebars <'static>, instance_metrics: file_server::metrics::PerInstance, + gauges: RwLock , client: Client, hidden_path: Option , } @@ -210,6 +214,9 @@ pub async fn run_server ( }, handlebars, instance_metrics, + gauges: RwLock::new (file_server::metrics::Gauges { + rss_mib: 7, + }), client, hidden_path, }); From e8d94da661f62a76adc49209322f58a23dc91732 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 17:44:03 +0000 Subject: [PATCH 139/208] :recycle: refactor: change PerInstance to Startup --- crates/ptth_file_server_bin/src/main.rs | 4 ++-- crates/ptth_server/src/file_server/metrics.rs | 9 +++++---- crates/ptth_server/src/file_server/mod.rs | 8 ++++---- crates/ptth_server/src/lib.rs | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 5921497..561072b 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -40,7 +40,7 @@ pub struct Config { struct ServerState <'a> { config: Config, handlebars: handlebars::Handlebars <'a>, - instance_metrics: metrics::PerInstance, + instance_metrics: metrics::Startup, hidden_path: Option , } @@ -105,7 +105,7 @@ async fn main () -> Result <(), anyhow::Error> { let handlebars = file_server::load_templates (&PathBuf::new ())?; - let instance_metrics = metrics::PerInstance::new ( + let instance_metrics = metrics::Startup::new ( config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) ); diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs index 9db8d51..eb6ae0c 100644 --- a/crates/ptth_server/src/file_server/metrics.rs +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -12,7 +12,7 @@ fn serialize_ulid (t: &Ulid, s: S) // They don't change after that. #[derive (Debug, serde::Serialize)] -pub struct PerInstance { +pub struct Startup { // D-Bus machine ID, if we're on Linux pub machine_id: Option , @@ -28,6 +28,7 @@ pub struct PerInstance { #[serde (serialize_with = "serialize_ulid")] pub instance_id: Ulid, + // System UTC pub startup_utc: DateTime , } @@ -54,7 +55,7 @@ fn get_machine_id () -> Option { Some (s) } -impl PerInstance { +impl Startup { #[must_use] pub fn new (server_name: String) -> Self { @@ -74,8 +75,8 @@ mod tests { #[test] fn ulid_null () { - let a = PerInstance::new ("bogus".to_string ()); - let b = PerInstance::new ("bogus".to_string ()); + let a = Startup::new ("bogus".to_string ()); + let b = Startup::new ("bogus".to_string ()); assert_ne! (a.instance_id, b.instance_id); } diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 37ee6a0..78481c4 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -89,7 +89,7 @@ struct DirEntryHtml { #[derive (Serialize)] struct DirHtml <'a> { #[serde (flatten)] - instance_metrics: &'a metrics::PerInstance, + instance_metrics: &'a metrics::Startup, path: Cow <'a, str>, entries: Vec , @@ -187,7 +187,7 @@ async fn read_dir_entry_json (entry: DirEntry) -> Option async fn serve_root ( handlebars: &Handlebars <'static>, - instance_metrics: &metrics::PerInstance + instance_metrics: &metrics::Startup ) -> Result { let s = handlebars.render ("file_server_root", &instance_metrics)?; @@ -232,7 +232,7 @@ async fn serve_dir_json ( #[instrument (level = "debug", skip (handlebars, instance_metrics, dir))] async fn serve_dir_html ( handlebars: &Handlebars <'static>, - instance_metrics: &metrics::PerInstance, + instance_metrics: &metrics::Startup, path: Cow <'_, str>, mut dir: ReadDir ) -> Result @@ -358,7 +358,7 @@ async fn serve_file ( #[instrument (level = "debug", skip (handlebars, headers, instance_metrics))] pub async fn serve_all ( handlebars: &Handlebars <'static>, - instance_metrics: &metrics::PerInstance, + instance_metrics: &metrics::Startup, root: &Path, method: Method, uri: &str, diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 0bf214e..0f2ae0b 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -54,7 +54,7 @@ pub fn password_is_bad (mut password: String) -> bool { struct ServerState { config: Config, handlebars: Handlebars <'static>, - instance_metrics: file_server::metrics::PerInstance, + instance_metrics: file_server::metrics::Startup, gauges: RwLock , client: Client, hidden_path: Option , @@ -205,7 +205,7 @@ pub async fn run_server ( .build ().map_err (ServerError::CantBuildHttpClient)?; let handlebars = file_server::load_templates (&asset_root)?; - let instance_metrics = file_server::metrics::PerInstance::new (config_file.name); + let instance_metrics = file_server::metrics::Startup::new (config_file.name); let state = Arc::new (ServerState { config: Config { From 009601e1367b9bafb42e79c531d8570e0fd7c815 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:04:19 +0000 Subject: [PATCH 140/208] :recycle: refactor: move file server config into file_server --- crates/ptth_file_server_bin/src/main.rs | 9 ++------- crates/ptth_server/src/file_server/mod.rs | 10 +++++++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 561072b..006ca74 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -32,13 +32,8 @@ use ptth_server::{ load_toml, }; -#[derive (Default)] -pub struct Config { - pub file_server_root: Option , -} - struct ServerState <'a> { - config: Config, + config: file_server::Config, handlebars: handlebars::Handlebars <'a>, instance_metrics: metrics::Startup, hidden_path: Option , @@ -112,7 +107,7 @@ async fn main () -> Result <(), anyhow::Error> { debug! ("{:?}", instance_metrics); let state = Arc::new (ServerState { - config: Config { + config: file_server::Config { file_server_root: config_file.file_server_root, }, handlebars, diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 78481c4..3c51859 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -9,7 +9,10 @@ use std::{ collections::HashMap, convert::{Infallible, TryFrom}, io::SeekFrom, - path::Path, + path::{ + Path, + PathBuf, + }, }; use handlebars::Handlebars; @@ -53,6 +56,11 @@ mod emoji { pub const ERROR: &str = "\u{26a0}\u{fe0f}"; } +#[derive (Default)] +pub struct Config { + pub file_server_root: Option , +} + #[derive (Serialize)] struct DirEntryJson { name: String, From 31750d30fc732dac79990b4e1ab92729d284f221 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:06:21 +0000 Subject: [PATCH 141/208] :recycle: refactor: use new file_server::Config in ptth_server --- crates/ptth_server/src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 0f2ae0b..c581bf6 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -73,7 +73,7 @@ async fn handle_one_req ( debug! ("Handling request {}", req_id); let default_root = PathBuf::from ("./"); - let file_server_root: &std::path::Path = state.config.file_server_root + let file_server_root: &std::path::Path = state.config.file_server.file_server_root .as_ref () .unwrap_or (&default_root); @@ -172,7 +172,7 @@ impl ConfigFile { #[derive (Default)] pub struct Config { pub relay_url: String, - pub file_server_root: Option , + pub file_server: file_server::Config, } pub async fn run_server ( @@ -210,7 +210,9 @@ pub async fn run_server ( let state = Arc::new (ServerState { config: Config { relay_url: config_file.relay_url, - file_server_root: config_file.file_server_root, + file_server: file_server::Config { + file_server_root: config_file.file_server_root, + }, }, handlebars, instance_metrics, From 1aff4389bdcc85f788b254da634ee074ca56e183 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:10:11 +0000 Subject: [PATCH 142/208] :recycle: refactor: move file server runtime state into file_server --- crates/ptth_file_server_bin/src/main.rs | 12 +++--------- crates/ptth_server/src/file_server/mod.rs | 7 +++++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 006ca74..6cce5cb 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -28,18 +28,12 @@ use ptth_server::{ file_server::{ self, metrics, + State, }, load_toml, }; -struct ServerState <'a> { - config: file_server::Config, - handlebars: handlebars::Handlebars <'a>, - instance_metrics: metrics::Startup, - hidden_path: Option , -} - -async fn handle_all (req: Request , state: Arc >) +async fn handle_all (req: Request , state: Arc >) -> Result , anyhow::Error> { use std::str::FromStr; @@ -106,7 +100,7 @@ async fn main () -> Result <(), anyhow::Error> { debug! ("{:?}", instance_metrics); - let state = Arc::new (ServerState { + let state = Arc::new (State { config: file_server::Config { file_server_root: config_file.file_server_root, }, diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 3c51859..5252534 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -61,6 +61,13 @@ pub struct Config { pub file_server_root: Option , } +pub struct State <'a> { + pub config: Config, + pub handlebars: handlebars::Handlebars <'a>, + pub instance_metrics: metrics::Startup, + pub hidden_path: Option , +} + #[derive (Serialize)] struct DirEntryJson { name: String, From 88c3500fd0d6e5456e3ee71b01137fc303c59c1a Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:21:02 +0000 Subject: [PATCH 143/208] :recycle: refactor --- crates/ptth_server/src/lib.rs | 37 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index c581bf6..4c95767 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -51,20 +51,18 @@ pub fn password_is_bad (mut password: String) -> bool { ac.find (BAD_PASSWORDS).is_some () } -struct ServerState { +struct State <'a> { + file_server: file_server::State <'a>, config: Config, - handlebars: Handlebars <'static>, - instance_metrics: file_server::metrics::Startup, gauges: RwLock , client: Client, - hidden_path: Option , } // Unwrap a request from PTTH format and pass it into file_server. // When file_server responds, wrap it back up and stream it to the relay. async fn handle_one_req ( - state: &Arc , + state: &Arc >, wrapped_req: http_serde::WrappedRequest ) -> Result <(), ServerError> { @@ -73,18 +71,18 @@ async fn handle_one_req ( debug! ("Handling request {}", req_id); let default_root = PathBuf::from ("./"); - let file_server_root: &std::path::Path = state.config.file_server.file_server_root + let file_server_root: &std::path::Path = state.file_server.config.file_server_root .as_ref () .unwrap_or (&default_root); let response = file_server::serve_all ( - &state.handlebars, - &state.instance_metrics, + &state.file_server.handlebars, + &state.file_server.instance_metrics, file_server_root, parts.method, &parts.uri, &parts.headers, - state.hidden_path.as_deref () + state.file_server.hidden_path.as_deref () ).await?; let mut resp_req = state.client @@ -123,7 +121,7 @@ async fn handle_one_req ( } async fn handle_req_resp ( - state: &Arc , + state: &Arc >, req_resp: reqwest::Response ) -> Result <(), ServerError> { @@ -172,7 +170,6 @@ impl ConfigFile { #[derive (Default)] pub struct Config { pub relay_url: String, - pub file_server: file_server::Config, } pub async fn run_server ( @@ -207,20 +204,22 @@ pub async fn run_server ( let instance_metrics = file_server::metrics::Startup::new (config_file.name); - let state = Arc::new (ServerState { - config: Config { - relay_url: config_file.relay_url, - file_server: file_server::Config { + let state = Arc::new (State { + file_server: file_server::State { + config: file_server::Config { file_server_root: config_file.file_server_root, }, + handlebars, + instance_metrics, + hidden_path, + }, + config: Config { + relay_url: config_file.relay_url, }, - handlebars, - instance_metrics, gauges: RwLock::new (file_server::metrics::Gauges { rss_mib: 7, }), client, - hidden_path, }); let mut backoff_delay = 0; @@ -243,7 +242,7 @@ pub async fn run_server ( debug! ("http_listen"); - let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, state.instance_metrics.server_name)).send (); + let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, state.file_server.instance_metrics.server_name)).send (); let err_backoff_delay = std::cmp::min (30_000, backoff_delay * 2 + 500); From 4bd38180d0ddb60ff512095c31f5aff7b2104095 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:23:17 +0000 Subject: [PATCH 144/208] :recycle: refactor: remove lifetime on handlebars --- crates/ptth_file_server_bin/src/main.rs | 2 +- crates/ptth_server/src/file_server/mod.rs | 4 ++-- crates/ptth_server/src/lib.rs | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 6cce5cb..bc666ac 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -33,7 +33,7 @@ use ptth_server::{ load_toml, }; -async fn handle_all (req: Request , state: Arc >) +async fn handle_all (req: Request , state: Arc ) -> Result , anyhow::Error> { use std::str::FromStr; diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 5252534..d98d395 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -61,9 +61,9 @@ pub struct Config { pub file_server_root: Option , } -pub struct State <'a> { +pub struct State { pub config: Config, - pub handlebars: handlebars::Handlebars <'a>, + pub handlebars: handlebars::Handlebars <'static>, pub instance_metrics: metrics::Startup, pub hidden_path: Option , } diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 4c95767..1152d14 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -51,8 +51,8 @@ pub fn password_is_bad (mut password: String) -> bool { ac.find (BAD_PASSWORDS).is_some () } -struct State <'a> { - file_server: file_server::State <'a>, +struct State { + file_server: file_server::State, config: Config, gauges: RwLock , client: Client, @@ -62,7 +62,7 @@ struct State <'a> { // When file_server responds, wrap it back up and stream it to the relay. async fn handle_one_req ( - state: &Arc >, + state: &Arc , wrapped_req: http_serde::WrappedRequest ) -> Result <(), ServerError> { @@ -121,7 +121,7 @@ async fn handle_one_req ( } async fn handle_req_resp ( - state: &Arc >, + state: &Arc , req_resp: reqwest::Response ) -> Result <(), ServerError> { From 066c95dc077b930e190a28ca46f53270685fb910 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:38:39 +0000 Subject: [PATCH 145/208] :recycle: refactor: extract html.rs --- crates/ptth_server/src/file_server/html.rs | 196 ++++++++++++++++++++ crates/ptth_server/src/file_server/mod.rs | 174 +---------------- crates/ptth_server/src/file_server/tests.rs | 23 --- crates/ptth_server/src/lib.rs | 1 - 4 files changed, 204 insertions(+), 190 deletions(-) create mode 100644 crates/ptth_server/src/file_server/html.rs diff --git a/crates/ptth_server/src/file_server/html.rs b/crates/ptth_server/src/file_server/html.rs new file mode 100644 index 0000000..498b3d2 --- /dev/null +++ b/crates/ptth_server/src/file_server/html.rs @@ -0,0 +1,196 @@ +use std::borrow::Cow; + +use handlebars::Handlebars; +use serde::Serialize; +use tracing::instrument; + +use super::{ + FileServerError, + Response, + metrics, + pretty_print_bytes, +}; + +#[derive (Serialize)] +struct Dir <'a> { + #[serde (flatten)] + instance_metrics: &'a metrics::Startup, + + path: Cow <'a, str>, + entries: Vec , +} + +#[derive (Serialize)] +struct DirEntry { + icon: &'static str, + trailing_slash: &'static str, + + // Unfortunately file_name will allocate as long as some platforms + // (Windows!) aren't UTF-8. Cause I don't want to write separate code + // for such a small problem. + + file_name: String, + + // This could be a Cow with file_name if no encoding was done but + // it's simpler to allocate. + + encoded_file_name: String, + + size: Cow <'static, str>, + + error: bool, +} + +pub async fn serve_root ( + handlebars: &Handlebars <'static>, + instance_metrics: &metrics::Startup +) -> Result +{ + let s = handlebars.render ("file_server_root", &instance_metrics)?; + + Ok (serve_html (s)) +} + +#[instrument (level = "debug", skip (handlebars, instance_metrics, dir))] +pub async fn serve_dir ( + handlebars: &Handlebars <'static>, + instance_metrics: &metrics::Startup, + path: Cow <'_, str>, + mut dir: tokio::fs::ReadDir +) -> Result +{ + let mut entries = vec! []; + + while let Ok (Some (entry)) = dir.next_entry ().await { + entries.push (read_dir_entry (entry).await); + } + + entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name)); + + let s = handlebars.render ("file_server_dir", &Dir { + path, + entries, + instance_metrics, + })?; + + Ok (serve_html (s)) +} + +async fn read_dir_entry (entry: tokio::fs::DirEntry) -> DirEntry +{ + use percent_encoding::{ + CONTROLS, + utf8_percent_encode, + }; + + let file_name = match entry.file_name ().into_string () { + Ok (x) => x, + Err (_) => return DirEntry { + icon: emoji::ERROR, + trailing_slash: "", + file_name: "File / directory name is not UTF-8".into (), + encoded_file_name: "".into (), + size: "".into (), + error: true, + }, + }; + + let metadata = match entry.metadata ().await { + Ok (x) => x, + Err (_) => return DirEntry { + icon: emoji::ERROR, + trailing_slash: "", + file_name: "Could not fetch metadata".into (), + encoded_file_name: "".into (), + size: "".into (), + error: true, + }, + }; + + let (trailing_slash, icon, size) = { + let t = metadata.file_type (); + + if t.is_dir () { + ("/", emoji::FOLDER, "".into ()) + } + else { + ("", get_icon (&file_name), pretty_print_bytes (metadata.len ()).into ()) + } + }; + + let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string (); + + DirEntry { + icon, + trailing_slash: &trailing_slash, + file_name, + encoded_file_name, + size, + error: false, + } +} + +pub fn serve_html (s: String) -> Response { + let mut resp = Response::default (); + resp + .header ("content-type".to_string (), "text/html; charset=UTF-8".to_string ().into_bytes ()) + .body_bytes (s.into_bytes ()) + ; + resp +} + +fn get_icon (file_name: &str) -> &'static str { + if + file_name.ends_with (".mp4") || + file_name.ends_with (".avi") || + file_name.ends_with (".mkv") || + file_name.ends_with (".webm") + { + emoji::VIDEO + } + else if + file_name.ends_with (".jpg") || + file_name.ends_with (".jpeg") || + file_name.ends_with (".png") || + file_name.ends_with (".bmp") + { + emoji::PICTURE + } + else { + emoji::FILE + } +} + +mod emoji { + pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; + pub const PICTURE: &str = "\u{1f4f7}"; + pub const FILE: &str = "\u{1f4c4}"; + pub const FOLDER: &str = "\u{1f4c1}"; + pub const ERROR: &str = "\u{26a0}\u{fe0f}"; +} + +#[cfg (test)] +mod tests { + #[test] + fn icons () { + let video = "🎞️"; + let picture = "📷"; + let file = "📄"; + + for (input, expected) in vec! [ + ("copying_is_not_theft.mp4", video), + ("copying_is_not_theft.avi", video), + ("copying_is_not_theft.mkv", video), + ("copying_is_not_theft.webm", video), + ("lolcats.jpg", picture), + ("lolcats.jpeg", picture), + ("lolcats.png", picture), + ("lolcats.bmp", picture), + ("ptth.log", file), + ("README.md", file), + ("todo.txt", file), + ].into_iter () { + assert_eq! (super::get_icon (input), expected); + } + } +} diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index d98d395..9c0afbc 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -4,7 +4,6 @@ #![allow (clippy::enum_glob_use)] use std::{ - borrow::Cow, cmp::min, collections::HashMap, convert::{Infallible, TryFrom}, @@ -42,20 +41,13 @@ use ptth_core::{ pub mod errors; pub mod metrics; +mod html; mod internal; mod markdown; mod range; use errors::FileServerError; -mod emoji { - pub const VIDEO: &str = "\u{1f39e}\u{fe0f}"; - pub const PICTURE: &str = "\u{1f4f7}"; - pub const FILE: &str = "\u{1f4c4}"; - pub const FOLDER: &str = "\u{1f4c1}"; - pub const ERROR: &str = "\u{26a0}\u{fe0f}"; -} - #[derive (Default)] pub struct Config { pub file_server_root: Option , @@ -68,122 +60,16 @@ pub struct State { pub hidden_path: Option , } -#[derive (Serialize)] -struct DirEntryJson { - name: String, - size: u64, - is_dir: bool, -} - #[derive (Serialize)] struct DirJson { entries: Vec , } #[derive (Serialize)] -struct DirEntryHtml { - icon: &'static str, - trailing_slash: &'static str, - - // Unfortunately file_name will allocate as long as some platforms - // (Windows!) aren't UTF-8. Cause I don't want to write separate code - // for such a small problem. - - file_name: String, - - // This could be a Cow with file_name if no encoding was done but - // it's simpler to allocate. - - encoded_file_name: String, - - size: Cow <'static, str>, - - error: bool, -} - -#[derive (Serialize)] -struct DirHtml <'a> { - #[serde (flatten)] - instance_metrics: &'a metrics::Startup, - - path: Cow <'a, str>, - entries: Vec , -} - -fn get_icon (file_name: &str) -> &'static str { - if - file_name.ends_with (".mp4") || - file_name.ends_with (".avi") || - file_name.ends_with (".mkv") || - file_name.ends_with (".webm") - { - emoji::VIDEO - } - else if - file_name.ends_with (".jpg") || - file_name.ends_with (".jpeg") || - file_name.ends_with (".png") || - file_name.ends_with (".bmp") - { - emoji::PICTURE - } - else { - emoji::FILE - } -} - -async fn read_dir_entry_html (entry: DirEntry) -> DirEntryHtml -{ - use percent_encoding::{ - CONTROLS, - utf8_percent_encode, - }; - - let file_name = match entry.file_name ().into_string () { - Ok (x) => x, - Err (_) => return DirEntryHtml { - icon: emoji::ERROR, - trailing_slash: "", - file_name: "File / directory name is not UTF-8".into (), - encoded_file_name: "".into (), - size: "".into (), - error: true, - }, - }; - - let metadata = match entry.metadata ().await { - Ok (x) => x, - Err (_) => return DirEntryHtml { - icon: emoji::ERROR, - trailing_slash: "", - file_name: "Could not fetch metadata".into (), - encoded_file_name: "".into (), - size: "".into (), - error: true, - }, - }; - - let (trailing_slash, icon, size) = { - let t = metadata.file_type (); - - if t.is_dir () { - ("/", emoji::FOLDER, "".into ()) - } - else { - ("", get_icon (&file_name), pretty_print_bytes (metadata.len ()).into ()) - } - }; - - let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string (); - - DirEntryHtml { - icon, - trailing_slash: &trailing_slash, - file_name, - encoded_file_name, - size, - error: false, - } +struct DirEntryJson { + name: String, + size: u64, + is_dir: bool, } async fn read_dir_entry_json (entry: DirEntry) -> Option @@ -200,25 +86,6 @@ async fn read_dir_entry_json (entry: DirEntry) -> Option }) } -async fn serve_root ( - handlebars: &Handlebars <'static>, - instance_metrics: &metrics::Startup -) -> Result -{ - let s = handlebars.render ("file_server_root", &instance_metrics)?; - - Ok (serve_html (s)) -} - -fn serve_html (s: String) -> Response { - let mut resp = Response::default (); - resp - .header ("content-type".to_string (), "text/html; charset=UTF-8".to_string ().into_bytes ()) - .body_bytes (s.into_bytes ()) - ; - resp -} - async fn serve_dir_json ( mut dir: ReadDir ) -> Result @@ -244,31 +111,6 @@ async fn serve_dir_json ( Ok (response) } -#[instrument (level = "debug", skip (handlebars, instance_metrics, dir))] -async fn serve_dir_html ( - handlebars: &Handlebars <'static>, - instance_metrics: &metrics::Startup, - path: Cow <'_, str>, - mut dir: ReadDir -) -> Result -{ - let mut entries = vec! []; - - while let Ok (Some (entry)) = dir.next_entry ().await { - entries.push (read_dir_entry_html (entry).await); - } - - entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name)); - - let s = handlebars.render ("file_server_dir", &DirHtml { - path, - entries, - instance_metrics, - })?; - - Ok (serve_html (s)) -} - #[instrument (level = "debug", skip (f))] async fn serve_file ( mut f: File, @@ -419,14 +261,14 @@ pub async fn serve_all ( }, InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"), - Root => serve_root (handlebars, instance_metrics).await?, + Root => html::serve_root (handlebars, instance_metrics).await?, ServeDir (internal::ServeDirParams { path, dir, format }) => match format { OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?, - OutputFormat::Html => serve_dir_html (handlebars, instance_metrics, path.to_string_lossy (), dir.into_inner ()).await?, + OutputFormat::Html => html::serve_dir (handlebars, instance_metrics, path.to_string_lossy (), dir.into_inner ()).await?, }, ServeFile (internal::ServeFileParams { file, @@ -443,7 +285,7 @@ pub async fn serve_all ( serve_error (code, e.to_string ()) }, - MarkdownPreview (s) => serve_html (s), + MarkdownPreview (s) => html::serve_html (s), }) } diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index 9f096be..c616a29 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -9,29 +9,6 @@ use std::{ use maplit::*; use tokio::runtime::Runtime; -#[test] -fn icons () { - let video = "🎞️"; - let picture = "📷"; - let file = "📄"; - - for (input, expected) in vec! [ - ("copying_is_not_theft.mp4", video), - ("copying_is_not_theft.avi", video), - ("copying_is_not_theft.mkv", video), - ("copying_is_not_theft.webm", video), - ("lolcats.jpg", picture), - ("lolcats.jpeg", picture), - ("lolcats.png", picture), - ("lolcats.bmp", picture), - ("ptth.log", file), - ("README.md", file), - ("todo.txt", file), - ].into_iter () { - assert_eq! (super::get_icon (input), expected); - } -} - #[test] fn pretty_print_bytes () { for (input_after, expected_before, expected_after) in vec! [ diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 1152d14..809edb8 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -14,7 +14,6 @@ use std::{ }; use futures::FutureExt; -use handlebars::Handlebars; use reqwest::Client; use serde::Deserialize; use tokio::{ From 47788f56e087d9aaa76e3521cd537f513b2e63d9 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:40:49 +0000 Subject: [PATCH 146/208] :recycle: refactor: use `use X as Y` --- crates/ptth_server/src/file_server/html.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/ptth_server/src/file_server/html.rs b/crates/ptth_server/src/file_server/html.rs index 498b3d2..325cb30 100644 --- a/crates/ptth_server/src/file_server/html.rs +++ b/crates/ptth_server/src/file_server/html.rs @@ -3,6 +3,10 @@ use std::borrow::Cow; use handlebars::Handlebars; use serde::Serialize; use tracing::instrument; +use tokio::fs::{ + DirEntry as FsDirEntry, + ReadDir as FsReadDir, +}; use super::{ FileServerError, @@ -56,7 +60,7 @@ pub async fn serve_dir ( handlebars: &Handlebars <'static>, instance_metrics: &metrics::Startup, path: Cow <'_, str>, - mut dir: tokio::fs::ReadDir + mut dir: FsReadDir ) -> Result { let mut entries = vec! []; @@ -76,7 +80,7 @@ pub async fn serve_dir ( Ok (serve_html (s)) } -async fn read_dir_entry (entry: tokio::fs::DirEntry) -> DirEntry +async fn read_dir_entry (entry: FsDirEntry) -> DirEntry { use percent_encoding::{ CONTROLS, From bc361fa876b277280eee648c7f631d3345495971 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:52:53 +0000 Subject: [PATCH 147/208] :heavy_plus_sign: update: add heim for measuring process RSS --- Cargo.lock | 490 +++++++++++++++++- crates/ptth_server/Cargo.toml | 2 + crates/ptth_server/src/file_server/errors.rs | 9 + crates/ptth_server/src/file_server/metrics.rs | 15 + 4 files changed, 515 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3bb3d07..659cf6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,122 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "async-channel" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59740d83946db6a5af71ae25ddf9562c2b176b2ca42cf99a455f09f4a220d6b9" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb877970c7b440ead138f6321a3b5395d6061183af779340b65e20c0fede9146" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "vec-arena", +] + +[[package]] +name = "async-fs" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b3ca4f8ff117c37c278a2f7415ce9be55560b846b5bc4412aaa5d29c1c3dae2" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-io" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9315f8f07556761c3e48fec2e6b276004acf426e6dc068b2c2251854d65ee0fd" +dependencies = [ + "concurrent-queue", + "fastrand", + "futures-lite", + "libc", + "log", + "nb-connect", + "once_cell", + "parking", + "polling", + "vec-arena", + "waker-fn", + "winapi 0.3.9", +] + +[[package]] +name = "async-lock" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1996609732bde4a9988bc42125f55f2af5f3c36370e27c778d5191a4a1b63bfb" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-net" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06de475c85affe184648202401d7622afb32f0f74e02192857d0201a16defbe5" +dependencies = [ + "async-io", + "blocking", + "fastrand", + "futures-lite", +] + +[[package]] +name = "async-process" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8cea09c1fb10a317d1b5af8024eeba256d6554763e85ecd90ff8df31c7bbda" +dependencies = [ + "async-io", + "blocking", + "cfg-if 0.1.10", + "event-listener", + "futures-lite", + "once_cell", + "signal-hook", + "winapi 0.3.9", +] + +[[package]] +name = "async-task" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" + +[[package]] +name = "async-trait" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + [[package]] name = "atty" version = "0.2.14" @@ -135,6 +251,20 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "blocking" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + [[package]] name = "bumpalo" version = "3.4.0" @@ -159,6 +289,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + [[package]] name = "cc" version = "1.0.66" @@ -217,6 +353,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "console_error_panic_hook" version = "0.1.6" @@ -293,10 +438,30 @@ version = "3.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b57a92e9749e10f25a171adcebfafe72991d45e7ec2dcb853e8f83d9dafaeb08" dependencies = [ - "nix", + "nix 0.18.0", "winapi 0.3.9", ] +[[package]] +name = "darwin-libproc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc629b7cf42586fee31dae31f9ab73fa5ff5f0170016aa61be5fcbc12a90c516" +dependencies = [ + "darwin-libproc-sys", + "libc", + "memchr", +] + +[[package]] +name = "darwin-libproc-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef0aa083b94c54aa4cfd9bbfd37856714c139d1dc511af80270558c7ba3b4816" +dependencies = [ + "libc", +] + [[package]] name = "dashmap" version = "3.11.10" @@ -341,12 +506,27 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + [[package]] name = "fake-simd" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "fastrand" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3" +dependencies = [ + "instant", +] + [[package]] name = "fnv" version = "1.0.7" @@ -448,6 +628,21 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" +[[package]] +name = "futures-lite" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4481d0cd0de1d204a4fa55e7d45f07b1d958abcb06714b3446438e2eff695fb" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite 0.2.0", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.8" @@ -475,6 +670,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.8" @@ -558,6 +759,12 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "h2" version = "0.2.7" @@ -607,6 +814,125 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heim" +version = "0.1.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a653442b9bdd11a77d3753a60443c60c4437d3acac8e6c3d4a6a9acd7cceed" +dependencies = [ + "heim-common", + "heim-process", + "heim-runtime", +] + +[[package]] +name = "heim-common" +version = "0.1.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767e6e47cf88abe7c9a5ebb4df82f180d30d9c0ba0269b6d166482461765834" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation", + "futures-core", + "futures-util", + "lazy_static", + "libc", + "mach", + "nix 0.19.1", + "pin-utils", + "uom", + "winapi 0.3.9", +] + +[[package]] +name = "heim-cpu" +version = "0.1.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba5fb13a3b90581d22b4edf99e87c54316444622ae123d36816a227a7caa6df" +dependencies = [ + "cfg-if 1.0.0", + "futures", + "glob", + "heim-common", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "ntapi", + "smol", + "winapi 0.3.9", +] + +[[package]] +name = "heim-host" +version = "0.1.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9abf6cd02bc4f6e6aa31a7f80702a2d0e574f4f2c6156a93c3550eb036304722" +dependencies = [ + "cfg-if 1.0.0", + "heim-common", + "heim-runtime", + "lazy_static", + "libc", + "log", + "mach", + "ntapi", + "platforms", + "winapi 0.3.9", +] + +[[package]] +name = "heim-net" +version = "0.1.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d13afa5e9b71c813c1e087bb27f51ae87d3a6d68a2bdd045bae4322dfae4948b" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "heim-common", + "heim-runtime", + "libc", + "macaddr", + "nix 0.19.1", +] + +[[package]] +name = "heim-process" +version = "0.1.1-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "386870aac75d29b817fe709f1ef4303553e0af369e3c71d04beceeb50ae2cf39" +dependencies = [ + "async-trait", + "cfg-if 1.0.0", + "darwin-libproc", + "futures", + "heim-common", + "heim-cpu", + "heim-host", + "heim-net", + "heim-runtime", + "lazy_static", + "libc", + "mach", + "memchr", + "ntapi", + "ordered-float", + "smol", + "winapi 0.3.9", +] + +[[package]] +name = "heim-runtime" +version = "0.1.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ec7e5238c8f0dd0cc60914d31a5a7aadd4cde74c966a76c1caed1f5224e9b8" +dependencies = [ + "futures", + "futures-timer", + "once_cell", + "smol", +] + [[package]] name = "hermit-abi" version = "0.1.17" @@ -707,6 +1033,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -790,6 +1125,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "macaddr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "maplit" version = "1.0.2" @@ -915,6 +1265,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nb-connect" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8123a81538e457d44b933a02faf885d3fe8408806b23fa700e8f01c6c3a98998" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "net2" version = "0.2.37" @@ -938,6 +1298,27 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -948,6 +1329,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -1012,6 +1404,21 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-float" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dacdec97876ef3ede8c50efc429220641a0b11ba0048b4b0c357bccbc47c5204" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1125,6 +1532,25 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +[[package]] +name = "platforms" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc77a3fc329982cbf3ea772aa265b742a550998bad65747c630406ee52dac425" + +[[package]] +name = "polling" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a7bc6b2a29e632e45451c941832803a18cce6781db04de8a04696cdca8bde4" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "log", + "wepoll-sys", + "winapi 0.3.9", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1261,6 +1687,7 @@ dependencies = [ "chrono", "futures", "handlebars", + "heim", "http", "lazy_static", "maplit", @@ -1281,6 +1708,7 @@ dependencies = [ "tracing-futures", "tracing-subscriber", "ulid", + "uom", ] [[package]] @@ -1701,6 +2129,16 @@ dependencies = [ "loom", ] +[[package]] +name = "signal-hook" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604508c1418b99dfe1925ca9224829bb2a8a9a04dda655cc01fcad46f4ab05ed" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.2.2" @@ -1722,6 +2160,24 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" +[[package]] +name = "smol" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cf3b5351f3e783c1d79ab5fc604eeed8b8ae9abd36b166e8b87a089efd85e4" +dependencies = [ + "async-channel", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-net", + "async-process", + "blocking", + "futures-lite", + "once_cell", +] + [[package]] name = "socket2" version = "0.3.17" @@ -2102,6 +2558,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "uom" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e76503e636584f1e10b9b3b9498538279561adcef5412927ba00c2b32c4ce5ed" +dependencies = [ + "num-rational", + "num-traits", + "typenum", +] + [[package]] name = "url" version = "2.2.0" @@ -2120,6 +2587,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" +[[package]] +name = "vec-arena" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eafc1b9b2dfc6f5529177b62cf806484db55b32dc7c9658a118e11bbeb33061d" + [[package]] name = "vec_map" version = "0.8.2" @@ -2132,6 +2605,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "want" version = "0.3.0" @@ -2256,6 +2735,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wepoll-sys" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" +dependencies = [ + "cc", +] + [[package]] name = "winapi" version = "0.2.8" diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index 4f5b160..967bb15 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -15,6 +15,7 @@ blake3 = "0.3.7" chrono = {version = "0.4.19", features = ["serde"]} futures = "0.3.7" handlebars = "3.5.1" +heim = { version = "0.1.0-rc.1", features = ["process"] } http = "0.2.1" lazy_static = "1.4.0" percent-encoding = "2.1.0" @@ -32,6 +33,7 @@ tracing-futures = "0.2.4" tracing-subscriber = "0.2.15" toml = "0.5.7" ulid = "0.4.1" +uom = "0.30.0" always_equal = { path = "../always_equal" } ptth_core = { path = "../ptth_core" } diff --git a/crates/ptth_server/src/file_server/errors.rs b/crates/ptth_server/src/file_server/errors.rs index dbb9127..7be513c 100644 --- a/crates/ptth_server/src/file_server/errors.rs +++ b/crates/ptth_server/src/file_server/errors.rs @@ -25,4 +25,13 @@ pub enum FileServerError { #[error ("Invalid URI")] InvalidUri (#[from] http::uri::InvalidUri), + + #[error ("Heim process error")] + HeimProcess, +} + +impl From for FileServerError { + fn from (_: heim::process::ProcessError) -> Self { + FileServerError::HeimProcess + } } diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs index eb6ae0c..5165022 100644 --- a/crates/ptth_server/src/file_server/metrics.rs +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -37,6 +37,21 @@ pub struct Gauges { pub rss_mib: u64, } +impl Gauges { + pub async fn new () -> Result { + use heim::process; + use uom::si::information::mebibyte; + + let our_process = process::current ().await?; + let mem = our_process.memory ().await?; + let rss_mib = mem.rss ().get:: (); + + Ok (Gauges { + rss_mib, + }) + } +} + fn get_machine_id () -> Option { use std::{ fs::File, From 64ac4baaa841ee6c6e0620beec850cd6af97cb52 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 18:58:14 +0000 Subject: [PATCH 148/208] :heavy_plus_sign: update: print RSS in MiB at startup --- crates/ptth_file_server_bin/src/main.rs | 2 +- crates/ptth_server/src/file_server/metrics.rs | 17 +++++++++++++---- crates/ptth_server/src/lib.rs | 4 +--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index bc666ac..e2496a0 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -98,7 +98,7 @@ async fn main () -> Result <(), anyhow::Error> { config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) ); - debug! ("{:?}", instance_metrics); + let gauges = metrics::Gauges::new ().await?; let state = Arc::new (State { config: file_server::Config { diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs index 5165022..657c39e 100644 --- a/crates/ptth_server/src/file_server/metrics.rs +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, Utc}; +use tracing::debug; use ulid::Ulid; fn serialize_ulid (t: &Ulid, s: S) @@ -46,9 +47,13 @@ impl Gauges { let mem = our_process.memory ().await?; let rss_mib = mem.rss ().get:: (); - Ok (Gauges { + let x = Gauges { rss_mib, - }) + }; + + debug! ("metric gauges: {:?}", x); + + Ok (x) } } @@ -74,13 +79,17 @@ impl Startup { #[must_use] pub fn new (server_name: String) -> Self { - Self { + let x = Self { machine_id: get_machine_id (), git_version: None, server_name, instance_id: ulid::Ulid::new (), startup_utc: Utc::now (), - } + }; + + debug! ("metrics at startup: {:?}", x); + + x } } diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 809edb8..d8f222b 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -215,9 +215,7 @@ pub async fn run_server ( config: Config { relay_url: config_file.relay_url, }, - gauges: RwLock::new (file_server::metrics::Gauges { - rss_mib: 7, - }), + gauges: RwLock::new (file_server::metrics::Gauges::new ().await?), client, }); From b2b0bbc8fc4d492d00c77263e281c69ed837a92e Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 19:00:22 +0000 Subject: [PATCH 149/208] :recycle: refactor: rename --- crates/ptth_file_server_bin/src/main.rs | 6 +++--- crates/ptth_server/src/file_server/mod.rs | 10 +++++----- crates/ptth_server/src/lib.rs | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index e2496a0..0fcd807 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -56,7 +56,7 @@ async fn handle_all (req: Request , state: Arc ) let ptth_resp = file_server::serve_all ( &state.handlebars, - &state.instance_metrics, + &state.metrics_startup, file_server_root, ptth_req.method, &ptth_req.uri, @@ -94,7 +94,7 @@ async fn main () -> Result <(), anyhow::Error> { let handlebars = file_server::load_templates (&PathBuf::new ())?; - let instance_metrics = metrics::Startup::new ( + let metrics_startup = metrics::Startup::new ( config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) ); @@ -105,7 +105,7 @@ async fn main () -> Result <(), anyhow::Error> { file_server_root: config_file.file_server_root, }, handlebars, - instance_metrics, + metrics_startup, hidden_path: Some (path), }); diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 9c0afbc..5835cc8 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -56,7 +56,7 @@ pub struct Config { pub struct State { pub config: Config, pub handlebars: handlebars::Handlebars <'static>, - pub instance_metrics: metrics::Startup, + pub metrics_startup: metrics::Startup, pub hidden_path: Option , } @@ -212,10 +212,10 @@ async fn serve_file ( // When it returns, prettify it as HTML or JSON based on what the client // asked for. -#[instrument (level = "debug", skip (handlebars, headers, instance_metrics))] +#[instrument (level = "debug", skip (handlebars, headers, metrics_startup))] pub async fn serve_all ( handlebars: &Handlebars <'static>, - instance_metrics: &metrics::Startup, + metrics_startup: &metrics::Startup, root: &Path, method: Method, uri: &str, @@ -261,14 +261,14 @@ pub async fn serve_all ( }, InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"), - Root => html::serve_root (handlebars, instance_metrics).await?, + Root => html::serve_root (handlebars, metrics_startup).await?, ServeDir (internal::ServeDirParams { path, dir, format }) => match format { OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?, - OutputFormat::Html => html::serve_dir (handlebars, instance_metrics, path.to_string_lossy (), dir.into_inner ()).await?, + OutputFormat::Html => html::serve_dir (handlebars, metrics_startup, path.to_string_lossy (), dir.into_inner ()).await?, }, ServeFile (internal::ServeFileParams { file, diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index d8f222b..0858133 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -76,7 +76,7 @@ async fn handle_one_req ( let response = file_server::serve_all ( &state.file_server.handlebars, - &state.file_server.instance_metrics, + &state.file_server.metrics_startup, file_server_root, parts.method, &parts.uri, @@ -201,7 +201,7 @@ pub async fn run_server ( .build ().map_err (ServerError::CantBuildHttpClient)?; let handlebars = file_server::load_templates (&asset_root)?; - let instance_metrics = file_server::metrics::Startup::new (config_file.name); + let metrics_startup = file_server::metrics::Startup::new (config_file.name); let state = Arc::new (State { file_server: file_server::State { @@ -209,7 +209,7 @@ pub async fn run_server ( file_server_root: config_file.file_server_root, }, handlebars, - instance_metrics, + metrics_startup, hidden_path, }, config: Config { @@ -239,7 +239,7 @@ pub async fn run_server ( debug! ("http_listen"); - let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, state.file_server.instance_metrics.server_name)).send (); + let req_req = state.client.get (&format! ("{}/http_listen/{}", state.config.relay_url, state.file_server.metrics_startup.server_name)).send (); let err_backoff_delay = std::cmp::min (30_000, backoff_delay * 2 + 500); From e5103d48bd635252d1f992b090d0f9443fca7ef7 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 19:35:32 +0000 Subject: [PATCH 150/208] :heavy_plus_sign: update: add ArcSwap to update gauges lock-free --- Cargo.lock | 8 +++++++ crates/ptth_file_server_bin/Cargo.toml | 1 + crates/ptth_file_server_bin/src/main.rs | 23 ++++++++++++++++++- crates/ptth_server/Cargo.toml | 1 + crates/ptth_server/src/file_server/metrics.rs | 2 ++ crates/ptth_server/src/file_server/mod.rs | 3 +++ crates/ptth_server/src/lib.rs | 9 +++++--- 7 files changed, 43 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 659cf6f..55a1d67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,6 +46,12 @@ version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4" +[[package]] +name = "arc-swap" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec5a4539a733493f412c4d0bb962748ea1f90f3dfdba9ff3ee18acbefc3b33f0" + [[package]] name = "arrayref" version = "0.3.6" @@ -1636,6 +1642,7 @@ name = "ptth_file_server" version = "0.1.0" dependencies = [ "anyhow", + "arc-swap", "handlebars", "http", "hyper", @@ -1682,6 +1689,7 @@ dependencies = [ "aho-corasick", "always_equal", "anyhow", + "arc-swap", "base64 0.12.3", "blake3", "chrono", diff --git a/crates/ptth_file_server_bin/Cargo.toml b/crates/ptth_file_server_bin/Cargo.toml index 04b67b4..af3cf68 100644 --- a/crates/ptth_file_server_bin/Cargo.toml +++ b/crates/ptth_file_server_bin/Cargo.toml @@ -9,6 +9,7 @@ license = "AGPL-3.0" [dependencies] anyhow = "1.0.34" +arc-swap = "1.1.0" handlebars = "3.5.1" http = "0.2.1" hyper = "0.13.8" diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 0fcd807..32a25cb 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -6,6 +6,7 @@ use std::{ sync::Arc, }; +use arc_swap::ArcSwap; use hyper::{ Body, Request, @@ -98,7 +99,26 @@ async fn main () -> Result <(), anyhow::Error> { config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) ); - let gauges = metrics::Gauges::new ().await?; + let metrics_gauge = Arc::new (ArcSwap::from_pointee (None)); + + let gauge_writer = Arc::clone (&metrics_gauge); + + tokio::spawn (async move { + let mut interval = tokio::time::interval (std::time::Duration::from_secs (2)); + loop { + interval.tick ().await; + + let new_gauges = match file_server::metrics::Gauges::new ().await { + Err (e) => { + error! ("Failed to update gauge metrics: {:?}", e); + continue; + }, + Ok (x) => x, + }; + let new_gauges = Arc::new (Some (new_gauges)); + gauge_writer.store (new_gauges); + } + }); let state = Arc::new (State { config: file_server::Config { @@ -106,6 +126,7 @@ async fn main () -> Result <(), anyhow::Error> { }, handlebars, metrics_startup, + metrics_gauge, hidden_path: Some (path), }); diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index 967bb15..d03c9c4 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -10,6 +10,7 @@ license = "AGPL-3.0" aho-corasick = "0.7.14" anyhow = "1.0.34" +arc-swap = "1.1.0" base64 = "0.12.3" blake3 = "0.3.7" chrono = {version = "0.4.19", features = ["serde"]} diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs index 657c39e..659f4e3 100644 --- a/crates/ptth_server/src/file_server/metrics.rs +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -35,6 +35,7 @@ pub struct Startup { #[derive (Debug, serde::Serialize)] pub struct Gauges { + pub utc: DateTime , pub rss_mib: u64, } @@ -48,6 +49,7 @@ impl Gauges { let rss_mib = mem.rss ().get:: (); let x = Gauges { + utc: Utc::now (), rss_mib, }; diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 5835cc8..8d0934a 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -12,8 +12,10 @@ use std::{ Path, PathBuf, }, + sync::Arc, }; +use arc_swap::ArcSwap; use handlebars::Handlebars; use serde::Serialize; use tokio::{ @@ -57,6 +59,7 @@ pub struct State { pub config: Config, pub handlebars: handlebars::Handlebars <'static>, pub metrics_startup: metrics::Startup, + pub metrics_gauge: Arc >>, pub hidden_path: Option , } diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 0858133..3901dd5 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -53,7 +53,6 @@ pub fn password_is_bad (mut password: String) -> bool { struct State { file_server: file_server::State, config: Config, - gauges: RwLock , client: Client, } @@ -179,8 +178,11 @@ pub async fn run_server ( ) -> Result <(), ServerError> { - use std::convert::TryInto; + use std::{ + convert::TryInto, + }; + use arc_swap::ArcSwap; use http::status::StatusCode; let asset_root = asset_root.unwrap_or_else (PathBuf::new); @@ -202,6 +204,7 @@ pub async fn run_server ( let handlebars = file_server::load_templates (&asset_root)?; let metrics_startup = file_server::metrics::Startup::new (config_file.name); + let metrics_gauge = Arc::new (ArcSwap::from_pointee (None)); let state = Arc::new (State { file_server: file_server::State { @@ -210,12 +213,12 @@ pub async fn run_server ( }, handlebars, metrics_startup, + metrics_gauge, hidden_path, }, config: Config { relay_url: config_file.relay_url, }, - gauges: RwLock::new (file_server::metrics::Gauges::new ().await?), client, }); From 4cb0911b7702534f2503197f3a8f613b203a18bd Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 19:46:30 +0000 Subject: [PATCH 151/208] :heavy_plus_sign: update: add RSS MiB to server root page --- crates/ptth_file_server_bin/src/main.rs | 1 + crates/ptth_server/src/file_server/html.rs | 16 ++++++++++++++-- crates/ptth_server/src/file_server/mod.rs | 3 ++- crates/ptth_server/src/lib.rs | 1 + handlebars/server/file_server_root.html | 8 +++++++- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 32a25cb..4363bc3 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -58,6 +58,7 @@ async fn handle_all (req: Request , state: Arc ) let ptth_resp = file_server::serve_all ( &state.handlebars, &state.metrics_startup, + &**state.metrics_gauge.load (), file_server_root, ptth_req.method, &ptth_req.uri, diff --git a/crates/ptth_server/src/file_server/html.rs b/crates/ptth_server/src/file_server/html.rs index 325cb30..9c2a58a 100644 --- a/crates/ptth_server/src/file_server/html.rs +++ b/crates/ptth_server/src/file_server/html.rs @@ -47,10 +47,22 @@ struct DirEntry { pub async fn serve_root ( handlebars: &Handlebars <'static>, - instance_metrics: &metrics::Startup + metrics_startup: &metrics::Startup, + metrics_gauges: &Option , ) -> Result { - let s = handlebars.render ("file_server_root", &instance_metrics)?; + #[derive (Serialize)] + struct RootHtml <'a> { + metrics_startup: &'a metrics::Startup, + metrics_gauges: &'a Option , + } + + let params = RootHtml { + metrics_startup, + metrics_gauges, + }; + + let s = handlebars.render ("file_server_root", ¶ms)?; Ok (serve_html (s)) } diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 8d0934a..f8d45e4 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -219,6 +219,7 @@ async fn serve_file ( pub async fn serve_all ( handlebars: &Handlebars <'static>, metrics_startup: &metrics::Startup, + metrics_gauges: &Option , root: &Path, method: Method, uri: &str, @@ -264,7 +265,7 @@ pub async fn serve_all ( }, InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"), - Root => html::serve_root (handlebars, metrics_startup).await?, + Root => html::serve_root (handlebars, metrics_startup, metrics_gauges).await?, ServeDir (internal::ServeDirParams { path, dir, diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 3901dd5..5646498 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -76,6 +76,7 @@ async fn handle_one_req ( let response = file_server::serve_all ( &state.file_server.handlebars, &state.file_server.metrics_startup, + &**state.file_server.metrics_gauge.load (), file_server_root, parts.method, &parts.uri, diff --git a/handlebars/server/file_server_root.html b/handlebars/server/file_server_root.html index a234e11..e852ecc 100644 --- a/handlebars/server/file_server_root.html +++ b/handlebars/server/file_server_root.html @@ -16,10 +16,16 @@ background-color: #ddd; } -{{server_name}} +{{metrics_startup.server_name}} +

{{metrics_startup.server_name}}

+ +

Gauges

+ +

RSS MiB: {{metrics_gauges.rss_mib}}

+
From 96106e68fcfe8944adedf7e1c6f18b17e94fbb47 Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 19:55:20 +0000 Subject: [PATCH 152/208] :recycle: refactor: use file_server::State directly more --- crates/ptth_file_server_bin/src/main.rs | 13 +++++-------- crates/ptth_server/src/file_server/html.rs | 10 ++++------ crates/ptth_server/src/file_server/mod.rs | 17 +++++++---------- crates/ptth_server/src/lib.rs | 10 +++------- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 4363bc3..a5c5554 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -56,14 +56,11 @@ async fn handle_all (req: Request , state: Arc ) .unwrap_or (&default_root); let ptth_resp = file_server::serve_all ( - &state.handlebars, - &state.metrics_startup, - &**state.metrics_gauge.load (), + &state, file_server_root, ptth_req.method, &ptth_req.uri, - &ptth_req.headers, - state.hidden_path.as_deref () + &ptth_req.headers ).await?; let mut resp = Response::builder () @@ -100,9 +97,9 @@ async fn main () -> Result <(), anyhow::Error> { config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) ); - let metrics_gauge = Arc::new (ArcSwap::from_pointee (None)); + let metrics_gauges = Arc::new (ArcSwap::from_pointee (None)); - let gauge_writer = Arc::clone (&metrics_gauge); + let gauge_writer = Arc::clone (&metrics_gauges); tokio::spawn (async move { let mut interval = tokio::time::interval (std::time::Duration::from_secs (2)); @@ -127,7 +124,7 @@ async fn main () -> Result <(), anyhow::Error> { }, handlebars, metrics_startup, - metrics_gauge, + metrics_gauges, hidden_path: Some (path), }); diff --git a/crates/ptth_server/src/file_server/html.rs b/crates/ptth_server/src/file_server/html.rs index 9c2a58a..0a1fdfa 100644 --- a/crates/ptth_server/src/file_server/html.rs +++ b/crates/ptth_server/src/file_server/html.rs @@ -46,9 +46,7 @@ struct DirEntry { } pub async fn serve_root ( - handlebars: &Handlebars <'static>, - metrics_startup: &metrics::Startup, - metrics_gauges: &Option , + state: &super::State, ) -> Result { #[derive (Serialize)] @@ -58,11 +56,11 @@ pub async fn serve_root ( } let params = RootHtml { - metrics_startup, - metrics_gauges, + metrics_startup: &state.metrics_startup, + metrics_gauges: &**state.metrics_gauges.load (), }; - let s = handlebars.render ("file_server_root", ¶ms)?; + let s = state.handlebars.render ("file_server_root", ¶ms)?; Ok (serve_html (s)) } diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index f8d45e4..3817ee4 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -59,7 +59,7 @@ pub struct State { pub config: Config, pub handlebars: handlebars::Handlebars <'static>, pub metrics_startup: metrics::Startup, - pub metrics_gauge: Arc >>, + pub metrics_gauges: Arc >>, pub hidden_path: Option , } @@ -215,16 +215,13 @@ async fn serve_file ( // When it returns, prettify it as HTML or JSON based on what the client // asked for. -#[instrument (level = "debug", skip (handlebars, headers, metrics_startup))] +#[instrument (level = "debug", skip (state, headers))] pub async fn serve_all ( - handlebars: &Handlebars <'static>, - metrics_startup: &metrics::Startup, - metrics_gauges: &Option , + state: &State, root: &Path, method: Method, uri: &str, - headers: &HashMap >, - hidden_path: Option <&Path> + headers: &HashMap > ) -> Result { @@ -245,7 +242,7 @@ pub async fn serve_all ( resp } - Ok (match internal::serve_all (root, method, uri, headers, hidden_path).await? { + Ok (match internal::serve_all (root, method, uri, headers, state.hidden_path.as_ref ().map (|p| p.as_path ())).await? { Favicon => serve_error (StatusCode::NotFound, "Not found\n"), Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden\n"), MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method\n"), @@ -265,14 +262,14 @@ pub async fn serve_all ( }, InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"), - Root => html::serve_root (handlebars, metrics_startup, metrics_gauges).await?, + Root => html::serve_root (state).await?, ServeDir (internal::ServeDirParams { path, dir, format }) => match format { OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?, - OutputFormat::Html => html::serve_dir (handlebars, metrics_startup, path.to_string_lossy (), dir.into_inner ()).await?, + OutputFormat::Html => html::serve_dir (&state.handlebars, &state.metrics_startup, path.to_string_lossy (), dir.into_inner ()).await?, }, ServeFile (internal::ServeFileParams { file, diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 5646498..81e619b 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -19,7 +19,6 @@ use serde::Deserialize; use tokio::{ sync::{ oneshot, - RwLock, }, time::delay_for, }; @@ -74,14 +73,11 @@ async fn handle_one_req ( .unwrap_or (&default_root); let response = file_server::serve_all ( - &state.file_server.handlebars, - &state.file_server.metrics_startup, - &**state.file_server.metrics_gauge.load (), + &state.file_server, file_server_root, parts.method, &parts.uri, &parts.headers, - state.file_server.hidden_path.as_deref () ).await?; let mut resp_req = state.client @@ -205,7 +201,7 @@ pub async fn run_server ( let handlebars = file_server::load_templates (&asset_root)?; let metrics_startup = file_server::metrics::Startup::new (config_file.name); - let metrics_gauge = Arc::new (ArcSwap::from_pointee (None)); + let metrics_gauges = Arc::new (ArcSwap::from_pointee (None)); let state = Arc::new (State { file_server: file_server::State { @@ -214,7 +210,7 @@ pub async fn run_server ( }, handlebars, metrics_startup, - metrics_gauge, + metrics_gauges, hidden_path, }, config: Config { From 1e5aa528c9532f601fd5c1399ef7342a9530d65c Mon Sep 17 00:00:00 2001 From: _ <> Date: Sun, 20 Dec 2020 20:10:29 +0000 Subject: [PATCH 153/208] :construction: wip: collecting CPU time used. --- crates/ptth_file_server_bin/src/main.rs | 2 +- crates/ptth_server/src/file_server/metrics.rs | 35 ++++++++++++++++++- crates/ptth_server/src/lib.rs | 2 +- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index a5c5554..9ae3f47 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -97,7 +97,7 @@ async fn main () -> Result <(), anyhow::Error> { config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) ); - let metrics_gauges = Arc::new (ArcSwap::from_pointee (None)); + let metrics_gauges = Arc::new (ArcSwap::default ()); let gauge_writer = Arc::clone (&metrics_gauges); diff --git a/crates/ptth_server/src/file_server/metrics.rs b/crates/ptth_server/src/file_server/metrics.rs index 659f4e3..f5666f1 100644 --- a/crates/ptth_server/src/file_server/metrics.rs +++ b/crates/ptth_server/src/file_server/metrics.rs @@ -33,24 +33,57 @@ pub struct Startup { pub startup_utc: DateTime , } +// Gauges are things we instananeously measure on a fixed interval. +// They are not read back and accumulated like counters. + #[derive (Debug, serde::Serialize)] pub struct Gauges { pub utc: DateTime , pub rss_mib: u64, + + // What's the difference? + pub cpu_time_user: f64, + pub cpu_time_system: f64, + + #[serde (skip)] + pub cpu_usage: heim::process::CpuUsage, } impl Gauges { pub async fn new () -> Result { + use tokio::join; use heim::process; - use uom::si::information::mebibyte; + use uom::si::{ + information::mebibyte, + ratio, + time::second, + }; let our_process = process::current ().await?; + + let cpu_time = our_process.cpu_time (); + let cpu_usage = our_process.cpu_usage (); + + let (cpu_time, cpu_usage) = join! ( + cpu_time, + cpu_usage, + ); + + let cpu_time = cpu_time?; + + let cpu_time_user = cpu_time.user ().get:: (); + let cpu_time_system = cpu_time.system ().get:: (); + let cpu_usage = cpu_usage?; + let mem = our_process.memory ().await?; let rss_mib = mem.rss ().get:: (); let x = Gauges { utc: Utc::now (), rss_mib, + cpu_time_user, + cpu_time_system, + cpu_usage, }; debug! ("metric gauges: {:?}", x); diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index 81e619b..56eac6b 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -201,7 +201,7 @@ pub async fn run_server ( let handlebars = file_server::load_templates (&asset_root)?; let metrics_startup = file_server::metrics::Startup::new (config_file.name); - let metrics_gauges = Arc::new (ArcSwap::from_pointee (None)); + let metrics_gauges = Arc::new (ArcSwap::default ()); let state = Arc::new (State { file_server: file_server::State { From f335644b03433f297bfaa84a1f302f895b66f1cb Mon Sep 17 00:00:00 2001 From: _ <_@_> Date: Sun, 20 Dec 2020 17:17:31 -0600 Subject: [PATCH 154/208] :heavy_plus_sign: update: measuring CPU usage every minute --- Cargo.lock | 1 + crates/ptth_file_server_bin/Cargo.toml | 1 + crates/ptth_file_server_bin/src/main.rs | 38 ++++++-- crates/ptth_server/src/file_server/html.rs | 4 +- crates/ptth_server/src/file_server/metrics.rs | 92 ++++++++----------- crates/ptth_server/src/file_server/mod.rs | 2 +- crates/ptth_server/src/lib.rs | 4 +- handlebars/server/file_server_root.html | 2 +- 8 files changed, 78 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 55a1d67..22360fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1653,6 +1653,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "uom", ] [[package]] diff --git a/crates/ptth_file_server_bin/Cargo.toml b/crates/ptth_file_server_bin/Cargo.toml index af3cf68..8d91c39 100644 --- a/crates/ptth_file_server_bin/Cargo.toml +++ b/crates/ptth_file_server_bin/Cargo.toml @@ -18,6 +18,7 @@ structopt = "0.3.20" tokio = { version = "0.2.22", features = ["full"] } tracing = "0.1.21" tracing-subscriber = "0.2.15" +uom = "0.30.0" ptth_core = { path = "../ptth_core" } ptth_server = { path = "../ptth_server" } diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index 9ae3f47..b92d6e9 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -97,24 +97,46 @@ async fn main () -> Result <(), anyhow::Error> { config_file.name.unwrap_or_else (|| "PTTH File Server".to_string ()) ); - let metrics_gauges = Arc::new (ArcSwap::default ()); + let metrics_interval = Arc::new (ArcSwap::default ()); - let gauge_writer = Arc::clone (&metrics_gauges); + let interval_writer = Arc::clone (&metrics_interval); tokio::spawn (async move { - let mut interval = tokio::time::interval (std::time::Duration::from_secs (2)); + use std::time::Duration; + + use uom::si::ratio::percent; + + let mut interval = tokio::time::interval (Duration::from_secs (60)); + + let mut counter = 0_u64; + let mut next_10_time = counter; + let mut metrics_at_last_10: Arc