diff --git a/Cargo.lock b/Cargo.lock index 36c91c3..0d49e91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1269,6 +1269,7 @@ dependencies = [ "reqwest", "rmp-serde", "serde", + "serde_json", "structopt", "thiserror", "tokio", diff --git a/crates/ptth_core/src/lib.rs b/crates/ptth_core/src/lib.rs index 69bd7d6..4d2440a 100644 --- a/crates/ptth_core/src/lib.rs +++ b/crates/ptth_core/src/lib.rs @@ -24,3 +24,20 @@ pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str> None } } + +#[cfg (test)] +mod tests { + use super::*; + #[test] + fn prefix () { + for (p, h, expected) in &[ + ("/files/", "/files/a", Some ("a")), + ("/files/", "/files/abc/def", Some ("abc/def")), + ("/files/", "/files", None), + ("/files/", "/not_files", None), + ("/files/", "/files/", Some ("")), + ] { + assert_eq! (prefix_match (*p, *h), *expected); + } + } +} diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs index cec4bb2..33acbd7 100644 --- a/crates/ptth_file_server_bin/src/main.rs +++ b/crates/ptth_file_server_bin/src/main.rs @@ -44,6 +44,7 @@ async fn handle_all (req: Request , state: Arc >) -> Result , anyhow::Error> { use std::str::FromStr; + use hyper::header::HeaderName; debug! ("req.uri () = {:?}", req.uri ()); @@ -74,7 +75,7 @@ async fn handle_all (req: Request , state: Arc >) .status (StatusCode::from (ptth_resp.parts.status_code)); for (k, v) in ptth_resp.parts.headers { - resp = resp.header (hyper::header::HeaderName::from_str (&k)?, v); + resp = resp.header (HeaderName::from_str (&k)?, v); } let body = ptth_resp.body.map_or_else (Body::empty, Body::wrap_stream); diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml index 0095a55..81345a8 100644 --- a/crates/ptth_server/Cargo.toml +++ b/crates/ptth_server/Cargo.toml @@ -22,6 +22,7 @@ regex = "1.4.1" reqwest = { version = "0.10.8", features = ["stream"] } rmp-serde = "0.14.4" serde = {version = "1.0.117", features = ["derive"]} +serde_json = "1.0.60" structopt = "0.3.20" thiserror = "1.0.22" tokio = { version = "0.2.22", features = ["full"] } diff --git a/crates/ptth_server/src/file_server/internal.rs b/crates/ptth_server/src/file_server/internal.rs index 695c0b3..9b5080a 100644 --- a/crates/ptth_server/src/file_server/internal.rs +++ b/crates/ptth_server/src/file_server/internal.rs @@ -42,10 +42,17 @@ use super::{ range, }; +#[derive (Debug, PartialEq)] +pub enum OutputFormat { + Json, + Html, +} + #[derive (Debug, PartialEq)] pub struct ServeDirParams { pub path: PathBuf, pub dir: AlwaysEqual , + pub format: OutputFormat, } #[derive (Debug, PartialEq)] @@ -59,11 +66,12 @@ pub struct ServeFileParams { pub enum Response { Favicon, Forbidden, - InvalidQuery, MethodNotAllowed, NotFound, RangeNotSatisfiable (u64), Redirect (String), + InvalidQuery, + Root, ServeDir (ServeDirParams), ServeFile (ServeFileParams), @@ -77,7 +85,8 @@ fn serve_dir ( path: &Path, dir: tokio::fs::ReadDir, full_path: PathBuf, - uri: &http::Uri + uri: &http::Uri, + format: OutputFormat ) -> Result { @@ -98,6 +107,7 @@ fn serve_dir ( Ok (Response::ServeDir (ServeDirParams { dir, path: full_path, + format, })) } @@ -161,6 +171,53 @@ async fn serve_file ( }) } +async fn serve_api ( + root: &Path, + uri: &http::Uri, + hidden_path: Option <&Path>, + path: &str +) +-> Result +{ + use Response::*; + + match prefix_match ("/v1/dir/", path) { + None => (), + Some (path) => { + let encoded_path = &path [0..]; + + let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; + let path = Path::new (&*path_s); + + let full_path = root.join (path); + + debug! ("full_path = {:?}", full_path); + + if let Some (hidden_path) = hidden_path { + if full_path == hidden_path { + return Ok (Forbidden); + } + } + + return if let Ok (dir) = read_dir (&full_path).await { + serve_dir ( + &path_s, + path, + dir, + full_path, + &uri, + OutputFormat::Json + ) + } + else { + Ok (NotFound) + }; + }, + }; + + Ok (NotFound) +} + pub async fn serve_all ( root: &Path, method: Method, @@ -186,22 +243,28 @@ pub async fn serve_all ( } }; - if uri.path () == "/favicon.ico" { + let path = uri.path (); + + if path == "/favicon.ico" { return Ok (Favicon); } - let path = match prefix_match ("/files", uri.path ()) { - Some (x) => x, - None => return Ok (Root), - }; - - if path == "" { - return Ok (Redirect ("files/".to_string ())); + if path == "/" { + return Ok (Root); } + if let Some (path) = prefix_match ("/api", path) { + return serve_api (root, &uri, hidden_path, path).await; + } + + let path = match prefix_match ("/files/", path) { + Some (x) => x, + None => return Ok (NotFound), + }; + // TODO: There is totally a dir traversal attack in here somewhere - let encoded_path = &path [1..]; + let encoded_path = &path [0..]; let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; let path = Path::new (&*path_s); @@ -222,7 +285,8 @@ pub async fn serve_all ( path, dir, full_path, - &uri + &uri, + OutputFormat::Html ) } else if let Ok (file) = File::open (&full_path).await { diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs index 4ff2678..40d9d84 100644 --- a/crates/ptth_server/src/file_server/mod.rs +++ b/crates/ptth_server/src/file_server/mod.rs @@ -58,7 +58,19 @@ pub struct ServerInfo { } #[derive (Serialize)] -struct TemplateDirEntry { +struct DirEntryJson { + name: String, + size: u64, + is_dir: bool, +} + +#[derive (Serialize)] +struct DirJson { + entries: Vec , +} + +#[derive (Serialize)] +struct DirEntryHtml { icon: &'static str, trailing_slash: &'static str, @@ -79,12 +91,12 @@ struct TemplateDirEntry { } #[derive (Serialize)] -struct TemplateDirPage <'a> { +struct DirHtml <'a> { #[serde (flatten)] server_info: &'a ServerInfo, path: Cow <'a, str>, - entries: Vec , + entries: Vec , } fn get_icon (file_name: &str) -> &'static str { @@ -109,7 +121,7 @@ fn get_icon (file_name: &str) -> &'static str { } } -async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry +async fn read_dir_entry_html (entry: DirEntry) -> DirEntryHtml { use percent_encoding::{ CONTROLS, @@ -118,7 +130,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let file_name = match entry.file_name ().into_string () { Ok (x) => x, - Err (_) => return TemplateDirEntry { + Err (_) => return DirEntryHtml { icon: emoji::ERROR, trailing_slash: "", file_name: "File / directory name is not UTF-8".into (), @@ -130,7 +142,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let metadata = match entry.metadata ().await { Ok (x) => x, - Err (_) => return TemplateDirEntry { + Err (_) => return DirEntryHtml { icon: emoji::ERROR, trailing_slash: "", file_name: "Could not fetch metadata".into (), @@ -153,7 +165,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string (); - TemplateDirEntry { + DirEntryHtml { icon, trailing_slash: &trailing_slash, file_name, @@ -163,6 +175,20 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry } } +async fn read_dir_entry_json (entry: DirEntry) -> Option +{ + let name = entry.file_name ().into_string ().ok ()?; + let metadata = entry.metadata ().await.ok ()?; + let is_dir = metadata.is_dir (); + let size = metadata.len (); + + Some (DirEntryJson { + name, + size, + is_dir, + }) +} + async fn serve_root ( handlebars: &Handlebars <'static>, server_info: &ServerInfo @@ -182,8 +208,33 @@ fn serve_html (s: String) -> Response { resp } +async fn serve_dir_json ( + mut dir: ReadDir +) -> Result +{ + let mut entries = vec! []; + + while let Ok (Some (entry)) = dir.next_entry ().await { + if let Some (entry) = read_dir_entry_json (entry).await { + entries.push (entry); + } + } + + entries.sort_unstable_by (|a, b| a.name.cmp (&b.name)); + + let dir = DirJson { + entries, + }; + + let mut response = Response::default (); + response.header ("content-type".to_string (), "application/json; charset=UTF-8".to_string ().into_bytes ()); + response.body_bytes (serde_json::to_string (&dir).unwrap ().into_bytes ()); + + Ok (response) +} + #[instrument (level = "debug", skip (handlebars, dir))] -async fn serve_dir ( +async fn serve_dir_html ( handlebars: &Handlebars <'static>, server_info: &ServerInfo, path: Cow <'_, str>, @@ -193,12 +244,12 @@ async fn serve_dir ( let mut entries = vec! []; while let Ok (Some (entry)) = dir.next_entry ().await { - entries.push (read_dir_entry (entry).await); + entries.push (read_dir_entry_html (entry).await); } entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name)); - let s = handlebars.render ("file_server_dir", &TemplateDirPage { + let s = handlebars.render ("file_server_dir", &DirHtml { path, entries, server_info, @@ -316,7 +367,10 @@ pub async fn serve_all ( ) -> Result { - use internal::Response::*; + use internal::{ + OutputFormat, + Response::*, + }; fn serve_error >> ( status_code: StatusCode, @@ -331,11 +385,10 @@ pub async fn serve_all ( } Ok (match internal::serve_all (root, method, uri, headers, hidden_path).await? { - Favicon => serve_error (StatusCode::NotFound, ""), - Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), - InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object"), - MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method"), - NotFound => serve_error (StatusCode::NotFound, "404 Not Found"), + Favicon => serve_error (StatusCode::NotFound, "Not found\n"), + Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden\n"), + MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method\n"), + NotFound => serve_error (StatusCode::NotFound, "404 Not Found\nAre you missing a trailing slash?\n"), RangeNotSatisfiable (file_len) => { let mut resp = Response::default (); resp.status_code (StatusCode::RangeNotSatisfiable) @@ -344,16 +397,22 @@ pub async fn serve_all ( }, Redirect (location) => { let mut resp = Response::default (); - resp.status_code (StatusCode::TemporaryRedirect); - resp.header ("location".to_string (), location.into_bytes ()); - resp.body_bytes (b"Redirecting...".to_vec ()); + resp.status_code (StatusCode::TemporaryRedirect) + .header ("location".to_string (), location.into_bytes ()); + resp.body_bytes (b"Redirecting...\n".to_vec ()); resp }, + InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"), + Root => serve_root (handlebars, server_info).await?, ServeDir (internal::ServeDirParams { path, dir, - }) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?, + format + }) => match format { + OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?, + OutputFormat::Html => serve_dir_html (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?, + }, ServeFile (internal::ServeFileParams { file, send_body, diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs index 0b78385..9f096be 100644 --- a/crates/ptth_server/src/file_server/tests.rs +++ b/crates/ptth_server/src/file_server/tests.rs @@ -101,7 +101,7 @@ fn file_server () { for (uri_path, expected) in vec! [ ("/", Root), - ("/files", Redirect ("files/".to_string ())), + ("/files", NotFound), ("/files/?", InvalidQuery), ("/files/src", Redirect ("src/".to_string ())), ("/files/src/?", InvalidQuery), diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs index ce63528..bccc824 100644 --- a/crates/ptth_server/src/lib.rs +++ b/crates/ptth_server/src/lib.rs @@ -15,9 +15,6 @@ use std::{ use futures::FutureExt; use handlebars::Handlebars; -use http::status::{ - StatusCode, -}; use reqwest::Client; use serde::Deserialize; use tokio::{ @@ -59,10 +56,70 @@ struct ServerState { hidden_path: Option , } -async fn handle_req_resp <'a> ( +async fn handle_one_req ( + state: &Arc , + wrapped_req: http_serde::WrappedRequest +) -> Result <(), ServerError> +{ + let (req_id, parts) = (wrapped_req.id, wrapped_req.req); + + debug! ("Handling request {}", req_id); + + let default_root = PathBuf::from ("./"); + let file_server_root: &std::path::Path = state.config.file_server_root + .as_ref () + .unwrap_or (&default_root); + + let response = file_server::serve_all ( + &state.handlebars, + &state.server_info, + file_server_root, + parts.method, + &parts.uri, + &parts.headers, + state.hidden_path.as_deref () + ).await?; + + let mut resp_req = state.client + .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) + .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?)); + + if let Some (length) = response.content_length { + resp_req = resp_req.header ("Content-Length", length.to_string ()); + } + if let Some (body) = response.body { + resp_req = resp_req.body (reqwest::Body::wrap_stream (body)); + } + + let req = resp_req.build ().map_err (ServerError::Step5Responding)?; + + debug! ("{:?}", req.headers ()); + + //println! ("Step 6"); + match state.client.execute (req).await { + Ok (r) => { + let status = r.status (); + let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?; + debug! ("{:?} {:?}", status, text); + }, + Err (e) => { + if e.is_request () { + warn! ("Error while POSTing response. Client probably hung up."); + } + else { + error! ("Err: {:?}", e); + } + }, + } + + Ok::<(), ServerError> (()) +} + +async fn handle_req_resp ( state: &Arc , req_resp: reqwest::Response -) -> Result <(), ServerError> { +) -> Result <(), ServerError> +{ //println! ("Step 1"); let body = req_resp.bytes ().await.map_err (ServerError::CantCollectWrappedRequests)?; @@ -83,58 +140,7 @@ async fn handle_req_resp <'a> ( // These have to detach, so we won't be able to catch the join errors. tokio::spawn (async move { - let (req_id, parts) = (wrapped_req.id, wrapped_req.req); - - debug! ("Handling request {}", req_id); - - let default_root = PathBuf::from ("./"); - let file_server_root: &std::path::Path = state.config.file_server_root - .as_ref () - .unwrap_or (&default_root); - - let response = file_server::serve_all ( - &state.handlebars, - &state.server_info, - file_server_root, - parts.method, - &parts.uri, - &parts.headers, - state.hidden_path.as_deref () - ).await?; - - let mut resp_req = state.client - .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id)) - .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?)); - - if let Some (length) = response.content_length { - resp_req = resp_req.header ("Content-Length", length.to_string ()); - } - if let Some (body) = response.body { - resp_req = resp_req.body (reqwest::Body::wrap_stream (body)); - } - - let req = resp_req.build ().map_err (ServerError::Step5Responding)?; - - debug! ("{:?}", req.headers ()); - - //println! ("Step 6"); - match state.client.execute (req).await { - Ok (r) => { - let status = r.status (); - let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?; - debug! ("{:?} {:?}", status, text); - }, - Err (e) => { - if e.is_request () { - warn! ("Error while POSTing response. Client probably hung up."); - } - else { - error! ("Err: {:?}", e); - } - }, - } - - Ok::<(), ServerError> (()) + handle_one_req (&state, wrapped_req).await }); } @@ -172,6 +178,8 @@ pub async fn run_server ( { use std::convert::TryInto; + use http::status::StatusCode; + let asset_root = asset_root.unwrap_or_else (PathBuf::new); if password_is_bad (config_file.api_key.clone ()) { diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md index 18b855b..e161f08 100644 --- a/issues/2020-12Dec/auth-route-YNQAQKJS.md +++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md @@ -37,10 +37,11 @@ stronger is ready. - (X) Accept scraper key for some testing endpoint - (X) (POC) Test with curl - (X) Clean up scraper endpoint -- (X) Add (almost) end-to-end tests for scraper endpoint -- ( ) Add tests for scraper endpoints -- ( ) Factor v1 API into v1 module -- ( ) Add real scraper endpoints +- (X) Add (almost) end-to-end tests for test scraper endpoint +- ( ) Thread server endpoints through relay scraper auth +- ( ) Add tests for other scraper endpoints +- (don't care) Factor v1 API into v1 module +- (X) Add real scraper endpoints - ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Impl DB reads - ( ) Remove scraper key from config file @@ -72,8 +73,8 @@ the old ones deprecated. Endpoints needed: - (X) Query server list -- ( ) Query directory in server -- ( ) GET file with byte range (identical to frontend file API) +- (X) Query directory in server +- (not needed) GET file with byte range (identical to frontend file API) These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it. For compatibility with wget spidering, I _might_ do XML or HTML that's