reactor
/
ptth
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

⭐ new: add JSON API in server for dir listings

main
_ 2020-12-15 05:15:17 +00:00
parent 11f4b0e65b
commit cda627fa4b
9 changed files with 249 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -1269,6 +1269,7 @@ dependencies = [
"reqwest", "reqwest",
"rmp-serde", "rmp-serde",
"serde", "serde",
"serde_json",
"structopt", "structopt",
"thiserror", "thiserror",
"tokio", "tokio",

17
crates/ptth_core/src/lib.rs
View File

@ -24,3 +24,20 @@ pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str>
None None
} }
} }
#[cfg (test)]
mod tests {
use super::*;
#[test]
fn prefix () {
for (p, h, expected) in &[
("/files/", "/files/a", Some ("a")),
("/files/", "/files/abc/def", Some ("abc/def")),
("/files/", "/files", None),
("/files/", "/not_files", None),
("/files/", "/files/", Some ("")),
] {
assert_eq! (prefix_match (*p, *h), *expected);
}
}
}

3
crates/ptth_file_server_bin/src/main.rs
View File

@ -44,6 +44,7 @@ async fn handle_all (req: Request <Body>, state: Arc <ServerState <'static>>)
-> Result <Response <Body>, anyhow::Error> -> Result <Response <Body>, anyhow::Error>
{ {
use std::str::FromStr; use std::str::FromStr;
use hyper::header::HeaderName;
debug! ("req.uri () = {:?}", req.uri ()); debug! ("req.uri () = {:?}", req.uri ());
@ -74,7 +75,7 @@ async fn handle_all (req: Request <Body>, state: Arc <ServerState <'static>>)
.status (StatusCode::from (ptth_resp.parts.status_code)); .status (StatusCode::from (ptth_resp.parts.status_code));
for (k, v) in ptth_resp.parts.headers { for (k, v) in ptth_resp.parts.headers {
resp = resp.header (hyper::header::HeaderName::from_str (&k)?, v); resp = resp.header (HeaderName::from_str (&k)?, v);
} }
let body = ptth_resp.body.map_or_else (Body::empty, Body::wrap_stream); let body = ptth_resp.body.map_or_else (Body::empty, Body::wrap_stream);

1
crates/ptth_server/Cargo.toml
View File

@ -22,6 +22,7 @@ regex = "1.4.1"
reqwest = { version = "0.10.8", features = ["stream"] } reqwest = { version = "0.10.8", features = ["stream"] }
rmp-serde = "0.14.4" rmp-serde = "0.14.4"
serde = {version = "1.0.117", features = ["derive"]} serde = {version = "1.0.117", features = ["derive"]}
serde_json = "1.0.60"
structopt = "0.3.20" structopt = "0.3.20"
thiserror = "1.0.22" thiserror = "1.0.22"
tokio = { version = "0.2.22", features = ["full"] } tokio = { version = "0.2.22", features = ["full"] }

88
crates/ptth_server/src/file_server/internal.rs
View File

@ -42,10 +42,17 @@ use super::{
range, range,
}; };
#[derive (Debug, PartialEq)]
pub enum OutputFormat {
Json,
Html,
}
#[derive (Debug, PartialEq)] #[derive (Debug, PartialEq)]
pub struct ServeDirParams { pub struct ServeDirParams {
pub path: PathBuf, pub path: PathBuf,
pub dir: AlwaysEqual <ReadDir>, pub dir: AlwaysEqual <ReadDir>,
pub format: OutputFormat,
} }
#[derive (Debug, PartialEq)] #[derive (Debug, PartialEq)]
@ -59,11 +66,12 @@ pub struct ServeFileParams {
pub enum Response { pub enum Response {
Favicon, Favicon,
Forbidden, Forbidden,
InvalidQuery,
MethodNotAllowed, MethodNotAllowed,
NotFound, NotFound,
RangeNotSatisfiable (u64), RangeNotSatisfiable (u64),
Redirect (String), Redirect (String),
InvalidQuery,
Root, Root,
ServeDir (ServeDirParams), ServeDir (ServeDirParams),
ServeFile (ServeFileParams), ServeFile (ServeFileParams),
@ -77,7 +85,8 @@ fn serve_dir (
path: &Path, path: &Path,
dir: tokio::fs::ReadDir, dir: tokio::fs::ReadDir,
full_path: PathBuf, full_path: PathBuf,
uri: &http::Uri uri: &http::Uri,
format: OutputFormat
) )
-> Result <Response, FileServerError> -> Result <Response, FileServerError>
{ {
@ -98,6 +107,7 @@ fn serve_dir (
Ok (Response::ServeDir (ServeDirParams { Ok (Response::ServeDir (ServeDirParams {
dir, dir,
path: full_path, path: full_path,
format,
})) }))
} }
@ -161,6 +171,53 @@ async fn serve_file (
}) })
} }
async fn serve_api (
root: &Path,
uri: &http::Uri,
hidden_path: Option <&Path>,
path: &str
)
-> Result <Response, FileServerError>
{
use Response::*;
match prefix_match ("/v1/dir/", path) {
None => (),
Some (path) => {
let encoded_path = &path [0..];
let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?;
let path = Path::new (&*path_s);
let full_path = root.join (path);
debug! ("full_path = {:?}", full_path);
if let Some (hidden_path) = hidden_path {
if full_path == hidden_path {
return Ok (Forbidden);
}
}
return if let Ok (dir) = read_dir (&full_path).await {
serve_dir (
&path_s,
path,
dir,
full_path,
&uri,
OutputFormat::Json
)
}
else {
Ok (NotFound)
};
},
};
Ok (NotFound)
}
pub async fn serve_all ( pub async fn serve_all (
root: &Path, root: &Path,
method: Method, method: Method,
@ -186,22 +243,28 @@ pub async fn serve_all (
} }
}; };
if uri.path () == "/favicon.ico" { let path = uri.path ();
if path == "/favicon.ico" {
return Ok (Favicon); return Ok (Favicon);
} }
let path = match prefix_match ("/files", uri.path ()) { if path == "/" {
Some (x) => x, return Ok (Root);
None => return Ok (Root),
};
if path == "" {
return Ok (Redirect ("files/".to_string ()));
} }
if let Some (path) = prefix_match ("/api", path) {
return serve_api (root, &uri, hidden_path, path).await;
}
let path = match prefix_match ("/files/", path) {
Some (x) => x,
None => return Ok (NotFound),
};
// TODO: There is totally a dir traversal attack in here somewhere // TODO: There is totally a dir traversal attack in here somewhere
let encoded_path = &path [1..]; let encoded_path = &path [0..];
let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?; let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?;
let path = Path::new (&*path_s); let path = Path::new (&*path_s);
@ -222,7 +285,8 @@ pub async fn serve_all (
path, path,
dir, dir,
full_path, full_path,
&uri &uri,
OutputFormat::Html
) )
} }
else if let Ok (file) = File::open (&full_path).await { else if let Ok (file) = File::open (&full_path).await {

99
crates/ptth_server/src/file_server/mod.rs
View File

@ -58,7 +58,19 @@ pub struct ServerInfo {
} }
#[derive (Serialize)] #[derive (Serialize)]
struct TemplateDirEntry { struct DirEntryJson {
name: String,
size: u64,
is_dir: bool,
}
#[derive (Serialize)]
struct DirJson {
entries: Vec <DirEntryJson>,
}
#[derive (Serialize)]
struct DirEntryHtml {
icon: &'static str, icon: &'static str,
trailing_slash: &'static str, trailing_slash: &'static str,
@ -79,12 +91,12 @@ struct TemplateDirEntry {
} }
#[derive (Serialize)] #[derive (Serialize)]
struct TemplateDirPage <'a> { struct DirHtml <'a> {
#[serde (flatten)] #[serde (flatten)]
server_info: &'a ServerInfo, server_info: &'a ServerInfo,
path: Cow <'a, str>, path: Cow <'a, str>,
entries: Vec <TemplateDirEntry>, entries: Vec <DirEntryHtml>,
} }
fn get_icon (file_name: &str) -> &'static str { fn get_icon (file_name: &str) -> &'static str {
@ -109,7 +121,7 @@ fn get_icon (file_name: &str) -> &'static str {
} }
} }
async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry async fn read_dir_entry_html (entry: DirEntry) -> DirEntryHtml
{ {
use percent_encoding::{ use percent_encoding::{
CONTROLS, CONTROLS,
@ -118,7 +130,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let file_name = match entry.file_name ().into_string () { let file_name = match entry.file_name ().into_string () {
Ok (x) => x, Ok (x) => x,
Err (_) => return TemplateDirEntry { Err (_) => return DirEntryHtml {
icon: emoji::ERROR, icon: emoji::ERROR,
trailing_slash: "", trailing_slash: "",
file_name: "File / directory name is not UTF-8".into (), file_name: "File / directory name is not UTF-8".into (),
@ -130,7 +142,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let metadata = match entry.metadata ().await { let metadata = match entry.metadata ().await {
Ok (x) => x, Ok (x) => x,
Err (_) => return TemplateDirEntry { Err (_) => return DirEntryHtml {
icon: emoji::ERROR, icon: emoji::ERROR,
trailing_slash: "", trailing_slash: "",
file_name: "Could not fetch metadata".into (), file_name: "Could not fetch metadata".into (),
@ -153,7 +165,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string (); let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string ();
TemplateDirEntry { DirEntryHtml {
icon, icon,
trailing_slash: &trailing_slash, trailing_slash: &trailing_slash,
file_name, file_name,
@ -163,6 +175,20 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
} }
} }
async fn read_dir_entry_json (entry: DirEntry) -> Option <DirEntryJson>
{
let name = entry.file_name ().into_string ().ok ()?;
let metadata = entry.metadata ().await.ok ()?;
let is_dir = metadata.is_dir ();
let size = metadata.len ();
Some (DirEntryJson {
name,
size,
is_dir,
})
}
async fn serve_root ( async fn serve_root (
handlebars: &Handlebars <'static>, handlebars: &Handlebars <'static>,
server_info: &ServerInfo server_info: &ServerInfo
@ -182,8 +208,33 @@ fn serve_html (s: String) -> Response {
resp resp
} }
async fn serve_dir_json (
mut dir: ReadDir
) -> Result <Response, FileServerError>
{
let mut entries = vec! [];
while let Ok (Some (entry)) = dir.next_entry ().await {
if let Some (entry) = read_dir_entry_json (entry).await {
entries.push (entry);
}
}
entries.sort_unstable_by (|a, b| a.name.cmp (&b.name));
let dir = DirJson {
entries,
};
let mut response = Response::default ();
response.header ("content-type".to_string (), "application/json; charset=UTF-8".to_string ().into_bytes ());
response.body_bytes (serde_json::to_string (&dir).unwrap ().into_bytes ());
Ok (response)
}
#[instrument (level = "debug", skip (handlebars, dir))] #[instrument (level = "debug", skip (handlebars, dir))]
async fn serve_dir ( async fn serve_dir_html (
handlebars: &Handlebars <'static>, handlebars: &Handlebars <'static>,
server_info: &ServerInfo, server_info: &ServerInfo,
path: Cow <'_, str>, path: Cow <'_, str>,
@ -193,12 +244,12 @@ async fn serve_dir (
let mut entries = vec! []; let mut entries = vec! [];
while let Ok (Some (entry)) = dir.next_entry ().await { while let Ok (Some (entry)) = dir.next_entry ().await {
entries.push (read_dir_entry (entry).await); entries.push (read_dir_entry_html (entry).await);
} }
entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name)); entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name));
let s = handlebars.render ("file_server_dir", &TemplateDirPage { let s = handlebars.render ("file_server_dir", &DirHtml {
path, path,
entries, entries,
server_info, server_info,
@ -316,7 +367,10 @@ pub async fn serve_all (
) )
-> Result <Response, FileServerError> -> Result <Response, FileServerError>
{ {
use internal::Response::*; use internal::{
OutputFormat,
Response::*,
};
fn serve_error <S: Into <Vec <u8>>> ( fn serve_error <S: Into <Vec <u8>>> (
status_code: StatusCode, status_code: StatusCode,
@ -331,11 +385,10 @@ pub async fn serve_all (
} }
Ok (match internal::serve_all (root, method, uri, headers, hidden_path).await? { Ok (match internal::serve_all (root, method, uri, headers, hidden_path).await? {
Favicon => serve_error (StatusCode::NotFound, ""), Favicon => serve_error (StatusCode::NotFound, "Not found\n"),
Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"), Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden\n"),
InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object"), MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method\n"),
MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method"), NotFound => serve_error (StatusCode::NotFound, "404 Not Found\nAre you missing a trailing slash?\n"),
NotFound => serve_error (StatusCode::NotFound, "404 Not Found"),
RangeNotSatisfiable (file_len) => { RangeNotSatisfiable (file_len) => {
let mut resp = Response::default (); let mut resp = Response::default ();
resp.status_code (StatusCode::RangeNotSatisfiable) resp.status_code (StatusCode::RangeNotSatisfiable)
@ -344,16 +397,22 @@ pub async fn serve_all (
}, },
Redirect (location) => { Redirect (location) => {
let mut resp = Response::default (); let mut resp = Response::default ();
resp.status_code (StatusCode::TemporaryRedirect); resp.status_code (StatusCode::TemporaryRedirect)
resp.header ("location".to_string (), location.into_bytes ()); .header ("location".to_string (), location.into_bytes ());
resp.body_bytes (b"Redirecting...".to_vec ()); resp.body_bytes (b"Redirecting...\n".to_vec ());
resp resp
}, },
InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"),
Root => serve_root (handlebars, server_info).await?, Root => serve_root (handlebars, server_info).await?,
ServeDir (internal::ServeDirParams { ServeDir (internal::ServeDirParams {
path, path,
dir, dir,
}) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?, format
}) => match format {
OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?,
OutputFormat::Html => serve_dir_html (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?,
},
ServeFile (internal::ServeFileParams { ServeFile (internal::ServeFileParams {
file, file,
send_body, send_body,

2
crates/ptth_server/src/file_server/tests.rs
View File

@ -101,7 +101,7 @@ fn file_server () {
for (uri_path, expected) in vec! [ for (uri_path, expected) in vec! [
("/", Root), ("/", Root),
("/files", Redirect ("files/".to_string ())), ("/files", NotFound),
("/files/?", InvalidQuery), ("/files/?", InvalidQuery),
("/files/src", Redirect ("src/".to_string ())), ("/files/src", Redirect ("src/".to_string ())),
("/files/src/?", InvalidQuery), ("/files/src/?", InvalidQuery),

122
crates/ptth_server/src/lib.rs
View File

@ -15,9 +15,6 @@ use std::{
use futures::FutureExt; use futures::FutureExt;
use handlebars::Handlebars; use handlebars::Handlebars;
use http::status::{
StatusCode,
};
use reqwest::Client; use reqwest::Client;
use serde::Deserialize; use serde::Deserialize;
use tokio::{ use tokio::{
@ -59,10 +56,70 @@ struct ServerState {
hidden_path: Option <PathBuf>, hidden_path: Option <PathBuf>,
} }
async fn handle_req_resp <'a> ( async fn handle_one_req (
state: &Arc <ServerState>,
wrapped_req: http_serde::WrappedRequest
) -> Result <(), ServerError>
{
let (req_id, parts) = (wrapped_req.id, wrapped_req.req);
debug! ("Handling request {}", req_id);
let default_root = PathBuf::from ("./");
let file_server_root: &std::path::Path = state.config.file_server_root
.as_ref ()
.unwrap_or (&default_root);
let response = file_server::serve_all (
&state.handlebars,
&state.server_info,
file_server_root,
parts.method,
&parts.uri,
&parts.headers,
state.hidden_path.as_deref ()
).await?;
let mut resp_req = state.client
.post (&format! ("{}/http_response/{}", state.config.relay_url, req_id))
.header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?));
if let Some (length) = response.content_length {
resp_req = resp_req.header ("Content-Length", length.to_string ());
}
if let Some (body) = response.body {
resp_req = resp_req.body (reqwest::Body::wrap_stream (body));
}
let req = resp_req.build ().map_err (ServerError::Step5Responding)?;
debug! ("{:?}", req.headers ());
//println! ("Step 6");
match state.client.execute (req).await {
Ok (r) => {
let status = r.status ();
let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?;
debug! ("{:?} {:?}", status, text);
},
Err (e) => {
if e.is_request () {
warn! ("Error while POSTing response. Client probably hung up.");
}
else {
error! ("Err: {:?}", e);
}
},
}
Ok::<(), ServerError> (())
}
async fn handle_req_resp (
state: &Arc <ServerState>, state: &Arc <ServerState>,
req_resp: reqwest::Response req_resp: reqwest::Response
) -> Result <(), ServerError> { ) -> Result <(), ServerError>
{
//println! ("Step 1"); //println! ("Step 1");
let body = req_resp.bytes ().await.map_err (ServerError::CantCollectWrappedRequests)?; let body = req_resp.bytes ().await.map_err (ServerError::CantCollectWrappedRequests)?;
@ -83,58 +140,7 @@ async fn handle_req_resp <'a> (
// These have to detach, so we won't be able to catch the join errors. // These have to detach, so we won't be able to catch the join errors.
tokio::spawn (async move { tokio::spawn (async move {
let (req_id, parts) = (wrapped_req.id, wrapped_req.req); handle_one_req (&state, wrapped_req).await
debug! ("Handling request {}", req_id);
let default_root = PathBuf::from ("./");
let file_server_root: &std::path::Path = state.config.file_server_root
.as_ref ()
.unwrap_or (&default_root);
let response = file_server::serve_all (
&state.handlebars,
&state.server_info,
file_server_root,
parts.method,
&parts.uri,
&parts.headers,
state.hidden_path.as_deref ()
).await?;
let mut resp_req = state.client
.post (&format! ("{}/http_response/{}", state.config.relay_url, req_id))
.header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?));
if let Some (length) = response.content_length {
resp_req = resp_req.header ("Content-Length", length.to_string ());
}
if let Some (body) = response.body {
resp_req = resp_req.body (reqwest::Body::wrap_stream (body));
}
let req = resp_req.build ().map_err (ServerError::Step5Responding)?;
debug! ("{:?}", req.headers ());
//println! ("Step 6");
match state.client.execute (req).await {
Ok (r) => {
let status = r.status ();
let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?;
debug! ("{:?} {:?}", status, text);
},
Err (e) => {
if e.is_request () {
warn! ("Error while POSTing response. Client probably hung up.");
}
else {
error! ("Err: {:?}", e);
}
},
}
Ok::<(), ServerError> (())
}); });
} }
@ -172,6 +178,8 @@ pub async fn run_server (
{ {
use std::convert::TryInto; use std::convert::TryInto;
use http::status::StatusCode;
let asset_root = asset_root.unwrap_or_else (PathBuf::new); let asset_root = asset_root.unwrap_or_else (PathBuf::new);
if password_is_bad (config_file.api_key.clone ()) { if password_is_bad (config_file.api_key.clone ()) {

13
issues/2020-12Dec/auth-route-YNQAQKJS.md
View File

@ -37,10 +37,11 @@ stronger is ready.
- (X) Accept scraper key for some testing endpoint - (X) Accept scraper key for some testing endpoint
- (X) (POC) Test with curl - (X) (POC) Test with curl
- (X) Clean up scraper endpoint - (X) Clean up scraper endpoint
- (X) Add (almost) end-to-end tests for scraper endpoint - (X) Add (almost) end-to-end tests for test scraper endpoint
- ( ) Add tests for scraper endpoints - ( ) Thread server endpoints through relay scraper auth
- ( ) Factor v1 API into v1 module - ( ) Add tests for other scraper endpoints
- ( ) Add real scraper endpoints - (don't care) Factor v1 API into v1 module
- (X) Add real scraper endpoints
- ( ) Manually create SQLite DB for scraper keys, add 1 hash - ( ) Manually create SQLite DB for scraper keys, add 1 hash
- ( ) Impl DB reads - ( ) Impl DB reads
- ( ) Remove scraper key from config file - ( ) Remove scraper key from config file
@ -72,8 +73,8 @@ the old ones deprecated.
Endpoints needed: Endpoints needed:
- (X) Query server list - (X) Query server list
- ( ) Query directory in server - (X) Query directory in server
- ( ) GET file with byte range (identical to frontend file API) - (not needed) GET file with byte range (identical to frontend file API)
These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it. These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it.
For compatibility with wget spidering, I _might_ do XML or HTML that's For compatibility with wget spidering, I _might_ do XML or HTML that's