reactor
/
ptth
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

⭐ new: add JSON API in server for dir listings

main
_ 2020-12-15 05:15:17 +00:00
parent 11f4b0e65b
commit cda627fa4b
9 changed files with 249 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -1269,6 +1269,7 @@ dependencies = [
"reqwest",
"rmp-serde",
"serde",
"serde_json",
"structopt",
"thiserror",
"tokio",

17
crates/ptth_core/src/lib.rs
View File

@ -24,3 +24,20 @@ pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str>
None
}
}
#[cfg (test)]
mod tests {
use super::*;
#[test]
fn prefix () {
for (p, h, expected) in &[
("/files/", "/files/a", Some ("a")),
("/files/", "/files/abc/def", Some ("abc/def")),
("/files/", "/files", None),
("/files/", "/not_files", None),
("/files/", "/files/", Some ("")),
] {
assert_eq! (prefix_match (*p, *h), *expected);
}
}
}

3
crates/ptth_file_server_bin/src/main.rs
View File

@ -44,6 +44,7 @@ async fn handle_all (req: Request <Body>, state: Arc <ServerState <'static>>)
-> Result <Response <Body>, anyhow::Error>
{
use std::str::FromStr;
use hyper::header::HeaderName;
debug! ("req.uri () = {:?}", req.uri ());
@ -74,7 +75,7 @@ async fn handle_all (req: Request <Body>, state: Arc <ServerState <'static>>)
.status (StatusCode::from (ptth_resp.parts.status_code));
for (k, v) in ptth_resp.parts.headers {
resp = resp.header (hyper::header::HeaderName::from_str (&k)?, v);
resp = resp.header (HeaderName::from_str (&k)?, v);
}
let body = ptth_resp.body.map_or_else (Body::empty, Body::wrap_stream);

1
crates/ptth_server/Cargo.toml
View File

@ -22,6 +22,7 @@ regex = "1.4.1"
reqwest = { version = "0.10.8", features = ["stream"] }
rmp-serde = "0.14.4"
serde = {version = "1.0.117", features = ["derive"]}
serde_json = "1.0.60"
structopt = "0.3.20"
thiserror = "1.0.22"
tokio = { version = "0.2.22", features = ["full"] }

88
crates/ptth_server/src/file_server/internal.rs
View File

@ -42,10 +42,17 @@ use super::{
range,
};
#[derive (Debug, PartialEq)]
pub enum OutputFormat {
Json,
Html,
}
#[derive (Debug, PartialEq)]
pub struct ServeDirParams {
pub path: PathBuf,
pub dir: AlwaysEqual <ReadDir>,
pub format: OutputFormat,
}
#[derive (Debug, PartialEq)]
@ -59,11 +66,12 @@ pub struct ServeFileParams {
pub enum Response {
Favicon,
Forbidden,
InvalidQuery,
MethodNotAllowed,
NotFound,
RangeNotSatisfiable (u64),
Redirect (String),
InvalidQuery,
Root,
ServeDir (ServeDirParams),
ServeFile (ServeFileParams),
@ -77,7 +85,8 @@ fn serve_dir (
path: &Path,
dir: tokio::fs::ReadDir,
full_path: PathBuf,
uri: &http::Uri
uri: &http::Uri,
format: OutputFormat
)
-> Result <Response, FileServerError>
{
@ -98,6 +107,7 @@ fn serve_dir (
Ok (Response::ServeDir (ServeDirParams {
dir,
path: full_path,
format,
}))
}
@ -161,6 +171,53 @@ async fn serve_file (
})
}
async fn serve_api (
root: &Path,
uri: &http::Uri,
hidden_path: Option <&Path>,
path: &str
)
-> Result <Response, FileServerError>
{
use Response::*;
match prefix_match ("/v1/dir/", path) {
None => (),
Some (path) => {
let encoded_path = &path [0..];
let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?;
let path = Path::new (&*path_s);
let full_path = root.join (path);
debug! ("full_path = {:?}", full_path);
if let Some (hidden_path) = hidden_path {
if full_path == hidden_path {
return Ok (Forbidden);
}
}
return if let Ok (dir) = read_dir (&full_path).await {
serve_dir (
&path_s,
path,
dir,
full_path,
&uri,
OutputFormat::Json
)
}
else {
Ok (NotFound)
};
},
};
Ok (NotFound)
}
pub async fn serve_all (
root: &Path,
method: Method,
@ -186,22 +243,28 @@ pub async fn serve_all (
}
};
if uri.path () == "/favicon.ico" {
let path = uri.path ();
if path == "/favicon.ico" {
return Ok (Favicon);
}
let path = match prefix_match ("/files", uri.path ()) {
Some (x) => x,
None => return Ok (Root),
};
if path == "" {
return Ok (Redirect ("files/".to_string ()));
if path == "/" {
return Ok (Root);
}
if let Some (path) = prefix_match ("/api", path) {
return serve_api (root, &uri, hidden_path, path).await;
}
let path = match prefix_match ("/files/", path) {
Some (x) => x,
None => return Ok (NotFound),
};
// TODO: There is totally a dir traversal attack in here somewhere
let encoded_path = &path [1..];
let encoded_path = &path [0..];
let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?;
let path = Path::new (&*path_s);
@ -222,7 +285,8 @@ pub async fn serve_all (
path,
dir,
full_path,
&uri
&uri,
OutputFormat::Html
)
}
else if let Ok (file) = File::open (&full_path).await {

99
crates/ptth_server/src/file_server/mod.rs
View File

@ -58,7 +58,19 @@ pub struct ServerInfo {
}
#[derive (Serialize)]
struct TemplateDirEntry {
struct DirEntryJson {
name: String,
size: u64,
is_dir: bool,
}
#[derive (Serialize)]
struct DirJson {
entries: Vec <DirEntryJson>,
}
#[derive (Serialize)]
struct DirEntryHtml {
icon: &'static str,
trailing_slash: &'static str,
@ -79,12 +91,12 @@ struct TemplateDirEntry {
}
#[derive (Serialize)]
struct TemplateDirPage <'a> {
struct DirHtml <'a> {
#[serde (flatten)]
server_info: &'a ServerInfo,
path: Cow <'a, str>,
entries: Vec <TemplateDirEntry>,
entries: Vec <DirEntryHtml>,
}
fn get_icon (file_name: &str) -> &'static str {
@ -109,7 +121,7 @@ fn get_icon (file_name: &str) -> &'static str {
}
}
async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
async fn read_dir_entry_html (entry: DirEntry) -> DirEntryHtml
{
use percent_encoding::{
CONTROLS,
@ -118,7 +130,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let file_name = match entry.file_name ().into_string () {
Ok (x) => x,
Err (_) => return TemplateDirEntry {
Err (_) => return DirEntryHtml {
icon: emoji::ERROR,
trailing_slash: "",
file_name: "File / directory name is not UTF-8".into (),
@ -130,7 +142,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let metadata = match entry.metadata ().await {
Ok (x) => x,
Err (_) => return TemplateDirEntry {
Err (_) => return DirEntryHtml {
icon: emoji::ERROR,
trailing_slash: "",
file_name: "Could not fetch metadata".into (),
@ -153,7 +165,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string ();
TemplateDirEntry {
DirEntryHtml {
icon,
trailing_slash: &trailing_slash,
file_name,
@ -163,6 +175,20 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
}
}
async fn read_dir_entry_json (entry: DirEntry) -> Option <DirEntryJson>
{
let name = entry.file_name ().into_string ().ok ()?;
let metadata = entry.metadata ().await.ok ()?;
let is_dir = metadata.is_dir ();
let size = metadata.len ();
Some (DirEntryJson {
name,
size,
is_dir,
})
}
async fn serve_root (
handlebars: &Handlebars <'static>,
server_info: &ServerInfo
@ -182,8 +208,33 @@ fn serve_html (s: String) -> Response {
resp
}
async fn serve_dir_json (
mut dir: ReadDir
) -> Result <Response, FileServerError>
{
let mut entries = vec! [];
while let Ok (Some (entry)) = dir.next_entry ().await {
if let Some (entry) = read_dir_entry_json (entry).await {
entries.push (entry);
}
}
entries.sort_unstable_by (|a, b| a.name.cmp (&b.name));
let dir = DirJson {
entries,
};
let mut response = Response::default ();
response.header ("content-type".to_string (), "application/json; charset=UTF-8".to_string ().into_bytes ());
response.body_bytes (serde_json::to_string (&dir).unwrap ().into_bytes ());
Ok (response)
}
#[instrument (level = "debug", skip (handlebars, dir))]
async fn serve_dir (
async fn serve_dir_html (
handlebars: &Handlebars <'static>,
server_info: &ServerInfo,
path: Cow <'_, str>,
@ -193,12 +244,12 @@ async fn serve_dir (
let mut entries = vec! [];
while let Ok (Some (entry)) = dir.next_entry ().await {
entries.push (read_dir_entry (entry).await);
entries.push (read_dir_entry_html (entry).await);
}
entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name));
let s = handlebars.render ("file_server_dir", &TemplateDirPage {
let s = handlebars.render ("file_server_dir", &DirHtml {
path,
entries,
server_info,
@ -316,7 +367,10 @@ pub async fn serve_all (
)
-> Result <Response, FileServerError>
{
use internal::Response::*;
use internal::{
OutputFormat,
Response::*,
};
fn serve_error <S: Into <Vec <u8>>> (
status_code: StatusCode,
@ -331,11 +385,10 @@ pub async fn serve_all (
}
Ok (match internal::serve_all (root, method, uri, headers, hidden_path).await? {
Favicon => serve_error (StatusCode::NotFound, ""),
Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"),
InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object"),
MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method"),
NotFound => serve_error (StatusCode::NotFound, "404 Not Found"),
Favicon => serve_error (StatusCode::NotFound, "Not found\n"),
Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden\n"),
MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method\n"),
NotFound => serve_error (StatusCode::NotFound, "404 Not Found\nAre you missing a trailing slash?\n"),
RangeNotSatisfiable (file_len) => {
let mut resp = Response::default ();
resp.status_code (StatusCode::RangeNotSatisfiable)
@ -344,16 +397,22 @@ pub async fn serve_all (
},
Redirect (location) => {
let mut resp = Response::default ();
resp.status_code (StatusCode::TemporaryRedirect);
resp.header ("location".to_string (), location.into_bytes ());
resp.body_bytes (b"Redirecting...".to_vec ());
resp.status_code (StatusCode::TemporaryRedirect)
.header ("location".to_string (), location.into_bytes ());
resp.body_bytes (b"Redirecting...\n".to_vec ());
resp
},
InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"),
Root => serve_root (handlebars, server_info).await?,
ServeDir (internal::ServeDirParams {
path,
dir,
}) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?,
format
}) => match format {
OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?,
OutputFormat::Html => serve_dir_html (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?,
},
ServeFile (internal::ServeFileParams {
file,
send_body,

2
crates/ptth_server/src/file_server/tests.rs
View File

@ -101,7 +101,7 @@ fn file_server () {
for (uri_path, expected) in vec! [
("/", Root),
("/files", Redirect ("files/".to_string ())),
("/files", NotFound),
("/files/?", InvalidQuery),
("/files/src", Redirect ("src/".to_string ())),
("/files/src/?", InvalidQuery),

60
crates/ptth_server/src/lib.rs
View File

@ -15,9 +15,6 @@ use std::{
use futures::FutureExt;
use handlebars::Handlebars;
use http::status::{
StatusCode,
};
use reqwest::Client;
use serde::Deserialize;
use tokio::{
@ -59,30 +56,11 @@ struct ServerState {
hidden_path: Option <PathBuf>,
}
async fn handle_req_resp <'a> (
async fn handle_one_req (
state: &Arc <ServerState>,
req_resp: reqwest::Response
) -> Result <(), ServerError> {
//println! ("Step 1");
let body = req_resp.bytes ().await.map_err (ServerError::CantCollectWrappedRequests)?;
let wrapped_reqs: Vec <http_serde::WrappedRequest> = match rmp_serde::from_read_ref (&body)
{
Ok (x) => x,
Err (e) => {
error! ("Can't parse wrapped requests: {:?}", e);
return Err (ServerError::CantParseWrappedRequests (e));
},
};
debug! ("Unwrapped {} requests", wrapped_reqs.len ());
for wrapped_req in wrapped_reqs {
let state = state.clone ();
// These have to detach, so we won't be able to catch the join errors.
tokio::spawn (async move {
wrapped_req: http_serde::WrappedRequest
) -> Result <(), ServerError>
{
let (req_id, parts) = (wrapped_req.id, wrapped_req.req);
debug! ("Handling request {}", req_id);
@ -135,6 +113,34 @@ async fn handle_req_resp <'a> (
}
Ok::<(), ServerError> (())
}
async fn handle_req_resp (
state: &Arc <ServerState>,
req_resp: reqwest::Response
) -> Result <(), ServerError>
{
//println! ("Step 1");
let body = req_resp.bytes ().await.map_err (ServerError::CantCollectWrappedRequests)?;
let wrapped_reqs: Vec <http_serde::WrappedRequest> = match rmp_serde::from_read_ref (&body)
{
Ok (x) => x,
Err (e) => {
error! ("Can't parse wrapped requests: {:?}", e);
return Err (ServerError::CantParseWrappedRequests (e));
},
};
debug! ("Unwrapped {} requests", wrapped_reqs.len ());
for wrapped_req in wrapped_reqs {
let state = state.clone ();
// These have to detach, so we won't be able to catch the join errors.
tokio::spawn (async move {
handle_one_req (&state, wrapped_req).await
});
}
@ -172,6 +178,8 @@ pub async fn run_server (
{
use std::convert::TryInto;
use http::status::StatusCode;
let asset_root = asset_root.unwrap_or_else (PathBuf::new);
if password_is_bad (config_file.api_key.clone ()) {

13
issues/2020-12Dec/auth-route-YNQAQKJS.md
View File

@ -37,10 +37,11 @@ stronger is ready.
- (X) Accept scraper key for some testing endpoint
- (X) (POC) Test with curl
- (X) Clean up scraper endpoint
- (X) Add (almost) end-to-end tests for scraper endpoint
- ( ) Add tests for scraper endpoints
- ( ) Factor v1 API into v1 module
- ( ) Add real scraper endpoints
- (X) Add (almost) end-to-end tests for test scraper endpoint
- ( ) Thread server endpoints through relay scraper auth
- ( ) Add tests for other scraper endpoints
- (don't care) Factor v1 API into v1 module
- (X) Add real scraper endpoints
- ( ) Manually create SQLite DB for scraper keys, add 1 hash
- ( ) Impl DB reads
- ( ) Remove scraper key from config file
@ -72,8 +73,8 @@ the old ones deprecated.
Endpoints needed:
- (X) Query server list
- ( ) Query directory in server
- ( ) GET file with byte range (identical to frontend file API)
- (X) Query directory in server
- (not needed) GET file with byte range (identical to frontend file API)
These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it.
For compatibility with wget spidering, I _might_ do XML or HTML that's