From cda627fa4bc1ff493aaec115675aaaa73b93c799 Mon Sep 17 00:00:00 2001
From: _ <>
Date: Tue, 15 Dec 2020 05:15:17 +0000
Subject: [PATCH] :star: new: add JSON API in server for dir listings
---
Cargo.lock | 1 +
crates/ptth_core/src/lib.rs | 17 +++
crates/ptth_file_server_bin/src/main.rs | 3 +-
crates/ptth_server/Cargo.toml | 1 +
.../ptth_server/src/file_server/internal.rs | 88 +++++++++++--
crates/ptth_server/src/file_server/mod.rs | 99 +++++++++++---
crates/ptth_server/src/file_server/tests.rs | 2 +-
crates/ptth_server/src/lib.rs | 122 ++++++++++--------
issues/2020-12Dec/auth-route-YNQAQKJS.md | 13 +-
9 files changed, 249 insertions(+), 97 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 36c91c3..0d49e91 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1269,6 +1269,7 @@ dependencies = [
"reqwest",
"rmp-serde",
"serde",
+ "serde_json",
"structopt",
"thiserror",
"tokio",
diff --git a/crates/ptth_core/src/lib.rs b/crates/ptth_core/src/lib.rs
index 69bd7d6..4d2440a 100644
--- a/crates/ptth_core/src/lib.rs
+++ b/crates/ptth_core/src/lib.rs
@@ -24,3 +24,20 @@ pub fn prefix_match <'a> (prefix: &str, hay: &'a str) -> Option <&'a str>
None
}
}
+
+#[cfg (test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn prefix () {
+ for (p, h, expected) in &[
+ ("/files/", "/files/a", Some ("a")),
+ ("/files/", "/files/abc/def", Some ("abc/def")),
+ ("/files/", "/files", None),
+ ("/files/", "/not_files", None),
+ ("/files/", "/files/", Some ("")),
+ ] {
+ assert_eq! (prefix_match (*p, *h), *expected);
+ }
+ }
+}
diff --git a/crates/ptth_file_server_bin/src/main.rs b/crates/ptth_file_server_bin/src/main.rs
index cec4bb2..33acbd7 100644
--- a/crates/ptth_file_server_bin/src/main.rs
+++ b/crates/ptth_file_server_bin/src/main.rs
@@ -44,6 +44,7 @@ async fn handle_all (req: Request
, state: Arc >)
-> Result , anyhow::Error>
{
use std::str::FromStr;
+ use hyper::header::HeaderName;
debug! ("req.uri () = {:?}", req.uri ());
@@ -74,7 +75,7 @@ async fn handle_all (req: Request , state: Arc >)
.status (StatusCode::from (ptth_resp.parts.status_code));
for (k, v) in ptth_resp.parts.headers {
- resp = resp.header (hyper::header::HeaderName::from_str (&k)?, v);
+ resp = resp.header (HeaderName::from_str (&k)?, v);
}
let body = ptth_resp.body.map_or_else (Body::empty, Body::wrap_stream);
diff --git a/crates/ptth_server/Cargo.toml b/crates/ptth_server/Cargo.toml
index 0095a55..81345a8 100644
--- a/crates/ptth_server/Cargo.toml
+++ b/crates/ptth_server/Cargo.toml
@@ -22,6 +22,7 @@ regex = "1.4.1"
reqwest = { version = "0.10.8", features = ["stream"] }
rmp-serde = "0.14.4"
serde = {version = "1.0.117", features = ["derive"]}
+serde_json = "1.0.60"
structopt = "0.3.20"
thiserror = "1.0.22"
tokio = { version = "0.2.22", features = ["full"] }
diff --git a/crates/ptth_server/src/file_server/internal.rs b/crates/ptth_server/src/file_server/internal.rs
index 695c0b3..9b5080a 100644
--- a/crates/ptth_server/src/file_server/internal.rs
+++ b/crates/ptth_server/src/file_server/internal.rs
@@ -42,10 +42,17 @@ use super::{
range,
};
+#[derive (Debug, PartialEq)]
+pub enum OutputFormat {
+ Json,
+ Html,
+}
+
#[derive (Debug, PartialEq)]
pub struct ServeDirParams {
pub path: PathBuf,
pub dir: AlwaysEqual ,
+ pub format: OutputFormat,
}
#[derive (Debug, PartialEq)]
@@ -59,11 +66,12 @@ pub struct ServeFileParams {
pub enum Response {
Favicon,
Forbidden,
- InvalidQuery,
MethodNotAllowed,
NotFound,
RangeNotSatisfiable (u64),
Redirect (String),
+ InvalidQuery,
+
Root,
ServeDir (ServeDirParams),
ServeFile (ServeFileParams),
@@ -77,7 +85,8 @@ fn serve_dir (
path: &Path,
dir: tokio::fs::ReadDir,
full_path: PathBuf,
- uri: &http::Uri
+ uri: &http::Uri,
+ format: OutputFormat
)
-> Result
{
@@ -98,6 +107,7 @@ fn serve_dir (
Ok (Response::ServeDir (ServeDirParams {
dir,
path: full_path,
+ format,
}))
}
@@ -161,6 +171,53 @@ async fn serve_file (
})
}
+async fn serve_api (
+ root: &Path,
+ uri: &http::Uri,
+ hidden_path: Option <&Path>,
+ path: &str
+)
+-> Result
+{
+ use Response::*;
+
+ match prefix_match ("/v1/dir/", path) {
+ None => (),
+ Some (path) => {
+ let encoded_path = &path [0..];
+
+ let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?;
+ let path = Path::new (&*path_s);
+
+ let full_path = root.join (path);
+
+ debug! ("full_path = {:?}", full_path);
+
+ if let Some (hidden_path) = hidden_path {
+ if full_path == hidden_path {
+ return Ok (Forbidden);
+ }
+ }
+
+ return if let Ok (dir) = read_dir (&full_path).await {
+ serve_dir (
+ &path_s,
+ path,
+ dir,
+ full_path,
+ &uri,
+ OutputFormat::Json
+ )
+ }
+ else {
+ Ok (NotFound)
+ };
+ },
+ };
+
+ Ok (NotFound)
+}
+
pub async fn serve_all (
root: &Path,
method: Method,
@@ -186,22 +243,28 @@ pub async fn serve_all (
}
};
- if uri.path () == "/favicon.ico" {
+ let path = uri.path ();
+
+ if path == "/favicon.ico" {
return Ok (Favicon);
}
- let path = match prefix_match ("/files", uri.path ()) {
- Some (x) => x,
- None => return Ok (Root),
- };
-
- if path == "" {
- return Ok (Redirect ("files/".to_string ()));
+ if path == "/" {
+ return Ok (Root);
}
+ if let Some (path) = prefix_match ("/api", path) {
+ return serve_api (root, &uri, hidden_path, path).await;
+ }
+
+ let path = match prefix_match ("/files/", path) {
+ Some (x) => x,
+ None => return Ok (NotFound),
+ };
+
// TODO: There is totally a dir traversal attack in here somewhere
- let encoded_path = &path [1..];
+ let encoded_path = &path [0..];
let path_s = percent_decode (encoded_path.as_bytes ()).decode_utf8 ().map_err (FileServerError::PathNotUtf8)?;
let path = Path::new (&*path_s);
@@ -222,7 +285,8 @@ pub async fn serve_all (
path,
dir,
full_path,
- &uri
+ &uri,
+ OutputFormat::Html
)
}
else if let Ok (file) = File::open (&full_path).await {
diff --git a/crates/ptth_server/src/file_server/mod.rs b/crates/ptth_server/src/file_server/mod.rs
index 4ff2678..40d9d84 100644
--- a/crates/ptth_server/src/file_server/mod.rs
+++ b/crates/ptth_server/src/file_server/mod.rs
@@ -58,7 +58,19 @@ pub struct ServerInfo {
}
#[derive (Serialize)]
-struct TemplateDirEntry {
+struct DirEntryJson {
+ name: String,
+ size: u64,
+ is_dir: bool,
+}
+
+#[derive (Serialize)]
+struct DirJson {
+ entries: Vec ,
+}
+
+#[derive (Serialize)]
+struct DirEntryHtml {
icon: &'static str,
trailing_slash: &'static str,
@@ -79,12 +91,12 @@ struct TemplateDirEntry {
}
#[derive (Serialize)]
-struct TemplateDirPage <'a> {
+struct DirHtml <'a> {
#[serde (flatten)]
server_info: &'a ServerInfo,
path: Cow <'a, str>,
- entries: Vec ,
+ entries: Vec ,
}
fn get_icon (file_name: &str) -> &'static str {
@@ -109,7 +121,7 @@ fn get_icon (file_name: &str) -> &'static str {
}
}
-async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
+async fn read_dir_entry_html (entry: DirEntry) -> DirEntryHtml
{
use percent_encoding::{
CONTROLS,
@@ -118,7 +130,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let file_name = match entry.file_name ().into_string () {
Ok (x) => x,
- Err (_) => return TemplateDirEntry {
+ Err (_) => return DirEntryHtml {
icon: emoji::ERROR,
trailing_slash: "",
file_name: "File / directory name is not UTF-8".into (),
@@ -130,7 +142,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let metadata = match entry.metadata ().await {
Ok (x) => x,
- Err (_) => return TemplateDirEntry {
+ Err (_) => return DirEntryHtml {
icon: emoji::ERROR,
trailing_slash: "",
file_name: "Could not fetch metadata".into (),
@@ -153,7 +165,7 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
let encoded_file_name = utf8_percent_encode (&file_name, CONTROLS).to_string ();
- TemplateDirEntry {
+ DirEntryHtml {
icon,
trailing_slash: &trailing_slash,
file_name,
@@ -163,6 +175,20 @@ async fn read_dir_entry (entry: DirEntry) -> TemplateDirEntry
}
}
+async fn read_dir_entry_json (entry: DirEntry) -> Option
+{
+ let name = entry.file_name ().into_string ().ok ()?;
+ let metadata = entry.metadata ().await.ok ()?;
+ let is_dir = metadata.is_dir ();
+ let size = metadata.len ();
+
+ Some (DirEntryJson {
+ name,
+ size,
+ is_dir,
+ })
+}
+
async fn serve_root (
handlebars: &Handlebars <'static>,
server_info: &ServerInfo
@@ -182,8 +208,33 @@ fn serve_html (s: String) -> Response {
resp
}
+async fn serve_dir_json (
+ mut dir: ReadDir
+) -> Result
+{
+ let mut entries = vec! [];
+
+ while let Ok (Some (entry)) = dir.next_entry ().await {
+ if let Some (entry) = read_dir_entry_json (entry).await {
+ entries.push (entry);
+ }
+ }
+
+ entries.sort_unstable_by (|a, b| a.name.cmp (&b.name));
+
+ let dir = DirJson {
+ entries,
+ };
+
+ let mut response = Response::default ();
+ response.header ("content-type".to_string (), "application/json; charset=UTF-8".to_string ().into_bytes ());
+ response.body_bytes (serde_json::to_string (&dir).unwrap ().into_bytes ());
+
+ Ok (response)
+}
+
#[instrument (level = "debug", skip (handlebars, dir))]
-async fn serve_dir (
+async fn serve_dir_html (
handlebars: &Handlebars <'static>,
server_info: &ServerInfo,
path: Cow <'_, str>,
@@ -193,12 +244,12 @@ async fn serve_dir (
let mut entries = vec! [];
while let Ok (Some (entry)) = dir.next_entry ().await {
- entries.push (read_dir_entry (entry).await);
+ entries.push (read_dir_entry_html (entry).await);
}
entries.sort_unstable_by (|a, b| a.file_name.cmp (&b.file_name));
- let s = handlebars.render ("file_server_dir", &TemplateDirPage {
+ let s = handlebars.render ("file_server_dir", &DirHtml {
path,
entries,
server_info,
@@ -316,7 +367,10 @@ pub async fn serve_all (
)
-> Result
{
- use internal::Response::*;
+ use internal::{
+ OutputFormat,
+ Response::*,
+ };
fn serve_error >> (
status_code: StatusCode,
@@ -331,11 +385,10 @@ pub async fn serve_all (
}
Ok (match internal::serve_all (root, method, uri, headers, hidden_path).await? {
- Favicon => serve_error (StatusCode::NotFound, ""),
- Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden"),
- InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object"),
- MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method"),
- NotFound => serve_error (StatusCode::NotFound, "404 Not Found"),
+ Favicon => serve_error (StatusCode::NotFound, "Not found\n"),
+ Forbidden => serve_error (StatusCode::Forbidden, "403 Forbidden\n"),
+ MethodNotAllowed => serve_error (StatusCode::MethodNotAllowed, "Unsupported method\n"),
+ NotFound => serve_error (StatusCode::NotFound, "404 Not Found\nAre you missing a trailing slash?\n"),
RangeNotSatisfiable (file_len) => {
let mut resp = Response::default ();
resp.status_code (StatusCode::RangeNotSatisfiable)
@@ -344,16 +397,22 @@ pub async fn serve_all (
},
Redirect (location) => {
let mut resp = Response::default ();
- resp.status_code (StatusCode::TemporaryRedirect);
- resp.header ("location".to_string (), location.into_bytes ());
- resp.body_bytes (b"Redirecting...".to_vec ());
+ resp.status_code (StatusCode::TemporaryRedirect)
+ .header ("location".to_string (), location.into_bytes ());
+ resp.body_bytes (b"Redirecting...\n".to_vec ());
resp
},
+ InvalidQuery => serve_error (StatusCode::BadRequest, "Query is invalid for this object\n"),
+
Root => serve_root (handlebars, server_info).await?,
ServeDir (internal::ServeDirParams {
path,
dir,
- }) => serve_dir (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?,
+ format
+ }) => match format {
+ OutputFormat::Json => serve_dir_json (dir.into_inner ()).await?,
+ OutputFormat::Html => serve_dir_html (handlebars, server_info, path.to_string_lossy (), dir.into_inner ()).await?,
+ },
ServeFile (internal::ServeFileParams {
file,
send_body,
diff --git a/crates/ptth_server/src/file_server/tests.rs b/crates/ptth_server/src/file_server/tests.rs
index 0b78385..9f096be 100644
--- a/crates/ptth_server/src/file_server/tests.rs
+++ b/crates/ptth_server/src/file_server/tests.rs
@@ -101,7 +101,7 @@ fn file_server () {
for (uri_path, expected) in vec! [
("/", Root),
- ("/files", Redirect ("files/".to_string ())),
+ ("/files", NotFound),
("/files/?", InvalidQuery),
("/files/src", Redirect ("src/".to_string ())),
("/files/src/?", InvalidQuery),
diff --git a/crates/ptth_server/src/lib.rs b/crates/ptth_server/src/lib.rs
index ce63528..bccc824 100644
--- a/crates/ptth_server/src/lib.rs
+++ b/crates/ptth_server/src/lib.rs
@@ -15,9 +15,6 @@ use std::{
use futures::FutureExt;
use handlebars::Handlebars;
-use http::status::{
- StatusCode,
-};
use reqwest::Client;
use serde::Deserialize;
use tokio::{
@@ -59,10 +56,70 @@ struct ServerState {
hidden_path: Option ,
}
-async fn handle_req_resp <'a> (
+async fn handle_one_req (
+ state: &Arc ,
+ wrapped_req: http_serde::WrappedRequest
+) -> Result <(), ServerError>
+{
+ let (req_id, parts) = (wrapped_req.id, wrapped_req.req);
+
+ debug! ("Handling request {}", req_id);
+
+ let default_root = PathBuf::from ("./");
+ let file_server_root: &std::path::Path = state.config.file_server_root
+ .as_ref ()
+ .unwrap_or (&default_root);
+
+ let response = file_server::serve_all (
+ &state.handlebars,
+ &state.server_info,
+ file_server_root,
+ parts.method,
+ &parts.uri,
+ &parts.headers,
+ state.hidden_path.as_deref ()
+ ).await?;
+
+ let mut resp_req = state.client
+ .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id))
+ .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?));
+
+ if let Some (length) = response.content_length {
+ resp_req = resp_req.header ("Content-Length", length.to_string ());
+ }
+ if let Some (body) = response.body {
+ resp_req = resp_req.body (reqwest::Body::wrap_stream (body));
+ }
+
+ let req = resp_req.build ().map_err (ServerError::Step5Responding)?;
+
+ debug! ("{:?}", req.headers ());
+
+ //println! ("Step 6");
+ match state.client.execute (req).await {
+ Ok (r) => {
+ let status = r.status ();
+ let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?;
+ debug! ("{:?} {:?}", status, text);
+ },
+ Err (e) => {
+ if e.is_request () {
+ warn! ("Error while POSTing response. Client probably hung up.");
+ }
+ else {
+ error! ("Err: {:?}", e);
+ }
+ },
+ }
+
+ Ok::<(), ServerError> (())
+}
+
+async fn handle_req_resp (
state: &Arc ,
req_resp: reqwest::Response
-) -> Result <(), ServerError> {
+) -> Result <(), ServerError>
+{
//println! ("Step 1");
let body = req_resp.bytes ().await.map_err (ServerError::CantCollectWrappedRequests)?;
@@ -83,58 +140,7 @@ async fn handle_req_resp <'a> (
// These have to detach, so we won't be able to catch the join errors.
tokio::spawn (async move {
- let (req_id, parts) = (wrapped_req.id, wrapped_req.req);
-
- debug! ("Handling request {}", req_id);
-
- let default_root = PathBuf::from ("./");
- let file_server_root: &std::path::Path = state.config.file_server_root
- .as_ref ()
- .unwrap_or (&default_root);
-
- let response = file_server::serve_all (
- &state.handlebars,
- &state.server_info,
- file_server_root,
- parts.method,
- &parts.uri,
- &parts.headers,
- state.hidden_path.as_deref ()
- ).await?;
-
- let mut resp_req = state.client
- .post (&format! ("{}/http_response/{}", state.config.relay_url, req_id))
- .header (ptth_core::PTTH_MAGIC_HEADER, base64::encode (rmp_serde::to_vec (&response.parts).map_err (ServerError::MessagePackEncodeResponse)?));
-
- if let Some (length) = response.content_length {
- resp_req = resp_req.header ("Content-Length", length.to_string ());
- }
- if let Some (body) = response.body {
- resp_req = resp_req.body (reqwest::Body::wrap_stream (body));
- }
-
- let req = resp_req.build ().map_err (ServerError::Step5Responding)?;
-
- debug! ("{:?}", req.headers ());
-
- //println! ("Step 6");
- match state.client.execute (req).await {
- Ok (r) => {
- let status = r.status ();
- let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?;
- debug! ("{:?} {:?}", status, text);
- },
- Err (e) => {
- if e.is_request () {
- warn! ("Error while POSTing response. Client probably hung up.");
- }
- else {
- error! ("Err: {:?}", e);
- }
- },
- }
-
- Ok::<(), ServerError> (())
+ handle_one_req (&state, wrapped_req).await
});
}
@@ -172,6 +178,8 @@ pub async fn run_server (
{
use std::convert::TryInto;
+ use http::status::StatusCode;
+
let asset_root = asset_root.unwrap_or_else (PathBuf::new);
if password_is_bad (config_file.api_key.clone ()) {
diff --git a/issues/2020-12Dec/auth-route-YNQAQKJS.md b/issues/2020-12Dec/auth-route-YNQAQKJS.md
index 18b855b..e161f08 100644
--- a/issues/2020-12Dec/auth-route-YNQAQKJS.md
+++ b/issues/2020-12Dec/auth-route-YNQAQKJS.md
@@ -37,10 +37,11 @@ stronger is ready.
- (X) Accept scraper key for some testing endpoint
- (X) (POC) Test with curl
- (X) Clean up scraper endpoint
-- (X) Add (almost) end-to-end tests for scraper endpoint
-- ( ) Add tests for scraper endpoints
-- ( ) Factor v1 API into v1 module
-- ( ) Add real scraper endpoints
+- (X) Add (almost) end-to-end tests for test scraper endpoint
+- ( ) Thread server endpoints through relay scraper auth
+- ( ) Add tests for other scraper endpoints
+- (don't care) Factor v1 API into v1 module
+- (X) Add real scraper endpoints
- ( ) Manually create SQLite DB for scraper keys, add 1 hash
- ( ) Impl DB reads
- ( ) Remove scraper key from config file
@@ -72,8 +73,8 @@ the old ones deprecated.
Endpoints needed:
- (X) Query server list
-- ( ) Query directory in server
-- ( ) GET file with byte range (identical to frontend file API)
+- (X) Query directory in server
+- (not needed) GET file with byte range (identical to frontend file API)
These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it.
For compatibility with wget spidering, I _might_ do XML or HTML that's