reactor
/
ptth
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

⭐ new: finish MVP for scraper auth. 

Adding a SQLite DB to properly track the keys is going to take a while. For
now I'll just keep them in the config file and give them 30-day expirations.
main
_ 2020-12-16 14:46:03 +00:00
parent cda627fa4b
commit 9ac44cfeb7
6 changed files with 161 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
crates/ptth_relay/src/config.rs
View File

@ -8,7 +8,10 @@ use std::{
path::Path,
};
use crate::errors::ConfigError;
use crate::{
errors::ConfigError,
key_validity::*,
};
// Stuff we need to load from the config file and use to
// set up the HTTP server
@ -28,7 +31,7 @@ pub mod file {
#[derive (Deserialize)]
pub struct DevMode {
pub scraper_key: Option <ScraperKey <Valid7Days>>,
}
// Stuff that's identical between the file and the runtime structures
@ -45,19 +48,25 @@ pub mod file {
#[derive (Deserialize)]
pub struct Config {
pub port: Option <u16>,
pub servers: Vec <Server>,
#[serde (flatten)]
pub iso: Isomorphic,
pub port: Option <u16>,
pub servers: Vec <Server>,
// Adding a DB will take a while, so I'm moving these out of dev mode.
pub scraper_keys: Vec <ScraperKey <Valid30Days>>,
}
}
// Stuff we actually need at runtime
pub struct Config {
pub iso: file::Isomorphic,
pub port: Option <u16>,
pub servers: HashMap <String, file::Server>,
pub iso: file::Isomorphic,
pub scraper_keys: HashMap <String, ScraperKey <Valid30Days>>,
}
impl TryFrom <file::Config> for Config {
@ -69,10 +78,18 @@ impl TryFrom <file::Config> for Config {
let servers = itertools::process_results (servers, |i| HashMap::from_iter (i))?;
let scraper_keys = if f.iso.enable_scraper_api {
HashMap::from_iter (f.scraper_keys.into_iter ().map (|key| (key.hash.encode_base64 (), key)))
}
else {
Default::default ()
};
Ok (Self {
iso: f.iso,
port: f.port,
servers,
iso: f.iso,
scraper_keys,
})
}
}

41
crates/ptth_relay/src/key_validity.rs
View File

@ -69,7 +69,7 @@ impl <'de> Deserialize <'de> for BlakeHashWrapper {
}
pub struct Valid7Days;
//pub struct Valid30Days;
pub struct Valid30Days;
//pub struct Valid90Days;
pub trait MaxValidDuration {
@ -82,11 +82,19 @@ impl MaxValidDuration for Valid7Days {
}
}
impl MaxValidDuration for Valid30Days {
fn dur () -> Duration {
Duration::days (30)
}
}
#[derive (Deserialize)]
pub struct ScraperKey <V: MaxValidDuration> {
name: String,
not_before: DateTime <Utc>,
not_after: DateTime <Utc>,
hash: BlakeHashWrapper,
pub hash: BlakeHashWrapper,
#[serde (default)]
_phantom: std::marker::PhantomData <V>,
@ -103,11 +111,12 @@ pub enum KeyValidity {
DurationNegative,
}
impl ScraperKey <Valid7Days> {
pub fn new (input: &[u8]) -> Self {
impl ScraperKey <Valid30Days> {
pub fn new_30_day <S: Into <String>> (name: S, input: &[u8]) -> Self {
let now = Utc::now ();
Self {
name: name.into (),
not_before: now,
not_after: now + Duration::days (7),
hash: BlakeHashWrapper::from_key (input),
@ -161,7 +170,8 @@ mod tests {
fn duration_negative () {
let zero_time = Utc::now ();
let key = ScraperKey::<Valid7Days> {
let key = ScraperKey::<Valid30Days> {
name: "automated testing".to_string (),
not_before: zero_time + Duration::days (1 + 2),
not_after: zero_time + Duration::days (1),
hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()),
@ -183,14 +193,15 @@ mod tests {
fn key_valid_too_long () {
let zero_time = Utc::now ();
let key = ScraperKey::<Valid7Days> {
let key = ScraperKey::<Valid30Days> {
name: "automated testing".to_string (),
not_before: zero_time + Duration::days (1),
not_after: zero_time + Duration::days (1 + 8),
not_after: zero_time + Duration::days (1 + 31),
hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()),
_phantom: Default::default (),
};
let err = DurationTooLong (Duration::days (7));
let err = DurationTooLong (Duration::days (30));
for (input, expected) in &[
(zero_time + Duration::days (0), err),
@ -205,9 +216,10 @@ mod tests {
fn normal_key () {
let zero_time = Utc::now ();
let key = ScraperKey::<Valid7Days> {
let key = ScraperKey::<Valid30Days> {
name: "automated testing".to_string (),
not_before: zero_time + Duration::days (1),
not_after: zero_time + Duration::days (1 + 7),
not_after: zero_time + Duration::days (1 + 30),
hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()),
_phantom: Default::default (),
};
@ -215,7 +227,7 @@ mod tests {
for (input, expected) in &[
(zero_time + Duration::days (0), ClockIsBehind),
(zero_time + Duration::days (2), Valid),
(zero_time + Duration::days (1 + 7), Expired),
(zero_time + Duration::days (1 + 30), Expired),
(zero_time + Duration::days (100), Expired),
] {
assert_eq! (key.is_valid (*input, "bad_password".as_bytes ()), *expected);
@ -226,9 +238,10 @@ mod tests {
fn wrong_key () {
let zero_time = Utc::now ();
let key = ScraperKey::<Valid7Days> {
let key = ScraperKey::<Valid30Days> {
name: "automated testing".to_string (),
not_before: zero_time + Duration::days (1),
not_after: zero_time + Duration::days (1 + 7),
not_after: zero_time + Duration::days (1 + 30),
hash: BlakeHashWrapper::from_key ("bad_password".as_bytes ()),
_phantom: Default::default (),
};
@ -236,7 +249,7 @@ mod tests {
for input in &[
zero_time + Duration::days (0),
zero_time + Duration::days (2),
zero_time + Duration::days (1 + 7),
zero_time + Duration::days (1 + 30),
zero_time + Duration::days (100),
] {
let validity = key.is_valid (*input, "badder_password".as_bytes ());

2
crates/ptth_relay/src/lib.rs
View File

@ -338,6 +338,8 @@ async fn handle_all (
server_endpoint::handle_listen (state, listen_code.into (), api_key.as_bytes ()).await
}
else if let Some (rest) = prefix_match ("/frontend/servers/", &path) {
// DRY T4H76LB3
if rest == "" {
Ok (handle_server_list (state, handlebars).await?)
}

80
crates/ptth_relay/src/scraper_api.rs
View File

@ -23,11 +23,21 @@ use tracing::{
use crate::{
RequestError,
error_reply,
key_validity::KeyValidity,
key_validity::*,
prefix_match,
relay_state::RelayState,
};
// Not sure if this is the best way to do a hard-coded string table, but
// it'll keep the tests up to date
mod strings {
pub const FORBIDDEN: &str = "403 Forbidden";
pub const NO_API_KEY: &str = "Can't auth as scraper without API key";
pub const UNKNOWN_API_VERSION: &str = "Unknown scraper API version";
pub const UNKNOWN_API_ENDPOINT: &str = "Unknown scraper API endpoint";
}
// JSON is probably Good Enough For Now, so I'll just make everything
// a struct and lazily serialize it right before leaving the
// top-level handle () fn.
@ -109,27 +119,25 @@ async fn api_v1 (
let api_key = req.headers ().get ("X-ApiKey");
let api_key = match api_key {
None => return Ok (error_reply (StatusCode::FORBIDDEN, "Can't run scraper without an API key")?),
None => return Ok (error_reply (StatusCode::FORBIDDEN, strings::NO_API_KEY)?),
Some (x) => x,
};
let bad_key = || error_reply (StatusCode::FORBIDDEN, "403 Forbidden");
let actual_hash = BlakeHashWrapper::from_key (api_key.as_bytes ());
let bad_key = || error_reply (StatusCode::FORBIDDEN, strings::FORBIDDEN);
{
let config = state.config.read ().await;
let dev_mode = match &config.iso.dev_mode {
None => return Ok (bad_key ()?),
let expected_key = match config.scraper_keys.get (&actual_hash.encode_base64 ()) {
Some (x) => x,
};
let expected_key = match &dev_mode.scraper_key {
None => return Ok (bad_key ()?),
Some (x) => x,
};
let now = chrono::Utc::now ();
// The hash in is_valid is redundant
match expected_key.is_valid (now, api_key.as_bytes ()) {
KeyValidity::Valid => (),
KeyValidity::WrongKey (bad_hash) => {
@ -150,8 +158,25 @@ async fn api_v1 (
let x = v1_server_list (&state).await;
Ok (error_reply (StatusCode::OK, &serde_json::to_string (&x).unwrap ())?)
}
else if let Some (rest) = prefix_match ("server/", path_rest) {
// DRY T4H76LB3
if let Some (idx) = rest.find ('/') {
let listen_code = String::from (&rest [0..idx]);
let path = String::from (&rest [idx..]);
let (parts, _) = req.into_parts ();
// This is ugly. I don't like having scraper_api know about the
// crate root.
Ok (crate::handle_http_request (parts, path, state, listen_code).await?)
}
else {
Ok (error_reply (StatusCode::BAD_REQUEST, "Bad URI format")?)
}
}
else {
Ok (error_reply (StatusCode::NOT_FOUND, "Unknown API endpoint")?)
Ok (error_reply (StatusCode::NOT_FOUND, strings::UNKNOWN_API_ENDPOINT)?)
}
}
@ -176,7 +201,7 @@ pub async fn handle (
api_v1 (req, state, rest).await
}
else {
Ok (error_reply (StatusCode::NOT_FOUND, "Unknown scraper API version")?)
Ok (error_reply (StatusCode::NOT_FOUND, strings::UNKNOWN_API_VERSION)?)
}
}
@ -203,7 +228,7 @@ mod tests {
// Expected
expected_status: StatusCode,
expected_headers: Vec <(&'static str, &'static str)>,
expected_body: &'static str,
expected_body: String,
}
impl TestCase {
@ -237,16 +262,16 @@ mod tests {
x
}
fn expected_body (&self, v: &'static str) -> Self {
fn expected_body (&self, v: String) -> Self {
let mut x = self.clone ();
x.expected_body = v;
x
}
fn expected (&self, sc: StatusCode, body: &'static str) -> Self {
fn expected (&self, sc: StatusCode, body: &str) -> Self {
self
.expected_status (sc)
.expected_body (body)
.expected_body (format! ("{}\n", body))
}
async fn test (&self) {
@ -260,14 +285,15 @@ mod tests {
let input = input.body (Body::empty ()).unwrap ();
let config_file = config::file::Config {
port: Some (4000),
servers: vec! [],
iso: config::file::Isomorphic {
enable_scraper_api: true,
dev_mode: self.valid_key.map (|key| config::file::DevMode {
scraper_key: Some (key_validity::ScraperKey::new (key.as_bytes ())),
}),
dev_mode: Default::default (),
},
port: Some (4000),
servers: vec! [],
scraper_keys: vec! [
self.valid_key.map (|x| key_validity::ScraperKey::new_30_day ("automated test", x.as_bytes ())),
].into_iter ().filter_map (|x| x).collect (),
};
let config = config::Config::try_from (config_file).expect ("Can't load config");
@ -292,7 +318,7 @@ mod tests {
let actual_body = actual_body.to_vec ();
let actual_body = String::from_utf8 (actual_body).expect ("Body should be UTF-8");
assert_eq! (&actual_body, self.expected_body);
assert_eq! (actual_body, self.expected_body);
}
}
@ -309,21 +335,21 @@ mod tests {
expected_headers: vec! [
("content-type", "text/plain"),
],
expected_body: "You're valid!\n",
expected_body: "You're valid!\n".to_string (),
};
for case in &[
base_case.clone (),
base_case.path_rest ("v9999/test")
.expected (StatusCode::NOT_FOUND, "Unknown scraper API version\n"),
.expected (StatusCode::NOT_FOUND, strings::UNKNOWN_API_VERSION),
base_case.valid_key (None)
.expected (StatusCode::FORBIDDEN, "403 Forbidden\n"),
.expected (StatusCode::FORBIDDEN, strings::FORBIDDEN),
base_case.input_key (Some ("borgus"))
.expected (StatusCode::FORBIDDEN, "403 Forbidden\n"),
.expected (StatusCode::FORBIDDEN, strings::FORBIDDEN),
base_case.path_rest ("v1/toast")
.expected (StatusCode::NOT_FOUND, "Unknown API endpoint\n"),
.expected (StatusCode::NOT_FOUND, strings::UNKNOWN_API_ENDPOINT),
base_case.input_key (None)
.expected (StatusCode::FORBIDDEN, "Can't run scraper without an API key\n"),
.expected (StatusCode::FORBIDDEN, strings::NO_API_KEY),
] {
case.test ().await;
}

50
issues/2020-12Dec/auth-route-YNQAQKJS.md
View File

@ -2,6 +2,52 @@
(Find this issue with `git grep YNQAQKJS`)
## Test curl commands
(In production the API key should be loaded from a file. Putting it in the
Bash command is bad, because it will be saved to Bash's history file. Putting
it in environment variables is slightly better)
`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/api/test`
Should return "You're valid!"
`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server_list`
Should return a JSON object listing all the servers.
`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/api/v1/dir/`
Proxies into the "aliens_wildland" server and retrieves a JSON object listing
the file server root. (The server must be running a new version of ptth_server
which can serve the JSON API)
`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/api/v1/dir/src/`
Same, but retrieves the listing for "/src".
`curl -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs`
There is no special API for retrieving files yet - But the existing server
API will be is proxied through the new scraper API on the relay.
`curl --head -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs`
PTTH supports HEAD requests. This request will yield a "204 No Content", with
the "content-length" header.
`curl -H "range: bytes=100-199" -H "X-ApiKey: $API_KEY" 127.0.0.1:4000/scraper/v1/server/aliens_wildland/files/src/tests.rs`
PTTH supports byte range requests. This request will skip 100 bytes into the
file, and read 100 bytes.
To avoid fence-post errors, most programming languages use half-open ranges.
e.g. `0..3` means "0, 1, 2". However, HTTP byte ranges are closed ranges.
e.g. `0..3` means "0, 1, 2, 3". So 100-199 means 199 is the last byte retrieved.
By polling with HEAD and byte range requests, a scraper client can approximate
`tail -f` behavior of a server-side file.
## Problem statement
PTTH has 2 auth routes:
@ -38,7 +84,7 @@ stronger is ready.
- (X) (POC) Test with curl
- (X) Clean up scraper endpoint
- (X) Add (almost) end-to-end tests for test scraper endpoint
- ( ) Thread server endpoints through relay scraper auth
- (X) Thread server endpoints through relay scraper auth
- ( ) Add tests for other scraper endpoints
- (don't care) Factor v1 API into v1 module
- (X) Add real scraper endpoints
@ -80,7 +126,7 @@ These will all be JSON for now since Python, Rust, C++, C#, etc. can handle it.
For compatibility with wget spidering, I _might_ do XML or HTML that's
machine-readable. We'll see.
## Open questions
## Decision journal
**Who generates the API key? The scraper client, or the PTTH relay server?**

16
src/tests.rs
View File

@ -36,6 +36,7 @@ fn end_to_end () {
let tripcode = BlakeHashWrapper::from_key (api_key.as_bytes ());
debug! ("Relay is expecting tripcode {}", tripcode.encode_base64 ());
let config_file = ptth_relay::config::file::Config {
iso: Default::default (),
port: None,
servers: vec! [
ptth_relay::config::file::Server {
@ -44,7 +45,7 @@ fn end_to_end () {
display_name: None,
},
],
iso: Default::default (),
scraper_keys: vec! [],
};
let config = ptth_relay::config::Config::try_from (config_file).expect ("Can't load config");
@ -143,16 +144,15 @@ fn scraper_endpoints () {
use ptth_relay::*;
let config_file = config::file::Config {
port: Some (4001),
servers: vec! [
],
iso: config::file::Isomorphic {
enable_scraper_api: true,
dev_mode: Some (config::file::DevMode {
scraper_key: Some (key_validity::ScraperKey::new (b"bogus")),
}),
dev_mode: Default::default (),
},
port: Some (4001),
servers: vec! [],
scraper_keys: vec! [
key_validity::ScraperKey::new_30_day ("automated testing", b"bogus")
],
};
let config = config::Config::try_from (config_file).expect ("Can't load config");