reactor
/
ptth
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

⬆️ deps: upgrade everything to Tokio 1.2.0 

And pretend I didn't completely goober the Docker build somehow
main
_ 2021-03-06 21:15:41 +00:00
parent e0f5b229a1
commit a980d151fc
30 changed files with 392 additions and 465 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

575
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

10
Cargo.toml
View File

@ -23,16 +23,16 @@ exclude = [
[dependencies] [dependencies]
anyhow = "1.0.34" anyhow = "1.0.38"
blake3 = "0.3.7" blake3 = "0.3.7"
tokio = { version = "0.2.22", features = ["full"] } reqwest = { version = "0.11.1", features = ["stream"] }
tracing-subscriber = "0.2.15" tokio = { version = "1.2.0", features = ["full"] }
tracing-subscriber = "0.2.16"
tracing = "0.1.25"
debug_proxy = { path = "crates/debug_proxy" } debug_proxy = { path = "crates/debug_proxy" }
ptth_relay = { path = "crates/ptth_relay" } ptth_relay = { path = "crates/ptth_relay" }
ptth_server = { path = "crates/ptth_server" } ptth_server = { path = "crates/ptth_server" }
reqwest = { version = "0.10.8", features = ["stream"] }
tracing = "0.1.21"
[dev-dependencies] [dev-dependencies]

3
Dockerfile
View File

@ -27,9 +27,6 @@ COPY ./Cargo.toml ./
COPY ./crates/always_equal/Cargo.toml ./crates/always_equal/ COPY ./crates/always_equal/Cargo.toml ./crates/always_equal/
COPY ./crates/ptth_core/Cargo.toml ./crates/ptth_core/ COPY ./crates/ptth_core/Cargo.toml ./crates/ptth_core/
COPY ./crates/ptth_relay/Cargo.toml ./crates/ptth_relay/ COPY ./crates/ptth_relay/Cargo.toml ./crates/ptth_relay/
# COPY ./crates/ptth_server/Cargo.toml ./crates/ptth_server/
# COPY ./crates/ptth_file_server_bin/Cargo.toml ./crates/ptth_file_server_bin/
# COPY ./tools/ptth_tail/Cargo.toml ./tools/ptth_tail/
# this build step will cache your dependencies # this build step will cache your dependencies
RUN cargo build --release -p ptth_relay RUN cargo build --release -p ptth_relay

8
crates/debug_proxy/Cargo.toml
View File

@ -9,10 +9,12 @@ license = "AGPL-3.0"
[dependencies] [dependencies]
anyhow = "1.0.34" anyhow = "1.0.34"
futures-util = "0.3.8"
http = "0.2.1" http = "0.2.1"
hyper = "0.13.8" hyper = { version = "0.14.4", features = ["server", "stream"] }
reqwest = { version = "0.10.8", features = ["stream"] } reqwest = "0.11.1"
tokio = { version = "0.2.22", features = ["full"] } tokio = "1.2.0"
tokio-stream = "0.1.3"
tracing = "0.1.21" tracing = "0.1.21"
tracing-subscriber = "0.2.15" tracing-subscriber = "0.2.15"
ulid = "0.4.1" ulid = "0.4.1"

11
crates/debug_proxy/src/lib.rs
View File

@ -3,6 +3,7 @@ use std::{
sync::Arc, sync::Arc,
}; };
use futures_util::StreamExt;
use hyper::{ use hyper::{
Body, Body,
Request, Request,
@ -16,12 +17,12 @@ use hyper::{
use reqwest::Client; use reqwest::Client;
use tokio::{ use tokio::{
spawn, spawn,
stream::StreamExt,
sync::{ sync::{
mpsc, mpsc,
oneshot, oneshot,
}, },
}; };
use tokio_stream::wrappers::ReceiverStream;
use ulid::Ulid; use ulid::Ulid;
struct State { struct State {
@ -51,7 +52,7 @@ async fn handle_all (req: Request <Body>, state: Arc <State>)
upstream_req = upstream_req.header (k, v); upstream_req = upstream_req.header (k, v);
} }
let (mut tx, rx) = mpsc::channel (1); let (tx, rx) = mpsc::channel (1);
spawn ({ spawn ({
let req_id = req_id.clone (); let req_id = req_id.clone ();
async move { async move {
@ -78,7 +79,7 @@ async fn handle_all (req: Request <Body>, state: Arc <State>)
} }
}); });
let upstream_resp = upstream_req.body (reqwest::Body::wrap_stream (rx)).send ().await?; let upstream_resp = upstream_req.body (reqwest::Body::wrap_stream (ReceiverStream::new (rx))).send ().await?;
let mut resp = Response::builder () let mut resp = Response::builder ()
.status (upstream_resp.status ()); .status (upstream_resp.status ());
@ -87,7 +88,7 @@ async fn handle_all (req: Request <Body>, state: Arc <State>)
resp = resp.header (k, v); resp = resp.header (k, v);
} }
let (mut tx, rx) = mpsc::channel (1); let (tx, rx) = mpsc::channel (1);
spawn (async move { spawn (async move {
let mut body = upstream_resp.bytes_stream (); let mut body = upstream_resp.bytes_stream ();
let mut bytes_transferred = 0; let mut bytes_transferred = 0;
@ -112,7 +113,7 @@ async fn handle_all (req: Request <Body>, state: Arc <State>)
Ok::<_, anyhow::Error> (()) Ok::<_, anyhow::Error> (())
}); });
Ok (resp.body (Body::wrap_stream (rx))?) Ok (resp.body (Body::wrap_stream (ReceiverStream::new (rx)))?)
} }
pub async fn run_proxy ( pub async fn run_proxy (

14
crates/ptth_core/Cargo.toml
View File

@ -11,11 +11,11 @@ repository = "https://github.com/ReactorScram/ptth"
[dependencies] [dependencies]
ctrlc = { version = "3.1.7", features = [ "termination" ] } ctrlc = { version = "3.1.8", features = [ "termination" ] }
futures = "0.3.7" futures = "0.3.7"
hyper = "0.13.8" hyper = "0.14.4"
serde = {version = "1.0.117", features = ["derive"]} serde = {version = "1.0.124", features = ["derive"]}
thiserror = "1.0.22" thiserror = "1.0.24"
tokio = { version = "0.2.22", features = ["full"] } tokio = { version = "1.2.0", features = ["full"] }
tracing = "0.1.21" tracing = "0.1.25"
tracing-futures = "0.2.4" tracing-futures = "0.2.5"

4
crates/ptth_core/src/graceful_shutdown.rs
View File

@ -9,7 +9,7 @@ use std::{
use futures::prelude::*; use futures::prelude::*;
use tokio::{ use tokio::{
sync::oneshot, sync::oneshot,
time::delay_for, time::sleep,
}; };
use crate::prelude::*; use crate::prelude::*;
@ -82,7 +82,7 @@ impl ForcedShutdown {
self.tx.send (()).expect ("Error forwarding graceful shutdown signal"); self.tx.send (()).expect ("Error forwarding graceful shutdown signal");
let timeout = 5; let timeout = 5;
debug! ("Starting graceful shutdown. Forcing shutdown in {} seconds", timeout); debug! ("Starting graceful shutdown. Forcing shutdown in {} seconds", timeout);
delay_for (Duration::from_secs (timeout)).await; sleep (Duration::from_secs (timeout)).await;
error! ("Forcing shutdown"); error! ("Forcing shutdown");
}; };

2
crates/ptth_core/src/http_serde.rs
View File

@ -175,7 +175,7 @@ impl Response {
self.content_length = b.len ().try_into ().ok (); self.content_length = b.len ().try_into ().ok ();
self.header ("content-length".to_string (), b.len ().to_string ().into_bytes ()); self.header ("content-length".to_string (), b.len ().to_string ().into_bytes ());
let (mut tx, rx) = tokio::sync::mpsc::channel (1); let (tx, rx) = tokio::sync::mpsc::channel (1);
tokio::spawn (async move { tokio::spawn (async move {
tx.send (Ok (b)).await.ok (); tx.send (Ok (b)).await.ok ();
}); });

19
crates/ptth_file_server_bin/Cargo.toml
View File

@ -8,16 +8,17 @@ license = "AGPL-3.0"
[dependencies] [dependencies]
anyhow = "1.0.34" anyhow = "1.0.38"
arc-swap = "1.1.0" arc-swap = "1.2.0"
handlebars = "3.5.1" handlebars = "3.5.3"
http = "0.2.1" http = "0.2.1"
hyper = "0.13.8" hyper = "0.14.4"
serde = {version = "1.0.117", features = ["derive"]} serde = {version = "1.0.124", features = ["derive"]}
structopt = "0.3.20" structopt = "0.3.21"
tokio = { version = "0.2.22", features = ["full"] } tokio = { version = "1.2.0", features = [] }
tracing = "0.1.21" tokio-stream = "0.1.3"
tracing-subscriber = "0.2.15" tracing = "0.1.25"
tracing-subscriber = "0.2.16"
uom = "0.30.0" uom = "0.30.0"
ptth_core = { path = "../ptth_core" } ptth_core = { path = "../ptth_core" }

5
crates/ptth_file_server_bin/src/main.rs
View File

@ -19,6 +19,7 @@ use hyper::{
StatusCode, StatusCode,
}; };
use serde::Deserialize; use serde::Deserialize;
use tokio_stream::wrappers::ReceiverStream;
use tracing::debug; use tracing::debug;
use ptth_core::{ use ptth_core::{
@ -70,7 +71,9 @@ async fn handle_all (req: Request <Body>, state: Arc <State>)
resp = resp.header (HeaderName::from_str (&k)?, v); resp = resp.header (HeaderName::from_str (&k)?, v);
} }
let body = ptth_resp.body.map_or_else (Body::empty, Body::wrap_stream); let body = ptth_resp.body.map_or_else (Body::empty, |body| {
Body::wrap_stream (ReceiverStream::new (body))
});
Ok (resp.body (body)?) Ok (resp.body (body)?)
} }

26
crates/ptth_forwarding/Cargo.toml
View File

@ -7,17 +7,19 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
anyhow = "1.0.34" anyhow = "1.0.38"
base64 = "0.12.3" base64 = "0.13.0"
clap = "2.33.3" clap = "2.33.3"
futures = "0.3.7" futures = "0.3.7"
reqwest = { version = "0.10.8", features = ["stream"] } futures-util = "0.3.8"
rmp-serde = "0.14.4" reqwest = { version = "0.11.1", features = ["stream"] }
serde = {version = "1.0.117", features = ["derive"]} rmp-serde = "0.15.4"
serde_json = "1.0.60" serde = {version = "1.0.124", features = ["derive"]}
structopt = "0.3.20" serde_json = "1.0.64"
thiserror = "1.0.22" structopt = "0.3.21"
tokio = { version = "0.2.22", features = ["full"] } thiserror = "1.0.24"
tracing = "0.1.21" tokio = { version = "1.2.0", features = [] }
tracing-futures = "0.2.4" tokio-stream = "0.1.3"
tracing-subscriber = "0.2.15" tracing = "0.1.25"
tracing-futures = "0.2.5"
tracing-subscriber = "0.2.16"

9
crates/ptth_forwarding/src/main.rs
View File

@ -1,4 +1,5 @@
use clap::{App, SubCommand}; use clap::{App, SubCommand};
use futures_util::StreamExt;
use reqwest::Client; use reqwest::Client;
use tokio::{ use tokio::{
io::{ io::{
@ -11,9 +12,9 @@ use tokio::{
tcp::OwnedReadHalf, tcp::OwnedReadHalf,
}, },
spawn, spawn,
stream::StreamExt,
sync::mpsc, sync::mpsc,
}; };
use tokio_stream::wrappers::ReceiverStream;
#[tokio::main] #[tokio::main]
async fn main () -> anyhow::Result <()> { async fn main () -> anyhow::Result <()> {
@ -37,7 +38,7 @@ async fn main () -> anyhow::Result <()> {
TcpStream::connect ("127.0.0.1:4010").await? TcpStream::connect ("127.0.0.1:4010").await?
} }
else if let Some (_matches) = matches.subcommand_matches ("client") { else if let Some (_matches) = matches.subcommand_matches ("client") {
let mut listener = TcpListener::bind ("127.0.0.1:4020").await?; let listener = TcpListener::bind ("127.0.0.1:4020").await?;
let (stream, _addr) = listener.accept ().await?; let (stream, _addr) = listener.accept ().await?;
stream stream
} }
@ -53,7 +54,7 @@ async fn main () -> anyhow::Result <()> {
handle_upstream (tcp_upstream, upstream_tx).await handle_upstream (tcp_upstream, upstream_tx).await
}); });
let upstream = client.post ("http://127.0.0.1:4003/").body (reqwest::Body::wrap_stream (upstream_rx)); let upstream = client.post ("http://127.0.0.1:4003/").body (reqwest::Body::wrap_stream (ReceiverStream::new (upstream_rx)));
spawn (async move { spawn (async move {
upstream.send ().await upstream.send ().await
@ -72,7 +73,7 @@ async fn main () -> anyhow::Result <()> {
async fn handle_upstream ( async fn handle_upstream (
mut tcp_upstream: OwnedReadHalf, mut tcp_upstream: OwnedReadHalf,
mut upstream_tx: mpsc::Sender <Result <Vec <u8>, anyhow::Error>> upstream_tx: mpsc::Sender <Result <Vec <u8>, anyhow::Error>>
) -> anyhow::Result <()> { ) -> anyhow::Result <()> {
loop { loop {
let mut buffer = vec! [0u8; 65_536]; let mut buffer = vec! [0u8; 65_536];

16
crates/ptth_forwarding_relay/Cargo.toml
View File

@ -7,12 +7,14 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
anyhow = "1.0.34" anyhow = "1.0.38"
futures = "0.3.7" futures = "0.3.7"
hyper = "0.13.8" futures-util = "0.3.8"
thiserror = "1.0.22" hyper = "0.14.4"
tokio = { version = "0.2.22", features = ["full"] } thiserror = "1.0.24"
tracing = "0.1.21" tokio = { version = "1.2.0", features = [] }
tracing-futures = "0.2.4" tokio-stream = "0.1.3"
tracing-subscriber = "0.2.15" tracing = "0.1.25"
tracing-futures = "0.2.5"
tracing-subscriber = "0.2.16"
ulid = "0.4.1" ulid = "0.4.1"

9
crates/ptth_forwarding_relay/src/main.rs
View File

@ -4,6 +4,7 @@ use std::{
time::Duration, time::Duration,
}; };
use futures_util::StreamExt;
use hyper::{ use hyper::{
Body, Body,
Method, Method,
@ -13,13 +14,13 @@ use hyper::{
}; };
use tokio::{ use tokio::{
spawn, spawn,
stream::StreamExt,
sync::{ sync::{
RwLock, RwLock,
mpsc, mpsc,
}, },
time::interval, time::interval,
}; };
use tokio_stream::wrappers::ReceiverStream;
use tracing::{ use tracing::{
info, trace, info, trace,
}; };
@ -150,7 +151,7 @@ impl HttpService {
}); });
Ok::<_, anyhow::Error> (Response::builder () Ok::<_, anyhow::Error> (Response::builder ()
.body (Body::wrap_stream (rx))?) .body (Body::wrap_stream (ReceiverStream::new (rx)))?)
} }
async fn handle_posts (req: Request <Body>, state: &RelayState) async fn handle_posts (req: Request <Body>, state: &RelayState)
@ -168,7 +169,7 @@ impl HttpService {
.body (Body::from ("hello\n"))?) .body (Body::from ("hello\n"))?)
} }
async fn handle_downstream (mut tx: mpsc::Sender <anyhow::Result <String>>) -> Result <(), anyhow::Error> { async fn handle_downstream (tx: mpsc::Sender <anyhow::Result <String>>) -> Result <(), anyhow::Error> {
let mut int = interval (Duration::from_secs (1)); let mut int = interval (Duration::from_secs (1));
let mut counter = 0u64; let mut counter = 0u64;
@ -178,8 +179,6 @@ impl HttpService {
tx.send (Ok::<_, anyhow::Error> (format! ("Counter: {}\n", counter))).await?; tx.send (Ok::<_, anyhow::Error> (format! ("Counter: {}\n", counter))).await?;
counter += 1; counter += 1;
} }
Ok (())
} }
} }

6
crates/ptth_kv/Cargo.toml
View File

@ -5,8 +5,8 @@ authors = ["Trish"]
edition = "2018" edition = "2018"
[dependencies] [dependencies]
anyhow = "1.0.34" anyhow = "1.0.38"
base64 = "0.13.0" base64 = "0.13.0"
hyper = "0.13.8" hyper = { version = "0.14.4", features = ["full"] }
thiserror = "1.0.22" thiserror = "1.0.22"
tokio = { version = "0.2.22", features = ["full"] } tokio = { version = "1.2.0", features = ["full"] }

2
crates/ptth_kv/src/main.rs
View File

@ -63,7 +63,7 @@ impl HttpService {
use std::net::SocketAddr; use std::net::SocketAddr;
use hyper::{ use hyper::{
server::Server, Server,
service::{ service::{
make_service_fn, make_service_fn,
service_fn, service_fn,

18
crates/ptth_relay/Cargo.toml
View File

@ -8,24 +8,26 @@ license = "AGPL-3.0"
[dependencies] [dependencies]
base64 = "0.12.3" base64 = "0.13.0"
blake3 = "0.3.7" blake3 = "0.3.7"
chrono = {version = "0.4.19", features = ["serde"]} chrono = { version = "0.4.19", features = ["serde"] }
clap = "2.33.3" clap = "2.33.3"
dashmap = "3.11.10" dashmap = "3.11.10"
futures = "0.3.7" futures = "0.3.7"
handlebars = "3.5.1" futures-util = "0.3.8"
http = "0.2.1" handlebars = "3.5.3"
hyper = "0.13.8" http = "0.2.3"
hyper = { version = "0.14.4", features = ["server", "stream"] }
itertools = "0.9.0" itertools = "0.9.0"
rand = "0.6.5" rand = "0.6.5"
rmp-serde = "0.14.4" rmp-serde = "0.14.4"
serde = {version = "1.0.117", features = ["derive"]} serde = { version = "1.0.117", features = ["derive"] }
serde_json = "1.0.60" serde_json = "1.0.60"
thiserror = "1.0.22" thiserror = "1.0.22"
tokio = { version = "0.2.22", features = ["full"] } tokio = { version = "1.2.0", features = [] }
tokio-stream = "0.1.3"
toml = "0.5.7" toml = "0.5.7"
tracing = "0.1.21" tracing = "0.1.25"
tracing-futures = "0.2.4" tracing-futures = "0.2.4"
tracing-subscriber = "0.2.15" tracing-subscriber = "0.2.15"
ulid = "0.4.1" ulid = "0.4.1"

10
crates/ptth_relay/src/config.rs
View File

@ -96,15 +96,7 @@ impl TryFrom <file::Config> for Config {
impl Config { impl Config {
pub async fn from_file (path: &Path) -> Result <Self, ConfigError> { pub async fn from_file (path: &Path) -> Result <Self, ConfigError> {
use tokio::prelude::*; let config_s = tokio::fs::read_to_string (path).await?;
let mut f = tokio::fs::File::open (path).await?;
let mut buffer = vec! [0_u8; 4096];
let bytes_read = f.read (&mut buffer).await?;
buffer.truncate (bytes_read);
let config_s = String::from_utf8 (buffer)?;
let new_config: file::Config = toml::from_str (&config_s)?; let new_config: file::Config = toml::from_str (&config_s)?;
Self::try_from (new_config) Self::try_from (new_config)

12
crates/ptth_relay/src/lib.rs
View File

@ -29,6 +29,7 @@ use chrono::{
Utc Utc
}; };
use dashmap::DashMap; use dashmap::DashMap;
use futures_util::StreamExt;
use handlebars::Handlebars; use handlebars::Handlebars;
use hyper::{ use hyper::{
Body, Body,
@ -47,6 +48,7 @@ use tokio::{
oneshot, oneshot,
}, },
}; };
use tokio_stream::wrappers::ReceiverStream;
use ptth_core::{ use ptth_core::{
http_serde, http_serde,
@ -166,7 +168,7 @@ async fn handle_http_request (
request_rendezvous.insert (watcher_code, new_rendezvous); request_rendezvous.insert (watcher_code, new_rendezvous);
} }
let timeout = tokio::time::delay_for (std::time::Duration::from_secs (30)); let timeout = tokio::time::sleep (std::time::Duration::from_secs (30));
let received = tokio::select! { let received = tokio::select! {
val = rx => val, val = rx => val,
@ -305,8 +307,6 @@ async fn handle_server_list (
async fn handle_endless_sink (req: Request <Body>) -> Result <Response <Body>, http::Error> async fn handle_endless_sink (req: Request <Body>) -> Result <Response <Body>, http::Error>
{ {
use tokio::stream::StreamExt;
let (_parts, mut body) = req.into_parts (); let (_parts, mut body) = req.into_parts ();
let mut bytes_received = 0; let mut bytes_received = 0;
@ -339,8 +339,6 @@ async fn handle_endless_source (gib: usize, throttle: Option <usize>)
let (tx, rx) = mpsc::channel (1); let (tx, rx) = mpsc::channel (1);
tokio::spawn (async move { tokio::spawn (async move {
let mut tx = tx;
let random_block = { let random_block = {
use rand::RngCore; use rand::RngCore;
@ -374,7 +372,7 @@ async fn handle_endless_source (gib: usize, throttle: Option <usize>)
Response::builder () Response::builder ()
.status (StatusCode::OK) .status (StatusCode::OK)
.header ("content-type", "application/octet-stream") .header ("content-type", "application/octet-stream")
.body (Body::wrap_stream (rx)) .body (Body::wrap_stream (ReceiverStream::new (rx)))
} }
#[instrument (level = "trace", skip (req, state, handlebars))] #[instrument (level = "trace", skip (req, state, handlebars))]
@ -564,7 +562,7 @@ pub async fn run_relay (
shutdown_oneshot.await.ok (); shutdown_oneshot.await.ok ();
state.shutdown_watch_tx.broadcast (true).expect ("Can't broadcast graceful shutdown"); state.shutdown_watch_tx.send (true).expect ("Can't broadcast graceful shutdown");
let mut response_rendezvous = state.response_rendezvous.write ().await; let mut response_rendezvous = state.response_rendezvous.write ().await;
let mut swapped = DashMap::default (); let mut swapped = DashMap::default ();

12
crates/ptth_relay/src/server_endpoint.rs
View File

@ -20,8 +20,8 @@ use tokio::{
mpsc, mpsc,
oneshot, oneshot,
}, },
time::delay_for,
}; };
use tokio_stream::wrappers::ReceiverStream;
use ptth_core::{ use ptth_core::{
http_serde, http_serde,
@ -115,7 +115,7 @@ pub async fn handle_listen (
Ok (Err (ShuttingDownError::ShuttingDown)) => Ok (error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon")?), Ok (Err (ShuttingDownError::ShuttingDown)) => Ok (error_reply (StatusCode::SERVICE_UNAVAILABLE, "Server is shutting down, try again soon")?),
Err (_) => Ok (error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error")?), Err (_) => Ok (error_reply (StatusCode::INTERNAL_SERVER_ERROR, "Server error")?),
}, },
_ = delay_for (Duration::from_secs (30)).fuse () => { _ = tokio::time::sleep (Duration::from_secs (30)).fuse () => {
trace! ("Timed out http_listen for server {}", watcher_code); trace! ("Timed out http_listen for server {}", watcher_code);
return Ok (error_reply (StatusCode::NO_CONTENT, "No requests now, long-poll again")?) return Ok (error_reply (StatusCode::NO_CONTENT, "No requests now, long-poll again")?)
} }
@ -151,12 +151,12 @@ pub async fn handle_response (
// Intercept the body packets here so we can check when the stream // Intercept the body packets here so we can check when the stream
// ends or errors out // ends or errors out
let (mut body_tx, body_rx) = mpsc::channel (2); let (body_tx, body_rx) = mpsc::channel (2);
let (body_finished_tx, body_finished_rx) = oneshot::channel (); let (body_finished_tx, body_finished_rx) = oneshot::channel ();
let mut shutdown_watch_rx = state.shutdown_watch_rx.clone (); let mut shutdown_watch_rx = state.shutdown_watch_rx.clone ();
let relay_task = spawn (async move { let relay_task = spawn (async move {
if shutdown_watch_rx.recv ().await == Some (false) { if *shutdown_watch_rx.borrow () == false {
loop { loop {
let item = body.next ().await; let item = body.next ().await;
@ -171,7 +171,7 @@ pub async fn handle_response (
body_finished_tx.send (ClientDisconnected).map_err (|_| LostServer)?; body_finished_tx.send (ClientDisconnected).map_err (|_| LostServer)?;
break; break;
}, },
_ = shutdown_watch_rx.recv ().fuse () => { _ = shutdown_watch_rx.changed ().fuse () => {
debug! ("Closing stream: relay is shutting down"); debug! ("Closing stream: relay is shutting down");
break; break;
}, },
@ -191,7 +191,7 @@ pub async fn handle_response (
Ok::<(), HandleHttpResponseError> (()) Ok::<(), HandleHttpResponseError> (())
}); });
let body = Body::wrap_stream (body_rx); let body = Body::wrap_stream (ReceiverStream::new (body_rx));
let tx = { let tx = {
let response_rendezvous = state.response_rendezvous.read ().await; let response_rendezvous = state.response_rendezvous.read ().await;

23
crates/ptth_server/Cargo.toml
View File

@ -8,10 +8,10 @@ license = "AGPL-3.0"
[dependencies] [dependencies]
aho-corasick = "0.7.14" aho-corasick = "0.7.15"
anyhow = "1.0.34" anyhow = "1.0.38"
arc-swap = "1.1.0" arc-swap = "1.2.0"
base64 = "0.12.3" base64 = "0.13.0"
blake3 = "0.3.7" blake3 = "0.3.7"
chrono = {version = "0.4.19", features = ["serde"]} chrono = {version = "0.4.19", features = ["serde"]}
futures = "0.3.7" futures = "0.3.7"
@ -22,16 +22,17 @@ lazy_static = "1.4.0"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"
pulldown-cmark = "0.8.0" pulldown-cmark = "0.8.0"
regex = "1.4.1" regex = "1.4.1"
reqwest = { version = "0.10.8", features = ["stream"] } reqwest = { version = "0.11.1", features = [] }
rmp-serde = "0.14.4" rmp-serde = "0.14.4"
serde = {version = "1.0.117", features = ["derive"]} serde = {version = "1.0.117", features = ["derive"]}
serde_json = "1.0.60" serde_json = "1.0.60"
structopt = "0.3.20" structopt = "0.3.20"
thiserror = "1.0.22" thiserror = "1.0.24"
tokio = { version = "0.2.22", features = ["full"] } tokio = { version = "1.2.0", features = [] }
tracing = "0.1.21" tokio-stream = "0.1.3"
tracing-futures = "0.2.4" tracing = "0.1.25"
tracing-subscriber = "0.2.15" tracing-futures = "0.2.5"
tracing-subscriber = "0.2.16"
toml = "0.5.7" toml = "0.5.7"
ulid = "0.4.1" ulid = "0.4.1"
uom = "0.30.0" uom = "0.30.0"
@ -42,4 +43,4 @@ ptth_core = { path = "../ptth_core" }
[dev-dependencies] [dev-dependencies]
maplit = "1.0.2" maplit = "1.0.2"
rand = "0.6.5" rand = "0.8.3"

2
crates/ptth_server/src/bin/ptth_server.rs
View File

@ -41,7 +41,7 @@ fn main () -> Result <(), anyhow::Error> {
return Ok (()); return Ok (());
} }
let mut rt = runtime::Runtime::new ()?; let rt = runtime::Runtime::new ()?;
rt.block_on (async move { rt.block_on (async move {
run_server ( run_server (

2
crates/ptth_server/src/file_server/metrics.rs
View File

@ -82,8 +82,6 @@ impl Interval {
use heim::process; use heim::process;
use uom::si::{ use uom::si::{
information::mebibyte, information::mebibyte,
ratio,
time::second,
}; };
let our_process = process::current ().await?; let our_process = process::current ().await?;

6
crates/ptth_server/src/file_server/mod.rs
View File

@ -24,7 +24,10 @@ use tokio::{
File, File,
ReadDir, ReadDir,
}, },
io::AsyncReadExt, io::{
AsyncReadExt,
AsyncSeekExt,
},
sync::mpsc::{ sync::mpsc::{
channel, channel,
}, },
@ -141,7 +144,6 @@ async fn serve_file (
if should_send_body { if should_send_body {
tokio::spawn (async move { tokio::spawn (async move {
let mut tx = tx;
let mut bytes_sent = 0; let mut bytes_sent = 0;
let mut bytes_left = content_length; let mut bytes_left = content_length;

2
crates/ptth_server/src/file_server/tests.rs
View File

@ -64,7 +64,7 @@ fn file_server () {
use super::*; use super::*;
tracing_subscriber::fmt ().try_init ().ok (); tracing_subscriber::fmt ().try_init ().ok ();
let mut rt = Runtime::new ().expect ("Can't create runtime"); let rt = Runtime::new ().expect ("Can't create runtime");
rt.block_on (async { rt.block_on (async {
let file_server_root = PathBuf::from ("./"); let file_server_root = PathBuf::from ("./");

15
crates/ptth_server/src/lib.rs
View File

@ -20,8 +20,8 @@ use tokio::{
sync::{ sync::{
oneshot, oneshot,
}, },
time::delay_for,
}; };
use tokio_stream::wrappers::ReceiverStream;
use ptth_core::{ use ptth_core::{
http_serde, http_serde,
@ -88,7 +88,7 @@ async fn handle_one_req (
resp_req = resp_req.header ("Content-Length", length.to_string ()); resp_req = resp_req.header ("Content-Length", length.to_string ());
} }
if let Some (body) = response.body { if let Some (body) = response.body {
resp_req = resp_req.body (reqwest::Body::wrap_stream (body)); resp_req = resp_req.body (reqwest::Body::wrap_stream (ReceiverStream::new (body)));
} }
let req = resp_req.build ().map_err (ServerError::Step5Responding)?; let req = resp_req.build ().map_err (ServerError::Step5Responding)?;
@ -231,14 +231,15 @@ pub async fn run_server (
// TODO: Extract loop body to function? // TODO: Extract loop body to function?
if backoff_delay > 0 { if backoff_delay > 0 {
let mut delay = delay_for (Duration::from_millis (backoff_delay)).fuse (); let sleep = tokio::time::sleep (Duration::from_millis (backoff_delay));
tokio::pin! (sleep);
if futures::select! ( tokio::select! {
_ = delay => false, _ = &mut sleep => {},
_ = shutdown_oneshot => true, _ = &mut shutdown_oneshot => {
) {
info! ("Received graceful shutdown"); info! ("Received graceful shutdown");
break; break;
},
} }
} }

5
src/main.rs
View File

@ -10,7 +10,6 @@ use reqwest::Client;
use tokio::{ use tokio::{
spawn, spawn,
sync::oneshot, sync::oneshot,
time::delay_for,
}; };
use tracing::{ use tracing::{
debug, debug,
@ -89,7 +88,7 @@ async fn main () -> anyhow::Result <()> {
}) })
}; };
delay_for (Duration::from_millis (1000)).await; tokio::time::sleep (Duration::from_millis (1000)).await;
assert_eq! (relay_state.list_servers ().await, vec! [ assert_eq! (relay_state.list_servers ().await, vec! [
server_name.to_string (), server_name.to_string (),
@ -129,7 +128,7 @@ async fn main () -> anyhow::Result <()> {
assert_eq! (resp.status (), reqwest::StatusCode::NOT_FOUND); assert_eq! (resp.status (), reqwest::StatusCode::NOT_FOUND);
delay_for (Duration::from_secs (60)).await; tokio::time::sleep (Duration::from_secs (60)).await;
info! ("Shutting down end-to-end test"); info! ("Shutting down end-to-end test");

12
src/tests.rs
View File

@ -12,7 +12,6 @@ use tokio::{
runtime::Runtime, runtime::Runtime,
spawn, spawn,
sync::oneshot, sync::oneshot,
time::delay_for,
}; };
use reqwest::Client; use reqwest::Client;
@ -24,7 +23,7 @@ use tracing::{debug, info};
async fn wait_for_any_server (relay_state: &ptth_relay::RelayState) { async fn wait_for_any_server (relay_state: &ptth_relay::RelayState) {
for _ in 0..50 { for _ in 0..50 {
delay_for (Duration::from_millis (100)).await; tokio::time::sleep (Duration::from_millis (100)).await;
if ! relay_state.list_servers ().await.is_empty () { if ! relay_state.list_servers ().await.is_empty () {
break; break;
} }
@ -181,7 +180,7 @@ fn end_to_end () {
// and we don't care if another test already installed a subscriber. // and we don't care if another test already installed a subscriber.
//tracing_subscriber::fmt ().try_init ().ok (); //tracing_subscriber::fmt ().try_init ().ok ();
let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); let rt = Runtime::new ().expect ("Can't create runtime for testing");
// Spawn the root task // Spawn the root task
rt.block_on (async { rt.block_on (async {
@ -197,7 +196,6 @@ fn end_to_end () {
proxy_port, proxy_port,
relay_port, relay_port,
}; };
let relay_url = testing_config.relay_url ();
let testing_relay = TestingRelay::new (&testing_config).await; let testing_relay = TestingRelay::new (&testing_config).await;
let testing_server = TestingServer::new (&testing_config).await; let testing_server = TestingServer::new (&testing_config).await;
@ -222,7 +220,7 @@ fn end_to_end () {
#[test] #[test]
fn debug_proxy () { fn debug_proxy () {
tracing_subscriber::fmt ().try_init ().ok (); tracing_subscriber::fmt ().try_init ().ok ();
let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); let rt = Runtime::new ().expect ("Can't create runtime for testing");
rt.block_on (async { rt.block_on (async {
let relay_port = 4002; let relay_port = 4002;
@ -278,7 +276,7 @@ fn debug_proxy () {
#[test] #[test]
fn scraper_endpoints () { fn scraper_endpoints () {
let mut rt = Runtime::new ().expect ("Can't create runtime for testing"); let rt = Runtime::new ().expect ("Can't create runtime for testing");
rt.block_on (async { rt.block_on (async {
use ptth_relay::*; use ptth_relay::*;
@ -334,7 +332,7 @@ fn scraper_endpoints () {
}, },
}; };
delay_for (Duration::from_millis (200)).await; tokio::time::sleep (Duration::from_millis (200)).await;
} }
let resp = resp.expect ("Reqwest repeatedly failed to connect to the relay"); let resp = resp.expect ("Reqwest repeatedly failed to connect to the relay");
let resp = resp.bytes ().await.expect ("Couldn't check if relay is up"); let resp = resp.bytes ().await.expect ("Couldn't check if relay is up");

11
tools/ptth_tail/Cargo.toml
View File

@ -8,14 +8,15 @@ license = "AGPL-3.0"
[dependencies] [dependencies]
anyhow = "1.0.34" anyhow = "1.0.38"
tracing = "0.1.21" futures-util = "0.3.8"
tracing-subscriber = "0.2.15" tracing = "0.1.25"
tracing-subscriber = "0.2.16"
[dependencies.reqwest] [dependencies.reqwest]
version = "0.10.8" version = "0.11.1"
features = ["stream"] features = ["stream"]
[dependencies.tokio] [dependencies.tokio]
version = "0.2.22" version = "1.2.0"
features = ["full"] features = ["full"]

4
tools/ptth_tail/src/main.rs
View File

@ -7,12 +7,12 @@ use std::{
time::Duration, time::Duration,
}; };
use futures_util::StreamExt;
use reqwest::{ use reqwest::{
Client, Client,
StatusCode, StatusCode,
}; };
use tokio::{ use tokio::{
stream::StreamExt,
time::interval, time::interval,
}; };
use tracing::{ use tracing::{
@ -75,6 +75,4 @@ async fn main () -> Result <(), anyhow::Error> {
total_bytes_received += u64::try_from (chunk.len ())?; total_bytes_received += u64::try_from (chunk.len ())?;
} }
} }
Ok (())
} }