reactor
/
ptth
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

🐛 bug: looks like the memory leak for large files might have been in Tokio or reqwest.

main
_ 2022-04-11 16:56:59 -05:00
parent 39bb4d0137
commit 8c731c4cef
8 changed files with 221 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

255
Cargo.lock generated
View File

@ -272,7 +272,7 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8"
dependencies = [
"sct",
"sct 0.6.1",
]
[[package]]
@ -546,14 +546,14 @@ checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
dependencies = [
"cfg-if",
"libc",
"wasi",
"wasi 0.10.0+wasi-snapshot-preview1",
]
[[package]]
name = "h2"
version = "0.3.6"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c06815895acec637cd6ed6e9662c935b866d20a106f8361892893a7d9234964"
checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57"
dependencies = [
"bytes",
"fnv",
@ -564,7 +564,7 @@ dependencies = [
"indexmap",
"slab",
"tokio",
"tokio-util",
"tokio-util 0.7.1",
"tracing",
]
@ -620,7 +620,7 @@ checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b"
dependencies = [
"bytes",
"fnv",
"itoa",
"itoa 0.4.8",
]
[[package]]
@ -661,9 +661,9 @@ dependencies = [
"http-body",
"httparse",
"httpdate",
"itoa",
"itoa 0.4.8",
"pin-project-lite",
"socket2 0.4.2",
"socket2 0.4.4",
"tokio",
"tower-service",
"tracing",
@ -672,17 +672,15 @@ dependencies = [
[[package]]
name = "hyper-rustls"
version = "0.22.1"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac"
dependencies = [
"futures-util",
"http",
"hyper",
"log",
"rustls",
"rustls 0.20.4",
"tokio",
"tokio-rustls",
"webpki",
]
[[package]]
@ -719,15 +717,6 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "instant"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "716d3d89f35ac6a34fd0eed635395f4c3b76fa889338a4632e5231a8684216bd"
dependencies = [
"cfg-if",
]
[[package]]
name = "ipnet"
version = "2.3.1"
@ -749,6 +738,12 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "itoa"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
[[package]]
name = "js-sys"
version = "0.3.55"
@ -766,15 +761,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.103"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6"
checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259"
[[package]]
name = "lock_api"
version = "0.4.5"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109"
checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b"
dependencies = [
"scopeguard",
]
@ -851,6 +846,20 @@ dependencies = [
"winapi",
]
[[package]]
name = "mio"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
dependencies = [
"libc",
"log",
"miow",
"ntapi",
"wasi 0.11.0+wasi-snapshot-preview1",
"winapi",
]
[[package]]
name = "miow"
version = "0.3.7"
@ -982,27 +991,25 @@ dependencies = [
[[package]]
name = "parking_lot"
version = "0.11.2"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.5"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
"windows-sys",
]
[[package]]
@ -1407,14 +1414,14 @@ dependencies = [
"futures",
"lazy_static",
"libc",
"mio",
"mio 0.7.13",
"quinn-proto",
"rustls",
"rustls 0.19.1",
"socket2 0.3.19",
"thiserror",
"tokio",
"tracing",
"webpki",
"webpki 0.21.4",
]
[[package]]
@ -1427,13 +1434,13 @@ dependencies = [
"ct-logs",
"rand",
"ring",
"rustls",
"rustls 0.19.1",
"rustls-native-certs",
"slab",
"thiserror",
"tinyvec",
"tracing",
"webpki",
"webpki 0.21.4",
]
[[package]]
@ -1543,15 +1550,16 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.11.4"
version = "0.11.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22"
checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb"
dependencies = [
"base64",
"bytes",
"encoding_rs",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"hyper",
@ -1565,12 +1573,15 @@ dependencies = [
"native-tls",
"percent-encoding",
"pin-project-lite",
"rustls",
"rustls 0.20.4",
"rustls-pemfile",
"serde",
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-native-tls",
"tokio-rustls",
"tokio-util 0.6.8",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
@ -1658,8 +1669,20 @@ dependencies = [
"base64",
"log",
"ring",
"sct",
"webpki",
"sct 0.6.1",
"webpki 0.21.4",
]
[[package]]
name = "rustls"
version = "0.20.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921"
dependencies = [
"log",
"ring",
"sct 0.7.0",
"webpki 0.22.0",
]
[[package]]
@ -1669,11 +1692,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092"
dependencies = [
"openssl-probe",
"rustls",
"rustls 0.19.1",
"schannel",
"security-framework",
]
[[package]]
name = "rustls-pemfile"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360"
dependencies = [
"base64",
]
[[package]]
name = "rusty_ulid"
version = "0.10.1"
@ -1727,6 +1759,16 @@ dependencies = [
"untrusted",
]
[[package]]
name = "sct"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "security-framework"
version = "2.3.1"
@ -1776,19 +1818,19 @@ version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
dependencies = [
"itoa",
"itoa 0.4.8",
"ryu",
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.0"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [
"form_urlencoded",
"itoa",
"itoa 1.0.1",
"ryu",
"serde",
]
@ -1861,9 +1903,9 @@ dependencies = [
[[package]]
name = "socket2"
version = "0.4.2"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516"
checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
dependencies = [
"libc",
"winapi",
@ -1975,7 +2017,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
dependencies = [
"libc",
"wasi",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi",
]
@ -1996,29 +2038,29 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.12.0"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc"
checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
dependencies = [
"autocfg",
"bytes",
"libc",
"memchr",
"mio",
"mio 0.8.2",
"num_cpus",
"once_cell",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.4.4",
"tokio-macros",
"winapi",
]
[[package]]
name = "tokio-macros"
version = "1.4.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "154794c8f499c2619acd19e839294703e9e32e7630ef5f46ea80d4ef0fbee5eb"
checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
dependencies = [
"proc-macro2",
"quote",
@ -2037,13 +2079,13 @@ dependencies = [
[[package]]
name = "tokio-rustls"
version = "0.22.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e"
dependencies = [
"rustls",
"rustls 0.20.4",
"tokio",
"webpki",
"webpki 0.22.0",
]
[[package]]
@ -2071,6 +2113,20 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
"pin-project-lite",
"tokio",
"tracing",
]
[[package]]
name = "toml"
version = "0.5.8"
@ -2310,6 +2366,12 @@ version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.78"
@ -2317,8 +2379,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce"
dependencies = [
"cfg-if",
"serde",
"serde_json",
"wasm-bindgen-macro",
]
@ -2399,12 +2459,22 @@ dependencies = [
]
[[package]]
name = "webpki-roots"
version = "0.21.1"
name = "webpki"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940"
checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
dependencies = [
"webpki",
"ring",
"untrusted",
]
[[package]]
name = "webpki-roots"
version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf"
dependencies = [
"webpki 0.22.0",
]
[[package]]
@ -2439,10 +2509,53 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "winreg"
version = "0.7.0"
name = "windows-sys"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825"
dependencies = [
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_msvc"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d"
[[package]]
name = "windows_i686_gnu"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed"
[[package]]
name = "windows_i686_msvc"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956"
[[package]]
name = "windows_x86_64_gnu"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4"
[[package]]
name = "windows_x86_64_msvc"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9"
[[package]]
name = "winreg"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
dependencies = [
"winapi",
]

14
crates/ptth_file_server_bin/src/lib.rs
View File

@ -1,6 +1,7 @@
#![warn (clippy::pedantic)]
use std::{
collections::*,
ffi::OsString,
net::SocketAddr,
path::{PathBuf},
@ -75,6 +76,7 @@ async fn handle_all (req: Request <Body>, state: Arc <FileServer>)
#[derive (Deserialize)]
struct ConfigFile {
file_server_root: Option <PathBuf>,
file_server_roots: Option <BTreeMap <String, PathBuf>>,
name: Option <String>,
}
@ -82,20 +84,25 @@ pub async fn main (_args: &[OsString]) -> anyhow::Result <()> {
let path = PathBuf::from ("./config/ptth_server.toml");
let file_server_root;
let file_server_roots;
let name;
match load_toml::load::<ConfigFile, _> (&path) {
Ok (config_file) => {
file_server_root = config_file.file_server_root;
file_server_roots = config_file.file_server_roots;
name = config_file.name;
},
_ => {
info! ("No ptth_server.toml file, using default configs");
file_server_root = None;
file_server_roots = None;
name = None;
},
};
let file_server_root = file_server_root.unwrap_or_else (|| PathBuf::from ("."));
let file_server_roots = file_server_roots.unwrap_or_else (|| Default::default ());
let name = name.unwrap_or_else (|| "PTTH File Server".to_string ());
info! ("file_server_root: {:?}", file_server_root);
@ -110,8 +117,13 @@ pub async fn main (_args: &[OsString]) -> anyhow::Result <()> {
file_server::metrics::Interval::monitor (interval_writer).await;
});
let config = file_server::Config {
file_server_root,
file_server_roots,
};
let state = Arc::new (FileServer::new (
file_server_root.unwrap_or_else (|| PathBuf::from (".")),
config,
&PathBuf::new (),
name,
metrics_interval,

4
crates/ptth_server/Cargo.toml
View File

@ -39,7 +39,7 @@ serde = {version = "1.0.117", features = ["derive"]}
serde_json = "1.0.60"
structopt = "0.3.20"
thiserror = "1.0.24"
tokio = { version = "1.8.1", features = [] }
tokio = { version = "1.17.0", features = [] }
tokio-stream = "0.1.3"
tracing = "0.1.25"
tracing-futures = "0.2.5"
@ -50,7 +50,7 @@ always_equal = { path = "../always_equal", version = "1.0.0" }
ptth_core = { path = "../ptth_core", version = "2.0.0" }
[dependencies.reqwest]
version = "0.11.1"
version = "0.11.10"
default-features = false
features = ["stream", "rustls-tls", "hyper-rustls"]

4
crates/ptth_server/src/file_server.rs
View File

@ -167,14 +167,16 @@ async fn serve_file (
let content_length = range.end - range.start;
let body = if decision.should_send_body {
debug! ("Sending range {}-{}", range.start, range.end);
trace! ("Sending range {}-{}", range.start, range.end);
let seek = SeekFrom::Start (range.start);
f.seek (seek).await?;
let (tx, rx) = channel (1);
tokio::spawn (async move {
debug! ("stream_file task begin");
stream_file (f, content_length, tx).await;
debug! ("stream_file task end");
});
Some (rx)

25
crates/ptth_server/src/lib.rs
View File

@ -140,7 +140,7 @@ async fn handle_one_req (
Ok (r) => {
let status = r.status ();
let text = r.text ().await.map_err (ServerError::Step7AfterResponse)?;
debug! ("http_response {} {:?} {:?}", req_id, status, text);
trace! ("http_response {} {:?} {:?}", req_id, status, text);
},
Err (e) => {
if e.is_request () {
@ -180,7 +180,10 @@ SH: Send + FnMut () -> H
let f = handler (parts);
let response = f.await?;
handle_one_req (&state, req_id, response).await
let output = handle_one_req (&state, req_id.clone (), response).await;
debug! ("Req {} task exiting", req_id);
output
});
}
@ -455,7 +458,7 @@ impl State {
}
}
debug! ("http_listen {}...", i);
trace! ("http_listen {}...", i);
let http_listen_fut = Self::http_listen (state);
@ -478,7 +481,7 @@ impl State {
Ok (x) => x,
};
debug! ("http_listen {} unwrapped {} requests", i, reqs.len ());
trace! ("http_listen {} unwrapped {} requests", i, reqs.len ());
// Unpack the requests, spawn them into new tasks, then loop back
// around.
@ -506,6 +509,7 @@ impl State {
pub mod executable {
use std::{
collections::*,
path::{Path, PathBuf},
};
use structopt::StructOpt;
@ -540,17 +544,17 @@ pub mod executable {
}
};
let file_server_roots = config_file.file_server_roots
.unwrap_or_else (|| Default::default ());
// `git grep JRY5NXZU` # duplicated code?
let config_file = super::ConfigFile {
name: opt.name.or (config_file.name).ok_or (anyhow::anyhow! ("`name` must be provided in command line or config file"))?,
api_key: config_file.api_key,
relay_url: opt.relay_url.or (config_file.relay_url).ok_or (anyhow::anyhow! ("`--relay-url` must be provided in command line or `relay_url` in config file"))?,
file_server_root: opt.file_server_root.or (config_file.file_server_root).unwrap_or_else (PathBuf::new),
file_server_roots: vec! [
("c", "C:/"),
("d", "D:/"),
].into_iter ()
.map (|(k, v)| (String::from (k), PathBuf::from (v)))
.collect (),
file_server_roots,
throttle_upload: opt.throttle_upload,
allow_any_client: true,
client_keys: Default::default (),
@ -606,6 +610,7 @@ pub mod executable {
pub api_key: String,
pub relay_url: Option <String>,
pub file_server_root: Option <PathBuf>,
pub file_server_roots: Option <BTreeMap <String, PathBuf>>,
}
fn gen_and_save_key (path: &Path) -> anyhow::Result <()> {

2
crates/ptth_server_gui/Cargo.toml
View File

@ -8,7 +8,7 @@ edition = "2018"
anyhow = "1.0.38"
fltk = "1.3.1"
serde = {version = "1.0.117", features = ["derive"]}
tokio = "1.4.0"
tokio = "1.17.0"
tracing = "0.1.25"
tracing-subscriber = "0.2.16"

2
crates/ptth_server_gui/src/main.rs
View File

@ -92,6 +92,8 @@ fn main ()
},
};
// `git grep JRY5NXZU` # duplicated code?
let config_file = ptth_server::ConfigFile {
name: gui.input_name.value ().to_string (),
api_key: gui.input_api_key.value ().to_string (),

2
src/main.rs
View File

@ -77,7 +77,7 @@ async fn main () -> anyhow::Result <()> {
let (stop_server_tx, stop_server_rx) = oneshot::channel ();
let task_server = {
spawn (async move {
ptth_server::run_server (config_file, stop_server_rx, None, None).await
ptth_server::run_server (config_file, stop_server_rx, None, None, None).await
})
};