2020-12-21 00:09:24 +00:00
|
|
|
use std::{
|
|
|
|
sync::Arc,
|
|
|
|
time::{Duration, Instant},
|
|
|
|
};
|
2020-12-20 23:32:13 +00:00
|
|
|
|
|
|
|
use arc_swap::ArcSwap;
|
2020-12-18 20:43:34 +00:00
|
|
|
use chrono::{DateTime, Utc};
|
2020-12-20 23:32:13 +00:00
|
|
|
use tracing::{
|
|
|
|
debug, error, trace,
|
|
|
|
};
|
2020-12-18 20:43:34 +00:00
|
|
|
use ulid::Ulid;
|
|
|
|
|
2020-12-20 23:17:31 +00:00
|
|
|
// Metrics are named for when they're updated:
|
|
|
|
// - Startup (Once, when the server state is initialized)
|
|
|
|
// - Interval (About once per minute)
|
|
|
|
// - Events (When a request is processed)
|
2020-12-18 20:43:34 +00:00
|
|
|
|
|
|
|
#[derive (Debug, serde::Serialize)]
|
2020-12-20 17:44:03 +00:00
|
|
|
pub struct Startup {
|
2020-12-18 20:43:34 +00:00
|
|
|
// D-Bus machine ID, if we're on Linux
|
|
|
|
pub machine_id: Option <String>,
|
|
|
|
|
|
|
|
// Git version that ptth_server was built from (unimplemented)
|
2020-12-18 23:41:52 +00:00
|
|
|
pub git_version: Option <String>,
|
2020-12-18 20:43:34 +00:00
|
|
|
|
|
|
|
// User-assigned and human-readable name for this server.
|
|
|
|
// Must be unique within a relay.
|
|
|
|
pub server_name: String,
|
|
|
|
|
|
|
|
// Random base64 instance ID. ptth_server generates this at process start.
|
|
|
|
// It's a fallback for detecting outages without relying on any clocks.
|
|
|
|
#[serde (serialize_with = "serialize_ulid")]
|
|
|
|
pub instance_id: Ulid,
|
|
|
|
|
2020-12-20 17:44:03 +00:00
|
|
|
// System UTC
|
2020-12-18 20:43:34 +00:00
|
|
|
pub startup_utc: DateTime <Utc>,
|
|
|
|
}
|
|
|
|
|
2020-12-20 17:41:00 +00:00
|
|
|
#[derive (Debug, serde::Serialize)]
|
2020-12-20 23:17:31 +00:00
|
|
|
pub struct Interval {
|
2020-12-21 00:09:24 +00:00
|
|
|
#[serde (skip)]
|
|
|
|
pub at: Instant,
|
2020-12-20 19:35:32 +00:00
|
|
|
pub utc: DateTime <Utc>,
|
2020-12-20 17:41:00 +00:00
|
|
|
pub rss_mib: u64,
|
2020-12-20 20:10:29 +00:00
|
|
|
|
|
|
|
#[serde (skip)]
|
|
|
|
pub cpu_usage: heim::process::CpuUsage,
|
2020-12-20 17:41:00 +00:00
|
|
|
}
|
|
|
|
|
2020-12-20 23:17:31 +00:00
|
|
|
pub struct Events {
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Startup {
|
|
|
|
#[must_use]
|
|
|
|
pub fn new (server_name: String) -> Self
|
|
|
|
{
|
|
|
|
let x = Self {
|
|
|
|
machine_id: get_machine_id (),
|
|
|
|
git_version: None,
|
|
|
|
server_name,
|
|
|
|
instance_id: ulid::Ulid::new (),
|
|
|
|
startup_utc: Utc::now (),
|
|
|
|
};
|
|
|
|
|
|
|
|
debug! ("metrics at startup: {:?}", x);
|
|
|
|
|
|
|
|
x
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn serialize_ulid <S: serde::Serializer> (t: &Ulid, s: S)
|
|
|
|
-> Result <S::Ok, S::Error>
|
|
|
|
{
|
|
|
|
let t = t.to_string ();
|
|
|
|
s.serialize_str (&t)
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Interval {
|
2020-12-20 18:52:53 +00:00
|
|
|
pub async fn new () -> Result <Self, super::FileServerError> {
|
2020-12-20 20:10:29 +00:00
|
|
|
use tokio::join;
|
2020-12-20 18:52:53 +00:00
|
|
|
use heim::process;
|
2020-12-20 20:10:29 +00:00
|
|
|
use uom::si::{
|
|
|
|
information::mebibyte,
|
|
|
|
ratio,
|
|
|
|
time::second,
|
|
|
|
};
|
2020-12-20 18:52:53 +00:00
|
|
|
|
|
|
|
let our_process = process::current ().await?;
|
2020-12-20 20:10:29 +00:00
|
|
|
|
|
|
|
let cpu_usage = our_process.cpu_usage ();
|
|
|
|
|
2020-12-20 23:17:31 +00:00
|
|
|
let (cpu_usage, mem) = join! {
|
2020-12-20 20:10:29 +00:00
|
|
|
cpu_usage,
|
2020-12-20 23:17:31 +00:00
|
|
|
our_process.memory ()
|
|
|
|
};
|
2020-12-20 20:10:29 +00:00
|
|
|
let cpu_usage = cpu_usage?;
|
2020-12-20 23:17:31 +00:00
|
|
|
let mem = mem?;
|
2020-12-20 18:52:53 +00:00
|
|
|
let rss_mib = mem.rss ().get::<mebibyte> ();
|
|
|
|
|
2020-12-20 23:17:31 +00:00
|
|
|
let x = Self {
|
2020-12-21 00:09:24 +00:00
|
|
|
at: Instant::now (),
|
2020-12-20 19:35:32 +00:00
|
|
|
utc: Utc::now (),
|
2020-12-20 18:52:53 +00:00
|
|
|
rss_mib,
|
2020-12-20 20:10:29 +00:00
|
|
|
cpu_usage,
|
2020-12-20 18:58:14 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
Ok (x)
|
2020-12-20 18:52:53 +00:00
|
|
|
}
|
2020-12-20 23:32:13 +00:00
|
|
|
|
|
|
|
pub async fn monitor (interval_writer: Arc <ArcSwap <Option <Interval>>>)
|
|
|
|
{
|
|
|
|
use uom::si::ratio::percent;
|
|
|
|
|
2020-12-20 23:34:55 +00:00
|
|
|
let mut interval = tokio::time::interval (Duration::from_secs (60));
|
2020-12-20 23:32:13 +00:00
|
|
|
|
2020-12-21 00:09:24 +00:00
|
|
|
#[derive (Default)]
|
|
|
|
struct Window {
|
|
|
|
window_length: u64,
|
|
|
|
next_interval: u64,
|
|
|
|
last_metrics: Arc <Option <Interval>>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Window {
|
|
|
|
fn new (window_length: u64) -> Self {
|
|
|
|
Window {
|
|
|
|
window_length,
|
|
|
|
next_interval: 0,
|
|
|
|
last_metrics: Default::default (),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn update (
|
|
|
|
&mut self,
|
|
|
|
counter: u64,
|
|
|
|
new_metrics: &Arc <Option <Interval>>
|
|
|
|
) {
|
|
|
|
if counter >= self.next_interval {
|
|
|
|
if let (Some (old), Some (new)) = (&*self.last_metrics, &**new_metrics) {
|
|
|
|
let diff = new.cpu_usage.clone () - old.cpu_usage.clone ();
|
|
|
|
trace! (
|
|
|
|
"CPU usage over {} s: {}%",
|
|
|
|
(new.at - old.at).as_secs (),
|
|
|
|
diff.get::<percent> (),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
self.next_interval += self.window_length;
|
|
|
|
self.last_metrics = new_metrics.clone ();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-20 23:32:13 +00:00
|
|
|
let mut counter = 0_u64;
|
2020-12-21 00:09:24 +00:00
|
|
|
|
|
|
|
let mut windows = [1, 5, 10, 60, 1440]
|
|
|
|
.iter ()
|
|
|
|
.map (|len| Window::new (*len))
|
|
|
|
.collect::<Vec <_>> ();
|
2020-12-20 23:32:13 +00:00
|
|
|
|
|
|
|
loop {
|
|
|
|
interval.tick ().await;
|
|
|
|
|
|
|
|
let new_interval_metrics = match Interval::new ().await {
|
|
|
|
Err (e) => {
|
|
|
|
error! ("Failed to update interval metrics: {:?}", e);
|
|
|
|
continue;
|
|
|
|
},
|
|
|
|
Ok (x) => x,
|
|
|
|
};
|
|
|
|
let new_interval_metrics = Arc::new (Some (new_interval_metrics));
|
|
|
|
|
2020-12-21 00:09:24 +00:00
|
|
|
for window in windows.iter_mut () {
|
|
|
|
window.update (counter, &new_interval_metrics);
|
2020-12-20 23:32:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
interval_writer.store (new_interval_metrics);
|
|
|
|
counter += 1;
|
|
|
|
//trace! ("interval metrics 1");
|
|
|
|
}
|
|
|
|
}
|
2020-12-20 18:52:53 +00:00
|
|
|
}
|
|
|
|
|
2020-12-18 20:43:34 +00:00
|
|
|
fn get_machine_id () -> Option <String> {
|
|
|
|
use std::{
|
|
|
|
fs::File,
|
|
|
|
io::Read,
|
|
|
|
};
|
|
|
|
|
2020-12-18 23:41:52 +00:00
|
|
|
let mut buf = vec! [0; 1024];
|
2020-12-18 20:43:34 +00:00
|
|
|
let mut f = File::open ("/etc/machine-id").ok ()?;
|
|
|
|
let bytes_read = f.read (&mut buf).ok ()?;
|
|
|
|
|
|
|
|
buf.truncate (bytes_read);
|
|
|
|
|
|
|
|
let s = std::str::from_utf8 (&buf).ok ()?;
|
|
|
|
let s = s.trim_end ().to_string ();
|
|
|
|
|
|
|
|
Some (s)
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg (test)]
|
|
|
|
mod tests {
|
|
|
|
use super::*;
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn ulid_null () {
|
2020-12-20 17:44:03 +00:00
|
|
|
let a = Startup::new ("bogus".to_string ());
|
|
|
|
let b = Startup::new ("bogus".to_string ());
|
2020-12-18 20:43:34 +00:00
|
|
|
|
|
|
|
assert_ne! (a.instance_id, b.instance_id);
|
|
|
|
}
|
|
|
|
}
|