Added Prometheus instrumentation

I need to instrument the mobc library used for the Redis connection
pool, but that can be done later since I am somewhat tired. I don't
remember how much I've worked and I need a break... >.<

4 files changed, 213 insertions, 1 deletions

diff --git a/Cargo.lock b/Cargo.lock
index add964b..6f2acdc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3,6 +3,12 @@
 version = 3
 
 [[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
 name = "aead"
 version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -595,6 +601,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba"
 
 [[package]]
+name = "crc32fast"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
+[[package]]
 name = "crossbeam-queue"
 version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -832,6 +847,24 @@ dependencies = [
 ]
 
 [[package]]
+name = "flate2"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crc32fast",
+ "libc",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
 name = "form_urlencoded"
 version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1046,6 +1079,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
 name = "hkdf"
 version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1205,6 +1244,7 @@ dependencies = [
  "mobc-redis",
  "mockito",
  "newbase60",
+ "prometheus",
  "retainer",
  "serde",
  "serde_json",
@@ -1263,6 +1303,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6"
 
 [[package]]
+name = "lock_api"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb"
+dependencies = [
+ "scopeguard",
+]
+
+[[package]]
 name = "log"
 version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1353,6 +1402,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "miniz_oxide"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
+dependencies = [
+ "adler",
+ "autocfg",
+]
+
+[[package]]
 name = "mobc"
 version = "0.7.3"
 source = "git+https://github.com/kisik21/mobc#1c35dbef09651e8b457ec4273e5192640f0efee6"
@@ -1472,6 +1531,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
 
 [[package]]
+name = "parking_lot"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
+dependencies = [
+ "instant",
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
+dependencies = [
+ "cfg-if 1.0.0",
+ "instant",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "winapi",
+]
+
+[[package]]
 name = "percent-encoding"
 version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1633,6 +1717,43 @@ dependencies = [
 ]
 
 [[package]]
+name = "procfs"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab8809e0c18450a2db0f236d2a44ec0b4c1412d0eb936233579f0990faa5d5cd"
+dependencies = [
+ "bitflags",
+ "byteorder",
+ "flate2",
+ "hex",
+ "lazy_static",
+ "libc",
+]
+
+[[package]]
+name = "prometheus"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5986aa8d62380092d2f50f8b1cdba9cb9b6731ffd4b25b51fd126b6c3e05b99c"
+dependencies = [
+ "cfg-if 1.0.0",
+ "fnv",
+ "lazy_static",
+ "libc",
+ "memchr",
+ "parking_lot",
+ "procfs",
+ "protobuf",
+ "thiserror",
+]
+
+[[package]]
+name = "protobuf"
+version = "2.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db50e77ae196458ccd3dc58a31ea1a90b0698ab1b7928d89f644c25d72070267"
+
+[[package]]
 name = "quote"
 version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1791,6 +1912,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "redox_syscall"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
 name = "regex"
 version = "1.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1878,6 +2008,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
 [[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
+[[package]]
 name = "sct"
 version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 7418d90..8556c3b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,7 +40,6 @@ log = "^0.4.14"              # A lightweight logging facade for Rust
 markdown = "^0.3.0"          # Native Rust library for parsing Markdown and (outputting HTML)
 markup = "^0.12.0"           # HTML templating engine... ok also very funny about markdown and markup... i just realized the pun...
 newbase60 = "^0.1.3"         # A library that implements Tantek Çelik's New Base 60
-#prometheus = "^0.12.0"       # Prometheus instrumentation library for Rust applications
 retainer = "^0.2.2"          # Minimal async cache in Rust with support for key expirations
 serde_json = "^1.0.64"       # A JSON serialization file format
 serde_urlencoded = "^0.7.0"  # `x-www-form-urlencoded` meets Serde
@@ -68,6 +67,10 @@ version = "^0.7.0"
 default-features = false
 features = ["async-std-comp"]
 
+[dependencies.prometheus]    # Prometheus instrumentation library for Rust applications
+version = "^0.12.0"
+features = ["process"]
+
 [dependencies.serde]         # A generic serialization/deserialization framework
 version = "^1.0.125"
 features = ["derive"]
diff --git a/src/lib.rs b/src/lib.rs
index 6a62dcc..77a6c11 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -4,6 +4,7 @@ mod database;
 mod frontend;
 mod indieauth;
 mod micropub;
+mod metrics;
 
 use crate::indieauth::IndieAuthMiddleware;
 use crate::micropub::CORSMiddleware;
@@ -42,6 +43,7 @@ where
             .build())
     });
     app.at("/")
+        .with(CORSMiddleware {})
         .with(frontend::ErrorHandlerMiddleware {})
         .get(frontend::mainpage)
         .post(frontend::onboarding_receiver);
@@ -54,7 +56,13 @@ where
     app.at("/coffee")
         .with(frontend::ErrorHandlerMiddleware {})
         .get(frontend::coffee);
+    // TODO make sure the health check actually checks the backend or something
+    // otherwise it'll get false-negatives for application faults like resource
+    // exhaustion
     app.at("/health").get(|_| async { Ok("OK") });
+    app.at("/metrics").get(metrics::gather);
+
+    app.with(metrics::InstrumentationMiddleware {});
 
     app
 }
diff --git a/src/metrics.rs b/src/metrics.rs
new file mode 100644
index 0000000..0537b9d
--- /dev/null
+++ b/src/metrics.rs
@@ -0,0 +1,65 @@
+use tide::{Request, Response, Result, Next};
+use prometheus::{self, IntCounterVec, HistogramVec, TextEncoder, Encoder, register_int_counter_vec, register_histogram_vec};
+use lazy_static::lazy_static;
+use async_trait::async_trait;
+use std::time::{Instant, Duration};
+
+// Copied from https://docs.rs/prometheus/0.12.0/src/prometheus/histogram.rs.html#885-889
+#[inline]
+fn duration_to_seconds(d: Duration) -> f64 {
+    let nanos = f64::from(d.subsec_nanos()) / 1e9;
+    d.as_secs() as f64 + nanos
+}
+
+lazy_static! {
+    static ref HTTP_CONNS_COUNTER: IntCounterVec = register_int_counter_vec!(
+        "http_requests_total", "Number of processed HTTP requests",
+        &["code", "method", "url"]
+    ).unwrap();
+
+    static ref HTTP_REQUEST_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!(
+        "http_request_duration_seconds", "Duration of HTTP requests",
+        &["code", "method", "url"]
+    ).unwrap();
+}
+
+pub struct InstrumentationMiddleware {}
+
+
+#[async_trait]
+impl<S> tide::Middleware<S> for InstrumentationMiddleware
+where
+    S: Send + Sync + Clone + 'static,
+{
+    async fn handle(
+        &self,
+        req: Request<S>,
+        next: Next<'_, S>,
+    ) -> Result {
+        let url = req.url().to_string();
+        let method = req.method().to_string();
+        // Execute the request
+        let instant = Instant::now();
+        let res = next.run(req).await;
+        let elapsed = duration_to_seconds(instant.elapsed());
+        // Get the code from the response
+        let code = res.status().to_string();
+
+        HTTP_CONNS_COUNTER.with_label_values(&[&code, &method, &url]).inc();
+        HTTP_REQUEST_DURATION_HISTOGRAM.with_label_values(&[&code, &method, &url]).observe(elapsed);
+
+        Ok(res)
+    }
+}
+
+pub async fn gather<S>(_: Request<S>) -> Result
+where
+    S: Send + Sync + Clone
+{
+    let mut buffer: Vec<u8> = vec![];
+    let encoder = TextEncoder::new();
+    let metric_families = prometheus::gather();
+    encoder.encode(&metric_families, &mut buffer).unwrap();
+
+    Ok(Response::builder(200).body(buffer).build())
+}