From 31a0bdad439a4575c1686f690e9e72bd44dde472 Mon Sep 17 00:00:00 2001 From: Vika Date: Mon, 26 Aug 2024 15:22:29 +0300 Subject: Add HTTP fetcher cache It just does its thing in the background, potentially speeding up things. Maybe I could also use the underlying in-memory cache implementation (Moka) to speed up my database. I heard crates.io got some good results from that. --- Cargo.lock | 174 +++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 + src/frontend/onboarding.rs | 6 +- src/indieauth/mod.rs | 9 ++- src/lib.rs | 6 +- src/login.rs | 6 +- src/main.rs | 10 ++- src/micropub/mod.rs | 35 +++++++-- src/tokenauth.rs | 6 +- src/webmentions/mod.rs | 6 +- 10 files changed, 233 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be9a1b2..7edf736 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,6 +238,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.81" @@ -444,6 +455,15 @@ dependencies = [ "scoped-tls", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -715,6 +735,24 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -971,6 +1009,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "faker_rand" version = "0.1.1" @@ -1417,6 +1465,61 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-cache" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ffb12b95bb2a369fe47ca8924016c72c2fa0e6059ba98bd1516f558696c5a8" +dependencies = [ + "async-trait", + "bincode", + "http", + "http-cache-semantics", + "httpdate", + "moka", + "serde", + "url", +] + +[[package]] +name = "http-cache-reqwest" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be3e27c4e2e510571cbcc601407b639667146aa9a4e818d5cc1d97c8b4b27d61" +dependencies = [ + "anyhow", + "async-trait", + "http", + "http-cache", + "http-cache-semantics", + "reqwest", + "reqwest-middleware", + "serde", + "url", +] + +[[package]] +name = "http-cache-semantics" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92baf25cf0b8c9246baecf3a444546360a97b569168fdf92563ee6a47829920c" +dependencies = [ + "http", + "http-serde", + "serde", + "time", +] + +[[package]] +name = "http-serde" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" +dependencies = [ + "http", + "serde", +] + [[package]] name = "httparse" version = "1.9.4" @@ -1632,6 +1735,7 @@ dependencies = [ "futures-util", "hex", "html5ever 0.27.0", + "http-cache-reqwest", "hyper", "kittybox-frontend-renderer", "kittybox-indieauth", @@ -1647,6 +1751,7 @@ dependencies = [ "redis", "relative-path", "reqwest", + "reqwest-middleware", "serde", "serde_json", "serde_urlencoded", @@ -2020,6 +2125,30 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "moka" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e0d88686dc561d743b40de8269b26eaf0dc58781bde087b0984646602021d08" +dependencies = [ + "async-lock", + "async-trait", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "event-listener", + "futures-util", + "once_cell", + "parking_lot", + "quanta", + "rustc_version", + "smallvec", + "tagptr", + "thiserror", + "triomphe", + "uuid", +] + [[package]] name = "multer" version = "3.1.0" @@ -2561,6 +2690,21 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "quanta" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quinn" version = "0.11.2" @@ -2697,6 +2841,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "raw-cpuid" +version = "11.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "redis" version = "0.26.0" @@ -2836,6 +2989,21 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest-middleware" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest", + "serde", + "thiserror", + "tower-service", +] + [[package]] name = "ring" version = "0.17.8" @@ -3785,6 +3953,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" version = "3.10.1" diff --git a/Cargo.toml b/Cargo.toml index b5346b4..6d56c89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,8 @@ webauthn = { version = "0.5.0", package = "webauthn-rs", features = ["danger-all base64 = "0.22.1" html5ever = "0.27.0" mime = "0.3.17" +http-cache-reqwest = { version = "0.14.0", default-features = false, features = ["manager-moka"] } +reqwest-middleware = "0.3.3" [dependencies.tokio] version = "^1.29.1" features = ["full", "tracing"] # TODO determine if my app doesn't need some features diff --git a/src/frontend/onboarding.rs b/src/frontend/onboarding.rs index be1669f..a8d2ae6 100644 --- a/src/frontend/onboarding.rs +++ b/src/frontend/onboarding.rs @@ -53,7 +53,7 @@ async fn onboard( db: D, user_uid: url::Url, data: OnboardingData, - http: reqwest::Client, + http: reqwest_middleware::ClientWithMiddleware, jobset: Arc>>, ) -> Result<(), FrontendError> { // Create a user to pass to the backend @@ -126,7 +126,7 @@ async fn onboard( pub async fn post( State(db): State, Host(host): Host, - State(http): State, + State(http): State, State(jobset): State>>>, Json(data): Json, ) -> axum::response::Response { @@ -165,7 +165,7 @@ pub fn router() -> axum::routing::MethodRouter where S: Storage + FromRef + 'static, Arc>>: FromRef, - reqwest::Client: FromRef, + reqwest_middleware::ClientWithMiddleware: FromRef, St: Clone + Send + Sync + 'static, { axum::routing::get(get) diff --git a/src/indieauth/mod.rs b/src/indieauth/mod.rs index 322a0e2..b3db77f 100644 --- a/src/indieauth/mod.rs +++ b/src/indieauth/mod.rs @@ -131,7 +131,7 @@ async fn authorization_endpoint_get( Host(host): Host, Query(request): Query, State(db): State, - State(http): State, + State(http): State, State(auth): State ) -> Response { let me: url::Url = format!("https://{host}/").parse().unwrap(); @@ -148,7 +148,10 @@ async fn authorization_endpoint_get( tracing::debug!("Sending request to {} to fetch metadata", request.client_id); let metadata_request = http.get(request.client_id.clone()) .header("Accept", "application/json, text/html"); - match metadata_request.send().await.and_then(|res| res.error_for_status()) { + match metadata_request.send().await + .and_then(|res| res.error_for_status() + .map_err(reqwest_middleware::Error::Reqwest)) + { Ok(response) if response.headers().typed_get::().to_owned().map(mime::Mime::from).map(|m| m.type_() == "text" && m.subtype() == "html").unwrap_or(false) => { let url = response.url().clone(); let text = response.text().await.unwrap(); @@ -847,7 +850,7 @@ pub fn router() -> axum::Router where S: Storage + FromRef + 'static, A: AuthBackend + FromRef, - reqwest::Client: FromRef, + reqwest_middleware::ClientWithMiddleware: FromRef, St: Clone + Send + Sync + 'static { use axum::routing::{Router, get, post}; diff --git a/src/lib.rs b/src/lib.rs index 596ffc0..5fe3b18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,7 +36,7 @@ Q: JobQueue + Sized pub storage: S, pub media_store: M, pub job_queue: Q, - pub http: reqwest::Client, + pub http: reqwest_middleware::ClientWithMiddleware, pub background_jobs: Arc>>, pub cookie_key: Key, pub session_store: SessionStore @@ -156,7 +156,7 @@ where A: AuthBackend, S: Storage, M: MediaStore, Q: JobQueue FromRef> for reqwest::Client +impl FromRef> for reqwest_middleware::ClientWithMiddleware where A: AuthBackend, S: Storage, M: MediaStore, Q: JobQueue { fn from_ref(input: &AppState) -> Self { @@ -291,7 +291,7 @@ A: AuthBackend + 'static + FromRef, S: Storage + 'static + FromRef, M: MediaStore + 'static + FromRef, Q: kittybox_util::queue::JobQueue + FromRef, -reqwest::Client: FromRef, +reqwest_middleware::ClientWithMiddleware: FromRef, Arc>>: FromRef, crate::SessionStore: FromRef, axum_extra::extract::cookie::Key: FromRef, diff --git a/src/login.rs b/src/login.rs index bfa84b3..e105bcc 100644 --- a/src/login.rs +++ b/src/login.rs @@ -48,7 +48,7 @@ struct LoginForm { async fn post( Host(host): Host, mut cookies: SignedCookieJar, - State(http): State, + State(http): State, Form(form): Form, ) -> axum::response::Response { let code_verifier = kittybox_indieauth::PKCEVerifier::new(); @@ -204,7 +204,7 @@ async fn callback( Host(host): Host, Query(result): Query, cookie_jar: SignedCookieJar, - State(http): State, + State(http): State, State(session_store): State, ) -> axum::response::Response { let client_id: url::Url = format!("https://{}/.kittybox/login/client_metadata", host).parse().unwrap(); @@ -355,7 +355,7 @@ pub fn router() -> axum::routing::Router where St: Clone + Send + Sync + 'static, cookie::Key: FromRef, - reqwest::Client: FromRef, + reqwest_middleware::ClientWithMiddleware: FromRef, crate::SessionStore: FromRef, S: Storage + FromRef + Send + Sync + 'static, { diff --git a/src/main.rs b/src/main.rs index f272a63..3aee6c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -92,7 +92,7 @@ async fn main() { let jobset: Arc>> = Default::default(); let session_store: kittybox::SessionStore = Default::default(); - let http: reqwest::Client = { + let http: reqwest_middleware::ClientWithMiddleware = { #[allow(unused_mut)] let mut builder = reqwest::Client::builder() .user_agent(concat!( @@ -137,7 +137,13 @@ async fn main() { builder = builder.danger_accept_invalid_certs(true); } - builder.build().unwrap() + reqwest_middleware::ClientBuilder::new(builder.build().unwrap()) + .with(http_cache_reqwest::Cache(http_cache_reqwest::HttpCache { + mode: http_cache_reqwest::CacheMode::Default, + manager: http_cache_reqwest::MokaManager::default(), + options: http_cache_reqwest::HttpCacheOptions::default(), + })) + .build() }; let backend_type = backend_uri.scheme(); diff --git a/src/micropub/mod.rs b/src/micropub/mod.rs index 08150d2..8f95085 100644 --- a/src/micropub/mod.rs +++ b/src/micropub/mod.rs @@ -82,7 +82,7 @@ fn populate_reply_context( async fn background_processing( db: D, mf2: serde_json::Value, - http: reqwest::Client, + http: reqwest_middleware::ClientWithMiddleware, ) -> () { // TODO: Post-processing the post (aka second write pass) // - [x] Download rich reply contexts @@ -232,7 +232,7 @@ pub(crate) async fn _post( uid: String, mf2: serde_json::Value, db: D, - http: reqwest::Client, + http: reqwest_middleware::ClientWithMiddleware, jobset: Arc>>, ) -> Result { // Here, we have the following guarantees: @@ -501,7 +501,7 @@ async fn dispatch_body( #[tracing::instrument(skip(db, http))] pub(crate) async fn post( State(db): State, - State(http): State, + State(http): State, State(jobset): State>>>, TypedHeader(content_type): TypedHeader, user: User, @@ -655,7 +655,7 @@ pub fn router() -> axum::routing::Metho where S: Storage + FromRef + 'static, A: AuthBackend + FromRef, - reqwest::Client: FromRef, + reqwest_middleware::ClientWithMiddleware: FromRef, Arc>>: FromRef { axum::routing::get(query::) @@ -758,7 +758,14 @@ mod tests { }; let (uid, mf2) = super::normalize_mf2(post, &user); - let err = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new()))) + let err = super::_post( + &user, uid, mf2, db.clone(), + reqwest_middleware::ClientWithMiddleware::new( + reqwest::Client::new(), + Box::default() + ), + Arc::new(Mutex::new(tokio::task::JoinSet::new())) + ) .await .unwrap_err(); @@ -788,7 +795,14 @@ mod tests { }; let (uid, mf2) = super::normalize_mf2(post, &user); - let err = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new()))) + let err = super::_post( + &user, uid, mf2, db.clone(), + reqwest_middleware::ClientWithMiddleware::new( + reqwest::Client::new(), + Box::default() + ), + Arc::new(Mutex::new(tokio::task::JoinSet::new())) + ) .await .unwrap_err(); @@ -816,7 +830,14 @@ mod tests { }; let (uid, mf2) = super::normalize_mf2(post, &user); - let res = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new()))) + let res = super::_post( + &user, uid, mf2, db.clone(), + reqwest_middleware::ClientWithMiddleware::new( + reqwest::Client::new(), + Box::default() + ), + Arc::new(Mutex::new(tokio::task::JoinSet::new())) + ) .await .unwrap(); diff --git a/src/tokenauth.rs b/src/tokenauth.rs index 244a045..414454a 100644 --- a/src/tokenauth.rs +++ b/src/tokenauth.rs @@ -173,7 +173,7 @@ where let Extension(TokenEndpoint(token_endpoint)): Extension = Extension::from_request(req).await.unwrap(); - let Extension(http): Extension = + let Extension(http): Extension = Extension::from_request(req).await.unwrap(); match http @@ -253,8 +253,8 @@ mod tests { } #[inline] - fn get_http_client() -> reqwest::Client { - reqwest::Client::new() + fn get_http_client() -> reqwest_middleware::ClientWithMiddleware { + reqwest_middleware::ClientWithMiddleware::new() } fn request>>( diff --git a/src/webmentions/mod.rs b/src/webmentions/mod.rs index d5a617e..22701b4 100644 --- a/src/webmentions/mod.rs +++ b/src/webmentions/mod.rs @@ -102,7 +102,7 @@ enum Error { Storage(StorageError) } -async fn process_webmentions_from_queue, S: Storage + 'static>(queue: Q, db: S, http: reqwest::Client) -> Result> { +async fn process_webmentions_from_queue, S: Storage + 'static>(queue: Q, db: S, http: reqwest_middleware::ClientWithMiddleware) -> Result> { use futures_util::StreamExt; use self::queue::Job; @@ -177,11 +177,11 @@ pub fn supervised_webmentions_task SupervisedTask -where reqwest::Client: FromRef +where reqwest_middleware::ClientWithMiddleware: FromRef { let queue = Q::from_ref(state); let storage = S::from_ref(state); - let http = reqwest::Client::from_ref(state); + let http = reqwest_middleware::ClientWithMiddleware::from_ref(state); supervisor::, _, _>(move || process_webmentions_from_queue( queue.clone(), storage.clone(), http.clone() ), cancellation_token) -- cgit 1.4.1