From 31a0bdad439a4575c1686f690e9e72bd44dde472 Mon Sep 17 00:00:00 2001 From: Vika Date: Mon, 26 Aug 2024 15:22:29 +0300 Subject: Add HTTP fetcher cache It just does its thing in the background, potentially speeding up things. Maybe I could also use the underlying in-memory cache implementation (Moka) to speed up my database. I heard crates.io got some good results from that. --- src/frontend/onboarding.rs | 6 +++--- src/indieauth/mod.rs | 9 ++++++--- src/lib.rs | 6 +++--- src/login.rs | 6 +++--- src/main.rs | 10 ++++++++-- src/micropub/mod.rs | 35 ++++++++++++++++++++++++++++------- src/tokenauth.rs | 6 +++--- src/webmentions/mod.rs | 6 +++--- 8 files changed, 57 insertions(+), 27 deletions(-) (limited to 'src') diff --git a/src/frontend/onboarding.rs b/src/frontend/onboarding.rs index be1669f..a8d2ae6 100644 --- a/src/frontend/onboarding.rs +++ b/src/frontend/onboarding.rs @@ -53,7 +53,7 @@ async fn onboard( db: D, user_uid: url::Url, data: OnboardingData, - http: reqwest::Client, + http: reqwest_middleware::ClientWithMiddleware, jobset: Arc>>, ) -> Result<(), FrontendError> { // Create a user to pass to the backend @@ -126,7 +126,7 @@ async fn onboard( pub async fn post( State(db): State, Host(host): Host, - State(http): State, + State(http): State, State(jobset): State>>>, Json(data): Json, ) -> axum::response::Response { @@ -165,7 +165,7 @@ pub fn router() -> axum::routing::MethodRouter where S: Storage + FromRef + 'static, Arc>>: FromRef, - reqwest::Client: FromRef, + reqwest_middleware::ClientWithMiddleware: FromRef, St: Clone + Send + Sync + 'static, { axum::routing::get(get) diff --git a/src/indieauth/mod.rs b/src/indieauth/mod.rs index 322a0e2..b3db77f 100644 --- a/src/indieauth/mod.rs +++ b/src/indieauth/mod.rs @@ -131,7 +131,7 @@ async fn authorization_endpoint_get( Host(host): Host, Query(request): Query, State(db): State, - State(http): State, + State(http): State, State(auth): State ) -> Response { let me: url::Url = format!("https://{host}/").parse().unwrap(); @@ -148,7 +148,10 @@ async fn authorization_endpoint_get( tracing::debug!("Sending request to {} to fetch metadata", request.client_id); let metadata_request = http.get(request.client_id.clone()) .header("Accept", "application/json, text/html"); - match metadata_request.send().await.and_then(|res| res.error_for_status()) { + match metadata_request.send().await + .and_then(|res| res.error_for_status() + .map_err(reqwest_middleware::Error::Reqwest)) + { Ok(response) if response.headers().typed_get::().to_owned().map(mime::Mime::from).map(|m| m.type_() == "text" && m.subtype() == "html").unwrap_or(false) => { let url = response.url().clone(); let text = response.text().await.unwrap(); @@ -847,7 +850,7 @@ pub fn router() -> axum::Router where S: Storage + FromRef + 'static, A: AuthBackend + FromRef, - reqwest::Client: FromRef, + reqwest_middleware::ClientWithMiddleware: FromRef, St: Clone + Send + Sync + 'static { use axum::routing::{Router, get, post}; diff --git a/src/lib.rs b/src/lib.rs index 596ffc0..5fe3b18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,7 +36,7 @@ Q: JobQueue + Sized pub storage: S, pub media_store: M, pub job_queue: Q, - pub http: reqwest::Client, + pub http: reqwest_middleware::ClientWithMiddleware, pub background_jobs: Arc>>, pub cookie_key: Key, pub session_store: SessionStore @@ -156,7 +156,7 @@ where A: AuthBackend, S: Storage, M: MediaStore, Q: JobQueue FromRef> for reqwest::Client +impl FromRef> for reqwest_middleware::ClientWithMiddleware where A: AuthBackend, S: Storage, M: MediaStore, Q: JobQueue { fn from_ref(input: &AppState) -> Self { @@ -291,7 +291,7 @@ A: AuthBackend + 'static + FromRef, S: Storage + 'static + FromRef, M: MediaStore + 'static + FromRef, Q: kittybox_util::queue::JobQueue + FromRef, -reqwest::Client: FromRef, +reqwest_middleware::ClientWithMiddleware: FromRef, Arc>>: FromRef, crate::SessionStore: FromRef, axum_extra::extract::cookie::Key: FromRef, diff --git a/src/login.rs b/src/login.rs index bfa84b3..e105bcc 100644 --- a/src/login.rs +++ b/src/login.rs @@ -48,7 +48,7 @@ struct LoginForm { async fn post( Host(host): Host, mut cookies: SignedCookieJar, - State(http): State, + State(http): State, Form(form): Form, ) -> axum::response::Response { let code_verifier = kittybox_indieauth::PKCEVerifier::new(); @@ -204,7 +204,7 @@ async fn callback( Host(host): Host, Query(result): Query, cookie_jar: SignedCookieJar, - State(http): State, + State(http): State, State(session_store): State, ) -> axum::response::Response { let client_id: url::Url = format!("https://{}/.kittybox/login/client_metadata", host).parse().unwrap(); @@ -355,7 +355,7 @@ pub fn router() -> axum::routing::Router where St: Clone + Send + Sync + 'static, cookie::Key: FromRef, - reqwest::Client: FromRef, + reqwest_middleware::ClientWithMiddleware: FromRef, crate::SessionStore: FromRef, S: Storage + FromRef + Send + Sync + 'static, { diff --git a/src/main.rs b/src/main.rs index f272a63..3aee6c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -92,7 +92,7 @@ async fn main() { let jobset: Arc>> = Default::default(); let session_store: kittybox::SessionStore = Default::default(); - let http: reqwest::Client = { + let http: reqwest_middleware::ClientWithMiddleware = { #[allow(unused_mut)] let mut builder = reqwest::Client::builder() .user_agent(concat!( @@ -137,7 +137,13 @@ async fn main() { builder = builder.danger_accept_invalid_certs(true); } - builder.build().unwrap() + reqwest_middleware::ClientBuilder::new(builder.build().unwrap()) + .with(http_cache_reqwest::Cache(http_cache_reqwest::HttpCache { + mode: http_cache_reqwest::CacheMode::Default, + manager: http_cache_reqwest::MokaManager::default(), + options: http_cache_reqwest::HttpCacheOptions::default(), + })) + .build() }; let backend_type = backend_uri.scheme(); diff --git a/src/micropub/mod.rs b/src/micropub/mod.rs index 08150d2..8f95085 100644 --- a/src/micropub/mod.rs +++ b/src/micropub/mod.rs @@ -82,7 +82,7 @@ fn populate_reply_context( async fn background_processing( db: D, mf2: serde_json::Value, - http: reqwest::Client, + http: reqwest_middleware::ClientWithMiddleware, ) -> () { // TODO: Post-processing the post (aka second write pass) // - [x] Download rich reply contexts @@ -232,7 +232,7 @@ pub(crate) async fn _post( uid: String, mf2: serde_json::Value, db: D, - http: reqwest::Client, + http: reqwest_middleware::ClientWithMiddleware, jobset: Arc>>, ) -> Result { // Here, we have the following guarantees: @@ -501,7 +501,7 @@ async fn dispatch_body( #[tracing::instrument(skip(db, http))] pub(crate) async fn post( State(db): State, - State(http): State, + State(http): State, State(jobset): State>>>, TypedHeader(content_type): TypedHeader, user: User, @@ -655,7 +655,7 @@ pub fn router() -> axum::routing::Metho where S: Storage + FromRef + 'static, A: AuthBackend + FromRef, - reqwest::Client: FromRef, + reqwest_middleware::ClientWithMiddleware: FromRef, Arc>>: FromRef { axum::routing::get(query::) @@ -758,7 +758,14 @@ mod tests { }; let (uid, mf2) = super::normalize_mf2(post, &user); - let err = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new()))) + let err = super::_post( + &user, uid, mf2, db.clone(), + reqwest_middleware::ClientWithMiddleware::new( + reqwest::Client::new(), + Box::default() + ), + Arc::new(Mutex::new(tokio::task::JoinSet::new())) + ) .await .unwrap_err(); @@ -788,7 +795,14 @@ mod tests { }; let (uid, mf2) = super::normalize_mf2(post, &user); - let err = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new()))) + let err = super::_post( + &user, uid, mf2, db.clone(), + reqwest_middleware::ClientWithMiddleware::new( + reqwest::Client::new(), + Box::default() + ), + Arc::new(Mutex::new(tokio::task::JoinSet::new())) + ) .await .unwrap_err(); @@ -816,7 +830,14 @@ mod tests { }; let (uid, mf2) = super::normalize_mf2(post, &user); - let res = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new()))) + let res = super::_post( + &user, uid, mf2, db.clone(), + reqwest_middleware::ClientWithMiddleware::new( + reqwest::Client::new(), + Box::default() + ), + Arc::new(Mutex::new(tokio::task::JoinSet::new())) + ) .await .unwrap(); diff --git a/src/tokenauth.rs b/src/tokenauth.rs index 244a045..414454a 100644 --- a/src/tokenauth.rs +++ b/src/tokenauth.rs @@ -173,7 +173,7 @@ where let Extension(TokenEndpoint(token_endpoint)): Extension = Extension::from_request(req).await.unwrap(); - let Extension(http): Extension = + let Extension(http): Extension = Extension::from_request(req).await.unwrap(); match http @@ -253,8 +253,8 @@ mod tests { } #[inline] - fn get_http_client() -> reqwest::Client { - reqwest::Client::new() + fn get_http_client() -> reqwest_middleware::ClientWithMiddleware { + reqwest_middleware::ClientWithMiddleware::new() } fn request>>( diff --git a/src/webmentions/mod.rs b/src/webmentions/mod.rs index d5a617e..22701b4 100644 --- a/src/webmentions/mod.rs +++ b/src/webmentions/mod.rs @@ -102,7 +102,7 @@ enum Error { Storage(StorageError) } -async fn process_webmentions_from_queue, S: Storage + 'static>(queue: Q, db: S, http: reqwest::Client) -> Result> { +async fn process_webmentions_from_queue, S: Storage + 'static>(queue: Q, db: S, http: reqwest_middleware::ClientWithMiddleware) -> Result> { use futures_util::StreamExt; use self::queue::Job; @@ -177,11 +177,11 @@ pub fn supervised_webmentions_task SupervisedTask -where reqwest::Client: FromRef +where reqwest_middleware::ClientWithMiddleware: FromRef { let queue = Q::from_ref(state); let storage = S::from_ref(state); - let http = reqwest::Client::from_ref(state); + let http = reqwest_middleware::ClientWithMiddleware::from_ref(state); supervisor::, _, _>(move || process_webmentions_from_queue( queue.clone(), storage.clone(), http.clone() ), cancellation_token) -- cgit 1.4.1