Add HTTP fetcher cache

It just does its thing in the background, potentially speeding up
things. Maybe I could also use the underlying in-memory cache
implementation (Moka) to speed up my database. I heard crates.io got
some good results from that.

8 files changed, 57 insertions, 27 deletions

diff --git a/src/frontend/onboarding.rs b/src/frontend/onboarding.rs
index be1669f..a8d2ae6 100644
--- a/src/frontend/onboarding.rs
+++ b/src/frontend/onboarding.rs
@@ -53,7 +53,7 @@ async fn onboard<D: Storage + 'static>(
     db: D,
     user_uid: url::Url,
     data: OnboardingData,
-    http: reqwest::Client,
+    http: reqwest_middleware::ClientWithMiddleware,
     jobset: Arc<Mutex<JoinSet<()>>>,
 ) -> Result<(), FrontendError> {
     // Create a user to pass to the backend
@@ -126,7 +126,7 @@ async fn onboard<D: Storage + 'static>(
 pub async fn post<D: Storage + 'static>(
     State(db): State<D>,
     Host(host): Host,
-    State(http): State<reqwest::Client>,
+    State(http): State<reqwest_middleware::ClientWithMiddleware>,
     State(jobset): State<Arc<Mutex<JoinSet<()>>>>,
     Json(data): Json<OnboardingData>,
 ) -> axum::response::Response {
@@ -165,7 +165,7 @@ pub fn router<St, S>() -> axum::routing::MethodRouter<St>
 where
     S: Storage + FromRef<St> + 'static,
     Arc<Mutex<JoinSet<()>>>: FromRef<St>,
-    reqwest::Client: FromRef<St>,
+    reqwest_middleware::ClientWithMiddleware: FromRef<St>,
     St: Clone + Send + Sync + 'static,
 {
     axum::routing::get(get)
diff --git a/src/indieauth/mod.rs b/src/indieauth/mod.rs
index 322a0e2..b3db77f 100644
--- a/src/indieauth/mod.rs
+++ b/src/indieauth/mod.rs
@@ -131,7 +131,7 @@ async fn authorization_endpoint_get<A: AuthBackend, D: Storage + 'static>(
     Host(host): Host,
     Query(request): Query<AuthorizationRequest>,
     State(db): State<D>,
-    State(http): State<reqwest::Client>,
+    State(http): State<reqwest_middleware::ClientWithMiddleware>,
     State(auth): State<A>
 ) -> Response {
     let me: url::Url = format!("https://{host}/").parse().unwrap();
@@ -148,7 +148,10 @@ async fn authorization_endpoint_get<A: AuthBackend, D: Storage + 'static>(
         tracing::debug!("Sending request to {} to fetch metadata", request.client_id);
         let metadata_request = http.get(request.client_id.clone())
             .header("Accept", "application/json, text/html");
-        match metadata_request.send().await.and_then(|res| res.error_for_status()) {
+        match metadata_request.send().await
+            .and_then(|res| res.error_for_status()
+                .map_err(reqwest_middleware::Error::Reqwest))
+        {
             Ok(response) if response.headers().typed_get::<ContentType>().to_owned().map(mime::Mime::from).map(|m| m.type_() == "text" && m.subtype() == "html").unwrap_or(false) => {
                 let url = response.url().clone();
                 let text = response.text().await.unwrap();
@@ -847,7 +850,7 @@ pub fn router<St, A, S>() -> axum::Router<St>
 where
     S: Storage + FromRef<St> + 'static,
     A: AuthBackend + FromRef<St>,
-    reqwest::Client: FromRef<St>,
+    reqwest_middleware::ClientWithMiddleware: FromRef<St>,
     St: Clone + Send + Sync + 'static
 {
     use axum::routing::{Router, get, post};
diff --git a/src/lib.rs b/src/lib.rs
index 596ffc0..5fe3b18 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -36,7 +36,7 @@ Q: JobQueue<webmentions::Webmention> + Sized
     pub storage: S,
     pub media_store: M,
     pub job_queue: Q,
-    pub http: reqwest::Client,
+    pub http: reqwest_middleware::ClientWithMiddleware,
     pub background_jobs: Arc<Mutex<JoinSet<()>>>,
     pub cookie_key: Key,
     pub session_store: SessionStore
@@ -156,7 +156,7 @@ where A: AuthBackend, S: Storage, M: MediaStore, Q: JobQueue<webmentions::Webmen
     }
 }
 
-impl<A, S, M, Q> FromRef<AppState<A, S, M, Q>> for reqwest::Client
+impl<A, S, M, Q> FromRef<AppState<A, S, M, Q>> for reqwest_middleware::ClientWithMiddleware
 where A: AuthBackend, S: Storage, M: MediaStore, Q: JobQueue<webmentions::Webmention>
 {
     fn from_ref(input: &AppState<A, S, M, Q>) -> Self {
@@ -291,7 +291,7 @@ A: AuthBackend + 'static + FromRef<St>,
 S: Storage + 'static + FromRef<St>,
 M: MediaStore + 'static + FromRef<St>,
 Q: kittybox_util::queue::JobQueue<crate::webmentions::Webmention> + FromRef<St>,
-reqwest::Client: FromRef<St>,
+reqwest_middleware::ClientWithMiddleware: FromRef<St>,
 Arc<Mutex<JoinSet<()>>>: FromRef<St>,
 crate::SessionStore: FromRef<St>,
 axum_extra::extract::cookie::Key: FromRef<St>,
diff --git a/src/login.rs b/src/login.rs
index bfa84b3..e105bcc 100644
--- a/src/login.rs
+++ b/src/login.rs
@@ -48,7 +48,7 @@ struct LoginForm {
 async fn post(
     Host(host): Host,
     mut cookies: SignedCookieJar,
-    State(http): State<reqwest::Client>,
+    State(http): State<reqwest_middleware::ClientWithMiddleware>,
     Form(form): Form<LoginForm>,
 ) -> axum::response::Response {
     let code_verifier = kittybox_indieauth::PKCEVerifier::new();
@@ -204,7 +204,7 @@ async fn callback(
     Host(host): Host,
     Query(result): Query<AuthorizationResponse>,
     cookie_jar: SignedCookieJar,
-    State(http): State<reqwest::Client>,
+    State(http): State<reqwest_middleware::ClientWithMiddleware>,
     State(session_store): State<crate::SessionStore>,
 ) -> axum::response::Response {
     let client_id: url::Url = format!("https://{}/.kittybox/login/client_metadata", host).parse().unwrap();
@@ -355,7 +355,7 @@ pub fn router<St, S>() -> axum::routing::Router<St>
 where
     St: Clone + Send + Sync + 'static,
     cookie::Key: FromRef<St>,
-    reqwest::Client: FromRef<St>,
+    reqwest_middleware::ClientWithMiddleware: FromRef<St>,
     crate::SessionStore: FromRef<St>,
     S: Storage + FromRef<St> + Send + Sync + 'static,
 {
diff --git a/src/main.rs b/src/main.rs
index f272a63..3aee6c3 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -92,7 +92,7 @@ async fn main() {
     let jobset: Arc<Mutex<JoinSet<()>>> = Default::default();
     let session_store: kittybox::SessionStore = Default::default();
 
-    let http: reqwest::Client = {
+    let http: reqwest_middleware::ClientWithMiddleware = {
         #[allow(unused_mut)]
         let mut builder = reqwest::Client::builder()
             .user_agent(concat!(
@@ -137,7 +137,13 @@ async fn main() {
             builder = builder.danger_accept_invalid_certs(true);
         }
 
-        builder.build().unwrap()
+        reqwest_middleware::ClientBuilder::new(builder.build().unwrap())
+            .with(http_cache_reqwest::Cache(http_cache_reqwest::HttpCache {
+                mode: http_cache_reqwest::CacheMode::Default,
+                manager: http_cache_reqwest::MokaManager::default(),
+                options: http_cache_reqwest::HttpCacheOptions::default(),
+            }))
+            .build()
     };
 
     let backend_type = backend_uri.scheme();
diff --git a/src/micropub/mod.rs b/src/micropub/mod.rs
index 08150d2..8f95085 100644
--- a/src/micropub/mod.rs
+++ b/src/micropub/mod.rs
@@ -82,7 +82,7 @@ fn populate_reply_context(
 async fn background_processing<D: 'static + Storage>(
     db: D,
     mf2: serde_json::Value,
-    http: reqwest::Client,
+    http: reqwest_middleware::ClientWithMiddleware,
 ) -> () {
     // TODO: Post-processing the post (aka second write pass)
     // - [x] Download rich reply contexts
@@ -232,7 +232,7 @@ pub(crate) async fn _post<D: 'static + Storage>(
     uid: String,
     mf2: serde_json::Value,
     db: D,
-    http: reqwest::Client,
+    http: reqwest_middleware::ClientWithMiddleware,
     jobset: Arc<Mutex<JoinSet<()>>>,
 ) -> Result<Response, MicropubError> {
     // Here, we have the following guarantees:
@@ -501,7 +501,7 @@ async fn dispatch_body(
 #[tracing::instrument(skip(db, http))]
 pub(crate) async fn post<D: Storage + 'static, A: AuthBackend>(
     State(db): State<D>,
-    State(http): State<reqwest::Client>,
+    State(http): State<reqwest_middleware::ClientWithMiddleware>,
     State(jobset): State<Arc<Mutex<JoinSet<()>>>>,
     TypedHeader(content_type): TypedHeader<ContentType>,
     user: User<A>,
@@ -655,7 +655,7 @@ pub fn router<A, S, St: Send + Sync + Clone + 'static>() -> axum::routing::Metho
 where
     S: Storage + FromRef<St> + 'static,
     A: AuthBackend + FromRef<St>,
-    reqwest::Client: FromRef<St>,
+    reqwest_middleware::ClientWithMiddleware: FromRef<St>,
     Arc<Mutex<JoinSet<()>>>: FromRef<St>
 {
     axum::routing::get(query::<S, A>)
@@ -758,7 +758,14 @@ mod tests {
         };
         let (uid, mf2) = super::normalize_mf2(post, &user);
 
-        let err = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new())))
+        let err = super::_post(
+            &user, uid, mf2, db.clone(),
+            reqwest_middleware::ClientWithMiddleware::new(
+                reqwest::Client::new(),
+                Box::default()
+            ),
+            Arc::new(Mutex::new(tokio::task::JoinSet::new()))
+        )
             .await
             .unwrap_err();
 
@@ -788,7 +795,14 @@ mod tests {
         };
         let (uid, mf2) = super::normalize_mf2(post, &user);
 
-        let err = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new())))
+        let err = super::_post(
+            &user, uid, mf2, db.clone(),
+            reqwest_middleware::ClientWithMiddleware::new(
+                reqwest::Client::new(),
+                Box::default()
+            ),
+            Arc::new(Mutex::new(tokio::task::JoinSet::new()))
+        )
             .await
             .unwrap_err();
 
@@ -816,7 +830,14 @@ mod tests {
         };
         let (uid, mf2) = super::normalize_mf2(post, &user);
 
-        let res = super::_post(&user, uid, mf2, db.clone(), reqwest::Client::new(), Arc::new(Mutex::new(tokio::task::JoinSet::new())))
+        let res = super::_post(
+            &user, uid, mf2, db.clone(),
+            reqwest_middleware::ClientWithMiddleware::new(
+                reqwest::Client::new(),
+                Box::default()
+            ),
+            Arc::new(Mutex::new(tokio::task::JoinSet::new()))
+        )
             .await
             .unwrap();
 
diff --git a/src/tokenauth.rs b/src/tokenauth.rs
index 244a045..414454a 100644
--- a/src/tokenauth.rs
+++ b/src/tokenauth.rs
@@ -173,7 +173,7 @@ where
         let Extension(TokenEndpoint(token_endpoint)): Extension<TokenEndpoint> =
             Extension::from_request(req).await.unwrap();
 
-        let Extension(http): Extension<reqwest::Client> =
+        let Extension(http): Extension<reqwest_middleware::ClientWithMiddleware> =
             Extension::from_request(req).await.unwrap();
 
         match http
@@ -253,8 +253,8 @@ mod tests {
     }
 
     #[inline]
-    fn get_http_client() -> reqwest::Client {
-        reqwest::Client::new()
+    fn get_http_client() -> reqwest_middleware::ClientWithMiddleware {
+        reqwest_middleware::ClientWithMiddleware::new()
     }
 
     fn request<A: Into<Option<&'static str>>>(
diff --git a/src/webmentions/mod.rs b/src/webmentions/mod.rs
index d5a617e..22701b4 100644
--- a/src/webmentions/mod.rs
+++ b/src/webmentions/mod.rs
@@ -102,7 +102,7 @@ enum Error<Q: std::error::Error + std::fmt::Debug + Send + 'static> {
     Storage(StorageError)
 }
 
-async fn process_webmentions_from_queue<Q: JobQueue<Webmention>, S: Storage + 'static>(queue: Q, db: S, http: reqwest::Client) -> Result<std::convert::Infallible, Error<Q::Error>> {
+async fn process_webmentions_from_queue<Q: JobQueue<Webmention>, S: Storage + 'static>(queue: Q, db: S, http: reqwest_middleware::ClientWithMiddleware) -> Result<std::convert::Infallible, Error<Q::Error>> {
     use futures_util::StreamExt;
     use self::queue::Job;
 
@@ -177,11 +177,11 @@ pub fn supervised_webmentions_task<St: Send + Sync + 'static, S: Storage + FromR
     state: &St,
     cancellation_token: tokio_util::sync::CancellationToken
 ) -> SupervisedTask
-where reqwest::Client: FromRef<St>
+where reqwest_middleware::ClientWithMiddleware: FromRef<St>
 {
     let queue = Q::from_ref(state);
     let storage = S::from_ref(state);
-    let http = reqwest::Client::from_ref(state);
+    let http = reqwest_middleware::ClientWithMiddleware::from_ref(state);
     supervisor::<Error<Q::Error>, _, _>(move || process_webmentions_from_queue(
         queue.clone(), storage.clone(), http.clone()
     ), cancellation_token)