From 3b96490a0ac57590396f4ae72d61311052659071 Mon Sep 17 00:00:00 2001 From: Vika Date: Thu, 6 May 2021 17:41:57 +0300 Subject: Deduplicated the fetched posts to save on bandwidth and time --- src/micropub/post.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/micropub/post.rs b/src/micropub/post.rs index 5b85ad4..4d208c2 100644 --- a/src/micropub/post.rs +++ b/src/micropub/post.rs @@ -226,6 +226,8 @@ async fn post_process_new_post(req: Request>, po // as possible without making them wait for potentially slow downstream websites to load // 1.1. Collect the list of contextually-significant post to load context from. // This will include reply-tos, liked, reposted and bookmarked content + // + // TODO: Fetch links mentioned in a post, since we need to send webmentions to those as mentions let mut contextually_significant_posts: Vec = vec![]; for prop in &["in-reply-to", "like-of", "repost-of", "bookmark-of"] { if let Some(array) = post["properties"][prop].as_array() { @@ -238,7 +240,11 @@ async fn post_process_new_post(req: Request>, po ); } } - // 1.2. Fetch the posts with their bodies and save them in a new Vec<(surf::Url, String)> + // 1.2. Deduplicate the list + contextually_significant_posts.sort_unstable(); + contextually_significant_posts.dedup(); + + // 1.3. Fetch the posts with their bodies and save them in a new Vec<(surf::Url, String)> let posts_with_bodies: Vec<(surf::Url, String)> = stream::iter(contextually_significant_posts.into_iter()) .filter_map(|v: surf::Url| async move { if let Ok(res) = http.get(&v).send().await { @@ -259,7 +265,7 @@ async fn post_process_new_post(req: Request>, po } }) .collect().await; - // 1.3. Parse the bodies and include them in relevant places on the MF2 struct + // 1.4. Parse the bodies and include them in relevant places on the MF2 struct // This requires an MF2 parser, and there are none for Rust at the moment. // // TODO: integrate https://gitlab.com/vikanezrimaya/mf2-parser when it's ready -- cgit 1.4.1