Deduplicated the fetched posts to save on bandwidth and time

1 files changed, 8 insertions, 2 deletions

diff --git a/src/micropub/post.rs b/src/micropub/post.rs
index 5b85ad4..4d208c2 100644
--- a/src/micropub/post.rs
+++ b/src/micropub/post.rs
@@ -226,6 +226,8 @@ async fn post_process_new_post<S: Storage>(req: Request<ApplicationState<S>>, po
     //    as possible without making them wait for potentially slow downstream websites to load
     // 1.1. Collect the list of contextually-significant post to load context from.
     //      This will include reply-tos, liked, reposted and bookmarked content
+    //
+    //      TODO: Fetch links mentioned in a post, since we need to send webmentions to those as mentions
     let mut contextually_significant_posts: Vec<surf::Url> = vec![];
     for prop in &["in-reply-to", "like-of", "repost-of", "bookmark-of"] {
         if let Some(array) = post["properties"][prop].as_array() {
@@ -238,7 +240,11 @@ async fn post_process_new_post<S: Storage>(req: Request<ApplicationState<S>>, po
             );
         }
     }
-    // 1.2. Fetch the posts with their bodies and save them in a new Vec<(surf::Url, String)>
+    // 1.2. Deduplicate the list
+    contextually_significant_posts.sort_unstable();
+    contextually_significant_posts.dedup();
+
+    // 1.3. Fetch the posts with their bodies and save them in a new Vec<(surf::Url, String)>
     let posts_with_bodies: Vec<(surf::Url, String)> = stream::iter(contextually_significant_posts.into_iter())
         .filter_map(|v: surf::Url| async move {
             if let Ok(res) = http.get(&v).send().await {
@@ -259,7 +265,7 @@ async fn post_process_new_post<S: Storage>(req: Request<ApplicationState<S>>, po
             }
         })
         .collect().await;
-    // 1.3. Parse the bodies and include them in relevant places on the MF2 struct
+    // 1.4. Parse the bodies and include them in relevant places on the MF2 struct
     //      This requires an MF2 parser, and there are none for Rust at the moment.
     //
     // TODO: integrate https://gitlab.com/vikanezrimaya/mf2-parser when it's ready