diff options
| -rw-r--r-- | .vscode/settings.json | 3 | ||||
| -rw-r--r-- | src/bin/kittybox_bulk_import.rs | 36 | ||||
| -rw-r--r-- | src/bin/pyindieblog_to_kittybox.rs | 59 | ||||
| -rw-r--r-- | src/database/mod.rs | 10 | ||||
| -rw-r--r-- | src/frontend/mod.rs | 25 | ||||
| -rw-r--r-- | src/indieauth.rs | 25 | ||||
| -rw-r--r-- | src/lib.rs | 17 | ||||
| -rw-r--r-- | src/main.rs | 2 | ||||
| -rw-r--r-- | src/micropub/mod.rs | 10 | ||||
| -rw-r--r-- | src/micropub/post.rs | 116 |
10 files changed, 198 insertions, 105 deletions
diff --git a/.vscode/settings.json b/.vscode/settings.json index 7213b6b..c4febcd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { - "rust-client.disableRustup": true + "rust-client.disableRustup": true, + "nixEnvSelector.suggestion": false } diff --git a/src/bin/kittybox_bulk_import.rs b/src/bin/kittybox_bulk_import.rs index a5252b7..7e1f6af 100644 --- a/src/bin/kittybox_bulk_import.rs +++ b/src/bin/kittybox_bulk_import.rs @@ -1,6 +1,6 @@ -use std::io; +use anyhow::{anyhow, bail, Context, Result}; use std::fs::File; -use anyhow::{anyhow, Context, Result, bail}; +use std::io; #[async_std::main] async fn main() -> Result<()> { @@ -8,18 +8,22 @@ async fn main() -> Result<()> { if args.iter().skip(1).any(|s| s == "--help") { println!("Usage: {} <url> [file]", args[0]); println!("\nIf launched with no arguments, reads from stdin."); - println!("\nUse KITTYBOX_AUTH_TOKEN environment variable to authorize to the Micropub endpoint."); + println!( + "\nUse KITTYBOX_AUTH_TOKEN environment variable to authorize to the Micropub endpoint." + ); std::process::exit(0); } - let token = std::env::var("KITTYBOX_AUTH_TOKEN").map_err(|_| anyhow!("No auth token found! Use KITTYBOX_AUTH_TOKEN env variable."))?; + let token = std::env::var("KITTYBOX_AUTH_TOKEN") + .map_err(|_| anyhow!("No auth token found! Use KITTYBOX_AUTH_TOKEN env variable."))?; let data: Vec<serde_json::Value> = (if args.len() == 2 || (args.len() == 3 && args[2] == "-") { serde_json::from_reader(io::stdin()) } else if args.len() == 3 { serde_json::from_reader(File::open(&args[2]).with_context(|| "Error opening input file")?) } else { bail!("See `{} --help` for usage.", args[0]); - }).with_context(|| "Error while loading the input file")?; + }) + .with_context(|| "Error while loading the input file")?; let url = surf::Url::parse(&args[1])?; let client = surf::Client::new(); @@ -27,13 +31,25 @@ async fn main() -> Result<()> { let iter = data.into_iter(); for post in iter { - println!("Processing {}...", post["properties"]["url"][0].as_str().or_else(|| post["properties"]["published"][0].as_str().or_else(|| post["properties"]["name"][0].as_str().or(Some("<unidentified post>")))).unwrap()); - match client.post(&url) - .body(surf::http::Body::from_string( - serde_json::to_string(&post)?)) + println!( + "Processing {}...", + post["properties"]["url"][0] + .as_str() + .or_else(|| post["properties"]["published"][0] + .as_str() + .or_else(|| post["properties"]["name"][0] + .as_str() + .or(Some("<unidentified post>")))) + .unwrap() + ); + match client + .post(&url) + .body(surf::http::Body::from_string(serde_json::to_string(&post)?)) .header("Content-Type", "application/json") .header("Authorization", format!("Bearer {}", &token)) - .send().await { + .send() + .await + { Ok(mut response) => { if response.status() == 201 || response.status() == 202 { println!("Posted at {}", response.header("location").unwrap().last()); diff --git a/src/bin/pyindieblog_to_kittybox.rs b/src/bin/pyindieblog_to_kittybox.rs index c932e0a..b4e2b97 100644 --- a/src/bin/pyindieblog_to_kittybox.rs +++ b/src/bin/pyindieblog_to_kittybox.rs @@ -1,45 +1,66 @@ -use std::collections::HashMap; -use std::fs::File; -use anyhow::{Result, Context, anyhow}; +use anyhow::{anyhow, Context, Result}; use mobc_redis::redis; use mobc_redis::redis::AsyncCommands; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs::File; #[derive(Default, Serialize, Deserialize)] struct PyindieblogData { posts: Vec<serde_json::Value>, - cards: Vec<serde_json::Value> + cards: Vec<serde_json::Value>, } #[async_std::main] async fn main() -> Result<()> { let mut args = std::env::args(); args.next(); // skip argv[0] which is the name - let redis_uri = args.next().ok_or_else(|| anyhow!("No Redis URI provided"))?; - let client = redis::Client::open(redis_uri.as_str()).with_context(|| format!("Failed to construct Redis client on {}", redis_uri))?; + let redis_uri = args + .next() + .ok_or_else(|| anyhow!("No Redis URI provided"))?; + let client = redis::Client::open(redis_uri.as_str()) + .with_context(|| format!("Failed to construct Redis client on {}", redis_uri))?; - let filename = args.next().ok_or_else(|| anyhow!("No filename provided for export"))?; + let filename = args + .next() + .ok_or_else(|| anyhow!("No filename provided for export"))?; let mut data: Vec<serde_json::Value>; let file = File::create(filename)?; - let mut conn = client.get_async_std_connection().await.with_context(|| "Failed to connect to the Redis server")?; + let mut conn = client + .get_async_std_connection() + .await + .with_context(|| "Failed to connect to the Redis server")?; - data = conn.hgetall::<&str, HashMap<String, String>>("posts").await? + data = conn + .hgetall::<&str, HashMap<String, String>>("posts") + .await? .values() - .map(|s| serde_json::from_str::<serde_json::Value>(s) - .with_context(|| format!("Failed to parse the following entry: {:?}", s))) + .map(|s| { + serde_json::from_str::<serde_json::Value>(s) + .with_context(|| format!("Failed to parse the following entry: {:?}", s)) + }) .collect::<std::result::Result<Vec<serde_json::Value>, anyhow::Error>>() .with_context(|| "Failed to export h-entries from pyindieblog")?; - data.extend(conn.hgetall::<&str, HashMap<String, String>>("hcards").await? - .values() - .map(|s| serde_json::from_str::<serde_json::Value>(s) - .with_context(|| format!("Failed to parse the following card: {:?}", s))) - .collect::<std::result::Result<Vec<serde_json::Value>, anyhow::Error>>() - .with_context(|| "Failed to export h-cards from pyindieblog")?); + data.extend( + conn.hgetall::<&str, HashMap<String, String>>("hcards") + .await? + .values() + .map(|s| { + serde_json::from_str::<serde_json::Value>(s) + .with_context(|| format!("Failed to parse the following card: {:?}", s)) + }) + .collect::<std::result::Result<Vec<serde_json::Value>, anyhow::Error>>() + .with_context(|| "Failed to export h-cards from pyindieblog")?, + ); - data.sort_by_key(|v| v["properties"]["published"][0].as_str().map(|s| s.to_string())); + data.sort_by_key(|v| { + v["properties"]["published"][0] + .as_str() + .map(|s| s.to_string()) + }); serde_json::to_writer(file, &data)?; diff --git a/src/database/mod.rs b/src/database/mod.rs index e0e4e7b..27c0025 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -198,7 +198,10 @@ mod tests { let alt_url = post["properties"]["url"][1].as_str().unwrap().to_string(); // Reading and writing - backend.put_post(&post, "https://fireburn.ru/").await.unwrap(); + backend + .put_post(&post, "https://fireburn.ru/") + .await + .unwrap(); if let Ok(Some(returned_post)) = backend.get_post(&key).await { assert!(returned_post.is_object()); assert_eq!( @@ -254,7 +257,10 @@ mod tests { }, "children": [] }); - backend.put_post(&feed, "https://fireburn.ru/").await.unwrap(); + backend + .put_post(&feed, "https://fireburn.ru/") + .await + .unwrap(); let chans = backend .get_channels(&crate::indieauth::User::new( "https://fireburn.ru/", diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs index 8155b2c..2cef026 100644 --- a/src/frontend/mod.rs +++ b/src/frontend/mod.rs @@ -705,7 +705,12 @@ struct OnboardingData { pub async fn onboarding_receiver<S: Storage>(mut req: Request<ApplicationState<S>>) -> Result { use serde_json::json; - <AsMut<tide::http::Request>>::as_mut(&mut req).url_mut().set_scheme("https"); + // This cannot error out as the URL must be valid. Or there is something horribly wrong + // and we shouldn't serve this request anyway. + <dyn AsMut<tide::http::Request>>::as_mut(&mut req) + .url_mut() + .set_scheme("https") + .unwrap(); let body = req.body_json::<OnboardingData>().await?; let backend = &req.state().storage; @@ -784,7 +789,12 @@ pub async fn coffee<S: Storage>(_: Request<ApplicationState<S>>) -> Result { } pub async fn mainpage<S: Storage>(mut req: Request<ApplicationState<S>>) -> Result { - <AsMut<tide::http::Request>>::as_mut(&mut req).url_mut().set_scheme("https"); + // This cannot error out as the URL must be valid. Or there is something horribly wrong + // and we shouldn't serve this request anyway. + <dyn AsMut<tide::http::Request>>::as_mut(&mut req) + .url_mut() + .set_scheme("https") + .unwrap(); let backend = &req.state().storage; let query = req.query::<QueryParams>()?; let authorization_endpoint = req.state().authorization_endpoint.to_string(); @@ -863,7 +873,12 @@ pub async fn render_post<S: Storage>(mut req: Request<ApplicationState<S>>) -> R let token_endpoint = req.state().token_endpoint.to_string(); let user: Option<String> = None; - <AsMut<tide::http::Request>>::as_mut(&mut req).url_mut().set_scheme("https"); + // This cannot error out as the URL must be valid. Or there is something horribly wrong + // and we shouldn't serve this request anyway. + <dyn AsMut<tide::http::Request>>::as_mut(&mut req) + .url_mut() + .set_scheme("https") + .unwrap(); #[cfg(any(not(debug_assertions), test))] let url = req.url(); #[cfg(all(debug_assertions, not(test)))] @@ -875,8 +890,8 @@ pub async fn render_post<S: Storage>(mut req: Request<ApplicationState<S>>) -> R let mut entry_url = req.url().clone(); entry_url.set_query(None); - let post = - get_post_from_database(&req.state().storage, entry_url.as_str(), query.after, &user).await?; + let post = get_post_from_database(&req.state().storage, entry_url.as_str(), query.after, &user) + .await?; let template: String = match post["type"][0] .as_str() diff --git a/src/indieauth.rs b/src/indieauth.rs index aea7e4d..f8f862b 100644 --- a/src/indieauth.rs +++ b/src/indieauth.rs @@ -166,14 +166,23 @@ where .build()) } Some(value) => { - match (&req.state().internal_token) { - Some(token) => if token == &value.last().to_string().split(" ").skip(1).collect::<String>() { - req.set_ext::<User>(User::new( - "", // no user ID here - "https://kittybox.fireburn.ru/", - "update delete undelete media kittybox_internal:do_what_thou_wilt" - )); - return Ok(next.run(req).await) + match &req.state().internal_token { + Some(token) => { + if token + == &value + .last() + .to_string() + .split(' ') + .skip(1) + .collect::<String>() + { + req.set_ext::<User>(User::new( + "", // no user ID here + "https://kittybox.fireburn.ru/", + "update delete undelete media kittybox_internal:do_what_thou_wilt", + )); + return Ok(next.run(req).await); + } } None => {} } diff --git a/src/lib.rs b/src/lib.rs index 398c3b2..6a62dcc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,8 +5,8 @@ mod frontend; mod indieauth; mod micropub; -use crate::micropub::CORSMiddleware; use crate::indieauth::IndieAuthMiddleware; +use crate::micropub::CORSMiddleware; #[derive(Clone)] pub struct ApplicationState<StorageBackend> @@ -64,7 +64,7 @@ pub async fn get_app_with_redis( authorization_endpoint: surf::Url, redis_uri: String, media_endpoint: Option<String>, - internal_token: Option<String> + internal_token: Option<String>, ) -> App<database::RedisStorage> { let app = tide::with_state(ApplicationState { token_endpoint, @@ -169,11 +169,14 @@ mod tests { .with_body(r#"{"me": "https://aaronparecki.com/", "client_id": "https://quill.p3k.io/", "scope": "create update delete media"}"#) .create(); - let mut response = app.post("/micropub") + let mut response = app + .post("/micropub") .header("Authorization", "Bearer awoo") .header("Content-Type", "application/json") .body(json!({ "action": "delete", "url": uid })) - .send().await.unwrap(); + .send() + .await + .unwrap(); println!("{}", response.body_string().await.unwrap()); assert_eq!(response.status(), 403); } @@ -215,7 +218,11 @@ mod tests { // Should be posted successfully, but... assert!(response.status() == 201 || response.status() == 202); // ...won't be available on a foreign URL - assert!(db.get_post("https://aaronparecki.com/posts/more-fake-news").await.unwrap().is_none()); + assert!(db + .get_post("https://aaronparecki.com/posts/more-fake-news") + .await + .unwrap() + .is_none()); let response = post_json(&app, json!({ "type": ["h-entry"], diff --git a/src/main.rs b/src/main.rs index eb7b538..0e57ed5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -68,7 +68,7 @@ async fn main() -> Result<(), std::io::Error> { authorization_endpoint, redis_uri, media_endpoint, - internal_token + internal_token, ) .await; app.listen(host).await diff --git a/src/micropub/mod.rs b/src/micropub/mod.rs index 84b9083..23f20c4 100644 --- a/src/micropub/mod.rs +++ b/src/micropub/mod.rs @@ -7,17 +7,21 @@ pub use post::post_handler; pub struct CORSMiddleware {} -use async_trait::async_trait; -use tide::{Next, Request, Result}; use crate::database; use crate::ApplicationState; +use async_trait::async_trait; +use tide::{Next, Request, Result}; #[async_trait] impl<B> tide::Middleware<ApplicationState<B>> for CORSMiddleware where B: database::Storage + Send + Sync + Clone, { - async fn handle(&self, req: Request<ApplicationState<B>>, next: Next<'_, ApplicationState<B>>) -> Result { + async fn handle( + &self, + req: Request<ApplicationState<B>>, + next: Next<'_, ApplicationState<B>>, + ) -> Result { let mut res = next.run(req).await; res.insert_header("Access-Control-Allow-Origin", "*"); diff --git a/src/micropub/post.rs b/src/micropub/post.rs index 8667451..f317da5 100644 --- a/src/micropub/post.rs +++ b/src/micropub/post.rs @@ -6,7 +6,7 @@ use core::iter::Iterator; use futures::stream; use futures::StreamExt; use http_types::Mime; -use log::{error, warn, info}; +use log::{error, info, warn}; use newbase60::num_to_sxg; use std::convert::TryInto; use std::str::FromStr; @@ -172,9 +172,9 @@ pub async fn new_post<S: Storage>( // where several users or identities share one Micropub server // (maybe a family website or a shitpost sideblog?) if !post["properties"]["uid"][0] - .as_str() - .unwrap() - .starts_with(user.me.as_str()) + .as_str() + .unwrap() + .starts_with(user.me.as_str()) || post["properties"]["channel"] .as_array() .unwrap() @@ -430,52 +430,60 @@ async fn post_process_new_post<S: Storage>( // We'll need the bodies here to get their endpoints let source = &uid; stream::iter(posts_with_bodies.into_iter()) - .filter_map(|(url, response, body): (surf::Url, surf::Response, String)| async move { - // Check Link headers first - // the first webmention endpoint will be returned - if let Some(values) = response.header("Link") { - let mut iter = values.iter().flat_map(|i| i.as_str().split(',')); - - for link in iter { - let mut split = link.split(";"); - - match split.next() { - Some(uri) => { - if let Some(uri) = uri.strip_prefix('<') { - if let Some(uri) = uri.strip_suffix('>') { - for prop in split { - let lowercased = prop.to_ascii_lowercase(); - if &lowercased == "rel=\"webmention\"" || &lowercased == "rel=webmention" { - if let Ok(endpoint) = url.join(uri) { - return Some((url, endpoint)); + .filter_map( + |(url, response, body): (surf::Url, surf::Response, String)| async move { + // Check Link headers first + // the first webmention endpoint will be returned + if let Some(values) = response.header("Link") { + let iter = values.iter().flat_map(|i| i.as_str().split(',')); + + for link in iter { + let mut split = link.split(';'); + + match split.next() { + Some(uri) => { + if let Some(uri) = uri.strip_prefix('<') { + if let Some(uri) = uri.strip_suffix('>') { + for prop in split { + let lowercased = prop.to_ascii_lowercase(); + if &lowercased == "rel=\"webmention\"" + || &lowercased == "rel=webmention" + { + if let Ok(endpoint) = url.join(uri) { + return Some((url, endpoint)); + } } } } } } - }, - None => continue + None => continue, + } } } - } - // TODO: Replace this function once the MF2 parser is ready - // A compliant parser's output format includes rels, - // we could just find a Webmention one in there - let pattern = easy_scraper::Pattern::new(r#"<link href="{url}" rel="webmention">"#) - .expect("Pattern for webmentions couldn't be parsed"); - let matches = pattern.matches(&body); - if matches.is_empty() { - return None; - } - let endpoint = &matches[0]["url"]; - if let Ok(endpoint) = url.join(endpoint) { - Some((url, endpoint)) - } else { - None - } - }) + // TODO: Replace this function once the MF2 parser is ready + // A compliant parser's output format includes rels, + // we could just find a Webmention one in there + let pattern = easy_scraper::Pattern::new(r#"<link href="{url}" rel="webmention">"#) + .expect("Pattern for webmentions couldn't be parsed"); + let matches = pattern.matches(&body); + if matches.is_empty() { + return None; + } + let endpoint = &matches[0]["url"]; + if let Ok(endpoint) = url.join(endpoint) { + Some((url, endpoint)) + } else { + None + } + }, + ) .map(|(target, endpoint)| async move { - info!("Sending webmention to {} about {}", source, &target.to_string()); + info!( + "Sending webmention to {} about {}", + source, + &target.to_string() + ); let response = http .post(&endpoint) .content_type("application/x-www-form-urlencoded") @@ -543,12 +551,14 @@ async fn process_json<S: Storage>( // authorization endpoint is supposed to reject any auth request trying to get this // scope. It is intended for TRUSTED external services that need to modify the // database while ignoring any access controls - if (url::Url::parse(url)?.origin().ascii_serialization() + "/") != user.me.as_str() && !user.check_scope("kittybox_internal:do_what_thou_wilt") { + if (url::Url::parse(url)?.origin().ascii_serialization() + "/") != user.me.as_str() + && !user.check_scope("kittybox_internal:do_what_thou_wilt") + { return error_json!( 403, "forbidden", "You're not allowed to delete someone else's posts." - ) + ); } if let Err(error) = req.state().storage.delete_post(&url).await { return Ok(error.into()); @@ -563,12 +573,14 @@ async fn process_json<S: Storage>( "You need an `update` scope to update posts." ); } - if (url::Url::parse(url)?.origin().ascii_serialization() + "/") != user.me.as_str() && !user.check_scope("kittybox_internal:do_what_thou_wilt") { + if (url::Url::parse(url)?.origin().ascii_serialization() + "/") != user.me.as_str() + && !user.check_scope("kittybox_internal:do_what_thou_wilt") + { return error_json!( 403, "forbidden", "You're not allowed to delete someone else's posts." - ) + ); } if let Err(error) = req.state().storage.update_post(&url, body.clone()).await { Ok(error.into()) @@ -637,12 +649,15 @@ async fn process_form<S: Storage>( } match form.iter().find(|(k, _)| k == "url") { Some((_, url)) => { - if (url::Url::parse(url)?.origin().ascii_serialization() + "/") != user.me.as_str() && !user.check_scope("kittybox_internal:do_what_thou_wilt") { + if (url::Url::parse(url)?.origin().ascii_serialization() + "/") + != user.me.as_str() + && !user.check_scope("kittybox_internal:do_what_thou_wilt") + { return error_json!( 403, "forbidden", "You're not allowed to delete someone else's posts." - ) + ); } if let Err(error) = req.state().storage.delete_post(&url).await { return error_json!(500, "database_error", error); @@ -791,11 +806,10 @@ mod tests { DateTime::parse_from_rfc3339(post["properties"]["published"][0].as_str().unwrap()) .expect("Couldn't parse date from rfc3339"); assert!( - post["properties"]["url"] + !post["properties"]["url"] .as_array() .expect("post['url'] is undefined") - .len() - > 0, + .is_empty(), "Post doesn't have any URLs" ); assert_eq!( |