diff options
Diffstat (limited to 'src/bin')
| -rw-r--r-- | src/bin/kittybox-check-webmention.rs | 152 | ||||
| -rw-r--r-- | src/bin/kittybox-indieauth-helper.rs | 233 | ||||
| -rw-r--r-- | src/bin/kittybox-mf2.rs | 49 | ||||
| -rw-r--r-- | src/bin/kittybox_bulk_import.rs | 66 | ||||
| -rw-r--r-- | src/bin/kittybox_database_converter.rs | 106 |
5 files changed, 606 insertions, 0 deletions
diff --git a/src/bin/kittybox-check-webmention.rs b/src/bin/kittybox-check-webmention.rs new file mode 100644 index 0000000..f02032c --- /dev/null +++ b/src/bin/kittybox-check-webmention.rs @@ -0,0 +1,152 @@ +use std::cell::{RefCell, Ref}; +use std::rc::Rc; + +use clap::Parser; +use microformats::types::PropertyValue; +use microformats::html5ever; +use microformats::html5ever::tendril::TendrilSink; + +#[derive(thiserror::Error, Debug)] +enum Error { + #[error("http request error: {0}")] + Http(#[from] reqwest::Error), + #[error("microformats error: {0}")] + Microformats(#[from] microformats::Error), + #[error("json error: {0}")] + Json(#[from] serde_json::Error), + #[error("url parse error: {0}")] + UrlParse(#[from] url::ParseError), +} + +use kittybox_util::MentionType; + +fn check_mention(document: impl AsRef<str>, base_url: &url::Url, link: &url::Url) -> Result<Option<MentionType>, Error> { + // First, check the document for MF2 markup + let document = microformats::from_html(document.as_ref(), base_url.clone())?; + + // Get an iterator of all items + let items_iter = document.items.iter() + .map(AsRef::as_ref) + .map(RefCell::borrow); + + for item in items_iter { + let props = item.properties.borrow(); + for (prop, interaction_type) in [ + ("in-reply-to", MentionType::Reply), ("like-of", MentionType::Like), + ("bookmark-of", MentionType::Bookmark), ("repost-of", MentionType::Repost) + ] { + if let Some(propvals) = props.get(prop) { + for val in propvals { + if let PropertyValue::Url(url) = val { + if url == link { + return Ok(Some(interaction_type)) + } + } + } + } + } + // Process `content` + if let Some(PropertyValue::Fragment(content)) = props.get("content") + .map(Vec::as_slice) + .unwrap_or_default() + .first() + { + let root = html5ever::parse_document(html5ever::rcdom::RcDom::default(), Default::default()) + .from_utf8() + .one(content.html.to_owned().as_bytes()) + .document; + + // This is a trick to unwrap recursion into a loop + // + // A list of unprocessed node is made. Then, in each + // iteration, the list is "taken" and replaced with an + // empty list, which is populated with nodes for the next + // iteration of the loop. + // + // Empty list means all nodes were processed. + let mut unprocessed_nodes: Vec<Rc<html5ever::rcdom::Node>> = root.children.borrow().iter().cloned().collect(); + while unprocessed_nodes.len() > 0 { + // "Take" the list out of its memory slot, replace it with an empty list + let nodes = std::mem::take(&mut unprocessed_nodes); + 'nodes_loop: for node in nodes.into_iter() { + // Add children nodes to the list for the next iteration + unprocessed_nodes.extend(node.children.borrow().iter().cloned()); + + if let html5ever::rcdom::NodeData::Element { ref name, ref attrs, .. } = node.data { + // If it's not `<a>`, skip it + if name.local != *"a" { continue; } + let mut is_mention: bool = false; + for attr in attrs.borrow().iter() { + if attr.name.local == *"rel" { + // Don't count `rel="nofollow"` links — a web crawler should ignore them + // and so for purposes of driving visitors they are useless + if attr.value + .as_ref() + .split([',', ' ']) + .any(|v| v == "nofollow") + { + // Skip the entire node. + continue 'nodes_loop; + } + } + // if it's not `<a href="...">`, skip it + if attr.name.local != *"href" { continue; } + // Be forgiving in parsing URLs, and resolve them against the base URL + if let Ok(url) = base_url.join(attr.value.as_ref()) { + if &url == link { + is_mention = true; + } + } + } + if is_mention { + return Ok(Some(MentionType::Mention)); + } + } + } + } + + } + } + + Ok(None) +} + +#[derive(Parser, Debug)] +#[clap( + name = "kittybox-check-webmention", + author = "Vika <vika@fireburn.ru>", + version = env!("CARGO_PKG_VERSION"), + about = "Verify an incoming webmention" +)] +struct Args { + #[clap(value_parser)] + url: url::Url, + #[clap(value_parser)] + link: url::Url +} + +#[tokio::main] +async fn main() -> Result<(), self::Error> { + let args = Args::parse(); + + let http: reqwest::Client = { + #[allow(unused_mut)] + let mut builder = reqwest::Client::builder() + .user_agent(concat!( + env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION") + )); + + builder.build().unwrap() + }; + + let response = http.get(args.url.clone()).send().await?; + let text = response.text().await?; + + if let Some(mention_type) = check_mention(text, &args.url, &args.link)? { + println!("{:?}", mention_type); + + Ok(()) + } else { + std::process::exit(1) + } +} diff --git a/src/bin/kittybox-indieauth-helper.rs b/src/bin/kittybox-indieauth-helper.rs new file mode 100644 index 0000000..3377ec3 --- /dev/null +++ b/src/bin/kittybox-indieauth-helper.rs @@ -0,0 +1,233 @@ +use kittybox_indieauth::{ + AuthorizationRequest, PKCEVerifier, + PKCEChallenge, PKCEMethod, GrantRequest, Scope, + AuthorizationResponse, TokenData, GrantResponse +}; +use clap::Parser; +use std::{borrow::Cow, io::Write}; + +const DEFAULT_CLIENT_ID: &str = "https://kittybox.fireburn.ru/indieauth-helper.html"; +const DEFAULT_REDIRECT_URI: &str = "http://localhost:60000/callback"; + +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("i/o error: {0}")] + IO(#[from] std::io::Error), + #[error("http request error: {0}")] + HTTP(#[from] reqwest::Error), + #[error("urlencoded encoding error: {0}")] + UrlencodedEncoding(#[from] serde_urlencoded::ser::Error), + #[error("url parsing error: {0}")] + UrlParse(#[from] url::ParseError), + #[error("indieauth flow error: {0}")] + IndieAuth(Cow<'static, str>) +} + +#[derive(Parser, Debug)] +#[clap( + name = "kittybox-indieauth-helper", + author = "Vika <vika@fireburn.ru>", + version = env!("CARGO_PKG_VERSION"), + about = "Retrieve an IndieAuth token for debugging", + long_about = None +)] +struct Args { + /// Profile URL to use for initiating IndieAuth metadata discovery. + #[clap(value_parser)] + me: url::Url, + /// Scopes to request for the token. + /// + /// All IndieAuth scopes are supported, including arbitrary custom scopes. + #[clap(short, long)] + scope: Vec<Scope>, + /// Client ID to use when requesting a token. + #[clap(short, long, value_parser, default_value = DEFAULT_CLIENT_ID)] + client_id: url::Url, + /// Redirect URI to declare. Note: This will break the flow, use only for testing UI. + #[clap(long, value_parser)] + redirect_uri: Option<url::Url> +} + +fn append_query_string<T: serde::Serialize>( + url: &url::Url, + query: T +) -> Result<url::Url, Error> { + let mut new_url = url.clone(); + let mut query = serde_urlencoded::to_string(query)?; + if let Some(old_query) = url.query() { + query.push('&'); + query.push_str(old_query); + } + new_url.set_query(Some(&query)); + + Ok(new_url) +} + +#[tokio::main] +async fn main() -> Result<(), Error> { + let args = Args::parse(); + + let http: reqwest::Client = { + #[allow(unused_mut)] + let mut builder = reqwest::Client::builder() + .user_agent(concat!( + env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION") + )); + + builder.build().unwrap() + }; + + let redirect_uri: url::Url = args.redirect_uri + .clone() + .unwrap_or_else(|| DEFAULT_REDIRECT_URI.parse().unwrap()); + + eprintln!("Checking .well-known for metadata..."); + let metadata = http.get(args.me.join("/.well-known/oauth-authorization-server")?) + .header("Accept", "application/json") + .send() + .await? + .json::<kittybox_indieauth::Metadata>() + .await?; + + let verifier = PKCEVerifier::new(); + + let authorization_request = AuthorizationRequest { + response_type: kittybox_indieauth::ResponseType::Code, + client_id: args.client_id.clone(), + redirect_uri: redirect_uri.clone(), + state: kittybox_indieauth::State::new(), + code_challenge: PKCEChallenge::new(&verifier, PKCEMethod::default()), + scope: Some(kittybox_indieauth::Scopes::new(args.scope)), + me: Some(args.me) + }; + + let indieauth_url = append_query_string( + &metadata.authorization_endpoint, + authorization_request + )?; + + eprintln!("Please visit the following URL in your browser:\n\n {}\n", indieauth_url.as_str()); + + if args.redirect_uri.is_some() { + eprintln!("Custom redirect URI specified, won't be able to catch authorization response."); + std::process::exit(0); + } + + // Prepare a callback + let (tx, rx) = tokio::sync::oneshot::channel::<AuthorizationResponse>(); + let server = { + use axum::{routing::get, extract::Query, response::IntoResponse}; + + let tx = std::sync::Arc::new(tokio::sync::Mutex::new(Some(tx))); + + let router = axum::Router::new() + .route("/callback", axum::routing::get( + move |query: Option<Query<AuthorizationResponse>>| async move { + if let Some(Query(response)) = query { + if let Some(tx) = tx.lock_owned().await.take() { + tx.send(response).unwrap(); + + (axum::http::StatusCode::OK, + [("Content-Type", "text/plain")], + "Thank you! This window can now be closed.") + .into_response() + } else { + (axum::http::StatusCode::BAD_REQUEST, + [("Content-Type", "text/plain")], + "Oops. The callback was already received. Did you click twice?") + .into_response() + } + } else { + axum::http::StatusCode::BAD_REQUEST.into_response() + } + } + )); + + use std::net::{SocketAddr, IpAddr, Ipv4Addr}; + + let server = hyper::server::Server::bind( + &SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST),60000) + ) + .serve(router.into_make_service()); + + tokio::task::spawn(server) + }; + + let authorization_response = rx.await.unwrap(); + + // Clean up after the server + tokio::task::spawn(async move { + // Wait for the server to settle -- it might need to send its response + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + // Abort the future -- this should kill the server + server.abort(); + }); + + eprintln!("Got authorization response: {:#?}", authorization_response); + eprint!("Checking issuer field..."); + std::io::stderr().lock().flush()?; + + if dbg!(authorization_response.iss.as_str()) == dbg!(metadata.issuer.as_str()) { + eprintln!(" Done"); + } else { + eprintln!(" Failed"); + #[cfg(not(debug_assertions))] + std::process::exit(1); + } + let grant_response: GrantResponse = http.post(metadata.token_endpoint) + .form(&GrantRequest::AuthorizationCode { + code: authorization_response.code, + client_id: args.client_id, + redirect_uri, + code_verifier: verifier + }) + .header("Accept", "application/json") + .send() + .await? + .json() + .await?; + + if let GrantResponse::AccessToken { + me, + profile, + access_token, + expires_in, + refresh_token, + token_type, + scope + } = grant_response { + eprintln!("Congratulations, {}, access token is ready! {}", + me.as_str(), + if let Some(exp) = expires_in { + format!("It expires in {exp} seconds.") + } else { + format!("It seems to have unlimited duration.") + } + ); + println!("{}", access_token); + if let Some(refresh_token) = refresh_token { + eprintln!("Save this refresh token, it will come in handy:"); + println!("{}", refresh_token); + }; + + if let Some(profile) = profile { + eprintln!("\nThe token endpoint returned some profile information:"); + if let Some(name) = profile.name { + eprintln!(" - Name: {name}") + } + if let Some(url) = profile.url { + eprintln!(" - URL: {url}") + } + if let Some(photo) = profile.photo { + eprintln!(" - Photo: {photo}") + } + if let Some(email) = profile.email { + eprintln!(" - Email: {email}") + } + } + + Ok(()) + } else { + return Err(Error::IndieAuth(Cow::Borrowed("IndieAuth token endpoint did not return an access token grant."))); + } +} diff --git a/src/bin/kittybox-mf2.rs b/src/bin/kittybox-mf2.rs new file mode 100644 index 0000000..4366cb8 --- /dev/null +++ b/src/bin/kittybox-mf2.rs @@ -0,0 +1,49 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +#[clap( + name = "kittybox-mf2", + author = "Vika <vika@fireburn.ru>", + version = env!("CARGO_PKG_VERSION"), + about = "Fetch HTML and turn it into MF2-JSON" +)] +struct Args { + #[clap(value_parser)] + url: url::Url, +} + +#[derive(thiserror::Error, Debug)] +enum Error { + #[error("http request error: {0}")] + Http(#[from] reqwest::Error), + #[error("microformats error: {0}")] + Microformats(#[from] microformats::Error), + #[error("json error: {0}")] + Json(#[from] serde_json::Error), + #[error("url parse error: {0}")] + UrlParse(#[from] url::ParseError), +} + +#[tokio::main] +async fn main() -> Result<(), Error> { + let args = Args::parse(); + + let http: reqwest::Client = { + #[allow(unused_mut)] + let mut builder = reqwest::Client::builder() + .user_agent(concat!( + env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION") + )); + + builder.build().unwrap() + }; + + let response = http.get(args.url.clone()).send().await?; + let text = response.text().await?; + + let mf2 = microformats::from_html(text.as_ref(), args.url)?; + + println!("{}", serde_json::to_string_pretty(&mf2)?); + + Ok(()) +} diff --git a/src/bin/kittybox_bulk_import.rs b/src/bin/kittybox_bulk_import.rs new file mode 100644 index 0000000..7e1f6af --- /dev/null +++ b/src/bin/kittybox_bulk_import.rs @@ -0,0 +1,66 @@ +use anyhow::{anyhow, bail, Context, Result}; +use std::fs::File; +use std::io; + +#[async_std::main] +async fn main() -> Result<()> { + let args = std::env::args().collect::<Vec<String>>(); + if args.iter().skip(1).any(|s| s == "--help") { + println!("Usage: {} <url> [file]", args[0]); + println!("\nIf launched with no arguments, reads from stdin."); + println!( + "\nUse KITTYBOX_AUTH_TOKEN environment variable to authorize to the Micropub endpoint." + ); + std::process::exit(0); + } + + let token = std::env::var("KITTYBOX_AUTH_TOKEN") + .map_err(|_| anyhow!("No auth token found! Use KITTYBOX_AUTH_TOKEN env variable."))?; + let data: Vec<serde_json::Value> = (if args.len() == 2 || (args.len() == 3 && args[2] == "-") { + serde_json::from_reader(io::stdin()) + } else if args.len() == 3 { + serde_json::from_reader(File::open(&args[2]).with_context(|| "Error opening input file")?) + } else { + bail!("See `{} --help` for usage.", args[0]); + }) + .with_context(|| "Error while loading the input file")?; + + let url = surf::Url::parse(&args[1])?; + let client = surf::Client::new(); + + let iter = data.into_iter(); + + for post in iter { + println!( + "Processing {}...", + post["properties"]["url"][0] + .as_str() + .or_else(|| post["properties"]["published"][0] + .as_str() + .or_else(|| post["properties"]["name"][0] + .as_str() + .or(Some("<unidentified post>")))) + .unwrap() + ); + match client + .post(&url) + .body(surf::http::Body::from_string(serde_json::to_string(&post)?)) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {}", &token)) + .send() + .await + { + Ok(mut response) => { + if response.status() == 201 || response.status() == 202 { + println!("Posted at {}", response.header("location").unwrap().last()); + } else { + println!("Error: {:?}", response.body_string().await); + } + } + Err(err) => { + println!("{}", err); + } + } + } + Ok(()) +} diff --git a/src/bin/kittybox_database_converter.rs b/src/bin/kittybox_database_converter.rs new file mode 100644 index 0000000..bc355c9 --- /dev/null +++ b/src/bin/kittybox_database_converter.rs @@ -0,0 +1,106 @@ +use anyhow::{anyhow, Context}; +use kittybox::database::FileStorage; +use kittybox::database::Storage; +use redis::{self, AsyncCommands}; +use std::collections::HashMap; + +/// Convert from a Redis storage to a new storage new_storage. +async fn convert_from_redis<S: Storage>(from: String, new_storage: S) -> anyhow::Result<()> { + let db = redis::Client::open(from).context("Failed to open the Redis connection")?; + + let mut conn = db + .get_async_std_connection() + .await + .context("Failed to connect to Redis")?; + + // Rebinding to convince the borrow checker we're not smuggling stuff outta scope + let storage = &new_storage; + + let mut stream = conn.hscan::<_, String>("posts").await?; + + while let Some(key) = stream.next_item().await { + let value = serde_json::from_str::<serde_json::Value>( + &stream + .next_item() + .await + .ok_or(anyhow!("Failed to find a corresponding value for the key"))?, + )?; + + println!("{}, {:?}", key, value); + + if value["see_other"].is_string() { + continue; + } + + let user = &(url::Url::parse(value["properties"]["uid"][0].as_str().unwrap()) + .unwrap() + .origin() + .ascii_serialization() + .clone() + + "/"); + if let Err(err) = storage.clone().put_post(&value, user).await { + eprintln!("Error saving post: {}", err); + } + } + + let mut stream: redis::AsyncIter<String> = conn.scan_match("settings_*").await?; + while let Some(key) = stream.next_item().await { + let mut conn = db + .get_async_std_connection() + .await + .context("Failed to connect to Redis")?; + let user = key.strip_prefix("settings_").unwrap(); + match conn + .hgetall::<&str, HashMap<String, String>>(&key) + .await + .context(format!("Failed getting settings from key {}", key)) + { + Ok(settings) => { + for (k, v) in settings.iter() { + if let Err(e) = storage + .set_setting(k, user, v) + .await + .with_context(|| format!("Failed setting {} for {}", k, user)) + { + eprintln!("{}", e); + } + } + } + Err(e) => { + eprintln!("{}", e); + } + } + } + + Ok(()) +} + +#[async_std::main] +async fn main() -> anyhow::Result<()> { + let mut args = std::env::args(); + args.next(); // skip argv[0] + let old_uri = args + .next() + .ok_or_else(|| anyhow!("No import source is provided."))?; + let new_uri = args + .next() + .ok_or_else(|| anyhow!("No import destination is provided."))?; + + let storage = if new_uri.starts_with("file:") { + let folder = new_uri.strip_prefix("file://").unwrap(); + let path = std::path::PathBuf::from(folder); + Box::new( + FileStorage::new(path) + .await + .context("Failed to construct the file storage")?, + ) + } else { + anyhow::bail!("Cannot construct the storage abstraction for destination storage. Check the storage type?"); + }; + + if old_uri.starts_with("redis") { + convert_from_redis(old_uri, *storage).await? + } + + Ok(()) +} |