From e559259686f984fdcce5669f0ab8c6dc27d76077 Mon Sep 17 00:00:00 2001 From: Vika Date: Wed, 27 Oct 2021 06:27:13 +0300 Subject: Deprecated Redis backend and added a database migration tool (untested, beware) --- src/bin/kittybox_database_converter.rs | 78 ++++++++++++++++++++++++++++++++++ src/bin/pyindieblog_to_kittybox.rs | 4 +- 2 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 src/bin/kittybox_database_converter.rs (limited to 'src/bin') diff --git a/src/bin/kittybox_database_converter.rs b/src/bin/kittybox_database_converter.rs new file mode 100644 index 0000000..ff28d49 --- /dev/null +++ b/src/bin/kittybox_database_converter.rs @@ -0,0 +1,78 @@ +use anyhow::{anyhow, Context}; +use redis::{self, AsyncCommands}; +use futures_util::StreamExt; +use kittybox::database::Storage; +use kittybox::database::FileStorage; +use std::collections::HashMap; + +/// Convert from a Redis storage to a new storage new_storage. +async fn convert_from_redis(from: String, new_storage: S) -> anyhow::Result<()> { + let db = redis::Client::open(from).context("Failed to open the Redis connection")?; + + let mut conn = db.get_async_std_connection().await.context("Failed to connect to Redis")?; + + // Rebinding to convince the borrow checker we're not smuggling stuff outta scope + let storage = &new_storage; + + conn.hscan::<_, String>("posts").await?.map(|it| async move { + let json = serde_json::from_str(&it); + (it, json) + }).for_each_concurrent(8, |it| async move { + let (orig, res): (String, serde_json::Result) = it.await; + match res { + Ok(json) => { + // XXX this assumes a trusted database that was created with Kittybox that checks for UID matching user token's prefix + let user = &(url::Url::parse(json["properties"]["uid"][0].as_str().unwrap()).unwrap().origin().ascii_serialization().clone() + "/"); + if let Err(err) = storage.clone().put_post(&json, user).await { + eprintln!("Error saving post: {}", err); + } + }, + Err(err) => { + eprintln!("Error: {} (rejected JSON follows below on stderr)", err); + eprintln!("{}", orig); + } + } + }).await; + + let mut stream: redis::AsyncIter = conn.scan_match("settings_*").await?; + while let Some(key) = stream.next_item().await { + let mut conn = db.get_async_std_connection().await.context("Failed to connect to Redis")?; + let user = key.strip_prefix("settings_").unwrap(); + match conn.hgetall::<&str, HashMap>(&key).await.context(format!("Failed getting settings from key {}", key)) { + Ok(settings) => { + for (k, v) in settings.iter() { + if let Err(e) = storage.set_setting(k, &user, v).await.with_context(|| format!("Failed setting {} for {}", k, user)) { + eprintln!("{}", e); + } + } + }, + Err(e) => { + eprintln!("{}", e); + } + } + } + + return Ok(()); +} + +#[async_std::main] +async fn main() -> anyhow::Result<()> { + let mut args = std::env::args(); + args.next(); // skip argv[0] + let old_uri = args.next().ok_or_else(|| anyhow!("No import source is provided."))?; + let new_uri = args.next().ok_or_else(|| anyhow!("No import destination is provided."))?; + + let storage = if new_uri.starts_with("file:") { + let folder = new_uri.strip_prefix("file://").unwrap(); + let path = std::path::PathBuf::from(folder); + Box::new(FileStorage::new(path).await.context("Failed to construct the file storage")?) + } else { + anyhow::bail!("Cannot construct the storage abstraction for destination storage. Check the storage type?"); + }; + + if old_uri.starts_with("redis") { + convert_from_redis(old_uri, *storage).await? + } + + Ok(()) +} diff --git a/src/bin/pyindieblog_to_kittybox.rs b/src/bin/pyindieblog_to_kittybox.rs index b4e2b97..303ca56 100644 --- a/src/bin/pyindieblog_to_kittybox.rs +++ b/src/bin/pyindieblog_to_kittybox.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, Context, Result}; -use mobc_redis::redis; -use mobc_redis::redis::AsyncCommands; +use redis; +use redis::AsyncCommands; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs::File; -- cgit 1.4.1