Smart Summary is now working!

There's no preferences dialog, so you can't really adjust the prompt
or the model it uses. The default settings work well for me. You may
want to tweak them depending on your model preferences and compute
budget. (Not many can afford to run Llama3-8B at high
quantization. Conversely, you might have a better GPU than me and wish
to run a 27B model or bigger.)

8 files changed, 338 insertions, 33 deletions

diff --git a/Cargo.lock b/Cargo.lock
index b5d907a..717c74d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -144,6 +144,7 @@ dependencies = [
  "microformats",
  "relm4",
  "relm4-icons",
+ "serde",
  "serde_json",
  "serde_urlencoded",
  "soup3",
diff --git a/Cargo.toml b/Cargo.toml
index 5e7f526..13b6aa2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,6 +19,7 @@ log = { version = "0.4.22", features = ["std"] }
 microformats = "0.9.1"
 relm4 = { version = "0.9.0", features = ["gnome_46", "adw", "css", "macros", "libadwaita"] }
 relm4-icons = { version = "0.9.0", features = ["icon-development-kit"] }
+serde = { version = "1.0.208", features = ["derive"] }
 serde_json = "1.0.125"
 serde_urlencoded = "0.7.1"
 soup3 = "0.7.0"
diff --git a/data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in b/data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in
index 81e0b13..4cec9d1 100644
--- a/data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in
+++ b/data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in
@@ -1,5 +1,53 @@
 <?xml version="1.0" encoding="utf-8"?>
 <schemalist>
   <schema path="/xyz/vikanezrimaya/kittybox/Bowl/" id="@app-id@" gettext-domain="@gettext-package@">
+    <key name="llm-endpoint" type="s">
+      <default>"http://localhost:11434/"</default>
+      <summary>LLM API endpoint</summary>
+      <description>
+        Ollama API endpoint used to query an LLM for Smart Summary.
+      </description>
+    </key>
+    <key name="smart-summary-model" type="s">
+      <default>"llama3.1:8b-instruct-q8_0"</default>
+      <summary>Smart Summary LLM</summary>
+      <!-- TRANSLATORS: please keep the link intact -->
+      <description>
+        <![CDATA[
+        The model that Ollama will load to produce
+        summaries. Available models can be seen at
+        <a href="https://ollama.com/library">Ollama library</a>.
+        ]]>
+      </description>
+    </key>
+    <key name="smart-summary-system-prompt" type="s">
+      <default>"You are a helpful AI assistant embedded into a blog authoring tool. You will be provided with a text to summarize. Reply only, strictly with a one-sentence summary of the provided text, and don't write anything else."</default>
+      <summary>LLM system prompt</summary>
+      <description>
+        The system prompt provided to the LLM. For best results, it
+        should instruct the LLM to provide a one-sentence summary of
+        the document it receives.
+
+        The default system prompt is tested for Llama 3.1-8B and
+        should work for posts written mainly in English. Performance
+        with other languages is untested.
+      </description>
+    </key>
+    <key name="smart-summary-prompt-prefix" type="s">
+      <default>"Summarize the following text:"</default>
+      <summary>Smart Summary prompt prefix</summary>
+      <description>
+        What the text is prefixed with when pasted into the LLM prompt.
+
+        Something like "Summarize this text:" works well.
+      </description>
+    </key>
+    <key name="smart-summary-prompt-suffix" type="s">
+      <default>""</default>
+      <summary>Smart Summary prompt suffix</summary>
+      <description>
+        Append this to the prompt after the article text.
+      </description>
+    </key>
   </schema>
 </schemalist>
diff --git a/po/bowl.pot b/po/bowl.pot
index 24e87b5..92da214 100644
--- a/po/bowl.pot
+++ b/po/bowl.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: bowl\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-09-01 18:02+0300\n"
+"POT-Creation-Date: 2024-09-04 15:59+0300\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -130,3 +130,52 @@ msgstr ""
 #: src/lib.rs:331
 msgid "Micropub access token for {}"
 msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:6
+msgid "LLM API endpoint"
+msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:7
+msgid "Ollama API endpoint used to query an LLM for Smart Summary."
+msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:13
+msgid "Smart Summary LLM"
+msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:25
+msgid "LLM system prompt"
+msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:26
+msgid ""
+"The system prompt provided to the LLM. For best results, it should instruct "
+"the LLM to provide a one-sentence summary of the document it receives. The "
+"default system prompt is tested for Llama 3.1-8B and should work for posts "
+"written mainly in English. Performance with other languages is untested."
+msgstr ""
+
+#. TRANSLATORS: please keep the link intact
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:15
+msgid ""
+"The model that Ollama will load to produce summaries. Available models can "
+"be seen at <a href=\"https://ollama.com/library\">Ollama library</a>."
+msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:38
+msgid "Smart Summary prompt prefix"
+msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:39
+msgid ""
+"What the text is prefixed with when pasted into the LLM prompt. Something "
+"like \"Summarize this text:\" works well."
+msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:47
+msgid "Smart Summary prompt suffix"
+msgstr ""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:48
+msgid "Append this to the prompt after the article text."
+msgstr ""
diff --git a/po/ru.po b/po/ru.po
index 29af65e..8ce26e5 100644
--- a/po/ru.po
+++ b/po/ru.po
@@ -29,7 +29,7 @@ msgstr "Минималистичная Micropub-утилита для напис
 #. Translators: Search terms to find this application. Do NOT translate or localize the semicolons! The list MUST also end with a semicolon!
 #: data/xyz.vikanezrimaya.kittybox.Bowl.desktop.in.in:10
 msgid "Micropub;IndieWeb;Kittybox;"
-msgstr "Miropub;IndieWeb;Kittybox"
+msgstr "Micropub;IndieWeb;Kittybox;"
 
 #. TRANSLATORS: please keep the newline and `<b>` tags
 #: src/components/smart_summary.rs:47
@@ -58,7 +58,7 @@ msgstr "Текст"
 
 #: src/components/post_editor.rs:280
 msgid "Visibility"
-msgstr ""
+msgstr "Видимость"
 
 #: src/components/post_editor.rs:493
 msgid "Smart Summary error: {}"
@@ -135,3 +135,60 @@ msgstr "Опубликовать"
 #: src/lib.rs:331
 msgid "Micropub access token for {}"
 msgstr "Токен доступа Micropub для {}"
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:6
+msgid "LLM API endpoint"
+msgstr "Точка API LLM"
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:7
+msgid "Ollama API endpoint used to query an LLM for Smart Summary."
+msgstr "API Ollama, которое используется, чтобы сгенерировать Умную Выжимку с помощью языковой модели."
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:13
+msgid "Smart Summary LLM"
+msgstr "Модель для Умной Выжимки"
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:22
+msgid "LLM system prompt"
+msgstr "Системная вводная для LLM"
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:23
+msgid ""
+"The system prompt provided to the LLM. For best results, it should instruct "
+"the LLM to provide a one-sentence summary of the document it receives. The "
+"default system prompt is tested for Llama 3.1-8B and should work for posts "
+"written mainly in English. Performance with other languages is untested."
+msgstr ""
+"Системная вводная, которая будет передана языковой модели. Для достижения "
+"наилучших результатов она должна содержать в себе указание для модели — "
+"описать суть документа одним предложением. Вводная, указанная по умолчанию, "
+"протестирована для Llama 3.1-8B и лучше всего работает со статьями на "
+"английском. Результаты для других языков не гарантированы."
+
+#. TRANSLATORS: please keep the link intact
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:15
+msgid ""
+"The model that Ollama will load to produce summaries. Available models can "
+"be seen at <a href=\"https://ollama.com/library\">Ollama library</a>."
+msgstr ""
+"Языковая модель, которую Ollama использует для извлечения содержания текста."
+"Доступные модели можно увидеть в <a href=\"https://ollama.com/library\">библиотеке Ollama</a>."
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:38
+msgid "Smart Summary prompt prefix"
+msgstr "Префикс вводной для Умной Выжимки"
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:39
+msgid ""
+"What the text is prefixed with when pasted into the LLM prompt. Something "
+"like \"Summarize this text:\" works well."
+msgstr "Что приписывается к началу текста для вводной языковой модели. Пример: "
+"\"Опиши смысл этого текста одним предложением:\""
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:47
+msgid "Smart Summary prompt suffix"
+msgstr "Суффикс вводной Умной Выжимки"
+
+#: data/xyz.vikanezrimaya.kittybox.Bowl.gschema.xml.in:48
+msgid "Append this to the prompt after the article text."
+msgstr "Что приписывается к вводной после текста статьи."
diff --git a/src/components/post_editor.rs b/src/components/post_editor.rs
index 8c81b6b..f2268ad 100644
--- a/src/components/post_editor.rs
+++ b/src/components/post_editor.rs
@@ -115,7 +115,7 @@ pub enum Input<E: std::error::Error + std::fmt::Debug + Send + 'static> {
 
 #[relm4::component(pub)]
 impl<E: std::error::Error + std::fmt::Debug + Send + 'static> Component for PostEditor<E> {
-    type Init = Option<Post>;
+    type Init = (<components::SmartSummaryButton as relm4::Component>::Init, Option<Post>);
     type Output = Option<Post>;
     type Input = Input<E>;
     type CommandOutput = ();
@@ -330,7 +330,11 @@ impl<E: std::error::Error + std::fmt::Debug + Send + 'static> Component for Post
         }
     }
 
-    fn init(init: Self::Init, root: Self::Root, sender: ComponentSender<Self>) -> ComponentParts<Self> {
+    fn init(
+        (http, init): Self::Init,
+        root: Self::Root,
+        sender: ComponentSender<Self>
+    ) -> ComponentParts<Self> {
         let mut model = Self {
             smart_summary_busy_guard: None,
             sending: false,
@@ -356,7 +360,7 @@ impl<E: std::error::Error + std::fmt::Debug + Send + 'static> Component for Post
             wide_layout: gtk::GridLayout::new(),
 
             smart_summary: components::SmartSummaryButton::builder()
-                .launch(())
+                .launch(http)
                 .forward(sender.input_sender(), Input::SmartSummary),
 
             tracker: Default::default(),
diff --git a/src/components/smart_summary.rs b/src/components/smart_summary.rs
index 9da67af..050a52c 100644
--- a/src/components/smart_summary.rs
+++ b/src/components/smart_summary.rs
@@ -1,15 +1,144 @@
+use futures::AsyncBufReadExt;
+use gio::prelude::SettingsExtManual;
+use soup::prelude::*;
 use adw::prelude::*;
 use gettextrs::*;
 use relm4::{gtk, prelude::{Component, ComponentParts}, ComponentSender};
 
+// All of this is incredibly minimalist.
+// This should be expanded later.
+#[derive(Debug, serde::Serialize)]
+struct OllamaRequest {
+    model: String,
+    prompt: String,
+    system: String,
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct OllamaChunk {
+    response: String,
+    done: bool,
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct OllamaError {
+    error: String
+}
+impl std::error::Error for OllamaError {}
+impl std::fmt::Display for OllamaError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.error.fmt(f)
+    }
+}
+
+#[derive(serde::Deserialize)]
+#[serde(untagged)]
+enum OllamaResult {
+    Ok(OllamaChunk),
+    Err(OllamaError),
+}
+
+impl From<OllamaResult> for Result<OllamaChunk, OllamaError> {
+    fn from(val: OllamaResult) -> Self {
+        match val {
+            OllamaResult::Ok(chunk) => Ok(chunk),
+            OllamaResult::Err(err) => Err(err)
+        }
+    }
+}
+
+
 #[derive(Debug, Default)]
 pub(crate) struct SmartSummaryButton {
-    busy: bool,
+    task: Option<relm4::JoinHandle<()>>,
+    waiting: bool,
+
+    http: soup::Session,
+}
+
+impl SmartSummaryButton {
+    async fn prompt_llm(
+        sender: relm4::Sender<Result<String, Error>>,
+        http: soup::Session,
+        endpoint: glib::Uri,
+        model: String,
+        system_prompt: String,
+        prompt_prefix: String,
+        mut prompt_suffix: String,
+        text: String,
+    ) {
+        let endpoint = endpoint.parse_relative("./api/generate", glib::UriFlags::NONE).unwrap();
+        log::debug!("endpoint: {}, model: {}", endpoint, model);
+        log::debug!("system prompt: {}", system_prompt);
+
+        let msg = soup::Message::from_uri(
+            "POST",
+            &endpoint
+        );
+
+        if !prompt_suffix.is_empty() {
+            prompt_suffix = String::from("\n\n") + &prompt_suffix;
+        }
+        msg.set_request_body_from_bytes(Some("application/json"),
+            Some(&glib::Bytes::from_owned(serde_json::to_vec(&OllamaRequest {
+                model, system: system_prompt, prompt: format!("{}\n\n{}{}", prompt_prefix, text, prompt_suffix),
+            }).unwrap()))
+        );
+
+        let mut stream = match http.send_future(&msg, glib::Priority::DEFAULT).await {
+            Ok(stream) => stream.into_async_buf_read(128),
+            Err(err) => {
+                let _ = sender.send(Err(err.into()));
+                return
+            }
+        };
+        log::debug!("response: {:?} ({})", msg.status(), msg.reason_phrase().unwrap_or_default());
+        let mut buffer = Vec::new();
+        const DELIM: u8 = b'\n';
+        loop {
+            let len = match stream.read_until(DELIM, &mut buffer).await {
+                Ok(len) => len,
+                Err(err) => {
+                    let _ = sender.send(Err(err.into()));
+                    return
+                }
+            };
+            log::debug!("Got chunk ({} bytes): {}", len, String::from_utf8_lossy(&buffer));
+            let response: Result<OllamaResult, serde_json::Error> = serde_json::from_slice(&buffer[..len]);
+            match response.map(Result::from) {
+                Ok(Ok(OllamaChunk { response: chunk, done })) => {
+                    if !chunk.is_empty() {
+                        sender.emit(Ok(chunk));
+                    }
+                    if done {
+                        sender.emit(Ok(String::new()));
+                        return
+                    }
+                },
+                Ok(Err(err)) => {
+                    sender.emit(Err(err.into()));
+                    return
+                }
+                Err(err) => {
+                    sender.emit(Err(err.into()));
+                    return
+                }
+            }
+            buffer.truncate(0);
+        }
+    }
 }
 
 #[derive(Debug, thiserror::Error)]
 pub(crate) enum Error {
-
+    #[error("glib error: {0}")]
+    Glib(#[from] glib::Error),
+    #[error("json error: {0}")]
+    Json(#[from] serde_json::Error),
+    #[error("ollama error: {0}")]
+    Ollama(#[from] OllamaError),
+    #[error("i/o error: {0}")]
+    Io(#[from] std::io::Error)
 }
 
 #[derive(Debug)]
@@ -33,7 +162,7 @@ impl Component for SmartSummaryButton {
     type Input = Input;
     type Output = Output;
 
-    type Init = ();
+    type Init = soup::Session;
     type CommandOutput = Result<String, Error>;
 
     view! {
@@ -42,11 +171,11 @@ impl Component for SmartSummaryButton {
         gtk::Button {
             connect_clicked => Input::ButtonPressed,
             #[watch]
-            set_sensitive: !model.busy,
+            set_sensitive: !(model.task.is_some() || model.waiting),
             // TRANSLATORS: please keep the newline and `<b>` tags
             set_tooltip_markup: Some(gettext("<b>Smart Summary</b>\nAsk a language model for a single-sentence summary.")).as_deref(),
 
-            if model.busy {
+            if model.task.is_some() || model.waiting {
                 gtk::Spinner { set_spinning: true }
             } else {
                 gtk::Label { set_markup: "✨" }
@@ -56,11 +185,14 @@ impl Component for SmartSummaryButton {
     }
 
     fn init(
-        _init: Self::Init,
+        init: Self::Init,
         root: Self::Root,
         sender: ComponentSender<Self>
     ) -> ComponentParts<Self> {
-        let model = Self::default();
+        let model = Self {
+            http: init,
+            ..Self::default()
+        };
         let widgets = view_output!();
 
         ComponentParts { model, widgets }
@@ -74,30 +206,41 @@ impl Component for SmartSummaryButton {
     ) {
         match msg {
             Input::Cancel => {
-                self.busy = false;
-                log::debug!("Parent component asked us to cancel.");
+                self.waiting = false;
+                if let Some(task) = self.task.take() {
+                    log::debug!("Parent component asked us to cancel.");
+                    task.abort();
+                } else {
+                    log::warn!("Parent component asked us to cancel, but we're not running a task.");
+                }
             },
             Input::ButtonPressed => if let Ok(()) = sender.output(Output::Start) {
-                self.busy = true;
+                self.waiting = true;
                 log::debug!("Requesting text to summarize from parent component...");
+                // TODO: set timeout in case parent component never replies
+                // This shouldn't happen, but I feel like we should handle this case.
             },
             Input::Text(text) => {
                 log::debug!("Would generate summary for the following text:\n{}", text);
 
-                sender.command(|sender, shutdown| shutdown.register(async move {
-                    tokio::time::sleep(std::time::Duration::from_millis(450)).await;
-
-                    for i in ["I'", "m ", "sorry,", " I", " am ", "unable", " to", " ", "write", " you ", "a summary.", " I", " am", " not ", "really ", "an ", "LLM."] {
-                        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
-
-                        if sender.send(Ok(i.to_string())).is_err() {
-                            return
-                        };
-                    }
-
-                    log::debug!("Done with the summary.");
-                    let _ = sender.send(Ok(String::new()));
-                }).drop_on_shutdown());
+                log::debug!("XDG_DATA_DIRS={:?}", std::env::var("XDG_DATA_DIRS"));
+                let settings = gio::Settings::new(crate::APPLICATION_ID);
+                // We shouldn't let the user record a bad setting anyway.
+                let endpoint = glib::Uri::parse(
+                    &settings.get::<String>("llm-endpoint"),
+                    glib::UriFlags::NONE,
+                ).unwrap();
+                let model = settings.get::<String>("smart-summary-model");
+                let system_prompt = settings.get::<String>("smart-summary-system-prompt");
+                let prompt_prefix = settings.get::<String>("smart-summary-prompt-prefix");
+                let prompt_suffix = settings.get::<String>("smart-summary-prompt-suffix");
+                let sender = sender.command_sender().clone();
+                relm4::spawn_local(Self::prompt_llm(
+                    sender, self.http.clone(),
+                    endpoint, model, system_prompt,
+                    prompt_prefix, prompt_suffix,
+                    text
+                ));
             }
         }
     }
@@ -105,11 +248,13 @@ impl Component for SmartSummaryButton {
     fn update_cmd(&mut self, msg: Self::CommandOutput, sender: ComponentSender<Self>, _root: &Self::Root) {
         match msg {
             Ok(chunk) if chunk.is_empty() => {
-                self.busy = false;
+                self.task = None;
+                self.waiting = false;
                 let _ = sender.output(Output::Done);
             },
             Err(err) => {
-                self.busy = false;
+                self.task = None;
+                self.waiting = false;
                 let _ = sender.output(Output::Error(err));
             }
             Ok(chunk) => {
diff --git a/src/lib.rs b/src/lib.rs
index 7f2a78d..30cba6f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -255,7 +255,7 @@ impl AsyncComponent for App {
             secret_schema,
 
             post_editor: components::PostEditor::builder()
-                .launch(None)
+                .launch((http.clone(), None))
                 .forward(sender.input_sender(), Self::Input::PostEditor),
             signin: components::SignIn::builder()
                 .launch((glib::Uri::parse(