re-implement a basic favicon scrapper

will get back at it once the rest is working

Cargo.toml

@@ -1,26 +1,27 @@
 [package]
+edition = "2018"
 name = "authenticator"
 version = "0.1.0"
-edition = "2018"
 
 [dependencies]
-log = "0.4"
-gettext-rs= { version = "0.5", features = ["gettext-system"] }
-pretty_env_logger = "0.4"
-lazy_static = "1.4"
 anyhow = "1.0"
-diesel = { version = "1.4", features = ["sqlite", "r2d2"] }
-diesel_migrations = {  version = "1.4" , features = ["sqlite"] }
-serde = "1.0"
-serde_json = "1.0"
-serde_derive = "1.0"
-nanoid = "0.3"
-reqwest = "0.10"
-url = "2.1"
-gtk-macros = "0.2"
 ashpd = "0.1"
-zbar-rust = "0.0"
+diesel = {version = "1.4", features = ["sqlite", "r2d2"]}
+diesel_migrations = {version = "1.4", features = ["sqlite"]}
+futures = "0.3"
+gettext-rs = {version = "0.5", features = ["gettext-system"]}
+gtk-macros = "0.2"
 image = "0.23"
+lazy_static = "1.4"
+log = "0.4"
+pretty_env_logger = "0.4"
+quick-xml = "0.20"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+surf = "2.1"
+url = "2.1"
+zbar-rust = "0.0"
 
 [dependencies.gtk]
 git = "https://github.com/gtk-rs/gtk4"
@@ -42,4 +43,3 @@ package = "libhandy4"
 
 [dependencies.gdk-pixbuf]
 git = "https://github.com/gtk-rs/gdk-pixbuf"
-

data/resources/ui/add_account.ui

@@ -57,6 +57,33 @@
                       <object class="GtkBox" id="main_container">
                         <property name="orientation">vertical</property>
                         <property name="spacing">42</property>
+                        <child>
+                          <object class="GtkStack" id="image_stack">
+                            <property name="transition-type">crossfade</property>
+                            <child>
+                              <object class="GtkStackPage">
+                                <property name="name">image</property>
+                                <property name="child">
+                                  <object class="GtkImage" id="image">
+                                    <property name="icon-name">image-missing</property>
+                                    <property name="pixel-size">96</property>
+                                  </object>
+                                </property>
+                              </object>
+                            </child>
+                            <child>
+                              <object class="GtkStackPage">
+                                <property name="name">loading</property>
+                                <property name="child">
+                                  <object class="GtkSpinner" id="spinner">
+                                    <property name="valign">center</property>
+                                    <property name="halign">center</property>
+                                  </object>
+                                </property>
+                              </object>
+                            </child>
+                          </object>
+                        </child>
                         <child>
                           <object class="GtkListBox" id="basic_list">
                             <property name="selection_mode">none</property>

data/resources/ui/window.ui.in

@@ -19,8 +19,6 @@
     </item>
   </menu>
   <object class="HdyApplicationWindow" id="window">
-    <property name="width_request">350</property>
-    <property name="height_request">500</property>
     <property name="default_width">360</property>
     <property name="default_height">550</property>
     <property name="icon-name">@app-id@</property>
@@ -69,11 +67,8 @@
                         <property name="valign">start</property>
                         <property name="hexpand">True</property>
                         <property name="vexpand">True</property>
-                        <property name="pixel_size">192</property>
+                        <property name="pixel_size">128</property>
                         <property name="icon-name">@app-id@</property>
-                        <style>
-                          <class name="empty-logo"/>
-                        </style>
                       </object>
                     </child>
                     <child>

src/meson.build

@@ -46,6 +46,7 @@ sources = files(
   'models/account.rs',
   'models/accounts.rs',
   'models/database.rs',
+  'models/favicon.rs',
   'models/mod.rs',
   'models/object_wrapper.rs',
   'models/provider.rs',

src/models/favicon.rs

@@ -0,0 +1,110 @@
+use quick_xml::events::{attributes::Attribute, BytesStart, Event};
+use url::Url;
+
+const SUPPORTED_RELS: [&[u8]; 7] = [
+    b"icon",
+    b"fluid-icon",
+    b"shortcut icon",
+    b"apple-touch-icon",
+    b"apple-touch-icon-precomposed",
+    b"fluid-icon",
+    b"alternate icon",
+];
+
+#[derive(Debug)]
+pub enum FaviconError {
+    Surf(surf::Error),
+    Url(url::ParseError),
+    NoResults,
+}
+
+impl From<surf::Error> for FaviconError {
+    fn from(e: surf::Error) -> Self {
+        Self::Surf(e)
+    }
+}
+
+impl From<url::ParseError> for FaviconError {
+    fn from(e: url::ParseError) -> Self {
+        Self::Url(e)
+    }
+}
+
+pub struct Favicon {
+    icons: Vec<Url>,
+}
+
+impl Favicon {}
+#[derive(Debug)]
+pub struct FaviconScrapper;
+
+impl FaviconScrapper {
+    pub async fn from_url(url: Url) -> Result<Vec<Url>, FaviconError> {
+        let mut res = surf::get(&url).await?;
+        let body = res.body_string().await?;
+        let mut reader = quick_xml::Reader::from_str(&body);
+
+        let icons = Self::from_reader(&mut reader, &url);
+
+        Ok(icons)
+    }
+
+    fn from_reader(reader: &mut quick_xml::Reader<&[u8]>, base_url: &Url) -> Vec<Url> {
+        let mut buf = Vec::new();
+        let mut urls = Vec::new();
+        loop {
+            match reader.read_event(&mut buf) {
+                Ok(Event::Start(ref e)) => match e.name() {
+                    b"link" => {
+                        if let Some(url) = Self::from_link(e, base_url) {
+                            urls.push(url);
+                        }
+                    }
+                    _ => (),
+                },
+                Ok(Event::Eof) => break,
+                Err(e) => warn!("Error at position {}: {:?}", reader.buffer_position(), e),
+                _ => (),
+            }
+        }
+        buf.clear();
+        urls
+    }
+
+    fn from_link(e: &BytesStart, base_url: &Url) -> Option<Url> {
+        let mut url = None;
+
+        let mut has_proper_rel = false;
+        for attr in e.html_attributes() {
+            match attr {
+                Ok(Attribute {
+                    key: b"href",
+                    value,
+                }) => {
+                    let href = String::from_utf8(value.into_owned()).unwrap();
+                    url = match Url::parse(&href) {
+                        Ok(url) => Some(url),
+                        Err(url::ParseError::RelativeUrlWithoutBase) => base_url.join(&href).ok(),
+                        Err(_) => None,
+                    };
+                    if has_proper_rel {
+                        break;
+                    }
+                }
+                Ok(Attribute { key: b"rel", value }) => {
+                    if SUPPORTED_RELS.contains(&value.into_owned().as_slice()) {
+                        has_proper_rel = true;
+                        if url.is_some() {
+                            break;
+                        }
+                    }
+                }
+                _ => (),
+            }
+        }
+        if has_proper_rel {
+            return url;
+        }
+        None
+    }
+}

src/models/mod.rs

@@ -2,6 +2,7 @@ mod account;
 mod accounts;
 mod algorithm;
 pub mod database;
+mod favicon;
 mod object_wrapper;
 mod provider;
 mod providers;
@@ -9,6 +10,7 @@ mod providers;
 pub use self::account::{Account, NewAccount};
 pub use self::accounts::AccountsModel;
 pub use self::algorithm::Algorithm;
+pub use self::favicon::{FaviconError, FaviconScrapper};
 pub use self::object_wrapper::ObjectWrapper;
 pub use self::provider::Provider;
 pub use self::providers::ProvidersModel;

src/models/provider.rs

@@ -1,7 +1,9 @@
 use super::algorithm::Algorithm;
 use crate::models::database;
+use crate::models::{FaviconError, FaviconScrapper};
 use anyhow::Result;
 use diesel::RunQueryDsl;
+use gio::FileExt;
 use glib::subclass;
 use glib::subclass::prelude::*;
 use glib::translate::*;
@@ -10,6 +12,7 @@ use glib::{StaticType, ToValue};
 use std::cell::{Cell, RefCell};
 use std::str::FromStr;
 use std::string::ToString;
+use url::Url;
 
 #[derive(Queryable, Hash, PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
 struct DiProvider {
@@ -240,6 +243,32 @@ impl Provider {
         .expect("Created provider is of wrong type")
     }
 
+    pub async fn favicon(&self) -> Result<gio::File, FaviconError> {
+        let website_url = Url::parse(&self.website().unwrap())?;
+        let favicons = FaviconScrapper::from_url(website_url).await?;
+
+        let icon_name = format!("{}_{}", self.id(), self.name().replace(' ', "_"));
+        let cache_path = glib::get_user_cache_dir()
+            .join("authenticator")
+            .join("favicons")
+            .join(icon_name);
+        let dest = gio::File::new_for_path(cache_path);
+
+        if let Some(favicon) = favicons.get(0) {
+            let mut res = surf::get(favicon).await?;
+            let body = res.body_bytes().await?;
+            dest.replace_contents(
+                &body,
+                None,
+                false,
+                gio::FileCreateFlags::REPLACE_DESTINATION,
+                gio::NONE_CANCELLABLE,
+            );
+            return Ok(dest);
+        }
+        Err(FaviconError::NoResults)
+    }
+
     pub fn id(&self) -> i32 {
         let priv_ = ProviderPriv::from_instance(self);
         priv_.id.get()

src/widgets/accounts/add.rs

@@ -3,31 +3,43 @@ use crate::helpers::qrcode;
 use crate::models::database::*;
 use crate::models::{Account, Algorithm, NewAccount, Provider, ProvidersModel};
 use anyhow::Result;
+use futures::future::FutureExt;
 use gio::prelude::*;
 use glib::StaticType;
-use glib::{signal::Inhibit, Sender};
+use glib::{signal::Inhibit, Receiver, Sender};
 use gtk::prelude::*;
 use libhandy::ComboRowExt;
 use std::cell::RefCell;
 use std::rc::Rc;
 
+pub enum AddAccountAction {
+    SetIcon(gio::File),
+}
+
 pub struct AddAccountDialog {
     pub widget: libhandy::Window,
     builder: gtk::Builder,
-    sender: Sender<Action>,
+    global_sender: Sender<Action>,
+    sender: Sender<AddAccountAction>,
+    receiver: RefCell<Option<Receiver<AddAccountAction>>>,
     model: Rc<ProvidersModel>,
     selected_provider: Rc<RefCell<Option<Provider>>>,
 }
 
 impl AddAccountDialog {
-    pub fn new(sender: Sender<Action>) -> Rc<Self> {
+    pub fn new(global_sender: Sender<Action>) -> Rc<Self> {
         let builder = gtk::Builder::from_resource("/com/belmoussaoui/Authenticator/add_account.ui");
         get_widget!(builder, libhandy::Window, add_dialog);
 
+        let (sender, r) = glib::MainContext::channel(glib::PRIORITY_DEFAULT);
+        let receiver = RefCell::new(Some(r));
+
         let add_account_dialog = Rc::new(Self {
             widget: add_dialog,
             builder,
+            global_sender,
             sender,
+            receiver,
             model: Rc::new(ProvidersModel::new()),
             selected_provider: Rc::new(RefCell::new(None)),
         });
@@ -71,12 +83,22 @@ impl AddAccountDialog {
             get_widget!(self.builder, gtk::Entry, @provider_website_entry).set_text(website);
         }
 
+        get_widget!(self.builder, gtk::Stack, @image_stack).set_visible_child_name("loading");
+        get_widget!(self.builder, gtk::Spinner, @spinner).start();
+
         unsafe {
             // This is safe because of the repr(u32)
             let selected_position: u32 = std::mem::transmute(provider.algorithm());
             get_widget!(self.builder, libhandy::ComboRow, @algorithm_comborow)
                 .set_selected(selected_position);
         }
+        let p = provider.clone();
+        let sender = self.sender.clone();
+        spawn!(async move {
+            if let Ok(file) = p.favicon().await {
+                send!(sender, AddAccountAction::SetIcon(file));
+            }
+        });
 
         get_widget!(self.builder, gtk::Entry, @token_entry)
             .set_property_secondary_icon_sensitive(provider.help_url().is_some());
@@ -131,7 +153,7 @@ impl AddAccountDialog {
                             get_widget!(builder, gtk::Entry, @username_entry).set_text(&username);
                         }
                         if let Some(ref provider) = otpauth.issuer {
-                            let provider = model.find_by_name(provider).unwrap();
+                            let provider = model.find_by_name(provider).    unwrap();
                             dialog.set_provider(provider);
                         }
                     }
@@ -142,6 +164,12 @@ impl AddAccountDialog {
     }
 
     fn setup_widgets(&self, dialog: Rc<Self>) {
+        let receiver = self.receiver.borrow_mut().take().unwrap();
+        receiver.attach(
+            None,
+            clone!(@strong dialog => move |action| dialog.do_action(action)),
+        );
+
         get_widget!(self.builder, gtk::EntryCompletion, provider_completion);
         provider_completion.set_model(Some(&self.model.completion_model()));
 
@@ -153,7 +181,7 @@ impl AddAccountDialog {
         algorithm_comborow.set_model(Some(&algorithms_model));
 
         provider_completion.connect_match_selected(
-            clone!(@strong dialog, @strong self.model as model => move |completion, store, iter| {
+            clone!(@strong dialog, @strong self.model as model => move |_, store, iter| {
                 let provider_id = store.get_value(iter, 0). get_some::<i32>().unwrap();
                 let provider = model.find_by_id(provider_id).unwrap();
                 dialog.set_provider(provider);
@@ -163,7 +191,7 @@ impl AddAccountDialog {
         );
 
         get_widget!(self.builder, gtk::Entry, token_entry);
-        token_entry.connect_icon_press(clone!(@strong dialog => move |entry, pos| {
+        token_entry.connect_icon_press(clone!(@strong dialog => move |_, pos| {
             if pos == gtk::EntryIconPosition::Secondary {
                 if let Some(ref provider) = dialog.selected_provider.borrow().clone() {
                    gio::AppInfo::launch_default_for_uri(&provider.help_url().unwrap(),  None::<&gio::AppLaunchContext>);
@@ -172,4 +200,16 @@ impl AddAccountDialog {
         }));
         get_widget!(self.builder, gtk::SpinButton, @period_spinbutton).set_value(30.0);
     }
+
+    fn do_action(&self, action: AddAccountAction) -> glib::Continue {
+        match action {
+            AddAccountAction::SetIcon(file) => {
+                get_widget!(self.builder, gtk::Image, @image)
+                    .set_from_file(file.get_path().unwrap());
+                get_widget!(self.builder, gtk::Spinner, @spinner).stop();
+                get_widget!(self.builder, gtk::Stack, @image_stack).set_visible_child_name("image");
+            }
+        };
+        glib::Continue(true)
+    }
 }