Disable icon downloads, accept optional query after icon href, format and clippy fixes

This commit is contained in:
Daniel García 2019-01-28 23:58:32 +01:00
parent bef1183c49
commit c0e350b734
No known key found for this signature in database
GPG key ID: FC8A7D14C3CD543A
4 changed files with 44 additions and 40 deletions

View file

@ -43,6 +43,12 @@
## It's recommended to also set 'ROCKET_CLI_COLORS=off' ## It's recommended to also set 'ROCKET_CLI_COLORS=off'
# LOG_FILE=/path/to/log # LOG_FILE=/path/to/log
## Disable icon downloading
## Set to true to disable icon downloading, this would still serve icons from $ICON_CACHE_FOLDER,
## but it won't produce any external network request. Needs to set $ICON_CACHE_TTL to 0,
## otherwise it will delete them and they won't be downloaded again.
# DISABLE_ICON_DOWNLOAD=false
## Controls if new users can register ## Controls if new users can register
# SIGNUPS_ALLOWED=true # SIGNUPS_ALLOWED=true

View file

@ -27,8 +27,8 @@ pub fn routes() -> Vec<Route> {
] ]
} }
const COOKIE_NAME: &'static str = "BWRS_ADMIN"; const COOKIE_NAME: &str = "BWRS_ADMIN";
const ADMIN_PATH: &'static str = "/admin"; const ADMIN_PATH: &str = "/admin";
#[derive(Serialize)] #[derive(Serialize)]
struct AdminTemplateData { struct AdminTemplateData {

View file

@ -1,32 +1,19 @@
use std::fs::{create_dir_all, remove_file, symlink_metadata, File}; use std::fs::{create_dir_all, remove_file, symlink_metadata, File};
use std::io::prelude::*; use std::io::prelude::*;
use std::time::SystemTime; use std::time::{Duration, SystemTime};
use rocket::http::ContentType; use rocket::http::ContentType;
use rocket::response::Content; use rocket::response::Content;
use rocket::Route; use rocket::Route;
use reqwest; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, ACCEPT_LANGUAGE, CACHE_CONTROL, PRAGMA, USER_AGENT};
use reqwest::Client; use reqwest::Client;
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT, ACCEPT_LANGUAGE, CACHE_CONTROL, PRAGMA, ACCEPT};
use std::time::Duration;
use crate::error::Error;
//use std::error::Error as StdError;
use crate::CONFIG;
//extern crate regex;
use regex::Regex; use regex::Regex;
//extern crate soup;
use soup::prelude::*; use soup::prelude::*;
use std::vec::Vec; use crate::error::Error;
#[derive(Debug)] use crate::CONFIG;
struct IconList {
priority: u8,
href: String,
}
pub fn routes() -> Vec<Route> { pub fn routes() -> Vec<Route> {
routes![icon] routes![icon]
@ -55,6 +42,10 @@ fn get_icon(domain: &str) -> Vec<u8> {
return icon; return icon;
} }
if CONFIG.disable_icon_download() {
return FALLBACK_ICON.to_vec();
}
// Get the icon, or fallback in case of error // Get the icon, or fallback in case of error
match download_icon(&domain) { match download_icon(&domain) {
Ok(icon) => { Ok(icon) => {
@ -129,6 +120,12 @@ fn icon_is_expired(path: &str) -> bool {
expired.unwrap_or(true) expired.unwrap_or(true)
} }
#[derive(Debug)]
struct IconList {
priority: u8,
href: String,
}
/// Returns a Result with a String which holds the preferend favicon location. /// Returns a Result with a String which holds the preferend favicon location.
/// There will always be a result with a string which will contain https://example.com/favicon.ico /// There will always be a result with a string which will contain https://example.com/favicon.ico
/// This does not mean that that location does exists, but it is the default location. /// This does not mean that that location does exists, but it is the default location.
@ -165,40 +162,39 @@ fn get_icon_url(domain: &str) -> Result<String, Error> {
let mut iconlist: Vec<IconList> = Vec::new(); let mut iconlist: Vec<IconList> = Vec::new();
let resp = client.get(&ssldomain).send().or_else(|_| client.get(&httpdomain).send()); let resp = client.get(&ssldomain).send().or_else(|_| client.get(&httpdomain).send());
if let Ok(mut content) = resp { if let Ok(content) = resp {
let body = content.text().unwrap(); // Extract the URL from the respose in case redirects occured (like @ gitlab.com)
// Extract the URL from te respose incase redirects occured (like @ gitlab.com) let url = content.url().origin().ascii_serialization();
let url = format!("{}://{}", content.url().scheme(), content.url().host().unwrap());
// Add the default favicon.ico to the list with the domain the content responded from. // Add the default favicon.ico to the list with the domain the content responded from.
iconlist.push(IconList { priority: 35, href: format!("{}{}", url, "/favicon.ico") }); iconlist.push(IconList { priority: 35, href: format!("{}/favicon.ico", url) });
let soup = Soup::new(&body); let soup = Soup::from_reader(content)?;
// Search for and filter // Search for and filter
let favicons = soup let favicons = soup
.tag("link") .tag("link")
.attr("rel", Regex::new(r"icon$|apple.*icon")?) // Only use icon rels .attr("rel", Regex::new(r"icon$|apple.*icon")?) // Only use icon rels
.attr("href", Regex::new(r"(?i)\w+(\.jp(e){0,1}g$|\.png$|\.ico$)")?) // Only allow specific extensions .attr("href", Regex::new(r"(?i)\w+\.(jpg|jpeg|png|ico)(\?.*)?$")?) // Only allow specific extensions
.find_all(); .find_all();
// Loop through all the found icons and determine it's priority // Loop through all the found icons and determine it's priority
for favicon in favicons { for favicon in favicons {
let favicon_sizes = favicon.get("sizes").unwrap_or("".to_string()).to_string(); let favicon_sizes = favicon.get("sizes").unwrap_or_default();
let favicon_href = fix_href(&favicon.get("href").unwrap_or("".to_string()).to_string(), &url); let href = fix_href(&favicon.get("href").unwrap_or_default(), &url);
let favicon_priority = get_icon_priority(&favicon_href, &favicon_sizes); let priority = get_icon_priority(&href, &favicon_sizes);
iconlist.push(IconList { priority: favicon_priority, href: favicon_href}) iconlist.push(IconList { priority, href })
} }
} else { } else {
// Add the default favicon.ico to the list with just the given domain // Add the default favicon.ico to the list with just the given domain
iconlist.push(IconList { priority: 35, href: format!("{}{}", ssldomain, "/favicon.ico") }); iconlist.push(IconList { priority: 35, href: format!("{}/favicon.ico", ssldomain) });
} }
// Sort the iconlist by priority // Sort the iconlist by priority
iconlist.sort_by_key(|x| x.priority); iconlist.sort_by_key(|x| x.priority);
// There always is an icon in the list, so no need to check if it exists, and just return the first one // There always is an icon in the list, so no need to check if it exists, and just return the first one
Ok(format!("{}", &iconlist[0].href)) Ok(iconlist.remove(0).href)
} }
/// Returns a Integer with the priority of the type of the icon which to prefer. /// Returns a Integer with the priority of the type of the icon which to prefer.
@ -215,10 +211,10 @@ fn get_icon_url(domain: &str) -> Result<String, Error> {
/// ``` /// ```
fn get_icon_priority(href: &str, sizes: &str) -> u8 { fn get_icon_priority(href: &str, sizes: &str) -> u8 {
// Check if there is a dimension set // Check if there is a dimension set
if ! sizes.is_empty() { if !sizes.is_empty() {
let dimensions : Vec<&str> = sizes.split("x").collect(); let dimensions: Vec<&str> = sizes.split('x').collect();
let width = dimensions[0].parse::<u16>().unwrap(); let width: u16 = dimensions[0].parse().unwrap();
let height = dimensions[1].parse::<u16>().unwrap(); let height: u16 = dimensions[1].parse().unwrap();
// Only allow square dimensions // Only allow square dimensions
if width == height { if width == height {
@ -270,22 +266,22 @@ fn fix_href(href: &str, url: &str) -> String {
format!("http:{}", href) format!("http:{}", href)
} }
// If the href_output just starts with a single / it does not have the host here at all. // If the href_output just starts with a single / it does not have the host here at all.
} else if ! href.starts_with("http") { } else if !href.starts_with("http") {
if href.starts_with("/") { if href.starts_with('/') {
format!("{}{}", url, href) format!("{}{}", url, href)
} else { } else {
format!("{}/{}", url, href) format!("{}/{}", url, href)
} }
// All seems oke, just return the given href // All seems oke, just return the given href
} else { } else {
format!("{}", href) href.to_string()
} }
} }
fn download_icon(domain: &str) -> Result<Vec<u8>, Error> { fn download_icon(domain: &str) -> Result<Vec<u8>, Error> {
let url = get_icon_url(&domain)?; let url = get_icon_url(&domain)?;
info!("Downloading icon for {} via {}...",domain, url); info!("Downloading icon for {} via {}...", domain, url);
let mut res = reqwest::get(&url)?; let mut res = reqwest::get(&url)?;
res = res.error_for_status()?; res = res.error_for_status()?;

View file

@ -56,6 +56,7 @@ make_config! {
extended_logging: bool, extended_logging: bool,
log_file: Option<String>, log_file: Option<String>,
disable_icon_download: bool,
signups_allowed: bool, signups_allowed: bool,
invitations_allowed: bool, invitations_allowed: bool,
admin_token: Option<String>, admin_token: Option<String>,
@ -166,6 +167,7 @@ impl Config {
extended_logging: get_env_or("EXTENDED_LOGGING", true), extended_logging: get_env_or("EXTENDED_LOGGING", true),
log_file: get_env("LOG_FILE"), log_file: get_env("LOG_FILE"),
disable_icon_download: get_env_or("DISABLE_ICON_DOWNLOAD", false),
signups_allowed: get_env_or("SIGNUPS_ALLOWED", true), signups_allowed: get_env_or("SIGNUPS_ALLOWED", true),
admin_token: get_env("ADMIN_TOKEN"), admin_token: get_env("ADMIN_TOKEN"),
invitations_allowed: get_env_or("INVITATIONS_ALLOWED", true), invitations_allowed: get_env_or("INVITATIONS_ALLOWED", true),