From c56b04fa5ab849e9fb96db5226b7cceeb9691db6 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 30 Jun 2023 18:13:46 +0530 Subject: [PATCH 01/11] feat: download published pow performance analytics --- src/api/v1/mod.rs | 2 + src/api/v1/routes.rs | 3 + src/api/v1/survey.rs | 177 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+) create mode 100644 src/api/v1/survey.rs diff --git a/src/api/v1/mod.rs b/src/api/v1/mod.rs index efac059c..b9b33644 100644 --- a/src/api/v1/mod.rs +++ b/src/api/v1/mod.rs @@ -14,6 +14,7 @@ pub mod meta; pub mod notifications; pub mod pow; mod routes; +pub mod survey; pub use routes::ROUTES; @@ -24,6 +25,7 @@ pub fn services(cfg: &mut ServiceConfig) { account::services(cfg); mcaptcha::services(cfg); notifications::services(cfg); + survey::services(cfg); } #[derive(Deserialize)] diff --git a/src/api/v1/routes.rs b/src/api/v1/routes.rs index 539a685b..6f5d98c6 100644 --- a/src/api/v1/routes.rs +++ b/src/api/v1/routes.rs @@ -11,6 +11,7 @@ use super::mcaptcha::routes::Captcha; use super::meta::routes::Meta; use super::notifications::routes::Notifications; use super::pow::routes::PoW; +use super::survey::routes::Survey; pub const ROUTES: Routes = Routes::new(); @@ -20,6 +21,7 @@ pub struct Routes { pub captcha: Captcha, pub meta: Meta, pub pow: PoW, + pub survey: Survey, pub notifications: Notifications, } @@ -32,6 +34,7 @@ impl Routes { meta: Meta::new(), pow: PoW::new(), notifications: Notifications::new(), + survey: Survey::new(), } } } diff --git a/src/api/v1/survey.rs b/src/api/v1/survey.rs new file mode 100644 index 00000000..f381d769 --- /dev/null +++ b/src/api/v1/survey.rs @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2023 Aravinth Manivannan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +use actix_web::web::ServiceConfig; +use actix_web::{web, HttpResponse, Responder}; +use serde::{Deserialize, Serialize}; + +use crate::errors::*; +use crate::AppData; + +pub fn services(cfg: &mut ServiceConfig) { + cfg.service(download); +} + +pub mod routes { + pub struct Survey { + pub download: &'static str, + } + + impl Survey { + pub const fn new() -> Self { + Self { + download: "/api/v1/survey/{survey_id}/get", + } + } + + pub fn get_download_route(&self, survey_id: &str, page: usize) -> String { + format!( + "{}?page={}", + self.download.replace("{survey_id}", survey_id), + page + ) + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] +pub struct Page { + pub page: usize, +} + +/// emits build details of the bninary +#[my_codegen::get(path = "crate::V1_API_ROUTES.survey.download")] +async fn download( + data: AppData, + page: web::Query, + psuedo_id: web::Path, +) -> ServiceResult { + const LIMIT: usize = 50; + let offset = LIMIT as isize * ((page.page as isize) - 1); + let offset = if offset < 0 { 0 } else { offset }; + let psuedo_id = psuedo_id.into_inner(); + let campaign_id = data + .db + .analytics_get_capmaign_id_from_psuedo_id(&psuedo_id.to_string()) + .await?; + let data = data + .db + .analytics_fetch(&campaign_id, LIMIT, offset as usize) + .await?; + Ok(HttpResponse::Ok().json(data)) +} + +#[cfg(test)] +pub mod tests { + use actix_web::{http::StatusCode, test, App}; + + use crate::tests::*; + use crate::*; + + #[actix_rt::test] + async fn survey_works_pg() { + let data = crate::tests::pg::get_data().await; + survey_works(data).await; + } + + #[actix_rt::test] + async fn survey_works_maria() { + let data = crate::tests::maria::get_data().await; + survey_works(data).await; + } + + pub async fn survey_works(data: ArcData) { + const NAME: &str = "survetuseranalytics"; + const PASSWORD: &str = "longpassworddomain"; + const EMAIL: &str = "survetuseranalytics@a.com"; + let data = &data; + + delete_user(data, NAME).await; + + register_and_signin(data, NAME, EMAIL, PASSWORD).await; + // create captcha + let (_, _signin_resp, key) = add_levels_util(data, NAME, PASSWORD).await; + let app = get_app!(data).await; + + let page = 1; + let tmp_id = uuid::Uuid::new_v4(); + let download_rotue = V1_API_ROUTES + .survey + .get_download_route(&tmp_id.to_string(), page); + + let download_req = test::call_service( + &app, + test::TestRequest::get().uri(&download_rotue).to_request(), + ) + .await; + assert_eq!(download_req.status(), StatusCode::NOT_FOUND); + + data.db + .analytics_create_psuedo_id_if_not_exists(&key.key) + .await + .unwrap(); + + let psuedo_id = data + .db + .analytics_get_psuedo_id_from_capmaign_id(&key.key) + .await + .unwrap(); + + for i in 0..60 { + println!("[{i}] Saving analytics"); + let analytics = db_core::CreatePerformanceAnalytics { + time: 0, + difficulty_factor: 0, + worker_type: "wasm".into(), + }; + data.db.analysis_save(&key.key, &analytics).await.unwrap(); + } + + for p in 1..3 { + let download_rotue = V1_API_ROUTES.survey.get_download_route(&psuedo_id, p); + println!("page={p}, download={download_rotue}"); + + let download_req = test::call_service( + &app, + test::TestRequest::get().uri(&download_rotue).to_request(), + ) + .await; + assert_eq!(download_req.status(), StatusCode::OK); + let analytics: Vec = + test::read_body_json(download_req).await; + if p == 1 { + assert_eq!(analytics.len(), 50); + } else if p == 2 { + assert_eq!(analytics.len(), 10); + } else { + assert_eq!(analytics.len(), 0); + } + } + + let download_rotue = V1_API_ROUTES.survey.get_download_route(&psuedo_id, 0); + data.db + .analytics_delete_all_records_for_campaign(&key.key) + .await + .unwrap(); + + let download_req = test::call_service( + &app, + test::TestRequest::get().uri(&download_rotue).to_request(), + ) + .await; + assert_eq!(download_req.status(), StatusCode::NOT_FOUND); + } +} From b6a670544919935691eb713119a2772b53cfe17f Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Tue, 17 Oct 2023 19:09:22 +0530 Subject: [PATCH 02/11] feat: read survey uploader's settings --- Cargo.lock | 172 ++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + config/default.toml | 4 ++ src/settings.rs | 7 ++ 4 files changed, 184 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 63a20634..b04f7153 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,6 +439,19 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "async-compression" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.53" @@ -1532,6 +1545,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.8.0" @@ -1559,6 +1583,43 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1651,6 +1712,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ipnet" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" + [[package]] name = "is-terminal" version = "0.4.9" @@ -1938,6 +2005,7 @@ dependencies = [ "openssl", "pretty_env_logger 0.4.0", "rand", + "reqwest", "rust-embed", "sailfish", "serde", @@ -2639,6 +2707,46 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +[[package]] +name = "reqwest" +version = "0.11.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +dependencies = [ + "async-compression", + "base64 0.21.2", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -3369,6 +3477,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.6.0" @@ -3572,6 +3701,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -3605,6 +3740,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.16.0" @@ -3779,6 +3920,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3810,6 +3960,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.84" @@ -4027,6 +4189,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index 66efa6c5..65533dbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,7 @@ lettre = { version = "0.10.0-rc.3", features = [ openssl = { version = "0.10.48", features = ["vendored"] } uuid = { version = "1.4.0", features = ["v4", "serde"] } +reqwest = { version = "0.11.18", features = ["json", "gzip"] } [dependencies.db-core] diff --git a/config/default.toml b/config/default.toml index fb06e174..0812afa6 100644 --- a/config/default.toml +++ b/config/default.toml @@ -66,3 +66,7 @@ url = "127.0.0.1" port = 10025 username = "admin" password = "password" + +[survey] +nodes = ["http://localhost:7001"] +rate_limit = 3600 # upload every hour diff --git a/src/settings.rs b/src/settings.rs index 4f467df4..d7194d8a 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -91,6 +91,12 @@ pub struct Redis { pub pool: u32, } +#[derive(Debug, Clone, Deserialize, Eq, PartialEq)] +pub struct Survey { + pub nodes: Vec, + pub rate_limit: u64, +} + #[derive(Debug, Clone, Deserialize, Eq, PartialEq)] pub struct Settings { pub debug: bool, @@ -99,6 +105,7 @@ pub struct Settings { pub allow_registration: bool, pub allow_demo: bool, pub database: Database, + pub survey: Survey, pub redis: Option, pub server: Server, pub captcha: Captcha, From 52c2c6e59879f8a069c198117695b35b8066890c Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Tue, 17 Oct 2023 19:09:36 +0530 Subject: [PATCH 03/11] feat: bootstrap survey uploader's endpoints --- src/api/v1/survey.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/v1/survey.rs b/src/api/v1/survey.rs index f381d769..bf16fe10 100644 --- a/src/api/v1/survey.rs +++ b/src/api/v1/survey.rs @@ -28,12 +28,14 @@ pub fn services(cfg: &mut ServiceConfig) { pub mod routes { pub struct Survey { pub download: &'static str, + pub secret: &'static str, } impl Survey { pub const fn new() -> Self { Self { download: "/api/v1/survey/{survey_id}/get", + secret: "/api/v1/survey/{survey_id}/secret", } } From 87785b38be304c4afe8f8dcc5d5fd09c5959250c Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Tue, 17 Oct 2023 19:09:48 +0530 Subject: [PATCH 04/11] feat: bootstrap survey upload job runner --- src/data.rs | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/src/data.rs b/src/data.rs index d9b2b1b1..98779937 100644 --- a/src/data.rs +++ b/src/data.rs @@ -4,8 +4,10 @@ // SPDX-License-Identifier: AGPL-3.0-or-later //! App data: redis cache, database connections, etc. -use std::sync::Arc; +use std::sync::{RwLock, Arc}; +use std::collections::HashMap; use std::thread; +use std::time::Duration; use actix::prelude::*; use argon2_creds::{Config, ConfigBuilder, PasswordPolicy}; @@ -28,7 +30,12 @@ use libmcaptcha::{ pow::Work, system::{System, SystemBuilder}, }; +use reqwest::Client; +use serde::{Serialize, Deserialize}; +use tokio::task::JoinHandle; +use tokio::time::sleep; +use crate::AppData; use crate::db::{self, BoxDB}; use crate::errors::ServiceResult; use crate::settings::Settings; @@ -242,6 +249,95 @@ impl Data { None } } + + async fn upload_survey_job(&self) -> ServiceResult<()> { + unimplemented!() + } + async fn register_survey(&self) -> ServiceResult<()> { + unimplemented!() + } +} + +#[async_trait::async_trait] +trait SurveyClientTrait { + async fn start_job(&self, data: AppData) -> ServiceResult>; + async fn register(&self, data: &AppData) -> ServiceResult<()>; +} + +#[derive(Clone, Debug, Default)] +struct SecretsStore { + store: Arc>> +} + +impl SecretsStore { + fn get(&self, key: &str) -> Option { + let r = self.store.read().unwrap(); + r.get(key).map(|x| x.to_owned()) + } + + fn set(&self, key: String, value: String) { + let mut w = self.store.write().unwrap(); + w.insert(key,value ); + drop(w); + } +} + + + +struct Survey { + settings: Settings, + client: Client, + secrets: SecretsStore, +} +impl Survey { + fn new(settings: Settings) -> Self { + Survey { + client: Client::new(), + settings, + secrets: SecretsStore::default(), + } + } +} + +#[async_trait::async_trait] +impl SurveyClientTrait for Survey { + async fn start_job(&self, data: AppData) -> ServiceResult> { + let fut = async move { + loop { + sleep(Duration::new(data.settings.survey.rate_limit, 0)).await; +// if let Err(e) = Self::delete_demo_user(&data).await { +// log::error!("Error while deleting demo user: {:?}", e); +// } +// if let Err(e) = Self::register_demo_user(&data).await { +// log::error!("Error while registering demo user: {:?}", e); +// } + } + }; + let handle = tokio::spawn(fut); + Ok(handle) + + } + async fn register(&self, data: &AppData) -> ServiceResult<()> { + let protocol = if self.settings.server.proxy_has_tls { + "https://" + } else { + "http://" + }; + #[derive(Serialize)] + struct MCaptchaInstance { + url: url::Url, + } + + let payload = MCaptchaInstance { + url: url::Url::parse(&format!("{protocol}{}", self.settings.server.domain))?, + }; + for url in self.settings.survey.nodes.iter() { + self.client.post(url.clone()).json(&payload).send().await.unwrap(); + } + Ok(()) + } + + } /// Mailer data type AsyncSmtpTransport From f933a30e7eb7d6cf6b011cbe59ff326316872b82 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Wed, 18 Oct 2023 12:28:02 +0530 Subject: [PATCH 05/11] feat: load survey keystore --- src/data.rs | 94 ++++-------------------------------- src/main.rs | 4 +- src/survey.rs | 121 +++++++++++++++++++++++++++++++++++++++++++++++ src/tests/mod.rs | 27 ++++++++++- 4 files changed, 157 insertions(+), 89 deletions(-) create mode 100644 src/survey.rs diff --git a/src/data.rs b/src/data.rs index 98779937..23cf1896 100644 --- a/src/data.rs +++ b/src/data.rs @@ -4,8 +4,8 @@ // SPDX-License-Identifier: AGPL-3.0-or-later //! App data: redis cache, database connections, etc. -use std::sync::{RwLock, Arc}; use std::collections::HashMap; +use std::sync::Arc; use std::thread; use std::time::Duration; @@ -31,15 +31,16 @@ use libmcaptcha::{ system::{System, SystemBuilder}, }; use reqwest::Client; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use tokio::task::JoinHandle; use tokio::time::sleep; -use crate::AppData; use crate::db::{self, BoxDB}; use crate::errors::ServiceResult; use crate::settings::Settings; use crate::stats::{Dummy, Real, Stats}; +use crate::survey::SecretsStore; +use crate::AppData; macro_rules! enum_system_actor { ($name:ident, $type:ident) => { @@ -173,6 +174,8 @@ pub struct Data { pub settings: Settings, /// stats recorder pub stats: Box, + /// survey secret store + pub survey_secrets: SecretsStore, } impl Data { @@ -187,7 +190,7 @@ impl Data { } #[cfg(not(tarpaulin_include))] /// create new instance of app data - pub async fn new(s: &Settings) -> Arc { + pub async fn new(s: &Settings, survey_secrets: SecretsStore) -> Arc { let creds = Self::get_creds(); let c = creds.clone(); @@ -216,6 +219,7 @@ impl Data { mailer: Self::get_mailer(s), settings: s.clone(), stats, + survey_secrets, }; #[cfg(not(debug_assertions))] @@ -258,87 +262,5 @@ impl Data { } } -#[async_trait::async_trait] -trait SurveyClientTrait { - async fn start_job(&self, data: AppData) -> ServiceResult>; - async fn register(&self, data: &AppData) -> ServiceResult<()>; -} - -#[derive(Clone, Debug, Default)] -struct SecretsStore { - store: Arc>> -} - -impl SecretsStore { - fn get(&self, key: &str) -> Option { - let r = self.store.read().unwrap(); - r.get(key).map(|x| x.to_owned()) - } - - fn set(&self, key: String, value: String) { - let mut w = self.store.write().unwrap(); - w.insert(key,value ); - drop(w); - } -} - - - -struct Survey { - settings: Settings, - client: Client, - secrets: SecretsStore, -} -impl Survey { - fn new(settings: Settings) -> Self { - Survey { - client: Client::new(), - settings, - secrets: SecretsStore::default(), - } - } -} - -#[async_trait::async_trait] -impl SurveyClientTrait for Survey { - async fn start_job(&self, data: AppData) -> ServiceResult> { - let fut = async move { - loop { - sleep(Duration::new(data.settings.survey.rate_limit, 0)).await; -// if let Err(e) = Self::delete_demo_user(&data).await { -// log::error!("Error while deleting demo user: {:?}", e); -// } -// if let Err(e) = Self::register_demo_user(&data).await { -// log::error!("Error while registering demo user: {:?}", e); -// } - } - }; - let handle = tokio::spawn(fut); - Ok(handle) - - } - async fn register(&self, data: &AppData) -> ServiceResult<()> { - let protocol = if self.settings.server.proxy_has_tls { - "https://" - } else { - "http://" - }; - #[derive(Serialize)] - struct MCaptchaInstance { - url: url::Url, - } - - let payload = MCaptchaInstance { - url: url::Url::parse(&format!("{protocol}{}", self.settings.server.domain))?, - }; - for url in self.settings.survey.nodes.iter() { - self.client.post(url.clone()).json(&payload).send().await.unwrap(); - } - Ok(()) - } - - -} - /// Mailer data type AsyncSmtpTransport pub type Mailer = AsyncSmtpTransport; diff --git a/src/main.rs b/src/main.rs index 2c44c8fd..55650eda 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,7 @@ mod routes; mod settings; mod static_assets; mod stats; +mod survey; #[cfg(test)] #[macro_use] mod tests; @@ -104,7 +105,8 @@ async fn main() -> std::io::Result<()> { ); let settings = Settings::new().unwrap(); - let data = Data::new(&settings).await; + let secrets = survey::SecretsStore::default(); + let data = Data::new(&settings, secrets).await; let data = actix_web::web::Data::new(data); let mut demo_user: Option = None; diff --git a/src/survey.rs b/src/survey.rs new file mode 100644 index 00000000..243d7ce9 --- /dev/null +++ b/src/survey.rs @@ -0,0 +1,121 @@ +// Copyright (C) 2022 Aravinth Manivannan +// SPDX-FileCopyrightText: 2023 Aravinth Manivannan +// +// SPDX-License-Identifier: AGPL-3.0-or-later +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::RwLock; +use std::time::Duration; + +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use tokio::task::JoinHandle; +use tokio::time::sleep; + +use crate::errors::*; +use crate::settings::Settings; +use crate::AppData; + +#[async_trait::async_trait] +trait SurveyClientTrait { + async fn start_job(&self, data: AppData) -> ServiceResult>; + async fn register(&self, data: &AppData) -> ServiceResult<()>; +} + +#[derive(Clone, Debug, Default)] +pub struct SecretsStore { + store: Arc>>, +} + +impl SecretsStore { + pub fn get(&self, key: &str) -> Option { + let r = self.store.read().unwrap(); + r.get(key).map(|x| x.to_owned()) + } + + pub fn rm(&self, key: &str) { + let mut w = self.store.write().unwrap(); + w.remove(key); + drop(w); + } + + pub fn set(&self, key: String, value: String) { + let mut w = self.store.write().unwrap(); + w.insert(key, value); + drop(w); + } +} + +struct Survey { + settings: Settings, + client: Client, + secrets: SecretsStore, +} +impl Survey { + fn new(settings: Settings, secrets: SecretsStore) -> Self { + Survey { + client: Client::new(), + settings, + secrets, + } + } +} + +#[async_trait::async_trait] +impl SurveyClientTrait for Survey { + async fn start_job(&self, data: AppData) -> ServiceResult> { + let fut = async move { + loop { + sleep(Duration::new(data.settings.survey.rate_limit, 0)).await; + // if let Err(e) = Self::delete_demo_user(&data).await { + // log::error!("Error while deleting demo user: {:?}", e); + // } + // if let Err(e) = Self::register_demo_user(&data).await { + // log::error!("Error while registering demo user: {:?}", e); + // } + } + }; + let handle = tokio::spawn(fut); + Ok(handle) + } + async fn register(&self, data: &AppData) -> ServiceResult<()> { + let protocol = if self.settings.server.proxy_has_tls { + "https://" + } else { + "http://" + }; + #[derive(Serialize)] + struct MCaptchaInstance { + url: url::Url, + secret: String, + } + + let this_instance_url = + url::Url::parse(&format!("{protocol}{}", self.settings.server.domain))?; + for url in self.settings.survey.nodes.iter() { + // mCaptcha/survey must send this token while uploading secret to authenticate itself + // this token must be sent to mCaptcha/survey with the registration payload + let secret_upload_auth_token = crate::api::v1::mcaptcha::get_random(20); + + let payload = MCaptchaInstance { + url: this_instance_url.clone(), + secret: secret_upload_auth_token.clone(), + }; + + // SecretsStore will store auth tokens generated by both mCaptcha/mCaptcha and + // mCaptcha/survey + // + // Storage schema: + // - mCaptcha/mCaptcha generated auth token: (, ) + // - mCaptcha/survey generated auth token (, Settings { @@ -30,6 +31,7 @@ pub mod pg { use crate::data::Data; use crate::settings::*; + use crate::survey::SecretsStore; use crate::ArcData; use super::get_settings; @@ -42,7 +44,7 @@ pub mod pg { settings.database.database_type = DBType::Postgres; settings.database.pool = 2; - Data::new(&settings).await + Data::new(&settings, SecretsStore::default()).await } } pub mod maria { @@ -50,6 +52,7 @@ pub mod maria { use crate::data::Data; use crate::settings::*; + use crate::survey::SecretsStore; use crate::ArcData; use super::get_settings; @@ -62,7 +65,7 @@ pub mod maria { settings.database.database_type = DBType::Maria; settings.database.pool = 2; - Data::new(&settings).await + Data::new(&settings, SecretsStore::default()).await } } //pub async fn get_data() -> ArcData { @@ -181,6 +184,26 @@ pub async fn signin( (creds, signin_resp) } +/// pub duplicate test +pub async fn bad_post_req_test_no_auth( + data: &ArcData, + url: &str, + payload: &T, + err: ServiceError, +) { + let app = get_app!(data).await; + + let resp = test::call_service(&app, post_request!(&payload, url).to_request()).await; + if resp.status() != err.status_code() { + let resp_err: ErrorToResponse = test::read_body_json(resp).await; + panic!("error {}", resp_err.error); + } + assert_eq!(resp.status(), err.status_code()); + let resp_err: ErrorToResponse = test::read_body_json(resp).await; + //println!("{}", txt.error); + assert_eq!(resp_err.error, format!("{}", err)); +} + /// pub duplicate test pub async fn bad_post_req_test( data: &ArcData, From d5617c7ec7b34368c4ba78b62964749f19808d99 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Wed, 18 Oct 2023 12:28:09 +0530 Subject: [PATCH 06/11] feat: upload secret route --- src/api/v1/survey.rs | 78 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/src/api/v1/survey.rs b/src/api/v1/survey.rs index bf16fe10..c751bd6a 100644 --- a/src/api/v1/survey.rs +++ b/src/api/v1/survey.rs @@ -23,6 +23,7 @@ use crate::AppData; pub fn services(cfg: &mut ServiceConfig) { cfg.service(download); + cfg.service(secret); } pub mod routes { @@ -35,7 +36,7 @@ pub mod routes { pub const fn new() -> Self { Self { download: "/api/v1/survey/{survey_id}/get", - secret: "/api/v1/survey/{survey_id}/secret", + secret: "/api/v1/survey/secret", } } @@ -76,25 +77,100 @@ async fn download( Ok(HttpResponse::Ok().json(data)) } +#[derive(Serialize, Deserialize)] +struct SurveySecretUpload { + secret: String, + auth_token: String, +} + +/// mCaptcha/survey upload secret route +#[my_codegen::post(path = "crate::V1_API_ROUTES.survey.secret")] +async fn secret( + data: AppData, + payload: web::Json, +) -> ServiceResult { + match data.survey_secrets.get(&payload.auth_token) { + Some(survey_instance_url) => { + let payload = payload.into_inner(); + data.survey_secrets.set(survey_instance_url, payload.secret); + data.survey_secrets.rm(&payload.auth_token); + Ok(HttpResponse::Ok()) + } + None => Err(ServiceError::WrongPassword), + } +} + #[cfg(test)] pub mod tests { use actix_web::{http::StatusCode, test, App}; + use super::*; + use crate::api::v1::mcaptcha::get_random; use crate::tests::*; use crate::*; #[actix_rt::test] async fn survey_works_pg() { let data = crate::tests::pg::get_data().await; + survey_registration_works(data.clone()).await; survey_works(data).await; } #[actix_rt::test] async fn survey_works_maria() { let data = crate::tests::maria::get_data().await; + survey_registration_works(data.clone()).await; survey_works(data).await; } + pub async fn survey_registration_works(data: ArcData) { + let data = &data; + let app = get_app!(data).await; + + let survey_instance_url = "http://survey_registration_works.survey.example.org"; + + let key = get_random(20); + + let msg = SurveySecretUpload { + auth_token: key.clone(), + secret: get_random(32), + }; + + // should fail with ServiceError::WrongPassword since auth token is not loaded into + // keystore + bad_post_req_test_no_auth( + data, + V1_API_ROUTES.survey.secret, + &msg, + errors::ServiceError::WrongPassword, + ) + .await; + + // load auth token into key store, should succeed + data.survey_secrets + .set(key.clone(), survey_instance_url.to_owned()); + let resp = test::call_service( + &app, + post_request!(&msg, V1_API_ROUTES.survey.secret).to_request(), + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + // uploaded secret must be in keystore + assert_eq!( + data.survey_secrets.get(survey_instance_url).unwrap(), + msg.secret + ); + + // should fail since mCaptcha/survey secret upload auth tokens are single-use + bad_post_req_test_no_auth( + data, + V1_API_ROUTES.survey.secret, + &msg, + errors::ServiceError::WrongPassword, + ) + .await; + } + pub async fn survey_works(data: ArcData) { const NAME: &str = "survetuseranalytics"; const PASSWORD: &str = "longpassworddomain"; From d4534c1c43e649c7fc87a0caf358ef9d6c4d2631 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 00:18:29 +0530 Subject: [PATCH 07/11] feat: define db method to get all psuedo IDs with pagination --- db/db-core/src/lib.rs | 3 ++ db/db-core/src/tests.rs | 7 ++++ ...0a7d88d8e7a7558775e238fe009f478003e46.json | 25 ++++++++++++++ db/db-sqlx-maria/src/lib.rs | 32 +++++++++++++++--- ...a0d6363939bdd87739b4292dd1d88e03ce6f7.json | 23 +++++++++++++ db/db-sqlx-postgres/src/lib.rs | 33 ++++++++++++++++--- 6 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 db/db-sqlx-maria/.sqlx/query-e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46.json create mode 100644 db/db-sqlx-postgres/.sqlx/query-d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7.json diff --git a/db/db-core/src/lib.rs b/db/db-core/src/lib.rs index 67c254da..a5fc7b20 100644 --- a/db/db-core/src/lib.rs +++ b/db/db-core/src/lib.rs @@ -289,6 +289,9 @@ pub trait MCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase { Err(e) => Err(e), } } + + /// Get all psuedo IDs + async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult>; } #[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] diff --git a/db/db-core/src/tests.rs b/db/db-core/src/tests.rs index 283e47b6..675e25d7 100644 --- a/db/db-core/src/tests.rs +++ b/db/db-core/src/tests.rs @@ -258,6 +258,12 @@ pub async fn database_works<'a, T: MCDatabase>( .analytics_get_psuedo_id_from_capmaign_id(c.key) .await .unwrap(); + assert_eq!( + vec![psuedo_id.clone()], + db.analytics_get_all_psuedo_ids(0).await.unwrap() + ); + assert!(db.analytics_get_all_psuedo_ids(1).await.unwrap().is_empty()); + db.analytics_create_psuedo_id_if_not_exists(c.key) .await .unwrap(); @@ -267,6 +273,7 @@ pub async fn database_works<'a, T: MCDatabase>( .await .unwrap() ); + assert_eq!( c.key, db.analytics_get_capmaign_id_from_psuedo_id(&psuedo_id) diff --git a/db/db-sqlx-maria/.sqlx/query-e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46.json b/db/db-sqlx-maria/.sqlx/query-e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46.json new file mode 100644 index 00000000..ec1265e6 --- /dev/null +++ b/db/db-sqlx-maria/.sqlx/query-e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46.json @@ -0,0 +1,25 @@ +{ + "db_name": "MySQL", + "query": "\n SELECT\n psuedo_id\n FROM\n mcaptcha_psuedo_campaign_id\n ORDER BY ID ASC LIMIT ? OFFSET ?;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "psuedo_id", + "type_info": { + "type": "VarString", + "flags": "NOT_NULL | UNIQUE_KEY | NO_DEFAULT_VALUE", + "char_set": 224, + "max_size": 400 + } + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false + ] + }, + "hash": "e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46" +} diff --git a/db/db-sqlx-maria/src/lib.rs b/db/db-sqlx-maria/src/lib.rs index 94c359a3..f0d12bc5 100644 --- a/db/db-sqlx-maria/src/lib.rs +++ b/db/db-sqlx-maria/src/lib.rs @@ -987,12 +987,8 @@ impl MCDatabase for Database { &self, captcha_id: &str, ) -> DBResult { - struct ID { - psuedo_id: String, - } - let res = sqlx::query_as!( - ID, + PsuedoID, "SELECT psuedo_id FROM mcaptcha_psuedo_campaign_id WHERE @@ -1069,6 +1065,28 @@ impl MCDatabase for Database { Ok(()) } + /// Get all psuedo IDs + async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult> { + const LIMIT: usize = 50; + let offset = LIMIT * page; + + let mut res = sqlx::query_as!( + PsuedoID, + " + SELECT + psuedo_id + FROM + mcaptcha_psuedo_campaign_id + ORDER BY ID ASC LIMIT ? OFFSET ?;", + LIMIT as i64, + offset as i64 + ) + .fetch_all(&self.pool) + .await + .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?; + + Ok(res.drain(0..).map(|r| r.psuedo_id).collect()) + } } #[derive(Clone)] @@ -1134,3 +1152,7 @@ impl From for Captcha { } } } + +struct PsuedoID { + psuedo_id: String, +} diff --git a/db/db-sqlx-postgres/.sqlx/query-d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7.json b/db/db-sqlx-postgres/.sqlx/query-d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7.json new file mode 100644 index 00000000..66def49c --- /dev/null +++ b/db/db-sqlx-postgres/.sqlx/query-d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n psuedo_id\n FROM\n mcaptcha_psuedo_campaign_id\n ORDER BY ID ASC LIMIT $1 OFFSET $2;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "psuedo_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7" +} diff --git a/db/db-sqlx-postgres/src/lib.rs b/db/db-sqlx-postgres/src/lib.rs index d06dfeeb..d0b2eb7b 100644 --- a/db/db-sqlx-postgres/src/lib.rs +++ b/db/db-sqlx-postgres/src/lib.rs @@ -994,12 +994,8 @@ impl MCDatabase for Database { &self, captcha_id: &str, ) -> DBResult { - struct ID { - psuedo_id: String, - } - let res = sqlx::query_as!( - ID, + PsuedoID, "SELECT psuedo_id FROM mcaptcha_psuedo_campaign_id WHERE @@ -1078,6 +1074,29 @@ impl MCDatabase for Database { Ok(()) } + + /// Get all psuedo IDs + async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult> { + const LIMIT: usize = 50; + let offset = LIMIT * page; + + let mut res = sqlx::query_as!( + PsuedoID, + " + SELECT + psuedo_id + FROM + mcaptcha_psuedo_campaign_id + ORDER BY ID ASC LIMIT $1 OFFSET $2;", + LIMIT as i64, + offset as i64 + ) + .fetch_all(&self.pool) + .await + .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?; + + Ok(res.drain(0..).map(|r| r.psuedo_id).collect()) + } } #[derive(Clone)] @@ -1125,6 +1144,10 @@ impl From for Notification { } } +struct PsuedoID { + psuedo_id: String, +} + #[derive(Clone)] struct InternaleCaptchaConfig { config_id: i32, From eab146b12167744b300838067cc84647d7bc8729 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 01:38:22 +0530 Subject: [PATCH 08/11] gc: get public hostname as config parameter --- config/default.toml | 3 ++- src/settings.rs | 50 +++++++++++++++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/config/default.toml b/config/default.toml index 0812afa6..1f69a48b 100644 --- a/config/default.toml +++ b/config/default.toml @@ -69,4 +69,5 @@ password = "password" [survey] nodes = ["http://localhost:7001"] -rate_limit = 3600 # upload every hour +rate_limit = 10 # upload every hour +instance_root_url = "http://localhost:7000" diff --git a/src/settings.rs b/src/settings.rs index d7194d8a..f5107def 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -95,6 +95,7 @@ pub struct Redis { pub struct Survey { pub nodes: Vec, pub rate_limit: u64, + pub instance_root_url: Url, } #[derive(Debug, Clone, Deserialize, Eq, PartialEq)] @@ -105,7 +106,7 @@ pub struct Settings { pub allow_registration: bool, pub allow_demo: bool, pub database: Database, - pub survey: Survey, + pub survey: Option, pub redis: Option, pub server: Server, pub captcha: Captcha, @@ -163,10 +164,9 @@ const ENV_VAR_CONFIG: [(&str, &str); 29] = [ ]; - const DEPRECATED_ENV_VARS: [(&str, &str); 23] = [ - ("debug","MCAPTCHA_DEBUG"), - ("commercial","MCAPTCHA_COMMERCIAL"), + ("debug", "MCAPTCHA_DEBUG"), + ("commercial", "MCAPTCHA_COMMERCIAL"), ("source_code", "MCAPTCHA_SOURCE_CODE"), ("allow_registration", "MCAPTCHA_ALLOW_REGISTRATION"), ("allow_demo", "MCAPTCHA_ALLOW_DEMO"), @@ -179,9 +179,18 @@ const DEPRECATED_ENV_VARS: [(&str, &str); 23] = [ ("server.proxy_has_tls", "MCAPTCHA_SERVER_PROXY_HAS_TLS"), ("captcha.salt", "MCAPTCHA_CAPTCHA_SALT"), ("captcha.gc", "MCAPTCHA_CAPTCHA_GC"), - ("captcha.default_difficulty_strategy.avg_traffic_difficulty", "MCAPTCHA_CAPTCHA_AVG_TRAFFIC_DIFFICULTY"), - ("captcha.default_difficulty_strategy.peak_sustainable_traffic_difficulty", "MCAPTCHA_CAPTCHA_PEAK_TRAFFIC_DIFFICULTY"), - ("captcha.default_difficulty_strategy.broke_my_site_traffic_difficulty", "MCAPTCHA_CAPTCHA_BROKE_MY_SITE_TRAFFIC"), + ( + "captcha.default_difficulty_strategy.avg_traffic_difficulty", + "MCAPTCHA_CAPTCHA_AVG_TRAFFIC_DIFFICULTY", + ), + ( + "captcha.default_difficulty_strategy.peak_sustainable_traffic_difficulty", + "MCAPTCHA_CAPTCHA_PEAK_TRAFFIC_DIFFICULTY", + ), + ( + "captcha.default_difficulty_strategy.broke_my_site_traffic_difficulty", + "MCAPTCHA_CAPTCHA_BROKE_MY_SITE_TRAFFIC", + ), ("smtp.from", "MCAPTCHA_SMTP_FROM"), ("smtp.reply", "MCAPTCHA_SMTP_REPLY_TO"), ("smtp.url", "MCAPTCHA_SMTP_URL"), @@ -244,7 +253,6 @@ impl Settings { } fn env_override(mut s: ConfigBuilder) -> ConfigBuilder { - for (parameter, env_var_name) in DEPRECATED_ENV_VARS.iter() { if let Ok(val) = env::var(env_var_name) { log::warn!( @@ -254,7 +262,6 @@ impl Settings { } } - for (parameter, env_var_name) in ENV_VAR_CONFIG.iter() { if let Ok(val) = env::var(env_var_name) { log::debug!( @@ -284,8 +291,6 @@ mod tests { use super::*; - - #[test] fn deprecated_env_override_works() { use crate::tests::get_settings; @@ -314,7 +319,11 @@ mod tests { /* top level */ helper!("MCAPTCHA_DEBUG", !init_settings.debug, debug); helper!("MCAPTCHA_COMMERCIAL", !init_settings.commercial, commercial); - helper!("MCAPTCHA_ALLOW_REGISTRATION", !init_settings.allow_registration, allow_registration); + helper!( + "MCAPTCHA_ALLOW_REGISTRATION", + !init_settings.allow_registration, + allow_registration + ); helper!("MCAPTCHA_ALLOW_DEMO", !init_settings.allow_demo, allow_demo); /* database_type */ @@ -371,8 +380,20 @@ mod tests { 999, captcha.default_difficulty_strategy.avg_traffic_difficulty ); - helper!("MCAPTCHA_CAPTCHA_PEAK_TRAFFIC_DIFFICULTY", 999 , captcha.default_difficulty_strategy.peak_sustainable_traffic_difficulty); - helper!("MCAPTCHA_CAPTCHA_BROKE_MY_SITE_TRAFFIC", 999 , captcha.default_difficulty_strategy.broke_my_site_traffic_difficulty); + helper!( + "MCAPTCHA_CAPTCHA_PEAK_TRAFFIC_DIFFICULTY", + 999, + captcha + .default_difficulty_strategy + .peak_sustainable_traffic_difficulty + ); + helper!( + "MCAPTCHA_CAPTCHA_BROKE_MY_SITE_TRAFFIC", + 999, + captcha + .default_difficulty_strategy + .broke_my_site_traffic_difficulty + ); /* SMTP */ @@ -407,7 +428,6 @@ mod tests { } } - #[test] fn env_override_works() { use crate::tests::get_settings; From 3d02f552413a556ab9c0b26a004f7b94de624581 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 01:39:19 +0530 Subject: [PATCH 09/11] fix: create psuedo id and setup publishing for those tht have opted in --- src/api/v1/mcaptcha/create.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/api/v1/mcaptcha/create.rs b/src/api/v1/mcaptcha/create.rs index 863b440e..6321f045 100644 --- a/src/api/v1/mcaptcha/create.rs +++ b/src/api/v1/mcaptcha/create.rs @@ -85,10 +85,17 @@ pub mod runner { data.db .add_captcha_levels(username, &key, &payload.levels) .await?; + + if payload.publish_benchmarks { + data.db.analytics_create_psuedo_id_if_not_exists(&key).await?;; + } + + let mcaptcha_config = MCaptchaDetails { name: payload.description.clone(), key, }; + Ok(mcaptcha_config) } } From 74364c4e170cf8a6129fd3df1ea59da164d50fdd Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 01:47:24 +0530 Subject: [PATCH 10/11] chore: lint --- src/api/v1/mcaptcha/create.rs | 5 +++-- src/api/v1/survey.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/api/v1/mcaptcha/create.rs b/src/api/v1/mcaptcha/create.rs index 6321f045..dd7c07cf 100644 --- a/src/api/v1/mcaptcha/create.rs +++ b/src/api/v1/mcaptcha/create.rs @@ -87,10 +87,11 @@ pub mod runner { .await?; if payload.publish_benchmarks { - data.db.analytics_create_psuedo_id_if_not_exists(&key).await?;; + data.db + .analytics_create_psuedo_id_if_not_exists(&key) + .await?; } - let mcaptcha_config = MCaptchaDetails { name: payload.description.clone(), key, diff --git a/src/api/v1/survey.rs b/src/api/v1/survey.rs index c751bd6a..45f1744a 100644 --- a/src/api/v1/survey.rs +++ b/src/api/v1/survey.rs @@ -35,7 +35,7 @@ pub mod routes { impl Survey { pub const fn new() -> Self { Self { - download: "/api/v1/survey/{survey_id}/get", + download: "/api/v1/survey/takeout/{survey_id}/get", secret: "/api/v1/survey/secret", } } From 960283324d3af998da6f98cc05b87732b5d5fb76 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 01:47:32 +0530 Subject: [PATCH 11/11] feat: schedule mCaptcha/survey registration and uploads --- config/default.toml | 8 +-- src/main.rs | 19 +++++- src/survey.rs | 160 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 146 insertions(+), 41 deletions(-) diff --git a/config/default.toml b/config/default.toml index 1f69a48b..72e0f7b9 100644 --- a/config/default.toml +++ b/config/default.toml @@ -67,7 +67,7 @@ port = 10025 username = "admin" password = "password" -[survey] -nodes = ["http://localhost:7001"] -rate_limit = 10 # upload every hour -instance_root_url = "http://localhost:7000" +#[survey] +#nodes = ["http://localhost:7001"] +#rate_limit = 10 # upload every hour +#instance_root_url = "http://localhost:7000" diff --git a/src/main.rs b/src/main.rs index 55650eda..c96b475c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,6 +46,7 @@ use static_assets::FileMap; pub use widget::WIDGET_ROUTES; use crate::demo::DemoUser; +use survey::SurveyClientTrait; lazy_static! { pub static ref SETTINGS: Settings = Settings::new().unwrap(); @@ -106,7 +107,7 @@ async fn main() -> std::io::Result<()> { let settings = Settings::new().unwrap(); let secrets = survey::SecretsStore::default(); - let data = Data::new(&settings, secrets).await; + let data = Data::new(&settings, secrets.clone()).await; let data = actix_web::web::Data::new(data); let mut demo_user: Option = None; @@ -119,6 +120,13 @@ async fn main() -> std::io::Result<()> { ); } + let (mut survey_upload_tx, mut survey_upload_handle) = (None, None); + if settings.survey.is_some() { + let survey_runner_ctx = survey::Survey::new(data.clone()); + let (x, y) = survey_runner_ctx.start_job().await.unwrap(); + (survey_upload_tx, survey_upload_handle) = (Some(x), Some(y)); + } + let ip = settings.server.get_ip(); println!("Starting server on: http://{ip}"); @@ -143,9 +151,18 @@ async fn main() -> std::io::Result<()> { .run() .await?; + if let Some(survey_upload_tx) = survey_upload_tx { + survey_upload_tx.send(()).unwrap(); + } + if let Some(demo_user) = demo_user { demo_user.abort(); } + + if let Some(survey_upload_handle) = survey_upload_handle { + survey_upload_handle.await.unwrap(); + } + Ok(()) } diff --git a/src/survey.rs b/src/survey.rs index 243d7ce9..b2ecfb5e 100644 --- a/src/survey.rs +++ b/src/survey.rs @@ -9,17 +9,21 @@ use std::time::Duration; use reqwest::Client; use serde::{Deserialize, Serialize}; +use tokio::sync::oneshot; use tokio::task::JoinHandle; use tokio::time::sleep; use crate::errors::*; use crate::settings::Settings; use crate::AppData; +use crate::V1_API_ROUTES; #[async_trait::async_trait] -trait SurveyClientTrait { - async fn start_job(&self, data: AppData) -> ServiceResult>; - async fn register(&self, data: &AppData) -> ServiceResult<()>; +pub trait SurveyClientTrait { + async fn start_job(&self) -> ServiceResult<(oneshot::Sender<()>, JoinHandle<()>)>; + async fn schedule_upload_job(&self) -> ServiceResult<()>; + async fn is_online(&self) -> ServiceResult; + async fn register(&self) -> ServiceResult<()>; } #[derive(Clone, Debug, Default)] @@ -46,60 +50,145 @@ impl SecretsStore { } } -struct Survey { - settings: Settings, +#[derive(Clone)] +pub struct Survey { client: Client, - secrets: SecretsStore, + app_ctx: AppData, } impl Survey { - fn new(settings: Settings, secrets: SecretsStore) -> Self { + pub fn new(app_ctx: AppData) -> Self { + if app_ctx.settings.survey.is_none() { + panic!("Survey uploader shouldn't be initialized it isn't configured, please report this bug") + } Survey { client: Client::new(), - settings, - secrets, + app_ctx, } } } #[async_trait::async_trait] impl SurveyClientTrait for Survey { - async fn start_job(&self, data: AppData) -> ServiceResult> { + async fn start_job(&self) -> ServiceResult<(oneshot::Sender<()>, JoinHandle<()>)> { + fn can_run(rx: &mut oneshot::Receiver<()>) -> bool { + match rx.try_recv() { + Err(oneshot::error::TryRecvError::Empty) => true, + _ => false, + } + } + + let (tx, mut rx) = oneshot::channel(); + let this = self.clone(); + let mut register = false; let fut = async move { loop { - sleep(Duration::new(data.settings.survey.rate_limit, 0)).await; - // if let Err(e) = Self::delete_demo_user(&data).await { - // log::error!("Error while deleting demo user: {:?}", e); - // } - // if let Err(e) = Self::register_demo_user(&data).await { - // log::error!("Error while registering demo user: {:?}", e); - // } + if !can_run(&mut rx) { + log::info!("Stopping survey uploads"); + break; + } + + if !register { + loop { + if this.is_online().await.unwrap() { + this.register().await.unwrap(); + register = true; + break; + } else { + sleep(Duration::new(1, 0)).await; + } + } + } + + for i in 0..this.app_ctx.settings.survey.as_ref().unwrap().rate_limit { + if !can_run(&mut rx) { + log::info!("Stopping survey uploads"); + break; + } + sleep(Duration::new(1, 0)).await; + } + let _ = this.schedule_upload_job().await; + + // for url in this.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() { + // if !can_run(&mut rx) { + // log::info!("Stopping survey uploads"); + // break; + // } + // log::info!("Uploading to survey instance {}", url); + // } } }; let handle = tokio::spawn(fut); - Ok(handle) + Ok((tx, handle)) } - async fn register(&self, data: &AppData) -> ServiceResult<()> { - let protocol = if self.settings.server.proxy_has_tls { - "https://" - } else { - "http://" - }; + async fn is_online(&self) -> ServiceResult { + let res = self + .client + .get(format!( + "http://{}{}", + self.app_ctx.settings.server.get_ip(), + V1_API_ROUTES.meta.health + )) + .send() + .await + .unwrap(); + Ok(res.status() == 200) + } + + async fn schedule_upload_job(&self) -> ServiceResult<()> { + log::debug!("Running upload job"); + #[derive(Serialize)] + struct Secret { + secret: String, + } + let mut page = 0; + loop { + let psuedo_ids = self.app_ctx.db.analytics_get_all_psuedo_ids(page).await?; + if psuedo_ids.is_empty() { + log::debug!("upload job complete, no more IDs to upload"); + break; + } + for id in psuedo_ids { + for url in self.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() { + if let Some(secret) = self.app_ctx.survey_secrets.get(url.as_str()) { + let payload = Secret { secret }; + + log::info!("Uploading to survey instance {} campaign {id}", url); + let mut url = url.clone(); + url.set_path(&format!("/mcaptcha/api/v1/{id}/upload")); + let resp = + self.client.post(url).json(&payload).send().await.unwrap(); + println!("{}", resp.text().await.unwrap()); + } + } + } + page += 1; + } + Ok(()) + } + + async fn register(&self) -> ServiceResult<()> { #[derive(Serialize)] struct MCaptchaInstance { url: url::Url, - secret: String, + auth_token: String, } - let this_instance_url = - url::Url::parse(&format!("{protocol}{}", self.settings.server.domain))?; - for url in self.settings.survey.nodes.iter() { + let this_instance_url = self + .app_ctx + .settings + .survey + .as_ref() + .unwrap() + .instance_root_url + .clone(); + for url in self.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() { // mCaptcha/survey must send this token while uploading secret to authenticate itself // this token must be sent to mCaptcha/survey with the registration payload let secret_upload_auth_token = crate::api::v1::mcaptcha::get_random(20); let payload = MCaptchaInstance { url: this_instance_url.clone(), - secret: secret_upload_auth_token.clone(), + auth_token: secret_upload_auth_token.clone(), }; // SecretsStore will store auth tokens generated by both mCaptcha/mCaptcha and @@ -108,13 +197,12 @@ impl SurveyClientTrait for Survey { // Storage schema: // - mCaptcha/mCaptcha generated auth token: (, ) // - mCaptcha/survey generated auth token (,