diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index 8619e3c47b..61e1acdc97 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -383,11 +383,6 @@ impl CommandApi { Ok(BlobObject::create_and_deduplicate(&ctx, file, file)?.to_abs_path()) } - async fn draft_self_report(&self, account_id: u32) -> Result { - let ctx = self.get_context(account_id).await?; - Ok(ctx.draft_self_report().await?.to_u32()) - } - /// Sets the given configuration key. async fn set_config(&self, account_id: u32, key: String, value: Option) -> Result<()> { let ctx = self.get_context(account_id).await?; @@ -880,9 +875,15 @@ impl CommandApi { /// **returns**: The chat ID of the joined chat, the UI may redirect to the this chat. /// A returned chat ID does not guarantee that the chat is protected or the belonging contact is verified. /// - async fn secure_join(&self, account_id: u32, qr: String) -> Result { + async fn secure_join( + &self, + account_id: u32, + qr: String, + source: Option, + uipath: Option, + ) -> Result { let ctx = self.get_context(account_id).await?; - let chat_id = securejoin::join_securejoin(&ctx, &qr).await?; + let chat_id = securejoin::join_securejoin_with_source(&ctx, &qr, source, uipath).await?; Ok(chat_id.to_u32()) } diff --git a/deltachat-jsonrpc/src/api/types/chat.rs b/deltachat-jsonrpc/src/api/types/chat.rs index 777defe763..052fff2385 100644 --- a/deltachat-jsonrpc/src/api/types/chat.rs +++ b/deltachat-jsonrpc/src/api/types/chat.rs @@ -1,7 +1,7 @@ use std::time::{Duration, SystemTime}; -use anyhow::{bail, Context as _, Result}; -use deltachat::chat::{self, get_chat_contacts, get_past_chat_contacts, ChatVisibility}; +use anyhow::{Context as _, Result, bail}; +use deltachat::chat::{self, ChatVisibility, get_chat_contacts, get_past_chat_contacts}; use deltachat::chat::{Chat, ChatId}; use deltachat::constants::Chattype; use deltachat::contact::{Contact, ContactId}; diff --git a/deltachat-jsonrpc/src/api/types/chat_list.rs b/deltachat-jsonrpc/src/api/types/chat_list.rs index b5d31a7913..1a9e851086 100644 --- a/deltachat-jsonrpc/src/api/types/chat_list.rs +++ b/deltachat-jsonrpc/src/api/types/chat_list.rs @@ -4,7 +4,7 @@ use deltachat::chatlist::get_last_message_for_chat; use deltachat::constants::*; use deltachat::contact::{Contact, ContactId}; use deltachat::{ - chat::{get_chat_contacts, ChatVisibility}, + chat::{ChatVisibility, get_chat_contacts}, chatlist::Chatlist, }; use num_traits::cast::ToPrimitive; diff --git a/deltachat-jsonrpc/src/api/types/http.rs b/deltachat-jsonrpc/src/api/types/http.rs index 9121a677ec..d370ba8f7c 100644 --- a/deltachat-jsonrpc/src/api/types/http.rs +++ b/deltachat-jsonrpc/src/api/types/http.rs @@ -16,7 +16,7 @@ pub struct HttpResponse { impl From for HttpResponse { fn from(response: CoreHttpResponse) -> Self { - use base64::{engine::general_purpose, Engine as _}; + use base64::{Engine as _, engine::general_purpose}; let blob = general_purpose::STANDARD_NO_PAD.encode(response.blob); let mimetype = response.mimetype; let encoding = response.encoding; diff --git a/src/config.rs b/src/config.rs index 4cc38285ba..06cc996d8d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -431,10 +431,23 @@ pub enum Config { /// used for signatures, encryption to self and included in `Autocrypt` header. KeyId, + /// Send statistics to Delta Chat's developers. + /// Can be exposed to the user as a setting. + SelfReporting, + + /// Last time statistics were sent to Delta Chat's developers + LastSelfReportSent, + /// This key is sent to the self_reporting bot so that the bot can recognize the user /// without storing the email address SelfReportingId, + /// The last message id that was already included in the previous report, + /// or that already existed before the user opted in. + /// Only messages with an id larger than this + /// will be counted in the next report. + SelfReportingLastMsgId, + /// MsgId of webxdc map integration. WebxdcIntegration, @@ -827,6 +840,10 @@ impl Context { .await?; } } + Config::SelfReporting => { + self.sql.set_raw_config(key.as_ref(), value).await?; + crate::self_reporting::set_last_msgid(self).await?; + } _ => { self.sql.set_raw_config(key.as_ref(), value).await?; } diff --git a/src/context.rs b/src/context.rs index b78a18d493..80eb4f2111 100644 --- a/src/context.rs +++ b/src/context.rs @@ -10,27 +10,22 @@ use std::time::Duration; use anyhow::{Context as _, Result, bail, ensure}; use async_channel::{self as channel, Receiver, Sender}; -use pgp::types::PublicKeyTrait; use ratelimit::Ratelimit; use tokio::sync::{Mutex, Notify, RwLock}; -use crate::chat::{ChatId, ProtectionStatus, get_chat_cnt}; +use crate::chat::{ChatId, get_chat_cnt}; use crate::chatlist_events; use crate::config::Config; -use crate::constants::{ - self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR, -}; -use crate::contact::{Contact, ContactId, import_vcard, mark_contact_id_as_verified}; +use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR}; +use crate::contact::{Contact, ContactId}; use crate::debug_logging::DebugLogging; -use crate::download::DownloadState; use crate::events::{Event, EventEmitter, EventType, Events}; use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{load_self_secret_key, self_fingerprint}; +use crate::key::self_fingerprint; use crate::log::{info, warn}; use crate::logged_debug_assert; use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; -use crate::message::{self, Message, MessageState, MsgId}; -use crate::param::{Param, Params}; +use crate::message::{self, MessageState, MsgId}; use crate::peer_channels::Iroh; use crate::push::PushSubscriber; use crate::quota::QuotaInfo; @@ -38,7 +33,7 @@ use crate::scheduler::{SchedulerState, convert_folder_meaning}; use crate::sql::Sql; use crate::stock_str::StockStrings; use crate::timesmearing::SmearedTimestamp; -use crate::tools::{self, create_id, duration_to_str, time, time_elapsed}; +use crate::tools::{self, duration_to_str, time, time_elapsed}; /// Builder for the [`Context`]. /// @@ -1067,6 +1062,24 @@ impl Context { .await? .unwrap_or_default(), ); + res.insert( + "self_reporting", + self.get_config_bool(Config::SelfReporting) + .await? + .to_string(), + ); + res.insert( + "last_self_report_sent", + self.get_config_i64(Config::LastSelfReportSent) + .await? + .to_string(), + ); + res.insert( + "self_reporting_last_msg_id", + self.get_config_i64(Config::SelfReportingLastMsgId) + .await? + .to_string(), + ); let elapsed = time_elapsed(&self.creation_time); res.insert("uptime", duration_to_str(elapsed)); @@ -1074,152 +1087,6 @@ impl Context { Ok(res) } - async fn get_self_report(&self) -> Result { - #[derive(Default)] - struct ChatNumbers { - protected: u32, - protection_broken: u32, - opportunistic_dc: u32, - opportunistic_mua: u32, - unencrypted_dc: u32, - unencrypted_mua: u32, - } - - let mut res = String::new(); - res += &format!("core_version {}\n", get_version_str()); - - let num_msgs: u32 = self - .sql - .query_get_value( - "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?", - (DC_CHAT_ID_TRASH,), - ) - .await? - .unwrap_or_default(); - res += &format!("num_msgs {num_msgs}\n"); - - let num_chats: u32 = self - .sql - .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ()) - .await? - .unwrap_or_default(); - res += &format!("num_chats {num_chats}\n"); - - let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len(); - res += &format!("db_size_bytes {db_size}\n"); - - let secret_key = &load_self_secret_key(self).await?.primary_key; - let key_created = secret_key.public_key().created_at().timestamp(); - res += &format!("key_created {key_created}\n"); - - // how many of the chats active in the last months are: - // - protected - // - protection-broken - // - opportunistic-encrypted and the contact uses Delta Chat - // - opportunistic-encrypted and the contact uses a classical MUA - // - unencrypted and the contact uses Delta Chat - // - unencrypted and the contact uses a classical MUA - let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3); - let chats = self - .sql - .query_map( - "SELECT c.protected, m.param, m.msgrmsg - FROM chats c - JOIN msgs m - ON c.id=m.chat_id - AND m.id=( - SELECT id - FROM msgs - WHERE chat_id=c.id - AND hidden=0 - AND download_state=? - AND to_id!=? - ORDER BY timestamp DESC, id DESC LIMIT 1) - WHERE c.id>9 - AND (c.blocked=0 OR c.blocked=2) - AND IFNULL(m.timestamp,c.created_timestamp) > ? - GROUP BY c.id", - (DownloadState::Done, ContactId::INFO, three_months_ago), - |row| { - let protected: ProtectionStatus = row.get(0)?; - let message_param: Params = - row.get::<_, String>(1)?.parse().unwrap_or_default(); - let is_dc_message: bool = row.get(2)?; - Ok((protected, message_param, is_dc_message)) - }, - |rows| { - let mut chats = ChatNumbers::default(); - for row in rows { - let (protected, message_param, is_dc_message) = row?; - let encrypted = message_param - .get_bool(Param::GuaranteeE2ee) - .unwrap_or(false); - - if protected == ProtectionStatus::Protected { - chats.protected += 1; - } else if protected == ProtectionStatus::ProtectionBroken { - chats.protection_broken += 1; - } else if encrypted { - if is_dc_message { - chats.opportunistic_dc += 1; - } else { - chats.opportunistic_mua += 1; - } - } else if is_dc_message { - chats.unencrypted_dc += 1; - } else { - chats.unencrypted_mua += 1; - } - } - Ok(chats) - }, - ) - .await?; - res += &format!("chats_protected {}\n", chats.protected); - res += &format!("chats_protection_broken {}\n", chats.protection_broken); - res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc); - res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua); - res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc); - res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua); - - let self_reporting_id = match self.get_config(Config::SelfReportingId).await? { - Some(id) => id, - None => { - let id = create_id(); - self.set_config(Config::SelfReportingId, Some(&id)).await?; - id - } - }; - res += &format!("self_reporting_id {self_reporting_id}"); - - Ok(res) - } - - /// Drafts a message with statistics about the usage of Delta Chat. - /// The user can inspect the message if they want, and then hit "Send". - /// - /// On the other end, a bot will receive the message and make it available - /// to Delta Chat's developers. - pub async fn draft_self_report(&self) -> Result { - const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); - let contact_id: ContactId = *import_vcard(self, SELF_REPORTING_BOT_VCARD) - .await? - .first() - .context("Self reporting bot vCard does not contain a contact")?; - mark_contact_id_as_verified(self, contact_id, ContactId::SELF).await?; - - let chat_id = ChatId::create_for_contact(self, contact_id).await?; - chat_id - .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id)) - .await?; - - let mut msg = Message::new_text(self.get_self_report().await?); - - chat_id.set_draft(self, Some(&mut msg)).await?; - - Ok(chat_id) - } - /// Get a list of fresh, unmuted messages in unblocked chats. /// /// The list starts with the most recent message diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 8850012841..5d6bad6069 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -6,9 +6,9 @@ use super::*; use crate::chat::{Chat, MuteDuration, get_chat_contacts, get_chat_msgs, send_msg, set_muted}; use crate::chatlist::Chatlist; use crate::constants::Chattype; -use crate::mimeparser::SystemMessage; +use crate::message::Message; use crate::receive_imf::receive_imf; -use crate::test_utils::{TestContext, get_chat_msg}; +use crate::test_utils::TestContext; use crate::tools::{SystemTime, create_outgoing_rfc724_mid}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -598,26 +598,6 @@ async fn test_get_next_msgs() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_draft_self_report() -> Result<()> { - let alice = TestContext::new_alice().await; - - let chat_id = alice.draft_self_report().await?; - let msg = get_chat_msg(&alice, chat_id, 0, 1).await; - assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); - - let chat = Chat::load_from_db(&alice, chat_id).await?; - assert!(chat.is_protected()); - - let mut draft = chat_id.get_draft(&alice).await?.unwrap(); - assert!(draft.text.starts_with("core_version")); - - // Test that sending into the protected chat works: - let _sent = alice.send_msg(chat_id, &mut draft).await; - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_cache_is_cleared_when_io_is_started() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/lib.rs b/src/lib.rs index 3c6402cbfe..64ac262cfd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -98,6 +98,7 @@ pub mod html; pub mod net; pub mod plaintext; mod push; +pub mod self_reporting; pub mod summary; mod debug_logging; diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 77da645eb2..ae7d744184 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -41,6 +41,7 @@ use crate::peer_channels::{add_gossip_peer_from_header, insert_topic_stub}; use crate::reaction::{Reaction, set_msg_reaction}; use crate::rusqlite::OptionalExtension; use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device}; +use crate::self_reporting::SELF_REPORTING_BOT_EMAIL; use crate::simplify; use crate::stock_str; use crate::sync::Sync::*; @@ -1731,7 +1732,11 @@ async fn add_parts( let state = if !mime_parser.incoming { MessageState::OutDelivered - } else if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent + } else if seen + || is_mdn + || chat_id_blocked == Blocked::Yes + || group_changes.silent + || mime_parser.from.addr == SELF_REPORTING_BOT_EMAIL // No check for `hidden` because only reactions are such and they should be `InFresh`. { MessageState::InSeen diff --git a/src/scheduler.rs b/src/scheduler.rs index 491840daff..3d5f32194e 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -25,6 +25,7 @@ use crate::imap::{FolderMeaning, Imap, session::Session}; use crate::location; use crate::log::{LogExt, error, info, warn}; use crate::message::MsgId; +use crate::self_reporting::maybe_send_self_report; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; use crate::tools::{self, duration_to_str, maybe_add_time_based_warnings, time, time_elapsed}; @@ -509,6 +510,7 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } }; + maybe_send_self_report(ctx).await.log_err(ctx).ok(); match ctx.get_config_bool(Config::FetchedExistingMsgs).await { Ok(fetched_existing_msgs) => { if !fetched_existing_msgs { diff --git a/src/securejoin.rs b/src/securejoin.rs index 4cba5407ea..995ec9e0b9 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -5,7 +5,6 @@ use deltachat_contact_tools::ContactAddress; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus, get_chat_id_by_grpid}; -use crate::chatlist_events; use crate::config::Config; use crate::constants::{Blocked, Chattype, NON_ALPHANUMERIC_WITHOUT_DOT}; use crate::contact::mark_contact_id_as_verified; @@ -15,6 +14,7 @@ use crate::e2ee::ensure_secret_key_exists; use crate::events::EventType; use crate::headerdef::HeaderDef; use crate::key::{DcKey, Fingerprint, load_self_public_key}; +use crate::log::LogExt as _; use crate::log::{error, info, warn}; use crate::logged_debug_assert; use crate::message::{Message, Viewtype}; @@ -24,6 +24,7 @@ use crate::qr::check_qr; use crate::securejoin::bob::JoinerProgress; use crate::sync::Sync::*; use crate::token; +use crate::{chatlist_events, self_reporting}; mod bob; mod qrinvite; @@ -145,12 +146,34 @@ async fn get_self_fingerprint(context: &Context) -> Result { /// /// The function returns immediately and the handshake will run in background. pub async fn join_securejoin(context: &Context, qr: &str) -> Result { - securejoin(context, qr).await.map_err(|err| { + join_securejoin_with_source(context, qr, None, None).await +} + +/// Take a scanned QR-code and do the setup-contact/join-group/invite handshake. +/// +/// This is the start of the process for the joiner. See the module and ffi documentation +/// for more details. +/// +/// The function returns immediately and the handshake will run in background. +pub async fn join_securejoin_with_source( + context: &Context, + qr: &str, + source: Option, + uipath: Option, +) -> Result { + let res = securejoin(context, qr).await.map_err(|err| { warn!(context, "Fatal joiner error: {:#}", err); // The user just scanned this QR code so has context on what failed. error!(context, "QR process failed"); err - }) + })?; + + self_reporting::count_securejoin(context, source, uipath) + .await + .log_err(context) + .ok(); + + Ok(res) } async fn securejoin(context: &Context, qr: &str) -> Result { diff --git a/src/self_reporting.rs b/src/self_reporting.rs new file mode 100644 index 0000000000..d41c96743f --- /dev/null +++ b/src/self_reporting.rs @@ -0,0 +1,530 @@ +//! TODO doc comment + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{Context as _, Result, ensure}; +use deltachat_derive::FromSql; +use pgp::types::PublicKeyTrait; +use serde::Serialize; + +use crate::chat::{self, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; +use crate::config::Config; +use crate::constants::{Chattype, DC_CHAT_ID_TRASH}; +use crate::contact::{ContactId, Origin, import_vcard, mark_contact_id_as_verified}; +use crate::context::{Context, get_version_str}; +use crate::key::load_self_public_keyring; +use crate::log::LogExt; +use crate::message::{Message, Viewtype}; +use crate::tools::{create_id, time}; + +pub(crate) const SELF_REPORTING_BOT_EMAIL: &str = "self_reporting@testrun.org"; +const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); + +#[derive(Serialize)] +struct Statistics { + core_version: String, + key_created: Vec, + self_reporting_id: String, + is_chatmail: bool, + contact_stats: Vec, + message_stats: MessageStats, + securejoin_source_stats: SecurejoinSourceStats, + securejoin_uipath_stats: SecurejoinUIPathStats, +} + +#[derive(Serialize, PartialEq)] +enum VerifiedStatus { + Direct, + Transitive, + TransitiveViaBot, + Opportunistic, + Unencrypted, +} + +#[derive(Serialize)] +struct ContactStat { + #[serde(skip_serializing)] + id: ContactId, + + verified: VerifiedStatus, + bot: bool, + direct_chat: bool, + last_seen: u64, + + #[serde(skip_serializing_if = "Option::is_none")] + transitive_chain: Option, + //new: bool, // TODO +} + +#[derive(Serialize)] +struct MessageStats { + to_verified: u32, + unverified_encrypted: u32, + unencrypted: u32, +} + +#[repr(u32)] +#[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] +enum SecurejoinSource { + Unknown = 0, + ExternalLink = 1, + InternalLink = 2, + Clipboard = 3, + ImageLoaded = 4, + Scan = 5, +} + +#[derive(Serialize)] +struct SecurejoinSourceStats { + unknown: u32, + external_link: u32, + internal_link: u32, + clipboard: u32, + image_loaded: u32, + scan: u32, +} + +#[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] +enum SecurejoinUIPath { + Unknown = 0, + QrIcon = 1, + NewContact = 2, +} + +#[derive(Serialize)] +struct SecurejoinUIPathStats { + other: u32, + qr_icon: u32, + new_contact: u32, +} + +/// Sends a message with statistics about the usage of Delta Chat, +/// if the last time such a message was sent +/// was more than a week ago. +/// +/// On the other end, a bot will receive the message and make it available +/// to Delta Chat's developers. +pub async fn maybe_send_self_report(context: &Context) -> Result> { + //#[cfg(target_os = "android")] TODO + if context.get_config_bool(Config::SelfReporting).await? { + let last_selfreport_time = context.get_config_i64(Config::LastSelfReportSent).await?; + let next_selfreport_time = last_selfreport_time.saturating_add(30); // TODO increase to 1 day or 1 week + if next_selfreport_time <= time() { + return Ok(Some(send_self_report(context).await?)); + } + } + Ok(None) +} + +async fn send_self_report(context: &Context) -> Result { + info!(context, "Sending self report."); + + // Setting this config at the beginning avoids endless loops when things do not + // work out for whatever reason. + context + .set_config_internal(Config::LastSelfReportSent, Some(&time().to_string())) + .await + .log_err(context) + .ok(); + + let chat_id = get_selfreporting_bot(context).await?; + + let mut msg = Message::new(Viewtype::File); + msg.set_text( + "The attachment contains anonymous usage statistics, \ +because you enabled this in the settings. \ +This helps us improve the security of Delta Chat. \ +See TODO[blog post] for more information." + .to_string(), + ); + + let self_report = get_self_report(context).await?; + + msg.set_file_from_bytes( + context, + "statistics.txt", + self_report.as_bytes(), + Some("text/plain"), + )?; + + crate::chat::send_msg(context, chat_id, &mut msg) + .await + .context("Failed to send self_reporting message") + .log_err(context) + .ok(); + + set_last_msgid(context).await?; + + Ok(chat_id) +} + +pub(crate) async fn set_last_msgid(context: &Context) -> Result<()> { + let last_msgid: u64 = context + .sql + .query_get_value("SELECT MAX(id) FROM msgs", ()) + .await? + .context("All messages are gone")?; + + context + .sql + .set_raw_config( + Config::SelfReportingLastMsgId.as_ref(), + Some(&last_msgid.to_string()), + ) + .await?; + + Ok(()) +} + +async fn get_self_report(context: &Context) -> Result { + let last_msgid = context + .get_config_u64(Config::SelfReportingLastMsgId) + .await?; + + let key_created: Vec = load_self_public_keyring(context) + .await? + .iter() + .map(|k| k.created_at().timestamp()) + .collect(); + + let self_reporting_id = match context.get_config(Config::SelfReportingId).await? { + Some(id) => id, + None => { + let id = create_id(); + context + .set_config_internal(Config::SelfReportingId, Some(&id)) + .await?; + id + } + }; + + let statistics = Statistics { + core_version: get_version_str().to_string(), + key_created, + self_reporting_id, + is_chatmail: context.is_chatmail().await?, + contact_stats: get_contact_stats(context).await?, + message_stats: get_message_stats(context, last_msgid).await?, + securejoin_source_stats: get_securejoin_source_stats(context).await?, + securejoin_uipath_stats: get_securejoin_uipath_stats(context).await?, + }; + + Ok(serde_json::to_string_pretty(&statistics)?) +} + +async fn get_selfreporting_bot(context: &Context) -> Result { + let contact_id: ContactId = *import_vcard(context, SELF_REPORTING_BOT_VCARD) + .await? + .first() + .context("Self reporting bot vCard does not contain a contact")?; + mark_contact_id_as_verified(context, contact_id, ContactId::SELF).await?; + + let chat_id = if let Some(res) = ChatId::lookup_by_contact(context, contact_id).await? { + // Already exists, no need to create. + res + } else { + let chat_id = ChatId::get_for_contact(context, contact_id).await?; + chat_id + .set_visibility(context, ChatVisibility::Archived) + .await?; + chat::set_muted(context, chat_id, MuteDuration::Forever).await?; + chat_id + }; + + chat_id + .set_protection( + context, + ProtectionStatus::Protected, + time(), + Some(contact_id), + ) + .await?; + + Ok(chat_id) +} + +async fn get_contact_stats(context: &Context) -> Result> { + let mut verified_by_map: BTreeMap = BTreeMap::new(); + let mut bot_ids: BTreeSet = BTreeSet::new(); + + let mut contacts: Vec = context + .sql + .query_map( + "SELECT id, fingerprint<>'', verifier, last_seen, is_bot FROM contacts c + WHERE id>9 AND origin>? AND addr<>?", + (Origin::Hidden, SELF_REPORTING_BOT_EMAIL), + |row| { + let id = row.get(0)?; + let is_encrypted: bool = row.get(1)?; + let verifier: ContactId = row.get(2)?; + let last_seen: u64 = row.get(3)?; + let bot: bool = row.get(4)?; + + let verified = match (is_encrypted, verifier) { + (true, ContactId::SELF) => VerifiedStatus::Direct, + (true, ContactId::UNDEFINED) => VerifiedStatus::Opportunistic, + (true, _) => VerifiedStatus::Transitive, // TransitiveViaBot will be filled later + (false, _) => VerifiedStatus::Unencrypted, + }; + + if verifier != ContactId::UNDEFINED { + verified_by_map.insert(id, verifier); + } + + if bot { + bot_ids.insert(id); + } + + Ok(ContactStat { + id, + verified, + bot, + direct_chat: false, // will be filled later + last_seen, + transitive_chain: None, // will be filled later + }) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + + // Fill TransitiveViaBot and transitive_chain + for contact in &mut contacts { + if contact.verified == VerifiedStatus::Transitive { + let mut transitive_chain: u32 = 0; + let mut has_bot = false; + let mut current_verifier_id = contact.id; + + while current_verifier_id != ContactId::SELF && transitive_chain < 100 { + current_verifier_id = match verified_by_map.get(¤t_verifier_id) { + Some(id) => *id, + None => { + // The chain ends here, probably because some verification was done + // before we started recording verifiers. + // It's unclear how long the chain really is. + transitive_chain = 0; + break; + } + }; + if bot_ids.contains(¤t_verifier_id) { + has_bot = true; + } + transitive_chain = transitive_chain.saturating_add(1); + } + + if transitive_chain > 0 { + contact.transitive_chain = Some(transitive_chain); + } + + if has_bot { + contact.verified = VerifiedStatus::TransitiveViaBot; + } + } + } + + // Fill direct_chat + for contact in &mut contacts { + let direct_chat = context + .sql + .exists( + "SELECT COUNT(*) + FROM chats_contacts cc INNER JOIN chats + WHERE cc.contact_id=? AND chats.type=?", + (contact.id, Chattype::Single), + ) + .await?; + contact.direct_chat = direct_chat; + } + + Ok(contacts) +} + +async fn get_message_stats(context: &Context, last_msgid: u64) -> Result { + ensure!( + last_msgid >= 9, + "Last_msgid < 9 would mean including 'special' messages in the report" + ); + + let selfreporting_bot_chat_id = get_selfreporting_bot(context).await?; + + let trans_fn = |t: &mut rusqlite::Transaction| { + t.pragma_update(None, "query_only", "0")?; + + // This table will hold all 'protected' chats. + // Protected chats guarantee that all messages in the chat + // are sent with verified encryption + // are marked with a green checkmark in the UI. + t.execute( + "CREATE TEMP TABLE temp.protected_chats ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + // id>9 because chat ids 0..9 are "special" chats like the trash chat. + t.execute( + "INSERT INTO temp.protected_chats + SELECT id FROM chats + WHERE protected=1 AND id>9", + (), + )?; + + // In the following SQL statements, + // - we always have the line + // `AND from_id=? AND chat_id<>? AND id>? AND hidden=0 AND chat_id<>?`. + // `from_id=?` is to count only outgoing messages. + // The rest excludes: + // - the chat with the self-reporting bot itself, + // - messages sent with the last report, or before the config was enabled + // - hidden system messages, which are not actually shown to the user + // - messages in the 'Trash' chat, which is an internal chat assigned to messages that are not shown to the user + // - `(param GLOB '*\nc=1*' OR param GLOB 'c=1*')` + // matches all messages that are end-to-end encrypted + let to_verified = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id IN temp.protected_chats + AND from_id=? AND chat_id<>? AND id>? AND hidden=0 AND chat_id<>?", + ( + ContactId::SELF, + selfreporting_bot_chat_id, + last_msgid, + DC_CHAT_ID_TRASH, + ), + |row| row.get(0), + )?; + + let unverified_encrypted = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id not IN temp.protected_chats + AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND from_id=? AND chat_id<>? AND id>? AND hidden=0 AND chat_id<>?", + ( + ContactId::SELF, + selfreporting_bot_chat_id, + last_msgid, + DC_CHAT_ID_TRASH, + ), + |row| row.get(0), + )?; + + let unencrypted = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id not IN temp.protected_chats + AND NOT (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND from_id=? AND chat_id<>? AND id>? AND hidden=0 AND chat_id<>?", + ( + ContactId::SELF, + selfreporting_bot_chat_id, + last_msgid, + DC_CHAT_ID_TRASH, + ), + |row| row.get(0), + )?; + + t.execute("DROP TABLE temp.protected_chats", ())?; + + Ok(MessageStats { + to_verified, + unverified_encrypted, + unencrypted, + }) + }; + + let query_only = true; + let message_stats: MessageStats = context.sql.transaction_ex(query_only, trans_fn).await?; + + Ok(message_stats) +} + +pub(crate) async fn count_securejoin( + context: &Context, + source: Option, + uipath: Option, +) -> Result<()> { + if context.get_config_bool(Config::SelfReporting).await? { + let source = source + .context("Missing securejoin source") + .log_err(context) + .unwrap_or(0); + + context + .sql + .execute( + "INSERT INTO stats_securejoin_sources VALUES (?, 1) + ON CONFLICT (source) DO UPDATE SET count=count+1;", + (source,), + ) + .await?; + + // We only get a UI path if the source is a QR code scan, + // a loaded image, or a link pasted from the QR code, + // so, no need to log an error if `uipath` is None: + let uipath = uipath.unwrap_or(0); + context + .sql + .execute( + "INSERT INTO stats_securejoin_uipaths VALUES (?, 1) + ON CONFLICT (uipath) DO UPDATE SET count=count+1;", + (uipath,), + ) + .await?; + } + Ok(()) +} + +async fn get_securejoin_source_stats(context: &Context) -> Result { + let map = context + .sql + .query_map( + "SELECT source, count FROM stats_securejoin_sources", + (), + |row| { + let source: SecurejoinSource = row.get(0)?; + let count: u32 = row.get(1)?; + Ok((source, count)) + }, + |rows| Ok(rows.collect::>>()?), + ) + .await?; + + let stats = SecurejoinSourceStats { + unknown: *map.get(&SecurejoinSource::Unknown).unwrap_or(&0), + external_link: *map.get(&SecurejoinSource::ExternalLink).unwrap_or(&0), + internal_link: *map.get(&SecurejoinSource::InternalLink).unwrap_or(&0), + clipboard: *map.get(&SecurejoinSource::Clipboard).unwrap_or(&0), + image_loaded: *map.get(&SecurejoinSource::ImageLoaded).unwrap_or(&0), + scan: *map.get(&SecurejoinSource::Scan).unwrap_or(&0), + }; + + Ok(stats) +} + +async fn get_securejoin_uipath_stats(context: &Context) -> Result { + let map = context + .sql + .query_map( + "SELECT uipath, count FROM stats_securejoin_uipaths", + (), + |row| { + let uipath: SecurejoinUIPath = row.get(0)?; + let count: u32 = row.get(1)?; + Ok((uipath, count)) + }, + |rows| Ok(rows.collect::>>()?), + ) + .await?; + + let stats = SecurejoinUIPathStats { + other: *map.get(&SecurejoinUIPath::Unknown).unwrap_or(&0), + qr_icon: *map.get(&SecurejoinUIPath::QrIcon).unwrap_or(&0), + new_contact: *map.get(&SecurejoinUIPath::NewContact).unwrap_or(&0), + }; + + Ok(stats) +} + +#[cfg(test)] +mod self_reporting_tests; diff --git a/src/self_reporting/self_reporting_tests.rs b/src/self_reporting/self_reporting_tests.rs new file mode 100644 index 0000000000..88eba7079f --- /dev/null +++ b/src/self_reporting/self_reporting_tests.rs @@ -0,0 +1,369 @@ +use std::time::Duration; + +use super::*; +use crate::chat::Chat; +use crate::mimeparser::SystemMessage; +use crate::securejoin::{get_securejoin_qr, join_securejoin, join_securejoin_with_source}; +use crate::test_utils::{TestContext, TestContextManager, get_chat_msg}; +use crate::tools::SystemTime; +use pretty_assertions::assert_eq; +use serde_json::{Number, Value}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_send_self_report() -> Result<()> { + let alice = &TestContext::new_alice().await; + + alice.set_config_bool(Config::SelfReporting, true).await?; + + let chat_id = maybe_send_self_report(&alice).await?.unwrap(); + let msg = get_chat_msg(&alice, chat_id, 0, 2).await; + assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); + + let chat = Chat::load_from_db(&alice, chat_id).await?; + assert!(chat.is_protected()); + + let msg = get_chat_msg(&alice, chat_id, 1, 2).await; + assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); + + let report = tokio::fs::read(msg.get_file(&alice).unwrap()).await?; + let report = std::str::from_utf8(&report)?; + println!("\nEmpty account:\n{}\n", report); + assert!(report.contains(r#""contact_stats": []"#)); + + let r: serde_json::Value = serde_json::from_str(&report)?; + assert_eq!( + r.get("contact_stats").unwrap(), + &serde_json::Value::Array(vec![]) + ); + assert_eq!(r.get("core_version").unwrap(), get_version_str()); + + assert_eq!(maybe_send_self_report(alice).await?, None); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_self_report_one_contact() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SelfReporting, true).await?; + + let report = get_self_report(alice).await?; + let r: serde_json::Value = serde_json::from_str(&report)?; + + tcm.send_recv_accept(bob, alice, "Hi!").await; + + let report = get_self_report(alice).await?; + println!("\nWith Bob:\n{report}\n"); + let r2: serde_json::Value = serde_json::from_str(&report)?; + + assert_eq!( + r.get("key_created").unwrap(), + r2.get("key_created").unwrap() + ); + assert_eq!( + r.get("self_reporting_id").unwrap(), + r2.get("self_reporting_id").unwrap() + ); + let contact_stats = r2.get("contact_stats").unwrap().as_array().unwrap(); + assert_eq!(contact_stats.len(), 1); + let contact_info = &contact_stats[0]; + assert_eq!( + contact_info.get("bot").unwrap(), + &serde_json::Value::Bool(false) + ); + assert_eq!( + contact_info.get("direct_chat").unwrap(), + &serde_json::Value::Bool(true) + ); + assert!(contact_info.get("transitive_chain").is_none(),); + assert_eq!( + contact_info.get("verified").unwrap(), + &serde_json::Value::String("Opportunistic".to_string()) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_message_stats() -> Result<()> { + fn check_report(report: &str, expected: &MessageStats) { + let key = "message_stats"; + let actual: serde_json::Value = serde_json::from_str(&report).unwrap(); + let actual = &actual[key]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SelfReporting, true).await?; + let email_chat = alice.create_email_chat(bob).await; + let encrypted_chat = alice.create_chat(bob).await; + + let mut expected = MessageStats { + to_verified: 0, + unverified_encrypted: 0, + unencrypted: 0, + }; + + check_report(&get_self_report(alice).await?, &expected); + + alice.send_text(email_chat.id, "foo").await; + expected.unencrypted += 1; + check_report(&get_self_report(alice).await?, &expected); + + alice.send_text(encrypted_chat.id, "foo").await; + expected.unverified_encrypted += 1; + check_report(&get_self_report(alice).await?, &expected); + + alice.send_text(encrypted_chat.id, "foo").await; + expected.unverified_encrypted += 1; + check_report(&get_self_report(alice).await?, &expected); + + tcm.execute_securejoin(alice, bob).await; + expected.to_verified = expected.unverified_encrypted; + expected.unverified_encrypted = 0; + check_report(&get_self_report(alice).await?, &expected); + + // Incoming messages are not counted: + let rcvd = tcm.send_recv(bob, alice, "bar").await; + check_report(&get_self_report(alice).await?, &expected); + + // Reactions are not counted: + crate::reaction::send_reaction(alice, rcvd.id, "👍") + .await + .unwrap(); + check_report(&get_self_report(alice).await?, &expected); + + tcm.section("Test that after actually sending a report, the message numbers are reset."); + let report_before_sending = get_self_report(alice).await.unwrap(); + + let report = send_and_read_self_report(alice).await; + // The report is supposed not to have changed yet + assert_eq!(report_before_sending, report); + + // Shift by 8 days so that the next report is due: + SystemTime::shift(Duration::from_secs(8 * 24 * 3600)); + + let report = send_and_read_self_report(alice).await; + assert_ne!(report_before_sending, report); + + expected = MessageStats { + to_verified: 0, + unverified_encrypted: 0, + unencrypted: 0, + }; + check_report(&report, &expected); + + tcm.section( + "Test that after sending a message again, the message statistics start to fill again.", + ); + SystemTime::shift(Duration::from_secs(8 * 24 * 3600)); + tcm.send_recv(alice, bob, "Hi").await; + expected.to_verified += 1; + check_report(&send_and_read_self_report(alice).await, &expected); + + Ok(()) +} + +async fn send_and_read_self_report(context: &TestContext) -> String { + let chat_id = maybe_send_self_report(&context).await.unwrap().unwrap(); + let msg = context.get_last_msg_in(chat_id).await; + assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); + + let report = tokio::fs::read(msg.get_file(&context).unwrap()) + .await + .unwrap(); + String::from_utf8(report).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_self_report_securejoin_source_stats() -> Result<()> { + async fn check_report(context: &TestContext, expected: &SecurejoinSourceStats) { + let report = get_self_report(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&report).unwrap(); + let actual = &actual["securejoin_source_stats"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SelfReporting, true).await?; + + let mut expected = SecurejoinSourceStats { + unknown: 0, + external_link: 0, + internal_link: 0, + clipboard: 0, + image_loaded: 0, + scan: 0, + }; + + check_report(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + join_securejoin(alice, &qr).await?; + expected.unknown += 1; + check_report(alice, &expected).await; + + join_securejoin(alice, &qr).await?; + expected.unknown += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + expected.clipboard += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source( + alice, + &qr, + Some(SecurejoinSource::ExternalLink as u32), + None, + ) + .await?; + expected.external_link += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source( + alice, + &qr, + Some(SecurejoinSource::InternalLink as u32), + None, + ) + .await?; + expected.internal_link += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::ImageLoaded as u32), None) + .await?; + expected.image_loaded += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Scan as u32), None).await?; + expected.scan += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + expected.clipboard += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + expected.clipboard += 1; + check_report(alice, &expected).await; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_self_report_securejoin_uipath_stats() -> Result<()> { + async fn check_report(context: &TestContext, expected: &SecurejoinUIPathStats) { + let report = get_self_report(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&report).unwrap(); + let actual = &actual["securejoin_uipath_stats"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SelfReporting, true).await?; + + let mut expected = SecurejoinUIPathStats { + other: 0, + qr_icon: 0, + new_contact: 0, + }; + + check_report(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + join_securejoin(alice, &qr).await?; + expected.other += 1; + check_report(alice, &expected).await; + + join_securejoin(alice, &qr).await?; + expected.other += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source( + alice, + &qr, + Some(0), + Some(SecurejoinUIPath::NewContact as u32), + ) + .await?; + expected.new_contact += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source( + alice, + &qr, + Some(0), + Some(SecurejoinUIPath::NewContact as u32), + ) + .await?; + expected.new_contact += 1; + check_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(0), Some(SecurejoinUIPath::QrIcon as u32)).await?; + expected.qr_icon += 1; + check_report(alice, &expected).await; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_self_report_is_chatmail() -> Result<()> { + let alice = &TestContext::new_alice().await; + alice.set_config_bool(Config::SelfReporting, true).await?; + + let r = get_self_report(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + assert_eq!(r.get("is_chatmail").unwrap().as_bool().unwrap(), false); + + alice.set_config_bool(Config::IsChatmail, true).await?; + + let r = get_self_report(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + assert_eq!(r.get("is_chatmail").unwrap().as_bool().unwrap(), true); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_self_report_key_creation_timestamp() -> Result<()> { + // Alice uses a pregenerated key. It was created at this timestamp: + const ALICE_KEY_CREATION_TIME: u128 = 1582855645; + + let alice = &TestContext::new_alice().await; + alice.set_config_bool(Config::SelfReporting, true).await?; + + let r = get_self_report(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + let key_created = r.get("key_created").unwrap().as_array().unwrap(); + assert_eq!( + key_created, + &vec![Value::Number( + Number::from_u128(ALICE_KEY_CREATION_TIME).unwrap() + )] + ); + + Ok(()) +} diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 653d74d09a..be54068749 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1251,6 +1251,30 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint); .await?; } + inc_and_check(&mut migration_version, 133)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE stats_securejoin_sources( + source INTEGER PRIMARY KEY, + count INTEGER NOT NULL DEFAULT 0 + ) STRICT", + migration_version, + ) + .await?; + } + + inc_and_check(&mut migration_version, 134)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE stats_securejoin_uipaths( + uipath INTEGER PRIMARY KEY, + count INTEGER NOT NULL DEFAULT 0 + ) STRICT", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await?