diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index f23883ee75..ffed80c1ff 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -375,9 +375,10 @@ impl CommandApi { Ok(BlobObject::create_and_deduplicate(&ctx, file, file)?.to_abs_path()) } + /// Deprecated 2025-04. Use the "self_reporting" config instead. async fn draft_self_report(&self, account_id: u32) -> Result { let ctx = self.get_context(account_id).await?; - Ok(ctx.draft_self_report().await?.to_u32()) + Ok(ctx.send_self_report().await?.to_u32()) } /// Sets the given configuration key. diff --git a/src/config.rs b/src/config.rs index 1fa11d49ec..3b69064ed1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -428,6 +428,13 @@ pub enum Config { /// used for signatures, encryption to self and included in `Autocrypt` header. KeyId, + /// Send statistics to Delta Chat's developers. + /// Can be exposed to the user as a setting. + SelfReporting, + + /// Last time statistics were sent to Delta Chat's developers + LastSelfReportSent, + /// This key is sent to the self_reporting bot so that the bot can recognize the user /// without storing the email address SelfReportingId, diff --git a/src/context.rs b/src/context.rs index be9ca15303..ded98b45ba 100644 --- a/src/context.rs +++ b/src/context.rs @@ -12,23 +12,23 @@ use anyhow::{bail, ensure, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; use pgp::types::PublicKeyTrait; use ratelimit::Ratelimit; +use serde::Serialize; use tokio::sync::{Mutex, Notify, RwLock}; -use crate::chat::{get_chat_cnt, ChatId, ProtectionStatus}; +use crate::chat::{get_chat_cnt, ChatId}; use crate::chatlist_events; use crate::config::Config; use crate::constants::{ - self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR, + self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR, }; -use crate::contact::{import_vcard, mark_contact_id_as_verified, Contact, ContactId}; +use crate::contact::{Contact, ContactId}; use crate::debug_logging::DebugLogging; -use crate::download::DownloadState; use crate::events::{Event, EventEmitter, EventType, Events}; use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{load_self_secret_key, self_fingerprint}; +use crate::key::{self_fingerprint, DcKey as _}; +use crate::log::LogExt; use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; -use crate::message::{self, Message, MessageState, MsgId}; -use crate::param::{Param, Params}; +use crate::message::{self, MessageState, MsgId}; use crate::peer_channels::Iroh; use crate::push::PushSubscriber; use crate::quota::QuotaInfo; @@ -36,7 +36,7 @@ use crate::scheduler::{convert_folder_meaning, SchedulerState}; use crate::sql::Sql; use crate::stock_str::StockStrings; use crate::timesmearing::SmearedTimestamp; -use crate::tools::{self, create_id, duration_to_str, time, time_elapsed}; +use crate::tools::{self, duration_to_str, time, time_elapsed}; /// Builder for the [`Context`]. /// @@ -1043,6 +1043,18 @@ impl Context { .await? .to_string(), ); + res.insert( + "self_reporting", + self.get_config_bool(Config::SelfReporting) + .await? + .to_string(), + ); + res.insert( + "last_self_report_sent", + self.get_config_i64(Config::LastSelfReportSent) + .await? + .to_string(), + ); let elapsed = time_elapsed(&self.creation_time); res.insert("uptime", duration_to_str(elapsed)); @@ -1050,152 +1062,6 @@ impl Context { Ok(res) } - async fn get_self_report(&self) -> Result { - #[derive(Default)] - struct ChatNumbers { - protected: u32, - protection_broken: u32, - opportunistic_dc: u32, - opportunistic_mua: u32, - unencrypted_dc: u32, - unencrypted_mua: u32, - } - - let mut res = String::new(); - res += &format!("core_version {}\n", get_version_str()); - - let num_msgs: u32 = self - .sql - .query_get_value( - "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?", - (DC_CHAT_ID_TRASH,), - ) - .await? - .unwrap_or_default(); - res += &format!("num_msgs {num_msgs}\n"); - - let num_chats: u32 = self - .sql - .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ()) - .await? - .unwrap_or_default(); - res += &format!("num_chats {num_chats}\n"); - - let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len(); - res += &format!("db_size_bytes {db_size}\n"); - - let secret_key = &load_self_secret_key(self).await?.primary_key; - let key_created = secret_key.public_key().created_at().timestamp(); - res += &format!("key_created {key_created}\n"); - - // how many of the chats active in the last months are: - // - protected - // - protection-broken - // - opportunistic-encrypted and the contact uses Delta Chat - // - opportunistic-encrypted and the contact uses a classical MUA - // - unencrypted and the contact uses Delta Chat - // - unencrypted and the contact uses a classical MUA - let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3); - let chats = self - .sql - .query_map( - "SELECT c.protected, m.param, m.msgrmsg - FROM chats c - JOIN msgs m - ON c.id=m.chat_id - AND m.id=( - SELECT id - FROM msgs - WHERE chat_id=c.id - AND hidden=0 - AND download_state=? - AND to_id!=? - ORDER BY timestamp DESC, id DESC LIMIT 1) - WHERE c.id>9 - AND (c.blocked=0 OR c.blocked=2) - AND IFNULL(m.timestamp,c.created_timestamp) > ? - GROUP BY c.id", - (DownloadState::Done, ContactId::INFO, three_months_ago), - |row| { - let protected: ProtectionStatus = row.get(0)?; - let message_param: Params = - row.get::<_, String>(1)?.parse().unwrap_or_default(); - let is_dc_message: bool = row.get(2)?; - Ok((protected, message_param, is_dc_message)) - }, - |rows| { - let mut chats = ChatNumbers::default(); - for row in rows { - let (protected, message_param, is_dc_message) = row?; - let encrypted = message_param - .get_bool(Param::GuaranteeE2ee) - .unwrap_or(false); - - if protected == ProtectionStatus::Protected { - chats.protected += 1; - } else if protected == ProtectionStatus::ProtectionBroken { - chats.protection_broken += 1; - } else if encrypted { - if is_dc_message { - chats.opportunistic_dc += 1; - } else { - chats.opportunistic_mua += 1; - } - } else if is_dc_message { - chats.unencrypted_dc += 1; - } else { - chats.unencrypted_mua += 1; - } - } - Ok(chats) - }, - ) - .await?; - res += &format!("chats_protected {}\n", chats.protected); - res += &format!("chats_protection_broken {}\n", chats.protection_broken); - res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc); - res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua); - res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc); - res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua); - - let self_reporting_id = match self.get_config(Config::SelfReportingId).await? { - Some(id) => id, - None => { - let id = create_id(); - self.set_config(Config::SelfReportingId, Some(&id)).await?; - id - } - }; - res += &format!("self_reporting_id {self_reporting_id}"); - - Ok(res) - } - - /// Drafts a message with statistics about the usage of Delta Chat. - /// The user can inspect the message if they want, and then hit "Send". - /// - /// On the other end, a bot will receive the message and make it available - /// to Delta Chat's developers. - pub async fn draft_self_report(&self) -> Result { - const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); - let contact_id: ContactId = *import_vcard(self, SELF_REPORTING_BOT_VCARD) - .await? - .first() - .context("Self reporting bot vCard does not contain a contact")?; - mark_contact_id_as_verified(self, contact_id, ContactId::SELF).await?; - - let chat_id = ChatId::create_for_contact(self, contact_id).await?; - chat_id - .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id)) - .await?; - - let mut msg = Message::new_text(self.get_self_report().await?); - - chat_id.set_draft(self, Some(&mut msg)).await?; - - Ok(chat_id) - } - /// Get a list of fresh, unmuted messages in unblocked chats. /// /// The list starts with the most recent message diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 111311aca2..0f429c55ea 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -591,26 +591,6 @@ async fn test_get_next_msgs() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_draft_self_report() -> Result<()> { - let alice = TestContext::new_alice().await; - - let chat_id = alice.draft_self_report().await?; - let msg = get_chat_msg(&alice, chat_id, 0, 1).await; - assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); - - let chat = Chat::load_from_db(&alice, chat_id).await?; - assert!(chat.is_protected()); - - let mut draft = chat_id.get_draft(&alice).await?.unwrap(); - assert!(draft.text.starts_with("core_version")); - - // Test that sending into the protected chat works: - let _sent = alice.send_msg(chat_id, &mut draft).await; - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_cache_is_cleared_when_io_is_started() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/lib.rs b/src/lib.rs index 3c6402cbfe..83eeca67b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -98,6 +98,7 @@ pub mod html; pub mod net; pub mod plaintext; mod push; +mod self_reporting; pub mod summary; mod debug_logging; diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 7080073152..02655a9737 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1672,7 +1672,11 @@ async fn add_parts( let state = if !mime_parser.incoming { MessageState::OutDelivered - } else if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent + } else if seen + || is_mdn + || chat_id_blocked == Blocked::Yes + || group_changes.silent + || mime_parser.from.addr == "self_reporting@testrun.org" // No check for `hidden` because only reactions are such and they should be `InFresh`. { MessageState::InSeen diff --git a/src/scheduler.rs b/src/scheduler.rs index 6f0ca6d149..50d5bdd51b 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -22,6 +22,7 @@ use crate::imap::{session::Session, FolderMeaning, Imap}; use crate::location; use crate::log::LogExt; use crate::message::MsgId; +use crate::self_reporting::maybe_send_self_report; use crate::smtp::{send_smtp_messages, Smtp}; use crate::sql; use crate::tools::{self, duration_to_str, maybe_add_time_based_warnings, time, time_elapsed}; @@ -505,6 +506,7 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } }; + maybe_send_self_report(ctx).await?; match ctx.get_config_bool(Config::FetchedExistingMsgs).await { Ok(fetched_existing_msgs) => { if !fetched_existing_msgs { diff --git a/src/self_reporting.rs b/src/self_reporting.rs new file mode 100644 index 0000000000..2e08ead8d0 --- /dev/null +++ b/src/self_reporting.rs @@ -0,0 +1,266 @@ +//! TODO doc comment + + +use anyhow::{Context as _, Result}; +use pgp::types::PublicKeyTrait; +use serde::Serialize; + +use crate::chat::{self, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; +use crate::config::Config; +use crate::constants::DC_CHAT_ID_TRASH; +use crate::contact::{import_vcard, mark_contact_id_as_verified, ContactId}; +use crate::context::{get_version_str, Context}; +use crate::download::DownloadState; +use crate::key::load_self_public_key; +use crate::log::LogExt; +use crate::message::{Message, Viewtype}; +use crate::param::{Param, Params}; +use crate::tools::{create_id, time}; + +#[derive(Serialize)] +struct Statistics { + core_version: String, + num_msgs: u32, + num_chats: u32, + db_size: u64, + key_created: i64, + chat_numbers: ChatNumbers, + self_reporting_id: String, +} +#[derive(Default, Serialize)] +struct ChatNumbers { + protected: u32, + protection_broken: u32, + opportunistic_dc: u32, + opportunistic_mua: u32, + unencrypted_dc: u32, + unencrypted_mua: u32, +} + +pub(crate) async fn maybe_send_self_report(context: &Context) -> Result<()> { + //#[cfg(target_os = "android")] TODO + if context.get_config_bool(Config::SelfReporting).await? { + match context.get_config_i64(Config::LastSelfReportSent).await { + Ok(last_selfreport_time) => { + let next_selfreport_time = last_selfreport_time.saturating_add(30); // TODO increase to 1 day or 1 week + if next_selfreport_time <= time() { + send_self_report(context).await?; + } + } + Err(err) => { + warn!(context, "Failed to get last self_reporting time: {}", err); + } + } + } + Ok(()) +} + +/// Drafts a message with statistics about the usage of Delta Chat. +/// The user can inspect the message if they want, and then hit "Send". +/// +/// On the other end, a bot will receive the message and make it available +/// to Delta Chat's developers. +async fn send_self_report(context: &Context) -> Result { + info!(context, "Sending self report."); + // Setting `Config::LastHousekeeping` at the beginning avoids endless loops when things do not + // work out for whatever reason or are interrupted by the OS. + context + .set_config_internal(Config::LastSelfReportSent, Some(&time().to_string())) + .await + .log_err(context) + .ok(); + + const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); + let contact_id: ContactId = *import_vcard(context, SELF_REPORTING_BOT_VCARD) + .await? + .first() + .context("Self reporting bot vCard does not contain a contact")?; + mark_contact_id_as_verified(context, contact_id, ContactId::SELF).await?; + + let chat_id = if let Some(res) = ChatId::lookup_by_contact(context, contact_id).await? { + // Already exists, no need to create. + res + } else { + let chat_id = ChatId::get_for_contact(context, contact_id).await?; + chat_id + .set_visibility(context, ChatVisibility::Archived) + .await?; + chat::set_muted(context, chat_id, MuteDuration::Forever).await?; + chat_id + }; + + chat_id + .set_protection( + context, + ProtectionStatus::Protected, + time(), + Some(contact_id), + ) + .await?; + + let mut msg = Message::new(Viewtype::File); + msg.set_text( + "The attachment contains anonymous usage statistics, \ +because you enabled this in the settings. \ +This helps us improve the security of Delta Chat. \ +See TODO[blog post] for more information." + .to_string(), + ); + msg.set_file_from_bytes( + context, + "statistics.txt", + get_self_report(context).await?.as_bytes(), + Some("text/plain"), + )?; + + crate::chat::send_msg(context, chat_id, &mut msg) + .await + .context("Failed to send self_reporting message") + .log_err(context) + .ok(); + + Ok(chat_id) +} + +async fn get_self_report(context: &Context) -> Result { + let num_msgs: u32 = context + .sql + .query_get_value( + "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?", + (DC_CHAT_ID_TRASH,), + ) + .await? + .unwrap_or_default(); + + let num_chats: u32 = context + .sql + .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ()) + .await? + .unwrap_or_default(); + + let db_size = tokio::fs::metadata(&context.sql.dbfile).await?.len(); + + let key_created = load_self_public_key(context) + .await? + .primary_key + .created_at() + .timestamp(); + + // how many of the chats active in the last months are: + // - protected + // - protection-broken + // - opportunistic-encrypted and the contact uses Delta Chat + // - opportunistic-encrypted and the contact uses a classical MUA + // - unencrypted and the contact uses Delta Chat + // - unencrypted and the contact uses a classical MUA + let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3); + let chat_numbers = context + .sql + .query_map( + "SELECT c.protected, m.param, m.msgrmsg + FROM chats c + JOIN msgs m + ON c.id=m.chat_id + AND m.id=( + SELECT id + FROM msgs + WHERE chat_id=c.id + AND hidden=0 + AND download_state=? + AND to_id!=? + ORDER BY timestamp DESC, id DESC LIMIT 1) + WHERE c.id>9 + AND (c.blocked=0 OR c.blocked=2) + AND IFNULL(m.timestamp,c.created_timestamp) > ? + GROUP BY c.id", + (DownloadState::Done, ContactId::INFO, three_months_ago), + |row| { + let protected: ProtectionStatus = row.get(0)?; + let message_param: Params = row.get::<_, String>(1)?.parse().unwrap_or_default(); + let is_dc_message: bool = row.get(2)?; + Ok((protected, message_param, is_dc_message)) + }, + |rows| { + let mut chats = ChatNumbers::default(); + for row in rows { + let (protected, message_param, is_dc_message) = row?; + let encrypted = message_param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or(false); + + if protected == ProtectionStatus::Protected { + chats.protected += 1; + } else if protected == ProtectionStatus::ProtectionBroken { + chats.protection_broken += 1; + } else if encrypted { + if is_dc_message { + chats.opportunistic_dc += 1; + } else { + chats.opportunistic_mua += 1; + } + } else if is_dc_message { + chats.unencrypted_dc += 1; + } else { + chats.unencrypted_mua += 1; + } + } + Ok(chats) + }, + ) + .await?; + + let self_reporting_id = match context.get_config(Config::SelfReportingId).await? { + Some(id) => id, + None => { + let id = create_id(); + context + .set_config_internal(Config::SelfReportingId, Some(&id)) + .await?; + id + } + }; + let statistics = Statistics { + core_version: get_version_str().to_string(), + num_msgs, + num_chats, + db_size, + key_created, + chat_numbers, + self_reporting_id, + }; + + Ok(serde_json::to_string_pretty(&statistics)?) +} + +#[cfg(test)] +mod self_reporting_tests { + use anyhow::Context as _; + use strum::IntoEnumIterator; + use tempfile::tempdir; + + use super::*; + use crate::chat::{get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, MuteDuration}; + use crate::chatlist::Chatlist; + use crate::constants::Chattype; + use crate::mimeparser::SystemMessage; + use crate::receive_imf::receive_imf; + use crate::test_utils::{get_chat_msg, TestContext}; + use crate::tools::{create_outgoing_rfc724_mid, SystemTime}; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_draft_self_report() -> Result<()> { + let alice = TestContext::new_alice().await; + + let chat_id = send_self_report(&alice).await?; + let msg = get_chat_msg(&alice, chat_id, 0, 2).await; + assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); + + let chat = Chat::load_from_db(&alice, chat_id).await?; + assert!(chat.is_protected()); + + let statistics_msg = get_chat_msg(&alice, chat_id, 1, 2).await; + assert_eq!(statistics_msg.get_filename().unwrap(), "statistics.txt"); + + Ok(()) + } +}