feat: receive pre-messages and adapt download on demand

fix python lint errors

receive pre-mesages, start with changes to imap loop.

refactor: move download code from `scheduler.rs` to `download.rs`, also
move `get_msg_id_by_rfc724_mid` to `MsgId::get_by_rfc724_mid`

`MAX_FETCH_MSG_SIZE` is no longer unused

Parse if it is a pre-message or full-message

start with receiving logic

get rid of `MsgId::get_by_rfc724_mid` because it was a duplicate of
`message::rfc724_mid_exists`

docs: add hint to `MimeMessage::from_bytes` stating that it has
side-effects.

receiving full message

send and receive `attachment_size` and set viewtype to text in
pre_message

metadata as struct in pre-message in header. And fill params that we can
already fill from the metadata. Also add a new api to check what
viewtype the message will have once downloaded.

api: jsonrpc: add `full_message_view_type` to `Message` and
`MessageInfo`

make PreMsgMetadata.to_header_value not consume self/PreMsgMetadata

add api to merge params

on download full message: merge new params into old params and remove
full-message metadata params

move tests to `src/tests/pre_messages.rs`

dynamically allocate test attachment bytes

fix detection of pre-messages. (it looked for the ChatFullMessageId
header in the unencrypted headers before)

fix setting dl state to avaiable on pre-messages

fix: save pre message with rfc724_mid of full message als disable
replacement for full messages

add some receiving tests and update test todo for premessage metadata

test: process full message before pre-message

test receive normal message

some serialization tests for PreMsgMetadata

remove outdated todo comment

test that pre-message contains message text

PreMsgMetadata: test_build_from_file_msg and test_build_from_file_msg

test: test_receive_pre_message_image

Test receiving the full message after receiving an edit after receiving
the pre-message

test_reaction_on_pre_message

test_full_download_after_trashed

test_webxdc_update_for_not_downloaded_instance

simplify fake webxdc generation in
test_webxdc_update_for_not_downloaded_instance

test_markseen_pre_msg

test_pre_msg_can_start_chat and test_full_msg_can_start_chat

test_download_later_keeps_message_order

test_chatlist_event_on_full_msg_download

fix download not working

log splitting into pre-message

add pre-message info to text when loading from db. this can be disabled
with config key `hide_pre_message_metadata_text` if ui wants to display
it in a prettier way.

update `download_limit` documentation

more logging: log size of pre and post messages

rename full message to Post-Message

split up the pre-message tests into multiple files

dedup test code by extracting code to create test messages into util
methods

remove post_message_view_type from api, now it is only used internally
for tests

remove `hide_pre_message_metadata_text` config option, as there
currently is no way to get the full message viewtype anymore

Update src/download.rs
resolve comment

use `parse_message_id` instead of removing `<>`parenthesis it manually

fix available_post_msgs gets no entries
handle forwarding and add a test for it.

convert comment to log warning event on unexpected download failure

add doc comment to `simple_imap_loop`

more logging

handle saving pre-message to self messages and test.

Author: Simon-Laux

deltachat-ffi/deltachat.h

@@ -488,11 +488,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    0=use IMAP IDLE if the server supports it.
  *                    This is a developer option used for testing polling used as an IDLE fallback.
  * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
- *                    For larger messages, only the header is downloaded and a placeholder is shown.
+ *                    For messages with large attachements, two messages are sent:
+ *                    a Pre-Message containing metadata and a Post-Message containing the attachment.
+ *                    Pre-Messages are always downloaded and show a placeholder message 
  *                    These messages can be downloaded fully using dc_download_full_msg() later.
- *                    The limit is compared against raw message sizes, including headers.
- *                    The actually used limit may be corrected
- *                    to not mess up with non-delivery-reports or read-receipts.
+ *                    Post-Messages are automatically downloaded if they are smaller than the download_limit.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
  * - `protect_autocrypt` = Enable Header Protection for Autocrypt header.
@@ -4310,7 +4310,8 @@ char*             dc_msg_get_webxdc_info      (const dc_msg_t* msg);
 
 /**
  * Get the size of the file. Returns the size of the file associated with a
- * message, if applicable.
+ * message, if applicable. 
+ * If message is a pre-message, then this returns size of the to be downloaded file.
  *
  * Typically, this is used to show the size of document files, e.g. a PDF.
  *

deltachat-jsonrpc/src/api/types/message.rs

@@ -92,6 +92,8 @@ pub struct MessageObject {
 
     file: Option<String>,
     file_mime: Option<String>,
+    /// Returns the size of the file in bytes, if applicable.
+    /// If message is a pre-message, then this returns size of the to be downloaded file.
     file_bytes: u64,
     file_name: Option<String>,
 

deltachat-rpc-client/tests/test_something.py

@@ -11,7 +11,7 @@
 import pytest
 
 from deltachat_rpc_client import Contact, EventType, Message, events
-from deltachat_rpc_client.const import DownloadState, MessageState
+from deltachat_rpc_client.const import MessageState
 from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS
 from deltachat_rpc_client.rpc import JsonRpcError
 

src/chat.rs

@@ -12,6 +12,7 @@ use std::time::Duration;
 use anyhow::{Context as _, Result, anyhow, bail, ensure};
 use chrono::TimeZone;
 use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line};
+use humansize::{BINARY, format_size};
 use mail_builder::mime::MimePart;
 use serde::{Deserialize, Serialize};
 use strum_macros::EnumIter;
@@ -2730,10 +2731,10 @@ async fn prepare_send_msg(
     Ok(row_ids)
 }
 
-/// Renders the message or Full-Message and Pre-Message.
+/// Renders the Message or splits it into Post-Message and Pre-Message.
 ///
-/// Pre-Message is a small message with metadata which announces a larger Full-Message.
-/// Full messages are not downloaded in the background.
+/// Pre-Message is a small message with metadata which announces a larger Post-Message.
+/// Post-Messages are not downloaded in the background.
 ///
 /// If pre-message is not nessesary this returns a normal message instead.
 async fn render_mime_message_and_pre_message(
@@ -2750,9 +2751,14 @@ async fn render_mime_message_and_pre_message(
             > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD;
 
     if needs_pre_message {
-        let mut mimefactory_full_msg = mimefactory.clone();
-        mimefactory_full_msg.set_as_full_message();
-        let rendered_msg = mimefactory_full_msg.render(context).await?;
+        info!(
+            context,
+            "Message is large and will be split into a pre- and a post-message.",
+        );
+
+        let mut mimefactory_post_msg = mimefactory.clone();
+        mimefactory_post_msg.set_as_post_message();
+        let rendered_msg = mimefactory_post_msg.render(context).await?;
 
         let mut mimefactory_pre_msg = mimefactory;
         mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg);
@@ -2856,6 +2862,21 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             }
         }?;
 
+    if let (post_msg, Some(pre_msg)) = (&rendered_msg, &rendered_pre_msg) {
+        info!(
+            context,
+            "Message Sizes: Pre-Message {}; Post-Message: {}",
+            format_size(pre_msg.message.len(), BINARY),
+            format_size(post_msg.message.len(), BINARY)
+        );
+    } else {
+        info!(
+            context,
+            "Message will be sent as normal message (no pre- and post message). Size: {}",
+            format_size(rendered_msg.message.len(), BINARY)
+        );
+    }
+
     if needs_encryption && !rendered_msg.is_encrypted {
         /* unrecoverable */
         message::set_msg_failed(
@@ -4319,6 +4340,14 @@ pub async fn forward_msgs_2ctx(
             msg.viewtype = Viewtype::Text;
         }
 
+        if msg.download_state != DownloadState::Done {
+            // we don't use Message.get_text() here,
+            // because it may change in future,
+            // when UI shows this info itself,
+            // then the additional_text will not be added in get_text anymore.
+            msg.text += &msg.additional_text;
+        }
+
         let param = &mut param;
         msg.param.steal(param, Param::File);
         msg.param.steal(param, Param::Filename);
@@ -4329,6 +4358,7 @@ pub async fn forward_msgs_2ctx(
         msg.param.steal(param, Param::ProtectQuote);
         msg.param.steal(param, Param::Quote);
         msg.param.steal(param, Param::Summary1);
+
         msg.in_reply_to = None;
 
         // do not leak data as group names; a default subject is generated by mimefactory
@@ -4395,31 +4425,42 @@ pub(crate) async fn save_copy_in_self_talk(
     msg.param.remove(Param::WebxdcDocumentTimestamp);
     msg.param.remove(Param::WebxdcSummary);
     msg.param.remove(Param::WebxdcSummaryTimestamp);
+    msg.param.remove(Param::PostMessageFileBytes);
+    msg.param.remove(Param::PostMessageViewtype);
+
+    if msg.download_state != DownloadState::Done {
+        // we don't use Message.get_text() here,
+        // because it may change in future,
+        // when UI shows this info itself,
+        // then the additional_text will not be added in get_text anymore.
+        msg.text += &msg.additional_text;
+    }
 
     if !msg.original_msg_id.is_unset() {
         bail!("message already saved.");
     }
 
-    let copy_fields = "from_id, to_id, timestamp_rcvd, type, txt,
+    let copy_fields = "from_id, to_id, timestamp_rcvd, type,
                        mime_modified, mime_headers, mime_compressed, mime_in_reply_to, subject, msgrmsg";
     let row_id = context
         .sql
         .insert(
             &format!(
                 "INSERT INTO msgs ({copy_fields},
                                    timestamp_sent,
-                                   chat_id, rfc724_mid, state, timestamp, param, starred)
+                                   txt, chat_id, rfc724_mid, state, timestamp, param, starred)
                  SELECT            {copy_fields},
                                    -- Outgoing messages on originating device
                                    -- have timestamp_sent == 0.
                                    -- We copy sort timestamp instead
                                    -- so UIs display the same timestamp
                                    -- for saved and original message.
                                    IIF(timestamp_sent == 0, timestamp, timestamp_sent),
-                                   ?, ?, ?, ?, ?, ?
+                                   ?, ?, ?, ?, ?, ?, ?
                  FROM msgs WHERE id=?;"
             ),
             (
+                msg.text,
                 dest_chat_id,
                 dest_rfc724_mid,
                 if msg.from_id == ContactId::SELF {

src/context.rs

@@ -609,7 +609,7 @@ impl Context {
                     convert_folder_meaning(self, folder_meaning).await?
                 {
                     connection
-                        .fetch_move_delete(self, &mut session, &watch_folder, folder_meaning)
+                        .fetch_move_delete(self, &mut session, true, &watch_folder, folder_meaning)
                         .await?;
                 }
             }
@@ -622,6 +622,12 @@ impl Context {
             {
                 warn!(self, "Failed to update quota: {err:#}.");
             }
+
+            // OPTIONAL TODO: if time left start downloading messages
+            // while (msg = download_when_normal_starts) {
+            //  if not time_left {break;}
+            //  connection.download_message(msg) }
+            // }
         }
 
         info!(
@@ -1080,13 +1086,6 @@ impl Context {
                 .await?
                 .unwrap_or_default(),
         );
-        res.insert(
-            "fail_on_receiving_full_msg",
-            self.sql
-                .get_raw_config("fail_on_receiving_full_msg")
-                .await?
-                .unwrap_or_default(),
-        );
         res.insert(
             "std_header_protection_composing",
             self.sql

src/context/context_tests.rs

@@ -297,7 +297,7 @@ async fn test_get_info_completeness() {
         "encrypted_device_token",
         "stats_last_update",
         "stats_last_old_contact_id",
-        "simulate_receive_imf_error",
+        "simulate_receive_imf_error", // only used in tests
     ];
     let t = TestContext::new().await;
     let info = t.get_info().await.unwrap();