Attempting to import valid contacts from Desktop (#250). Work on #251 …

…(almost done I think). Fix sorting of HTML search results. Update emoji list. Shrink searchidx.js some. Some refactoring (fileSize()). Warn when Desktop data directory is not found. Adjustments to CSS.
bepaald · Oct 14, 2024 · cb68139 · cb68139
1 parent 7e5242b
commit cb68139
Show file tree

Hide file tree

Showing 23 changed files with 3,433 additions and 3,293 deletions.
diff --git a/attachmentmetadata/attachmentmetadata.h b/attachmentmetadata/attachmentmetadata.h
@@ -21,13 +21,14 @@
 #define ATTACHMENTMETADATA_H_
 
 #include <string>
+#include <cstdint>
 
 struct AttachmentMetadata
 {
   int width;
   int height;
   std::string filetype;
-  long long int filesize;
+  uint64_t filesize;
   std::string hash;
   std::string filename;
   operator bool() const
@@ -37,7 +38,7 @@ struct AttachmentMetadata
 
   static AttachmentMetadata getAttachmentMetaData(std::string const &filename, bool skiphash = false);
   static AttachmentMetadata getAttachmentMetaData(std::string const &filename, unsigned char *data,
-                                                  long long int data_size, bool skiphash = false);
+                                                  uint64_t data_size, bool skiphash = false);
 };
 
 #endif
diff --git a/attachmentmetadata/getattachmentmetadata.cc b/attachmentmetadata/getattachmentmetadata.cc
@@ -24,7 +24,7 @@
 #include "../base64/base64.h"
 #include "../common_filesystem.h"
 
-AttachmentMetadata AttachmentMetadata::getAttachmentMetaData(std::string const &file, unsigned char *data, long long int data_size, bool skiphash) // static
+AttachmentMetadata AttachmentMetadata::getAttachmentMetaData(std::string const &file, unsigned char *data, uint64_t data_size, bool skiphash) // static
 {
   //struct AttachmentMetadata
   //{
@@ -67,7 +67,7 @@ AttachmentMetadata AttachmentMetadata::getAttachmentMetaData(std::string const &
   }
 
   // set buffer for file header
-  int bufsize = std::min(data_size, 30ll);
+  int bufsize = std::min(data_size, uint64_t(30));
   unsigned char *buf = data;
 
 
@@ -290,11 +290,17 @@ AttachmentMetadata AttachmentMetadata::getAttachmentMetaData(std::string const &
     return AttachmentMetadata{-1, -1, std::string(), 0, std::string(), std::string()};
   }
 
-  filestream.seekg(0, std::ios_base::end);
-  long long int file_size = filestream.tellg();
-  filestream.seekg(0, std::ios_base::beg);
+  //filestream.seekg(0, std::ios_base::end);
+  //long long int file_size = filestream.tellg();
+  //filestream.seekg(0, std::ios_base::beg);
+  uint64_t file_size = bepaald::fileSize(file);
+  if (file_size == static_cast<std::uintmax_t>(-1)) [[unlikely]]
+  {
+    Logger::warning("Failed to get filesize for attachment '", file, "'");
+    return AttachmentMetadata{-1, -1, std::string(), file_size, std::string(), file};
+  }
 
-  if (file_size == 0)
+  if (file_size == 0) [[unlikely]]
   {
     Logger::warning("Attachment '", file, "' is zero bytes");
     return AttachmentMetadata{-1, -1, std::string(), file_size, std::string(), file};

diff --git a/autoversion.h b/autoversion.h
@@ -20,6 +20,6 @@
 #ifndef VERSION_H_
 #define VERSION_H_
 
-#define VERSIONDATE "20241008.224008"
+#define VERSIONDATE "20241014.161022"
 
 #endif
diff --git a/common_filesystem.h b/common_filesystem.h
@@ -32,6 +32,7 @@ namespace bepaald
   inline bool createDir(std::string const &path);
   inline bool isEmpty(std::string const &path);
   inline bool clearDirectory(std::string const &path);
+  inline uint64_t fileSize(std::string const &path);
 }
 
 inline bool bepaald::fileOrDirExists(std::string const &path)
@@ -76,4 +77,10 @@ inline bool bepaald::clearDirectory(std::string const &path)
   return true;
 }
 
+inline uint64_t bepaald::fileSize(std::string const &path)
+{
+  std::error_code ec;
+  return std::filesystem::file_size(std::filesystem::path(path), ec);
+}
+
 #endif
diff --git a/desktopattachmentreader/getencryptedattachment.cc b/desktopattachmentreader/getencryptedattachment.cc
@@ -19,6 +19,8 @@
 
 #include "desktopattachmentreader.h"
 
+#include "../common_filesystem.h"
+
 int DesktopAttachmentReader::getAttachmentData(unsigned char **rawdata, bool verbose [[maybe_unused]])
 {
   // set AES+MAC key
@@ -30,7 +32,7 @@ int DesktopAttachmentReader::getAttachmentData(unsigned char **rawdata, bool ver
   unsigned char *mackey = key_data.get() + 32;
 
   // open file
-  std::ifstream file(d_path, std::ios_base::in | std::ios_base::binary);
+  std::ifstream file(std::filesystem::path(d_path), std::ios_base::in | std::ios_base::binary);
   if (!file.is_open())
   {
     Logger::error("Failed to open file '", d_path, "'");
@@ -39,9 +41,10 @@ int DesktopAttachmentReader::getAttachmentData(unsigned char **rawdata, bool ver
 
   // set iv/data length.
   int64_t iv_length = 16;
-  file.seekg(0, std::ios_base::end);
-  int64_t data_length = file.tellg() - static_cast<int64_t>(iv_length + mackey_length);
-  file.seekg(0, std::ios_base::beg);
+  //file.seekg(0, std::ios_base::end);
+  //int64_t data_length = file.tellg() - static_cast<int64_t>(iv_length + mackey_length);
+  //file.seekg(0, std::ios_base::beg);
+  int64_t data_length = bepaald::fileSize(d_path) - static_cast<int64_t>(iv_length + mackey_length);
 
   // set iv
   std::unique_ptr<unsigned char[]> iv(new unsigned char[iv_length]);

diff --git a/desktopdatabase/init.cc b/desktopdatabase/init.cc
@@ -23,7 +23,15 @@ bool DesktopDatabase::init()
 {
   // get directories
   if (d_configdir.empty() || d_databasedir.empty())
+  {
     std::tie(d_configdir, d_databasedir) = getDesktopDir();
+    if (d_configdir.empty() || d_databasedir.empty()) [[unlikely]]
+    {
+      Logger::warning("Failed to set default location of Signal Desktop data.");
+      Logger::warning_indent("Consider using `--desktopdir <DIR>' to specify manually.");
+      Logger::warning_indent("Attempting to continue, but this will likely cause errors.");
+    }
+  }
 
   // check if a wal (Write-Ahead Logging) file is present in path, and warn user to (cleanly) shut Signal Desktop down
   if (!d_ignorewal &&

diff --git a/dummybackup/dummybackup.h b/dummybackup/dummybackup.h
@@ -36,7 +36,7 @@ inline DummyBackup::DummyBackup(bool verbose, bool truncate, bool showprogress)
   :
   SignalBackup(verbose, truncate, showprogress)
 {
-  // set up required tables (database version 223
+  // set up required tables (database version 223)
 
   // MESSAGE
   if (!d_database.exec("CREATE TABLE IF NOT EXISTS message (_id INTEGER PRIMARY KEY AUTOINCREMENT, date_sent INTEGER NOT NULL, date_received INTEGER NOT NULL, date_server INTEGER DEFAULT -1, thread_id INTEGER NOT NULL REFERENCES thread (_id) ON DELETE CASCADE, from_recipient_id INTEGER NOT NULL REFERENCES recipient (_id) ON DELETE CASCADE, from_device_id INTEGER, to_recipient_id INTEGER NOT NULL REFERENCES recipient (_id) ON DELETE CASCADE, type INTEGER NOT NULL, body TEXT, read INTEGER DEFAULT 0, ct_l TEXT, exp INTEGER, m_type INTEGER, m_size INTEGER, st INTEGER, tr_id TEXT, subscription_id INTEGER DEFAULT -1, receipt_timestamp INTEGER DEFAULT -1, has_delivery_receipt INTEGER DEFAULT 0, has_read_receipt INTEGER DEFAULT 0, viewed INTEGER DEFAULT 0, mismatched_identities TEXT DEFAULT NULL, network_failures TEXT DEFAULT NULL, expires_in INTEGER DEFAULT 0, expire_started INTEGER DEFAULT 0, notified INTEGER DEFAULT 0, quote_id INTEGER DEFAULT 0, quote_author INTEGER DEFAULT 0, quote_body TEXT DEFAULT NULL, quote_missing INTEGER DEFAULT 0, quote_mentions BLOB DEFAULT NULL, quote_type INTEGER DEFAULT 0, shared_contacts TEXT DEFAULT NULL, unidentified INTEGER DEFAULT 0, link_previews TEXT DEFAULT NULL, view_once INTEGER DEFAULT 0, reactions_unread INTEGER DEFAULT 0, reactions_last_seen INTEGER DEFAULT -1, remote_deleted INTEGER DEFAULT 0, mentions_self INTEGER DEFAULT 0, notified_timestamp INTEGER DEFAULT 0, server_guid TEXT DEFAULT NULL, message_ranges BLOB DEFAULT NULL, story_type INTEGER DEFAULT 0, parent_story_id INTEGER DEFAULT 0, export_state BLOB DEFAULT NULL, exported INTEGER DEFAULT 0, scheduled_date INTEGER DEFAULT -1, latest_revision_id INTEGER DEFAULT NULL REFERENCES message (_id) ON DELETE CASCADE, original_message_id INTEGER DEFAULT NULL REFERENCES message (_id) ON DELETE CASCADE, revision_number INTEGER DEFAULT 0, message_extras BLOB DEFAULT NULL)"))
@@ -47,7 +47,8 @@ inline DummyBackup::DummyBackup(bool verbose, bool truncate, bool showprogress)
     return;
 
   // RECIPIENT
-  if (!d_database.exec("CREATE TABLE IF NOT EXISTS recipient ( _id INTEGER PRIMARY KEY AUTOINCREMENT, type INTEGER DEFAULT 0, e164 TEXT UNIQUE DEFAULT NULL, aci TEXT UNIQUE DEFAULT NULL, pni TEXT UNIQUE DEFAULT NULL CHECK (pni LIKE 'PNI:%'), username TEXT UNIQUE DEFAULT NULL, email TEXT UNIQUE DEFAULT NULL, group_id TEXT UNIQUE DEFAULT NULL, distribution_list_id INTEGER DEFAULT NULL, call_link_room_id TEXT DEFAULT NULL, registered INTEGER DEFAULT 0, unregistered_timestamp INTEGER DEFAULT 0, blocked INTEGER DEFAULT 0, hidden INTEGER DEFAULT 0, profile_key TEXT DEFAULT NULL, profile_key_credential TEXT DEFAULT NULL, profile_sharing INTEGER DEFAULT 0, profile_given_name TEXT DEFAULT NULL, profile_family_name TEXT DEFAULT NULL, profile_joined_name TEXT DEFAULT NULL, profile_avatar TEXT DEFAULT NULL, last_profile_fetch INTEGER DEFAULT 0, system_given_name TEXT DEFAULT NULL, system_family_name TEXT DEFAULT NULL, system_joined_name TEXT DEFAULT NULL, system_nickname TEXT DEFAULT NULL, system_photo_uri TEXT DEFAULT NULL, system_phone_label TEXT DEFAULT NULL, system_phone_type INTEGER DEFAULT -1, system_contact_uri TEXT DEFAULT NULL, system_info_pending INTEGER DEFAULT 0, notification_channel TEXT DEFAULT NULL, message_ringtone TEXT DEFAULT NULL, message_vibrate INTEGER DEFAULT 0, call_ringtone TEXT DEFAULT NULL, call_vibrate INTEGER DEFAULT 0, mute_until INTEGER DEFAULT 0, message_expiration_time INTEGER DEFAULT 0, sealed_sender_mode INTEGER DEFAULT 0, storage_service_id TEXT UNIQUE DEFAULT NULL, storage_service_proto TEXT DEFAULT NULL, mention_setting INTEGER DEFAULT 0, capabilities INTEGER DEFAULT 0, last_session_reset BLOB DEFAULT NULL, wallpaper BLOB DEFAULT NULL, wallpaper_uri TEXT DEFAULT NULL, about TEXT DEFAULT NULL, about_emoji TEXT DEFAULT NULL, extras BLOB DEFAULT NULL, groups_in_common INTEGER DEFAULT 0, avatar_color TEXT DEFAULT NULL, chat_colors BLOB DEFAULT NULL, custom_chat_colors_id INTEGER DEFAULT 0, badges BLOB DEFAULT NULL, needs_pni_signature INTEGER DEFAULT 0, reporting_token BLOB DEFAULT NULL , phone_number_sharing INTEGER DEFAULT 0, phone_number_discoverable INTEGER DEFAULT 0, pni_signature_verified INTEGER DEFAULT 0, nickname_given_name TEXT DEFAULT NULL, nickname_family_name TEXT DEFAULT NULL, nickname_joined_name TEXT DEFAULT NULL, note TEXT DEFAULT NULL)"))
+  // added message_expiration_time_version
+  if (!d_database.exec("CREATE TABLE IF NOT EXISTS recipient ( _id INTEGER PRIMARY KEY AUTOINCREMENT, type INTEGER DEFAULT 0, e164 TEXT UNIQUE DEFAULT NULL, aci TEXT UNIQUE DEFAULT NULL, pni TEXT UNIQUE DEFAULT NULL CHECK (pni LIKE 'PNI:%'), username TEXT UNIQUE DEFAULT NULL, email TEXT UNIQUE DEFAULT NULL, group_id TEXT UNIQUE DEFAULT NULL, distribution_list_id INTEGER DEFAULT NULL, call_link_room_id TEXT DEFAULT NULL, registered INTEGER DEFAULT 0, unregistered_timestamp INTEGER DEFAULT 0, blocked INTEGER DEFAULT 0, hidden INTEGER DEFAULT 0, profile_key TEXT DEFAULT NULL, profile_key_credential TEXT DEFAULT NULL, profile_sharing INTEGER DEFAULT 0, profile_given_name TEXT DEFAULT NULL, profile_family_name TEXT DEFAULT NULL, profile_joined_name TEXT DEFAULT NULL, profile_avatar TEXT DEFAULT NULL, last_profile_fetch INTEGER DEFAULT 0, system_given_name TEXT DEFAULT NULL, system_family_name TEXT DEFAULT NULL, system_joined_name TEXT DEFAULT NULL, system_nickname TEXT DEFAULT NULL, system_photo_uri TEXT DEFAULT NULL, system_phone_label TEXT DEFAULT NULL, system_phone_type INTEGER DEFAULT -1, system_contact_uri TEXT DEFAULT NULL, system_info_pending INTEGER DEFAULT 0, notification_channel TEXT DEFAULT NULL, message_ringtone TEXT DEFAULT NULL, message_vibrate INTEGER DEFAULT 0, call_ringtone TEXT DEFAULT NULL, call_vibrate INTEGER DEFAULT 0, mute_until INTEGER DEFAULT 0, message_expiration_time INTEGER DEFAULT 0, sealed_sender_mode INTEGER DEFAULT 0, storage_service_id TEXT UNIQUE DEFAULT NULL, storage_service_proto TEXT DEFAULT NULL, mention_setting INTEGER DEFAULT 0, capabilities INTEGER DEFAULT 0, last_session_reset BLOB DEFAULT NULL, wallpaper BLOB DEFAULT NULL, wallpaper_uri TEXT DEFAULT NULL, about TEXT DEFAULT NULL, about_emoji TEXT DEFAULT NULL, extras BLOB DEFAULT NULL, groups_in_common INTEGER DEFAULT 0, avatar_color TEXT DEFAULT NULL, chat_colors BLOB DEFAULT NULL, custom_chat_colors_id INTEGER DEFAULT 0, badges BLOB DEFAULT NULL, needs_pni_signature INTEGER DEFAULT 0, reporting_token BLOB DEFAULT NULL , phone_number_sharing INTEGER DEFAULT 0, phone_number_discoverable INTEGER DEFAULT 0, pni_signature_verified INTEGER DEFAULT 0, nickname_given_name TEXT DEFAULT NULL, nickname_family_name TEXT DEFAULT NULL, nickname_joined_name TEXT DEFAULT NULL, note TEXT DEFAULT NULL, message_expiration_time_version INTEGER DEFAULT 1 NOT NULL)"))
     return;
 
   // ATTACHMENT
@@ -78,6 +79,10 @@ inline DummyBackup::DummyBackup(bool verbose, bool truncate, bool showprogress)
   if (!d_database.exec("CREATE TABLE group_receipts (_id INTEGER PRIMARY KEY, mms_id INTEGER, address TEXT, status INTEGER, timestamp INTEGER, unidentified INTEGER DEFAULT 0)"))
     return;
 
+  // IDENTITIES // not really strictly necessary, but dtInsertRecipient now tries to fill this table...
+  if (!d_database.exec("CREATE TABLE identities (_id INTEGER PRIMARY KEY AUTOINCREMENT, address INTEGER UNIQUE, identity_key TEXT, first_use INTEGER DEFAULT 0, timestamp INTEGER DEFAULT 0, verified INTEGER DEFAULT 0, nonblocking_approval INTEGER DEFAULT 0)"))
+    return;
+
   // set database version
   d_databaseversion = 223;
   setColumnNames();

diff --git a/filedecryptor/filedecryptor.cc b/filedecryptor/filedecryptor.cc
@@ -19,6 +19,8 @@
 
 #include "filedecryptor.ih"
 
+#include "../common_filesystem.h"
+
 FileDecryptor::FileDecryptor(std::string const &filename, std::string const &passphrase, bool verbose, bool stoponerror, bool assumebadframesize, std::vector<long long int> const &editattachments)
   :
   CryptBase(verbose),
@@ -39,9 +41,10 @@ FileDecryptor::FileDecryptor(std::string const &filename, std::string const &pas
     return;
   }
 
-  file.seekg(0, std::ios_base::end);
-  d_filesize = file.tellg();
-  file.seekg(0);
+  //file.seekg(0, std::ios_base::end);
+  //d_filesize = file.tellg();
+  //file.seekg(0);
+  d_filesize = bepaald::fileSize(d_filename);
 
   // read first four bytes, they are the header size of the file:
   int32_t headerlength = getNextFrameBlockSize(file);

diff --git a/jsondatabase/jsondatabase.cc b/jsondatabase/jsondatabase.cc
@@ -19,6 +19,8 @@
 
 #include "jsondatabase.ih"
 
+#include "../common_filesystem.h"
+
 JsonDatabase::JsonDatabase(std::string const &jsonfile, bool verbose, bool truncate)
   :
   d_ok(false),
@@ -33,15 +35,16 @@ JsonDatabase::JsonDatabase(std::string const &jsonfile, bool verbose, bool trunc
     return;
   }
 
-  sourcefile.seekg(0, std::ios_base::end);
-  long long int datasize = sourcefile.tellg();
-  if (datasize <= 0)
+  //sourcefile.seekg(0, std::ios_base::end);
+  //long long int datasize = sourcefile.tellg();
+  //sourcefile.seekg(0, std::ios_base::beg);
+  uint64_t datasize = bepaald::fileSize(jsonfile);
+  if (datasize == 0 || datasize == static_cast<std::uintmax_t>(-1)) [[unlikely]]
   {
     Logger::error("Bad filesize (", datasize, ")");
     return;
   }
 
-  sourcefile.seekg(0, std::ios_base::beg);
   std::unique_ptr<char []> data(new char[datasize]);
 
   if (!sourcefile.read(data.get(), datasize))

diff --git a/rawfileattachmentreader/rawfileattachmentreader.h b/rawfileattachmentreader/rawfileattachmentreader.h
@@ -54,9 +54,10 @@ int RawFileAttachmentReader::getAttachment(FrameWithAttachment *frame, bool verb
     Logger::error("Failed to open file '", d_filename, "' for reading attachment");
     return 1;
   }
-  file.seekg(0, std::ios_base::end);
-  int64_t attachmentdata_size = file.tellg();
-  file.seekg(0, std::ios_base::beg);
+  //file.seekg(0, std::ios_base::end);
+  //int64_t attachmentdata_size = file.tellg();
+  //file.seekg(0, std::ios_base::beg);
+  uint64_t attachmentdata_size = bepaald::fileSize(d_filename);
 
   if (attachmentdata_size == 0) [[unlikely]]
     Logger::warning("Asked to read 0-byte attachment");