feat: Implemented a whatsapp and minor changes

This commit is contained in:
2024-12-08 20:04:45 +01:00
parent 7c23064afe
commit 05b31e426e
16 changed files with 840 additions and 83 deletions

View File

@ -2,7 +2,7 @@ use std::{collections::HashMap, process::exit};
use dialoguer::Select;
use crate::{services::{instagram::Instagram, service::Service}, types::{participant::Participant, project::Project, user::User}};
use crate::{services::{instagram::instagram::Instagram, service::Service, whatsapp::whatsapp::WhatsApp}, types::{participant::Participant, project::Project, user::User}};
pub async fn add(path: String) {
let mut raw_project = Project::load(path.clone()).await.unwrap_or_else(|e| {
@ -19,6 +19,9 @@ pub async fn add(path: String) {
Box::new(Instagram {
data: None
}),
Box::new(WhatsApp {
data: None
}),
];
let options: Vec<&'static str> = services.iter().map(|service| service.get_name()).collect();
@ -32,30 +35,41 @@ pub async fn add(path: String) {
let service = services.get(selected).unwrap();
let mut final_service = service.setup();
let _ = final_service.load().await;
println!("Loading the project file");
final_service.load().await.unwrap_or_else(|e| {
println!("Error while loading {} export: {}", final_service.get_name(), e);
exit(1);
});
let participants = final_service.get_participants();
if participants.len() > raw_project.project.users.len() {
eprintln!("You have {} users in your project but in the {} chat there are {} participants!", raw_project.project.users.len(), final_service.get_name(), participants.len());
println!("You need to have equal amount or more users in project that participants in chat!");
exit(1);
}
let mut user_map: HashMap<Participant, User> = HashMap::new();
println!("Now select what project user is coresponding to a {} participant", service.get_name());
for user in raw_project.project.users.clone() {
let parts: Vec<Participant> = participants.iter().filter(|participant| {
!user_map.contains_key(&participant)
for participant in participants {
let users: Vec<User> = raw_project.project.users.iter().filter(|user| {
!user_map.values().any(|x| x.name == user.name)
}).map(|e| e.clone()).collect();
let options: Vec<String> = parts.iter().map(|part| part.name.clone()).collect();
let options: Vec<String> = users.iter().map(|usr| usr.name.clone()).collect();
let selected = Select::new()
.with_prompt(format!("Select a {} participant for user '{}'", service.get_name(), user.name))
.with_prompt(format!("Select a {} user for participant '{}'", service.get_name(), participant.name))
.items(&options)
.default(0)
.interact()
.unwrap();
let selected_part = parts.get(selected).unwrap().clone();
let selected_user = users.get(selected).unwrap().clone();
user_map.insert(selected_part.clone(), user);
user_map.insert(participant, selected_user);
}
println!("Starting to merge messages...");

39
src/commands/export.rs Normal file
View File

@ -0,0 +1,39 @@
use std::process::exit;
use dialoguer::Select;
use tokio::{fs::File, io::AsyncWriteExt};
use crate::types::project::Project;
pub async fn export(path: String) {
let raw_project = Project::load(path.clone()).await.unwrap_or_else(|e| {
eprintln!("Error while loading a project: {}", e);
exit(1);
});
let options = vec!["HTML", "JSON (without media)"];
let selected = Select::new()
.with_prompt("Choose a output type")
.items(&options)
.default(0)
.interact()
.unwrap();
match selected {
0 => {
// HTML
}
1 => {
// JSON (without media)
let content = raw_project.project.to_projectjson();
let json_content = serde_json::to_string(&content).unwrap();
let mut file = File::create("./export.json").await.expect("Error while creating a file.");
let _ = file.write_all(json_content.as_bytes()).await;
}
_ => {
panic!("Invalid option");
}
};
}

View File

@ -1,2 +1,3 @@
pub mod users;
pub mod add;
pub mod export;

View File

@ -1,4 +1,4 @@
use std::process::exit;
use std::{collections::HashSet, process::exit};
use dialoguer::{Input, Select};
use tabled::Table;
@ -72,6 +72,21 @@ pub async fn menu(mut project_raw: ProjectRaw, path: String) {
.default(0)
.interact()
.unwrap();
let delete_name = options_users.get(option_remove).unwrap();
let mut participants_in_chat: HashSet<String> = HashSet::new();
for message in project.messages.clone() {
participants_in_chat.insert(message.sender_name);
for reaction in message.reactions {
participants_in_chat.insert(reaction.actor_name);
}
}
if participants_in_chat.contains(delete_name) {
eprintln!("This user has some messages in chat, therefore cannot be removed!")
}
project.users.remove(option_remove);

View File

@ -5,7 +5,7 @@ mod services;
use std::collections::HashSet;
use clap::{command, ArgGroup, Parser};
use commands::{add::add, users::users};
use commands::{add::add, export::export, users::users};
use types::project::Project;
#[derive(Parser, Debug)]
@ -15,7 +15,7 @@ use types::project::Project;
about = "Export and merge your messages between platforms",
author = "Jakub Žitník",
group = ArgGroup::new("command")
.args(&["new", "users", "add"])
.args(&["new", "users", "add", "export"])
.multiple(false)
.required(true)
)]
@ -28,6 +28,9 @@ struct Cli {
#[arg(long = "add", help = "Add messages from a service.", value_name = "PROJECT_FILE")]
add: Option<String>,
// #[arg(long = "export", help = "Export all the chats in specific format", value_name = "PROJECT_FILE")]
// export: Option<String>,
}
#[tokio::main]
@ -53,4 +56,8 @@ async fn main() {
if let Some(path) = cli.add {
add(path).await;
}
// if let Some(path) = cli.export {
// export(path).await;
// }
}

View File

@ -7,81 +7,15 @@ use encoding::{all::ISO_8859_1, EncoderTrap, Encoding};
use serde::Deserialize;
use tokio::{fs::File, io::AsyncReadExt};
use crate::types::{message::{Message, Photo}, participant::Participant, project::ProjectRaw, user::User};
use crate::{services::service::Service, types::{participant::Participant, project::ProjectRaw, user::User}};
use super::service::Service;
use super::types::{InstagramMessage, InstagramParticipant};
fn decode_str(input: &String) -> String {
pub fn decode_str(input: &String) -> String {
let vec = ISO_8859_1.encode(input.as_str(), EncoderTrap::Strict).unwrap();
String::from_utf8(vec).unwrap()
}
#[derive(Deserialize, Debug)]
pub struct InstagramMessage {
sender_name: String,
timestamp_ms: usize,
content: Option<String>,
photos: Option<Vec<InstagramPhoto>>
}
impl InstagramMessage {
pub fn conv(&self, media_index: &mut usize, project_path: PathBuf, user_map: &HashMap<Participant, User>) -> Message {
let text_content = if let Some(content) = &self.content {
Some(decode_str(content))
} else {
None
};
let photos: Vec<Photo> = if let Some(photos_list) = &self.photos {
photos_list.iter().map(|photo| photo.conv(&text_content, media_index, project_path.clone())).collect()
} else {
vec![]
};
let participant = Participant {
name: decode_str(&self.sender_name)
};
Message {
sender_name: user_map.get(&participant).unwrap().name.clone(),
text_content,
timestamp: self.timestamp_ms,
photos
}
}
}
#[derive(Deserialize, Debug)]
struct InstagramPhoto {
uri: String
}
impl InstagramPhoto {
pub fn conv(&self, text_content: &Option<String>, media_index: &mut usize, mut original_path: PathBuf) -> Photo {
*media_index += 1;
let img_path = (*media_index - 1).to_string();
for _ in 0..4 {
if !original_path.pop() {
break;
}
}
original_path.push(self.uri.clone());
Photo {
text_content: text_content.clone(),
img_path,
original_img_path: Some(original_path.display().to_string())
}
}
}
#[derive(Deserialize, Debug)]
pub struct InstagramParticipant {
pub name: String,
}
#[derive(Deserialize, Debug)]
struct InstagramDataJson {
messages: Vec<InstagramMessage>,
@ -212,7 +146,7 @@ impl Service for Instagram {
let moved = raw_project.project.push_msg(&msg);
if moved {
// Move the file to the media folder
// Move media files to the media folder
for photo in msg.photos.clone() {
let original_path = photo.original_img_path.unwrap().clone();
let path = format!("media/{}", photo.img_path);
@ -220,6 +154,20 @@ impl Service for Instagram {
media_files.push(photovalues);
}
for video in msg.videos.clone() {
let original_path = video.original_vid_path.unwrap().clone();
let path = format!("media/{}", video.vid_path);
let photovalues = (path, original_path);
media_files.push(photovalues);
}
for audio in msg.audio.clone() {
let original_path = audio.original_audio_path.unwrap().clone();
let path = format!("media/{}", audio.audio_path);
let photovalues = (path, original_path);
media_files.push(photovalues);
}
// Update the highest media index of the project if the message was merged
raw_project.project.media_index = media_index_clone;
}

View File

@ -0,0 +1,2 @@
pub mod types;
pub mod instagram;

View File

@ -0,0 +1,168 @@
use std::{collections::HashMap, path::PathBuf};
use serde::Deserialize;
use crate::types::{message::{Audio, Message, Photo, Reaction, Video}, participant::Participant, user::User};
use super::instagram::decode_str;
#[derive(Deserialize, Debug)]
pub struct InstagramParticipant {
pub name: String,
}
#[derive(Deserialize, Debug)]
pub struct InstagramMessage {
sender_name: String,
timestamp_ms: usize,
content: Option<String>,
photos: Option<Vec<InstagramPhoto>>,
videos: Option<Vec<InstagramVideo>>,
audio_files: Option<Vec<InstagramAudio>>,
reactions: Option<Vec<InstagramReaction>>
}
impl InstagramMessage {
pub fn conv(&self, media_index: &mut usize, project_path: PathBuf, user_map: &HashMap<Participant, User>) -> Message {
let text_content = if let Some(content) = &self.content {
Some(decode_str(content))
} else {
None
};
let photos: Vec<Photo> = if let Some(photos_list) = &self.photos {
photos_list.iter().map(|photo| photo.conv(&text_content, media_index, project_path.clone())).collect()
} else {
vec![]
};
let videos: Vec<Video> = if let Some(videos_list) = &self.videos {
videos_list.iter().map(|video| video.conv(&text_content, media_index, project_path.clone())).collect()
} else {
vec![]
};
let audio: Vec<Audio> = if let Some(audio_list) = &self.audio_files {
audio_list.iter().map(|audio| audio.conv(&text_content, media_index, project_path.clone())).collect()
} else {
vec![]
};
let reactions: Vec<Reaction> = if let Some(reaction_list) = &self.reactions {
reaction_list.iter().map(|reaction| reaction.conv()).collect()
} else {
vec![]
};
let participant = Participant {
name: decode_str(&self.sender_name)
};
let sender = user_map.get(&participant);
let sender_name = if let Some(sender_value) = sender {
sender_value.name.clone()
} else {
String::from("Unknown user")
};
Message {
sender_name,
text_content,
timestamp: self.timestamp_ms,
service: crate::types::message::Service::Instagram,
photos,
videos,
audio,
reactions
}
}
}
#[derive(Deserialize, Debug)]
pub struct InstagramReaction {
pub reaction: String,
pub actor: String,
pub timestamp: Option<usize>
}
impl InstagramReaction {
pub fn conv(&self) -> Reaction {
Reaction {
actor_name: decode_str(&self.actor),
content: decode_str(&self.reaction),
timestamp: self.timestamp.unwrap_or(0)
}
}
}
// Media files
#[derive(Deserialize, Debug)]
struct InstagramPhoto {
uri: String
}
#[derive(Deserialize, Debug)]
struct InstagramVideo {
uri: String
}
#[derive(Deserialize, Debug)]
struct InstagramAudio {
uri: String
}
impl InstagramPhoto {
pub fn conv(&self, text_content: &Option<String>, media_index: &mut usize, mut original_path: PathBuf) -> Photo {
*media_index += 1;
let img_path = (*media_index - 1).to_string();
for _ in 0..4 {
if !original_path.pop() {
break;
}
}
original_path.push(self.uri.clone());
Photo {
text_content: text_content.clone(),
img_path,
original_img_path: Some(original_path.display().to_string())
}
}
}
impl InstagramVideo {
pub fn conv(&self, text_content: &Option<String>, media_index: &mut usize, mut original_path: PathBuf) -> Video {
*media_index += 1;
let vid_path = (*media_index - 1).to_string();
for _ in 0..4 {
if !original_path.pop() {
break;
}
}
original_path.push(self.uri.clone());
Video {
text_content: text_content.clone(),
vid_path,
original_vid_path: Some(original_path.display().to_string())
}
}
}
impl InstagramAudio {
pub fn conv(&self, text_content: &Option<String>, media_index: &mut usize, mut original_path: PathBuf) -> Audio {
*media_index += 1;
let audio_path = (*media_index - 1).to_string();
for _ in 0..4 {
if !original_path.pop() {
break;
}
}
original_path.push(self.uri.clone());
Audio {
text_content: text_content.clone(),
audio_path,
original_audio_path: Some(original_path.display().to_string())
}
}
}

View File

@ -1,2 +1,4 @@
pub mod service;
pub mod instagram;
pub mod whatsapp;

View File

@ -0,0 +1,2 @@
pub mod whatsapp;
pub mod types;

View File

@ -0,0 +1,125 @@
use std::{collections::HashMap, path::PathBuf};
use serde::Deserialize;
use crate::types::{message::{Audio, Message, Photo, Video}, participant::Participant, user::User};
#[derive(Deserialize, Debug)]
pub struct WhatsAppMessage {
pub sender: String,
pub epoch: usize,
pub content: Option<String>,
pub photos: Vec<WhatsAppPhoto>,
pub videos: Vec<WhatsAppVideo>,
pub audio: Option<WhatsAppAudio>,
}
impl WhatsAppMessage {
pub fn conv(&self, media_index: &mut usize, project_path: PathBuf, user_map: &HashMap<Participant, User>) -> Message {
let photos: Vec<Photo> =
self.photos.iter().map(|photo| photo.conv(media_index, project_path.clone())).collect();
let videos: Vec<Video> =
self.videos.iter().map(|video| video.conv(media_index, project_path.clone())).collect();
let audio: Vec<Audio> = if let Some(audio_f) = &self.audio {
vec![audio_f.conv(media_index, project_path.clone())]
} else {
vec![]
};
let participant = Participant {
name: self.sender.clone()
};
let sender = user_map.get(&participant);
let sender_name = if let Some(sender_value) = sender {
sender_value.name.clone()
} else {
String::from("Unknown user")
};
Message {
sender_name,
text_content: self.content.clone(),
timestamp: self.epoch,
service: crate::types::message::Service::WhatsApp,
photos,
videos,
audio,
reactions: vec![]
}
}
}
// Media files
#[derive(Deserialize, Debug)]
pub struct WhatsAppPhoto {
pub uri: String,
pub text_content: String,
}
#[derive(Deserialize, Debug)]
pub struct WhatsAppVideo {
pub uri: String,
pub text_content: String,
}
#[derive(Deserialize, Debug)]
pub struct WhatsAppAudio {
pub uri: String,
pub text_content: String,
}
impl WhatsAppPhoto {
pub fn conv(&self, media_index: &mut usize, mut original_path: PathBuf) -> Photo {
*media_index += 1;
let img_path = (*media_index - 1).to_string();
original_path.push(self.uri.clone());
Photo {
text_content: Some(self.text_content.clone()),
img_path,
original_img_path: Some(original_path.display().to_string())
}
}
}
impl WhatsAppVideo {
pub fn conv(&self, media_index: &mut usize, mut original_path: PathBuf) -> Video {
*media_index += 1;
let vid_path = (*media_index - 1).to_string();
original_path.push(self.uri.clone());
Video {
text_content: Some(self.text_content.clone()),
vid_path,
original_vid_path: Some(original_path.display().to_string())
}
}
}
impl WhatsAppAudio {
pub fn conv(&self, media_index: &mut usize, mut original_path: PathBuf) -> Audio {
*media_index += 1;
let audio_path = (*media_index - 1).to_string();
original_path.push(self.uri.clone());
Audio {
text_content: Some(self.text_content.clone()),
audio_path,
original_audio_path: Some(original_path.display().to_string())
}
}
}
pub struct WhatsAppParticipant {
pub name: String
}
impl WhatsAppParticipant {
pub fn conv(&self) -> Participant {
Participant {
name: self.name.clone()
}
}
}

View File

@ -0,0 +1,232 @@
use std::{collections::{HashMap, HashSet}, fs, io, path::PathBuf, process::exit};
use chrono::NaiveDateTime;
use dialoguer::Input;
use async_trait::async_trait;
use regex::Regex;
use tokio::{fs::File, io::AsyncReadExt};
use crate::{services::service::Service, types::{participant::Participant, project::ProjectRaw, user::User}};
use super::types::{WhatsAppAudio, WhatsAppMessage, WhatsAppParticipant, WhatsAppPhoto, WhatsAppVideo};
pub fn parse_messages(input: &str) -> Vec<WhatsAppMessage> {
let mut messages = Vec::new();
let re = Regex::new(r"^(\d{2}/\d{2}/\d{2}), (\d{2}:\d{2}) - (.*?): (.*)").unwrap();
let mut current_message: Option<WhatsAppMessage> = None;
for line in input.lines() {
if let Some(captures) = re.captures(line) {
let date = captures.get(1).unwrap().as_str();
let time = captures.get(2).unwrap().as_str();
let sender = captures.get(3).unwrap().as_str().to_string();
let content = captures.get(4).unwrap().as_str();
// Parse date as DD/MM/YY and convert to epoch
let datetime_str = format!("{} {}", date, time); // DD/MM/YY HH:MM
let datetime = NaiveDateTime::parse_from_str(&datetime_str, "%d/%m/%y %H:%M").unwrap();
let epoch = datetime.and_utc().timestamp() as usize;
// Save the previous message if there is one
if let Some(msg) = current_message.take() {
messages.push(msg);
}
// Handle media messages
if content.starts_with("IMG") || content.starts_with("STK") {
current_message = Some(WhatsAppMessage {
sender,
epoch,
content: None,
photos: vec![WhatsAppPhoto {
uri: content.split(' ').next().unwrap().to_string(),
text_content: String::new(),
}],
videos: vec![],
audio: None,
});
} else if content.starts_with("VID") {
current_message = Some(WhatsAppMessage {
sender,
epoch,
content: None,
photos: vec![],
videos: vec![WhatsAppVideo {
uri: content.split(' ').next().unwrap().to_string(),
text_content: String::new(),
}],
audio: None,
});
} else if content.starts_with("PTT") {
messages.push(WhatsAppMessage {
sender,
epoch,
content: None,
photos: vec![],
videos: vec![],
audio: Some(WhatsAppAudio {
uri: content.split(' ').next().unwrap().to_string(),
text_content: String::new(),
}),
});
} else {
// Regular message
current_message = Some(WhatsAppMessage {
sender,
epoch,
content: Some(content.to_string()),
photos: vec![],
videos: vec![],
audio: None,
});
}
} else if let Some(msg) = current_message.as_mut() {
// Handle continuations or appending to media
if let Some(photo) = msg.photos.last_mut() {
photo.text_content.push('\n');
photo.text_content.push_str(line);
} else if let Some(video) = msg.videos.last_mut() {
video.text_content.push('\n');
video.text_content.push_str(line);
} else if let Some(content) = msg.content.as_mut() {
content.push('\n');
content.push_str(line);
}
}
}
// Add the last message if any
if let Some(msg) = current_message {
messages.push(msg);
}
messages
}
pub struct WhatsAppData {
pub project_path: String,
pub participants: Vec<WhatsAppParticipant>,
pub messages: Vec<WhatsAppMessage>,
}
pub struct WhatsApp {
pub data: Option<WhatsAppData>,
}
#[async_trait]
impl Service for WhatsApp {
fn get_name(&self) -> &'static str {
"WhatsApp"
}
fn setup(&self) -> Box<dyn Service> {
let export_path: String = Input::new()
.with_prompt("Enter path for your WhatsApp export. (extracted zip file)")
.interact()
.unwrap();
Box::new(WhatsApp {
data: Some(WhatsAppData {
project_path: export_path,
participants: vec![],
messages: vec![],
}),
})
}
async fn load(&mut self) -> io::Result<()> {
let data = self.data.as_mut().unwrap();
let project_path = &data.project_path;
let paths: Vec<PathBuf> = fs::read_dir(project_path).unwrap_or_else(|_| {
eprintln!("Folder not found!");
exit(1);
})
.filter_map(|entry| entry.ok())
.filter(|entry| {
if let Some(extension) = entry.path().extension() {
entry.path().is_file() && extension == "txt"
} else {
false
}
})
.map(|entry| entry.path()).collect();
if paths.len() != 1 {
eprintln!("The chat file was not found in the WhatsApp export");
exit(1);
}
let file_path = paths.get(0).unwrap();
let mut file = File::open(file_path).await?;
let mut contents = String::new();
file.read_to_string(&mut contents).await?;
let messages = parse_messages(&contents);
let mut participants: HashSet<String> = HashSet::new();
for message in &messages {
participants.insert(message.sender.clone());
}
data.participants = participants.into_iter().map(|p| {
WhatsAppParticipant {
name: p
}
}).collect();
data.messages = messages;
Ok(())
}
fn get_participants(&self) -> Vec<Participant> {
self.data.as_ref().unwrap().participants.iter().map(|part| part.conv()).collect()
}
async fn merge_messages(&self, raw_project: &mut ProjectRaw, user_map: &HashMap<Participant, User>) {
let data = self.data.as_ref().unwrap();
let path = PathBuf::from(data.project_path.clone());
let mut media_files: Vec<(String, String)> = vec![];
for message in &data.messages {
let mut media_index_clone = raw_project.project.media_index.clone();
let msg = message.conv(&mut media_index_clone, path.clone(), user_map);
let moved = raw_project.project.push_msg(&msg);
if moved {
// Move media files to the media folder
for photo in msg.photos.clone() {
let original_path = photo.original_img_path.unwrap().clone();
let path = format!("media/{}", photo.img_path);
let photovalues = (path, original_path);
media_files.push(photovalues);
}
for video in msg.videos.clone() {
let original_path = video.original_vid_path.unwrap().clone();
let path = format!("media/{}", video.vid_path);
let photovalues = (path, original_path);
media_files.push(photovalues);
}
for audio in msg.audio.clone() {
let original_path = audio.original_audio_path.unwrap().clone();
let path = format!("media/{}", audio.audio_path);
let photovalues = (path, original_path);
media_files.push(photovalues);
}
// Update the highest media index of the project if the message was merged
raw_project.project.media_index = media_index_clone;
}
}
let _ = raw_project.save_media_files(media_files).await;
}
}

View File

@ -5,7 +5,24 @@ pub struct Message {
pub sender_name: String,
pub timestamp: usize,
pub text_content: Option<String>,
pub photos: Vec<Photo>
pub photos: Vec<Photo>,
pub videos: Vec<Video>,
pub audio: Vec<Audio>,
pub service: Service,
pub reactions: Vec<Reaction>
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Reaction {
pub actor_name: String,
pub content: String,
pub timestamp: usize
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub enum Service {
Instagram,
WhatsApp,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
@ -16,3 +33,21 @@ pub struct Photo {
#[serde(skip)]
pub original_img_path: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Video {
pub text_content: Option<String>,
pub vid_path: String,
#[serde(skip)]
pub original_vid_path: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Audio {
pub text_content: Option<String>,
pub audio_path: String,
#[serde(skip)]
pub original_audio_path: Option<String>,
}

View File

@ -14,12 +14,25 @@ pub struct Project {
pub media_index: usize,
}
#[derive(Serialize)]
pub struct ProjectJson {
pub users: Vec<User>,
pub messages: Vec<Message>
}
pub struct ProjectRaw {
pub project: Project,
pub zip: ZipArchive<Cursor<Vec<u8>>>
}
impl Project {
pub fn to_projectjson(&self) -> ProjectJson {
ProjectJson {
users: self.users.clone(),
messages: self.messages.clone()
}
}
pub async fn load(pathstr: String) -> io::Result<ProjectRaw> {
let path = PathBuf::from(pathstr);