From f42a14b6c94d8b658e7e4c4db08f1955f5f93ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 02:16:34 -0300 Subject: [PATCH 01/25] #2686: Don't process "from" and "to" metadata for system messages --- .../java/iped/properties/ExtraProperties.java | 4 +- .../ufed/handler/InstantMessageHandler.java | 97 ++++++++++--------- 2 files changed, 55 insertions(+), 46 deletions(-) diff --git a/iped-api/src/main/java/iped/properties/ExtraProperties.java b/iped-api/src/main/java/iped/properties/ExtraProperties.java index ccf4fdcf83..e7ac70fb05 100644 --- a/iped-api/src/main/java/iped/properties/ExtraProperties.java +++ b/iped-api/src/main/java/iped/properties/ExtraProperties.java @@ -78,7 +78,9 @@ public class ExtraProperties { public static final String GROUP_ID = "GroupID"; - public static final String IS_GROUP_MESSAGE = "isGroupMessage"; + public static final String IS_GROUP_MESSAGE = COMMUNICATION_PREFIX + "isGroupMessage"; + public static final Property COMMUNICATION_SYSTEM_MESSAGE = Property.internalBoolean(COMMUNICATION_PREFIX + "isSystemMessage"); + public static final String MESSAGE_BODY = MESSAGE_PREFIX + "Body"; //$NON-NLS-1$ diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java index e3956ccb07..002281c058 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java @@ -34,6 +34,7 @@ import iped.parsers.ufed.model.ReplyMessageData; import iped.parsers.util.ConversationConstants; import iped.properties.BasicProps; +import iped.properties.ExtraProperties; import iped.search.IItemSearcher; /** @@ -64,62 +65,68 @@ protected void fillMetadata(String prefix, Metadata metadata) { } if (!model.getAttachments().isEmpty()) { - metadata.set(MESSAGE_ATTACHMENT_COUNT, Integer.toString(model.getAttachments().size())); + metadata.set(MESSAGE_ATTACHMENT_COUNT, model.getAttachments().size()); } - // Message -> Direction - model.getFrom().ifPresentOrElse(from -> { - if (from.isPhoneOwner()) { - metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); - } else { - metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); - } - }, () -> { - if (model.getTo().stream().filter(Party::isPhoneOwner).findAny().isPresent()) { - metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); - } - }); + if (model.isSystemMessage()) { + metadata.set(ExtraProperties.COMMUNICATION_SYSTEM_MESSAGE, true); - // Message -> From - model.getFrom().ifPresent(from -> { - new PartyHandler(from, model.getSource()).fillMetadata(COMMUNICATION_FROM, metadata); - }); + } else { - // Message -> To - if (model.getTo().size() == 1) { - new PartyHandler(model.getTo().get(0), model.getSource()).fillMetadata(COMMUNICATION_TO, metadata); + // Message -> Direction + model.getFrom().ifPresentOrElse(from -> { + if (from.isPhoneOwner()) { + metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + } else { + metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + } + }, () -> { + if (model.getTo().stream().filter(Party::isPhoneOwner).findAny().isPresent()) { + metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + } + }); - } else if (model.getChat() != null) { + // Message -> From + model.getFrom().ifPresent(from -> { + new PartyHandler(from, model.getSource()).fillMetadata(COMMUNICATION_FROM, metadata); + }); - // in case "To" is NOT set in InstantMessage, try to... - List otherParticipants; - if (model.getFrom().isPresent()) { + // Message -> To + if (model.getTo().size() == 1) { + new PartyHandler(model.getTo().get(0), model.getSource()).fillMetadata(COMMUNICATION_TO, metadata); - // ...get other participants (without "from") - otherParticipants = model.getChat().getParticipants().stream() // - .filter(p -> !p.equals(model.getFrom().get())) // - .collect(Collectors.toList()); // - } else { - // ...get other participants - otherParticipants = model.getChat().getParticipants(); - } + } else if (model.getChat() != null) { + + // in case "To" is NOT set in InstantMessage, try to... + List otherParticipants; + if (model.getFrom().isPresent()) { - if (otherParticipants.size() == 1) { - new PartyHandler(otherParticipants.get(0), model.getSource()).fillMetadata(COMMUNICATION_TO, metadata); + // ...get other participants (without "from") + otherParticipants = model.getChat().getParticipants().stream() // + .filter(p -> !p.equals(model.getFrom().get())) // + .collect(Collectors.toList()); // + } else { + // ...get other participants + otherParticipants = model.getChat().getParticipants(); + } - } else if (otherParticipants.size() > 2) { + if (otherParticipants.size() == 1) { + new PartyHandler(otherParticipants.get(0), model.getSource()).fillMetadata(COMMUNICATION_TO, metadata); - // there is more than one "To", so use chat information - metadata.add(COMMUNICATION_TO, new ChatHandler(model.getChat()).getTitle(false, false)); + } else if (otherParticipants.size() > 2) { - String chatId = model.getChat().getFieldId(); - if (StringUtils.isNotBlank(chatId)) { - metadata.add(COMMUNICATION_TO + ":id", chatId); - } + // there is more than one "To", so use chat information + metadata.add(COMMUNICATION_TO, new ChatHandler(model.getChat()).getTitle(false, false)); - String chatName = model.getChat().getName(); - if (StringUtils.isNotBlank(chatName)) { - metadata.add(COMMUNICATION_TO + ":name", chatName); + String chatId = model.getChat().getFieldId(); + if (StringUtils.isNotBlank(chatId)) { + metadata.add(COMMUNICATION_TO + ":id", chatId); + } + + String chatName = model.getChat().getName(); + if (StringUtils.isNotBlank(chatName)) { + metadata.add(COMMUNICATION_TO + ":name", chatName); + } } } } @@ -161,7 +168,7 @@ protected void fillMetadata(String prefix, Metadata metadata) { }); if (model.isSystemMessage()) { - metadata.set(UFED_META_PREFIX + "isSystemMessage", Boolean.toString(true)); + } if (model.getPosition() != null) { From 5cec445ae555d2dbe9a7d6ea9f5a5f07cf911208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 02:18:44 -0300 Subject: [PATCH 02/25] #2686: Use country code (cc) when find phone number (ph) in WhatsApp preferences XML --- .../java/iped/parsers/whatsapp/WAAccount.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAAccount.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAAccount.java index 47f751992f..66d7d54d12 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAAccount.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAAccount.java @@ -58,15 +58,16 @@ public static WAAccount getFromAndroidXml(InputStream is) throws SAXException, I } else { expr = xpath.compile("/map/string[@name=\"registration_jid\"]"); value = (String) expr.evaluate(doc, XPathConstants.STRING); - if (StringUtils.isBlank(value)) { - expr = xpath.compile("/map/string[@name=\"ph\"]"); - value = (String) expr.evaluate(doc, XPathConstants.STRING); - } if (StringUtils.isNotBlank(value)) { - if (!value.endsWith(waSuffix)) { - value += waSuffix; - } id = value; + } else { + expr = xpath.compile("/map/string[@name=\"cc\"]"); + String cc = (String) expr.evaluate(doc, XPathConstants.STRING); + expr = xpath.compile("/map/string[@name=\"ph\"]"); + String ph = (String) expr.evaluate(doc, XPathConstants.STRING); + if (StringUtils.isNoneBlank(cc, ph)) { + id = cc + ph + waSuffix; + } } } From 2d30869b01c328f1c74846a48872141e9bff4299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 02:22:19 -0300 Subject: [PATCH 03/25] #2686: Prefer retrieving the WhatsApp phone number via user ID rather than other sources --- .../chat/WhatsAppPartyStringBuilder.java | 21 +++++++++++++++---- .../parsers/ufed/handler/PartyHandler.java | 10 +++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java index 6056d13e9f..c7b37b253c 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java @@ -2,6 +2,9 @@ import org.apache.commons.lang3.StringUtils; +import iped.parsers.whatsapp.WAContact; +import iped.parsers.whatsapp.WhatsAppParser; + public class WhatsAppPartyStringBuilder extends PartyStringBuilder { private static final String PHONE_PREFIX = "+"; @@ -9,16 +12,26 @@ public class WhatsAppPartyStringBuilder extends PartyStringBuilder { @Override public String build() { - String phoneFromUserID = StringUtils.substringBefore(userId, '@'); - if (StringUtils.isNotBlank(phoneFromUserID)) { - phoneFromUserID = StringUtils.prependIfMissing(phoneFromUserID, PHONE_PREFIX); + String phoneFromUserID = null; + if (StringUtils.endsWithAny(userId, WAContact.waSuffix, WAContact.waStatusSuffix)) { + phoneFromUserID = StringUtils.substringBeforeLast(userId, "@"); + if (StringUtils.isNotBlank(phoneFromUserID)) { + phoneFromUserID = StringUtils.prependIfMissing(phoneFromUserID, PHONE_PREFIX); + } } + String phoneStr = StringUtils.firstNonBlank(phoneFromUserID, phoneNumber); + if (StringUtils.isNotBlank(phoneStr)) { + phoneStr = StringUtils.prependIfMissing(phoneStr, PHONE_PREFIX); + } if (StringUtils.isNotBlank(name)) { - if (StringUtils.isBlank(phoneStr) || StringUtils.equalsAny(name, phoneNumber, phoneFromUserID)) { + if (StringUtils.isBlank(phoneStr) || name.equals(WhatsAppParser.UNKNOWN_ACCOUNT)) { return name; } + if (StringUtils.equalsAny(name, phoneNumber, phoneFromUserID)) { + return StringUtils.prependIfMissing(name, PHONE_PREFIX); + } return name + " (" + phoneStr + ")"; } else if (StringUtils.isNotBlank(phoneStr)) { return phoneStr; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java index 9b9803a658..196847e3b5 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java @@ -18,7 +18,9 @@ import iped.data.IItemReader; import iped.parsers.chat.PartyStringBuilderFactory; +import iped.parsers.ufed.model.Chat; import iped.parsers.ufed.model.Party; +import iped.parsers.whatsapp.WAContact; import iped.properties.ExtraProperties; import iped.properties.MediaTypes; import iped.search.IItemSearcher; @@ -46,9 +48,13 @@ protected void fillMetadata(String prefix, Metadata metadata) { metadata.add(prefix, getTitle()); model.getReferencedContact().ifPresentOrElse(ref -> { - metadata.add(prefix + CONVERSATION_SUFFIX_ID, StringUtils.firstNonBlank(model.getIdentifier(), ref.getUserID())); + String id = StringUtils.firstNonBlank(model.getIdentifier(), ref.getUserID()); + String phoneNumber = StringUtils.containsIgnoreCase(source, Chat.SOURCE_WHATSAPP) + ? StringUtils.substringBeforeLast(id, WAContact.waSuffix) + : ref.getPhoneNumber(); + metadata.add(prefix + CONVERSATION_SUFFIX_ID, id); metadata.add(prefix + CONVERSATION_SUFFIX_NAME, StringUtils.firstNonBlank(model.getName(), ref.getName())); - metadata.add(prefix + CONVERSATION_SUFFIX_PHONE, ref.getPhoneNumber()); + metadata.add(prefix + CONVERSATION_SUFFIX_PHONE, phoneNumber); metadata.add(prefix + CONVERSATION_SUFFIX_USERNAME, ref.getUsername()); }, () -> { metadata.add(prefix + CONVERSATION_SUFFIX_ID, model.getIdentifier()); From a93aeb9c766e7671607c407e1e541f9d8d0199ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 02:25:04 -0300 Subject: [PATCH 04/25] #2686: Automatically calculate count metadata for properties listed in IndexItem.countMetadata --- .../java/iped/engine/task/index/IndexItem.java | 14 +++++++++++++- .../iped/parsers/ufed/handler/ChatHandler.java | 7 ------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/iped-engine/src/main/java/iped/engine/task/index/IndexItem.java b/iped-engine/src/main/java/iped/engine/task/index/IndexItem.java index dfeab6f552..2258f5f567 100644 --- a/iped-engine/src/main/java/iped/engine/task/index/IndexItem.java +++ b/iped-engine/src/main/java/iped/engine/task/index/IndexItem.java @@ -82,7 +82,6 @@ import iped.engine.preview.PreviewConstants; import iped.engine.preview.PreviewInputStreamFactory; import iped.engine.sleuthkit.SleuthkitInputStreamFactory; -import iped.engine.task.ImageThumbTask; import iped.engine.task.MinIOTask.MinIOInputInputStreamFactory; import iped.engine.task.ThumbTask; import iped.engine.task.similarity.ImageSimilarityTask; @@ -126,8 +125,12 @@ public class IndexItem extends BasicProps { public static final char EVENT_IDX_SEPARATOR2 = ','; public static final String EVENT_SEPARATOR = " | "; + private static final String COUNT_SUFFIX = ":count"; + static HashSet ignoredMetadata = new HashSet(); + static HashSet countMetadata = new HashSet<>(); + private static Map inputStreamFactories = new HashMap<>(); private static Map> typesMap = MetadataUtil.getMetadataTypes(); @@ -158,6 +161,9 @@ public class IndexItem extends BasicProps { // ocrCharCount is already copied to an extra attribute ignoredMetadata.add(OCRParser.OCR_CHAR_COUNT); + countMetadata.add(ExtraProperties.CONVERSATION_PARTICIPANTS); + countMetadata.add(ExtraProperties.CONVERSATION_ADMINS); + BasicProps.SET.add(ID_IN_SOURCE); BasicProps.SET.add(SOURCE_PATH); BasicProps.SET.add(SOURCE_DECODER); @@ -711,6 +717,12 @@ private static void addMetadataToDoc(Document doc, Metadata metadata, Set { new PartyHandler(p, model.getSource()).fillMetadata(CONVERSATION_PARTICIPANTS, metadata); }); - if (!model.getParticipants().isEmpty()) { - metadata.add(CONVERSATION_PARTICIPANTS + ":count", Integer.toString(model.getParticipants().size())); - } // Chat Owner Participant model.getPhoneOwnerParticipant().ifPresent(p -> { @@ -94,10 +91,6 @@ public void fillMetadata(String prefix, Metadata metadata) { model.getParticipants().stream().filter(Party::isGroupAdmin).forEach(p -> { new PartyHandler(p, model.getSource()).fillMetadata(CONVERSATION_ADMINS, metadata); }); - long groupAdminsCount = model.getParticipants().stream().filter(Party::isGroupAdmin).count(); - if (groupAdminsCount > 0) { - metadata.set(CONVERSATION_ADMINS + ":count", Long.toString(groupAdminsCount)); - } // Chat Owner is Group Admin if (model.getParticipants().stream().filter(Party::isGroupAdmin).filter(Party::isPhoneOwner).findAny().isPresent()) { From ca84ad90c8557bf550ebb9966f0fc3cf7d3b7279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 02:26:24 -0300 Subject: [PATCH 05/25] #2686: Move chat type constants to ConversationConstants --- .../main/java/iped/parsers/ufed/handler/ChatHandler.java | 9 +++++---- .../src/main/java/iped/parsers/ufed/model/Chat.java | 5 ----- .../java/iped/parsers/util/ConversationConstants.java | 4 ++++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/ChatHandler.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/ChatHandler.java index 74e7b35411..ec90201af2 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/ChatHandler.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/ChatHandler.java @@ -29,6 +29,7 @@ import iped.parsers.ufed.model.Chat; import iped.parsers.ufed.model.ContactPhoto; import iped.parsers.ufed.model.Party; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.Messages; import iped.parsers.whatsapp.WAContact; import iped.properties.MediaTypes; @@ -158,15 +159,15 @@ else if (StringUtils.isNotBlank(model.getServiceIdentifier())) if (StringUtils.isNotBlank(chatType)) { switch (chatType) { - case Chat.TYPE_ONEONONE: + case ConversationConstants.TYPE_ONEONONE: sb.append("Chat").append(' '); break; - case Chat.TYPE_GROUP: + case ConversationConstants.TYPE_GROUP: sb.append(Messages.getString("UfedChatParser.Group")).append(' '); break; - case Chat.TYPE_BROADCAST: + case ConversationConstants.TYPE_BROADCAST: if (model.getParticipants().size() == 1 && StringUtils.containsAnyIgnoreCase(source, Chat.SOURCE_TELEGRAM, Chat.SOURCE_WHATSAPP)) { sb.append(Messages.getString("UfedChatParser.Status")); } else if (Chat.SOURCE_TELEGRAM.equalsIgnoreCase(source)) { @@ -177,7 +178,7 @@ else if (StringUtils.isNotBlank(model.getServiceIdentifier())) sb.append(' '); break; - case Chat.TYPE_UNKNOWN: + case ConversationConstants.TYPE_UNKNOWN: break; default: diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/model/Chat.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/model/Chat.java index a383cdbf5e..5ebde73b60 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/model/Chat.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/model/Chat.java @@ -20,11 +20,6 @@ public class Chat extends BaseModel { private static final long serialVersionUID = -568791121571161053L; - public static final String TYPE_ONEONONE = "OneOnOne"; - public static final String TYPE_GROUP = "Group"; - public static final String TYPE_BROADCAST = "Broadcast"; - public static final String TYPE_UNKNOWN = "Unknown"; - public static final String SOURCE_WHATSAPP = "WhatsApp"; public static final String SOURCE_WHATSAPP_BUSINESS = "WhatsApp Business"; public static final String SOURCE_TELEGRAM = "Telegram"; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ConversationConstants.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ConversationConstants.java index 9b6e24df5b..972f2f7cfa 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ConversationConstants.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ConversationConstants.java @@ -5,4 +5,8 @@ public class ConversationConstants { public static final String DIRECTION_INCOMING = "Incoming"; public static final String DIRECTION_OUTGOING = "Outgoing"; + public static final String TYPE_ONEONONE = "OneOnOne"; + public static final String TYPE_GROUP = "Group"; + public static final String TYPE_BROADCAST = "Broadcast"; + public static final String TYPE_UNKNOWN = "Unknown"; } From 712ae07efdee05d557f517fe32fed3a95f3cf118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 02:28:05 -0300 Subject: [PATCH 06/25] #2686: Fix quoted messages CSS in WhatsApp internal parser --- .../src/main/java/iped/parsers/whatsapp/ReportGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java index 0f124d863b..2062e781c1 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java @@ -1263,7 +1263,7 @@ private String formatLocation(Message message) { private void printQuote(PrintWriter out, Message message, WAContactsDirectory contactsDirectory, WAAccount account) { - String quoteClass = message.isFromMe() ? "quoteTo" : "quoteFrom"; + String quoteClass = "quoteBlock " + (message.isFromMe() ? "quoteTo" : "quoteFrom"); Message messageQuote = message.getMessageQuote(); if (messageQuote != null) { From 67532f47507123856be653e2ff18bdc29d7594c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 02:30:15 -0300 Subject: [PATCH 07/25] #2686: Read admin and other account information in WhatsAppParser --- .../main/java/iped/parsers/whatsapp/Chat.java | 24 +++++++++++++++++-- .../java/iped/parsers/whatsapp/Extractor.java | 20 ++++++++++++---- .../parsers/whatsapp/ExtractorAndroid.java | 15 ++++++++---- .../parsers/whatsapp/ExtractorAndroidNew.java | 5 +--- .../iped/parsers/whatsapp/ExtractorIOS.java | 2 +- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Chat.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Chat.java index ebce23ab69..e6dbc0bc8d 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Chat.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Chat.java @@ -24,6 +24,10 @@ public class Chat { private String recoveredFrom; private Set groupMembers = new HashSet<>(); + private Set groupAdmins = new HashSet<>(); + + private WAAccount account; + private boolean isOwnerAdmin; public Chat(WAContact remote) { this.remote = remote; @@ -160,8 +164,24 @@ public Set getGroupMembers() { return groupMembers; } - public void setGroupMembers(Set groupmembers) { - this.groupMembers = groupmembers; + public Set getGroupAdmins() { + return groupAdmins; + } + + public WAAccount getAccount() { + return account; + } + + public void setAccount(WAAccount account) { + this.account = account; + } + + public boolean isOwnerAdmin() { + return isOwnerAdmin; + } + + public void setOwnerAdmin(boolean isOwnerAdmin) { + this.isOwnerAdmin = isOwnerAdmin; } public void setDeleted(boolean isDeleted) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Extractor.java index a52e12315f..c34c2bc370 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Extractor.java @@ -9,6 +9,7 @@ import java.util.List; public abstract class Extractor { + protected final File databaseFile; protected List chatList; protected final WAContactsDirectory contacts; @@ -43,17 +44,25 @@ public File getDatabaseFile() { protected abstract Connection getConnection() throws SQLException; - protected void setGroupMembers(Chat c, Connection conn, String SELECT_GROUP_MEMBERS) throws SQLException { + protected void setGroupMembers(Chat c, Connection conn, String selectGroupMembersQuery) throws SQLException { // adds all contacts that sent at least one message for (Message m : c.getMessages()) { - if (m.getRemoteResource() != null) + if (m.getRemoteResource() != null) { + if (c.isGroupOrChannelChat() && c.getRemote() != null && m.getRemoteResource().equals(c.getRemote().getFullId())) { + continue; // don't add the group itself as a member + } c.getGroupMembers().add(contacts.getContact(m.getRemoteResource())); + } } - if (SELECT_GROUP_MEMBERS == null) { + + // add account as member + c.getGroupMembers().add(contacts.getContact(account.getFullId())); + + if (selectGroupMembersQuery == null) { return; } // adds all contacts which is a member of the group now - try (PreparedStatement stmt = conn.prepareStatement(SELECT_GROUP_MEMBERS)) { + try (PreparedStatement stmt = conn.prepareStatement(selectGroupMembersQuery)) { stmt.setString(1, c.getRemote().getFullId()); try (ResultSet rs = stmt.executeQuery()) { @@ -61,6 +70,9 @@ protected void setGroupMembers(Chat c, Connection conn, String SELECT_GROUP_MEMB String memberId = rs.getString("member"); if (memberId != null && !memberId.trim().isEmpty()) { c.getGroupMembers().add(contacts.getContact(memberId)); + if (rs.getBoolean("admin")) { + c.getGroupAdmins().add(contacts.getContact(memberId)); + } } } } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java index 7f61acc52f..9439281a63 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java @@ -78,7 +78,8 @@ public abstract class ExtractorAndroid extends Extractor { private boolean hasMediaCaptionCol = false; private boolean hasSubjectCol = false; private boolean hasMediaDurationCol = false; - private boolean hasGroupParticiantsTable = true; + private boolean hasGroupParticipantsTable = true; + private boolean hasGroupParticipantUserTable = true; private boolean hasForwardedCol = false; private boolean hasQuoteCol = false; private SQLException parsingException = null; @@ -150,8 +151,8 @@ protected List extractChatList() throws WAExtractorException { hasSortTimestamp = databaseHasSortTimestamp(conn); hasChatView = databaseHasChatView(conn); hasThumbTable = SQLite3DBParser.containsTable("message_thumbnails", conn); //$NON-NLS-1$ - hasGroupParticiantsTable = SQLite3DBParser.containsTable("group_participants", conn); //$NON-NLS-1$ - hasEditVersionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "edit_version"); //$NON-NLS-1$ //$NON-NLS-2$ + hasGroupParticipantsTable = SQLite3DBParser.containsTable("group_participants", conn); //$NON-NLS-1$ + hasGroupParticipantUserTable = SQLite3DBParser.containsTable("group_participant_user", conn); //$NON-NLS-1$ hasEditVersionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "edit_version"); //$NON-NLS-1$ //$NON-NLS-2$ hasMediaCaptionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "media_caption"); //$NON-NLS-1$ //$NON-NLS-2$ hasMediaDurationCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "media_duration"); //$NON-NLS-1$ //$NON-NLS-2$ hasForwardedCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "forwarded"); //$NON-NLS-1$ //$NON-NLS-2$ @@ -218,7 +219,9 @@ protected List extractChatList() throws WAExtractorException { undeletedMessagesTable, hasThumbTable, hasEditVersionCol, firstTry)); if (c.isGroupChat()) { try { - setGroupMembers(c, conn, hasGroupParticiantsTable ? SELECT_GROUP_MEMBERS : null); + String query = hasGroupParticipantUserTable ? SELECT_GROUP_MEMBERS_2 + : hasGroupParticipantsTable ? SELECT_GROUP_MEMBERS : null; + setGroupMembers(c, conn, query); } catch (SQLException ex) { if (firstTry || !isSqliteCorruptException(ex)) { throw ex; @@ -779,7 +782,9 @@ private static String getMessagesQueryQuote(boolean hasThumbTable, boolean hasEd + "WHERE remoteId=? and status!=-1 ORDER BY timestamp"; //$NON-NLS-1$ // to address a field must use ` instead of ' - private static final String SELECT_GROUP_MEMBERS = "select gjid as 'group', jid as member FROM group_participants where `group`=?"; //$NON-NLS-1$ + private static final String SELECT_GROUP_MEMBERS = "select gjid as 'group', jid as member, admin FROM group_participants where `group`=?"; + static final String SELECT_GROUP_MEMBERS_2 = "select g._id as group_id, g.raw_string as group_name, u._id as user_id, u.raw_string as member, gp.rank > 0 as admin " + + "FROM group_participant_user gp inner join jid g on g._id=gp.group_jid_row_id inner join jid u on u._id=gp.user_jid_row_id where u.server='s.whatsapp.net' and u.type=0 and group_name=?"; private static final String SELECT_QUOTES_NO_THUMBS_TABLE = "SELECT mq._id AS id, mq.key_remote_jid " + "as remoteId, mq.remote_resource AS remoteResource, mq.status, mq.data, " diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java index 400b26fc89..cac49ebcf5 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java @@ -157,7 +157,7 @@ protected List extractChatList() throws WAExtractorException { for (Chat c : list) { Message.sort(c.getMessages()); if (c.isGroupChat()) { - setGroupMembers(c, conn, SELECT_GROUP_MEMBERS); + setGroupMembers(c, conn, ExtractorAndroid.SELECT_GROUP_MEMBERS_2); } } @@ -1238,7 +1238,4 @@ private static String getSelectBlockedQuery(Connection conn) throws SQLException + " left join chat chat1 on chat1.jid_row_id = log.jid_row_id" + " left join chat chat2 on chat2.jid_row_id = log.group_jid_row_id"; - private static final String SELECT_GROUP_MEMBERS = "select g._id as group_id, g.raw_string as group_name, u._id as user_id, u.raw_string as member " - + "FROM group_participant_user gp inner join jid g on g._id=gp.group_jid_row_id inner join jid u on u._id=gp.user_jid_row_id where u.server='s.whatsapp.net' and u.type=0 and group_name=?"; //$NON-NLS-1$ - } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java index c7026d4004..5c2c1b3b05 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java @@ -1805,7 +1805,7 @@ public void extractPushNames() { private static final String SELECT_PUSH_NAMES = "select ZJID as jid, ZPUSHNAME as pushname from ZWAPROFILEPUSHNAME"; - private static final String SELECT_GROUP_MEMBERS = "select CS.ZCONTACTJID as `group`, ZMEMBERJID as member from ZWAGROUPMEMBER GM " + private static final String SELECT_GROUP_MEMBERS = "select CS.ZCONTACTJID as `group`, ZMEMBERJID as member, ZISADMIN as admin from ZWAGROUPMEMBER GM " + "inner join ZWACHATSESSION CS on GM.ZCHATSESSION=CS.Z_PK where `group`=?"; private static final String SELECT_MESSAGES_USER = "SELECT ZWAMESSAGE.Z_PK AS id, ZCHATSESSION " //$NON-NLS-1$ From 42fc77f4f290d80d80b75631a1791d46307405f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 02:30:45 -0300 Subject: [PATCH 08/25] #2686: Ignore invalid hashes --- .../java/iped/parsers/whatsapp/Message.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java index 235e6aeb75..f2b3a27cf5 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java @@ -19,6 +19,7 @@ import iped.data.IItemReader; import iped.parsers.util.ChildPornHashLookup; +import iped.parsers.util.HashUtils; /** * @author Fabio Melo Pfeifer @@ -229,15 +230,15 @@ public String getMediaHash() { } public void setMediaHash(String mediaHash, boolean encoded) { - if (mediaHash == null || mediaHash.isEmpty()) - return; - if (encoded) { - byte[] hash = Base64.getDecoder().decode(mediaHash); - this.mediaHash = new String(Hex.encodeHex(hash, false)); - } else { - this.mediaHash = mediaHash; + if (HashUtils.isValidHash(mediaHash)) { + if (encoded) { + byte[] hash = Base64.getDecoder().decode(mediaHash); + this.mediaHash = new String(Hex.encodeHex(hash, false)); + } else { + this.mediaHash = mediaHash; + } + childPornSets = ChildPornHashLookup.lookupHashAndMerge(this.mediaHash, childPornSets); } - childPornSets = ChildPornHashLookup.lookupHashAndMerge(this.mediaHash, childPornSets); } public byte[] getThumbData() { From e8acaf778425e233658348a32ed616c7b16174e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 31 Oct 2025 03:07:17 -0300 Subject: [PATCH 09/25] #2686: Add "Conversation" and "Communication" metadata in WhatsAppParser --- .../java/iped/properties/ExtraProperties.java | 2 + .../java/iped/parsers/whatsapp/WAContact.java | 18 +-- .../iped/parsers/whatsapp/WhatsAppParser.java | 141 +++++++++--------- .../parsers/whatsapp/AbstractPkgTest.java | 27 ++-- .../parsers/whatsapp/WhatsAppParserTest.java | 82 ++++++++-- 5 files changed, 169 insertions(+), 101 deletions(-) diff --git a/iped-api/src/main/java/iped/properties/ExtraProperties.java b/iped-api/src/main/java/iped/properties/ExtraProperties.java index e7ac70fb05..090098e108 100644 --- a/iped-api/src/main/java/iped/properties/ExtraProperties.java +++ b/iped-api/src/main/java/iped/properties/ExtraProperties.java @@ -57,6 +57,8 @@ public class ExtraProperties { public static final Property CONVERSATION_MESSAGES_COUNT = Property.internalInteger(CONVERSATION_PREFIX + "messagesCount"); + public static final Property CONVERSATION_IS_EMPTY = Property.internalBoolean(CONVERSATION_PREFIX + "isEmpty"); + public static final String CONVERSATION_SUFFIX_ID = ":id"; public static final String CONVERSATION_SUFFIX_NAME = ":name"; public static final String CONVERSATION_SUFFIX_PHONE = ":phoneNumber"; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAContact.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAContact.java index 1c3e253665..eb0bf9cd8d 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAContact.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAContact.java @@ -1,5 +1,7 @@ package iped.parsers.whatsapp; +import org.apache.commons.lang3.StringUtils; + public class WAContact { public static final String waSuffix = "@s.whatsapp.net"; @@ -97,16 +99,7 @@ public void setNickName(String nickName) { } public String getName() { - if (displayName != null && !displayName.isBlank()) - return displayName; - else if (waName != null && !waName.isBlank()) - return waName; - else if (givenName != null && !givenName.isBlank()) - return givenName; - else if (nickName != null && !nickName.isBlank()) - return nickName; - else - return id; + return StringUtils.firstNonBlank(displayName, waName, givenName, nickName, id); } @@ -168,4 +161,9 @@ public int hashCode() { return this.getFullId().hashCode(); } + @Override + public String toString() { + // for debug + return getName() + " (" + getFullId() + ")"; + } } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java index b6410e6cfe..613b883020 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java @@ -49,6 +49,7 @@ import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.tika.config.Field; import org.apache.tika.exception.TikaException; @@ -70,10 +71,12 @@ import iped.data.IItem; import iped.data.IItemReader; import iped.io.SeekableInputStream; +import iped.parsers.chat.WhatsAppPartyStringBuilder; import iped.parsers.plist.detector.PListDetector; import iped.parsers.sqlite.SQLite3DBParser; import iped.parsers.sqlite.SQLite3Parser; import iped.parsers.standard.StandardParser; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.parsers.util.PhoneParsingConfig; import iped.parsers.vcard.VCardParser; @@ -131,6 +134,8 @@ public class WhatsAppParser extends SQLite3DBParser { public static final String DOWNLOAD_MEDIA_FILES_PROP = "downloadWhatsAppMediaProp"; + public static final String UNKNOWN_ACCOUNT = "unknownAccount"; + // TODO: Once #2286 is merged, use some property to identify WhatsApp Status chats/messages // private static final String STATUS_PROP = ExtraProperties.COMMUNICATION_PREFIX + "isStatus"; @@ -303,6 +308,7 @@ private void createReport(List chatList, IItemSearcher searcher, WAContact searchMediaFilesForMessagesInBatches(c.getMessages(), searcher, handler, extractor, dbPath, context, null); int frag = 0; int firstMsg = 0; + int messagesCount = (int) c.getMessages().stream().filter(m -> !m.isSystemMessage()).count(); ReportGenerator reportGenerator = new ReportGenerator(); reportGenerator.setMinChatSplitSize(this.minChatSplitSize); StringBuilder histFrag = new StringBuilder(); @@ -339,23 +345,36 @@ private void createReport(List chatList, IItemSearcher searcher, WAContact if (extractMessages && msgSubset.size() > 0) { chatMetadata.set(BasicProps.HASCHILD, Boolean.TRUE.toString()); } - if (account != null) { - String local = formatContact(account, cache); - chatMetadata.add(ExtraProperties.PARTICIPANTS, local); - } - if (c.isGroupChat()) { + if (c.isGroupOrChannelChat()) { for (WAContact member : c.getGroupMembers()) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, formatContact(member, cache)); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, member); } // string formatted as {creator's phone number}-{creation time}@g.us - chatMetadata.add(ExtraProperties.GROUP_ID, c.getRemote().getFullId()); + for (WAContact admin : c.getGroupAdmins()) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_ADMINS, admin); + } + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, c.isGroupChat() ? ConversationConstants.TYPE_GROUP : ConversationConstants.TYPE_BROADCAST); } else { if (c.getRemote() != null) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, formatContact(c.getRemote(), cache)); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, c.getRemote()); + } + if (account != null) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, account); } + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_ONEONONE); } - if (c.isBroadcast()) { - // TODO: chatMetadata.set(STATUS_PROP, Boolean.TRUE.toString()); + + // Communication:{ID, Name, Account, MessagesCount, isEmpty} + chatMetadata.set(ExtraProperties.CONVERSATION_ID, c.getRemote() != null ? c.getRemote().getFullId() : c.getPrintId()); + chatMetadata.set(ExtraProperties.CONVERSATION_NAME, c.getRemote() != null ? c.getRemote().getName() : ""); + if (account != null) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_ACCOUNT, account); + } + chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, messagesCount); + chatMetadata.set(ExtraProperties.CONVERSATION_IS_EMPTY, (messagesCount == 0)); + + if (c.isOwnerAdmin()) { + chatMetadata.set(ExtraProperties.CONVERSATION_IS_OWNER_ADMIN, true); } // Set created and modified dates based on the first and last messages dates @@ -366,16 +385,6 @@ private void createReport(List chatList, IItemSearcher searcher, WAContact chatMetadata.set(TikaCoreProperties.MODIFIED, last.getTimeStamp()); } - // "isEmpty" = the chat is empty or contains only system messages - boolean isEmpty = true; - for (Message m : msgSubset) { - if (!m.isSystemMessage()) { - isEmpty = false; - break; - } - } - chatMetadata.set(ExtraProperties.COMMUNICATION_PREFIX + "isEmpty", Boolean.valueOf(isEmpty).toString()); - ByteArrayInputStream chatStream = new ByteArrayInputStream(bytes); extractor.parseEmbedded(chatStream, handler, chatMetadata, false); bytes = nextBytes; @@ -391,6 +400,29 @@ private void createReport(List chatList, IItemSearcher searcher, WAContact } + private void addPartyFields(Metadata chatMetadata, String field, WAContact member) { + if (member == null) { + return; + } + String phoneNumber = null; + if (StringUtils.endsWithAny(member.getId(), WAContact.waSuffix, WAContact.waStatusSuffix)) { + phoneNumber = StringUtils.substringAfterLast(member.getId(), "@"); + } + chatMetadata.add(field, new WhatsAppPartyStringBuilder() + .withName(member.getName()) + .withUserId(member.getFullId()) + .withPhoneNumber(phoneNumber) + .build()); + + if (member instanceof WAAccount && ((WAAccount) member).isUnknown()) { + return; + } + + chatMetadata.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, member.getFullId()); + chatMetadata.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, member.getName()); + chatMetadata.add(field + ExtraProperties.CONVERSATION_SUFFIX_PHONE, phoneNumber); + } + private void parseWhatsAppMessages(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context, ExtractorFactory extFactory) throws IOException, SAXException, TikaException { @@ -417,6 +449,12 @@ private void parseWhatsAppMessages(InputStream stream, ContentHandler handler, M Extractor waExtractor = extFactory.createMessageExtractor(filePath, tempDbFile, contacts, account, recoverDeletedRecords); List chatList = waExtractor.getChatList(); + for (Chat chat : chatList) { + chat.setAccount(account); + if (chat.getGroupAdmins().stream().map(WAContact::getFullId).anyMatch(account.getFullId()::equals)) { + chat.setOwnerAdmin(true); + } + } createReport(chatList, searcher, contacts, handler, extractor, account, tis.getFile(), context); } catch (Exception e) { @@ -917,7 +955,7 @@ private String getInternationalPhone(String id) { } private WAAccount getUserAccount(IItemSearcher searcher, String dbPath, boolean isAndroid) { - WAAccount account = new WAAccount("unknownAccount"); + WAAccount account = new WAAccount(UNKNOWN_ACCOUNT); account.setUnknown(true); if (searcher != null) { StringBuilder query = new StringBuilder(); @@ -1000,33 +1038,6 @@ private static boolean isWABusiness(String path) { return path.contains(".w4b") || path.contains("WhatsApp Business"); } - private String formatContact(WAContact contact, Map cache) { - String result = cache.get(contact.getId()); - if (result == null) { - if (contact.getName() == null || contact.getName().isBlank()) { - result = contact.getFullId(); - } else if (contact.getName().strip().equals(contact.getId())) { - result = contact.getFullId(); - } else if (contact.getFullId().isBlank()) { - result = contact.getName().strip(); - } else { - result = contact.getName().strip() + " (" + contact.getFullId().strip() + ")"; //$NON-NLS-1$ //$NON-NLS-2$ - } - cache.put(contact.getId(), result); - } - return result; - } - - private void fillGroupRecipients(Metadata meta, Chat c) { - String to = c.isChannelChat() ? "Channel " : "Group "; - if (c.getSubject() != null) { - to += c.getSubject().strip(); - } - to += " (id:" + c.getId() + ")"; - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to); - meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true"); - } - private void extractMessages(String chatName, Chat c, List messages, WAAccount account, WAContactsDirectory contacts, int parentVirtualId, ContentHandler handler, EmbeddedDocumentExtractor extractor, Map cache) throws SAXException, IOException { @@ -1039,37 +1050,29 @@ private void extractMessages(String chatName, Chat c, List messages, WA meta.set(ExtraProperties.PARENT_VIRTUAL_ID, Integer.toString(parentVirtualId)); meta.set(ExtraProperties.PARENT_VIEW_POSITION, String.valueOf(m.getUniqueId())); meta.set(ExtraProperties.USER_ACCOUNT_TYPE, WHATSAPP); - meta.set(ExtraProperties.MESSAGE_DATE, m.getTimeStamp()); + meta.set(ExtraProperties.COMMUNICATION_DATE, m.getTimeStamp()); meta.set(TikaCoreProperties.CREATED, m.getTimeStamp()); meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); if (m.isDeleted()) { meta.set(ExtraProperties.DELETED, Boolean.toString(true)); } - if (!m.isSystemMessage()) { - String local = formatContact(account, cache); + if (m.isSystemMessage()) { + meta.set(ExtraProperties.COMMUNICATION_SYSTEM_MESSAGE, true); + } else { String remote = m.getRemoteResource(); - if (remote != null) { - WAContact contact = contacts.getContact(remote); - remote = contact == null ? remote : formatContact(contact, cache); - } + WAContact contact = remote != null ? contacts.getContact(remote) : null; if (m.isFromMe()) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, local); - if (c.isGroupOrChannelChat()) { - fillGroupRecipients(meta, c); - } else { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, remote); - } + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, account); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, c.isGroupOrChannelChat() ? c.getRemote() : contact); } else { - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, remote); - if (c.isGroupOrChannelChat()) { - fillGroupRecipients(meta, c); - } else { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, local); - } + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, contact); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, c.isGroupOrChannelChat() ? c.getRemote() : account); } - if (c.isBroadcast()) { - // TODO: meta.set(STATUS_PROP, Boolean.TRUE.toString()); + if (c.isGroupOrChannelChat()) { + meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); } } meta.set(ExtraProperties.MESSAGE_BODY, m.getData()); diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/AbstractPkgTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/AbstractPkgTest.java index b9823509fd..a27db67da2 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/AbstractPkgTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/AbstractPkgTest.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -35,11 +36,13 @@ protected ParseContext getContext(String resource) throws IOException { @SuppressWarnings("serial") protected static class EmbeddedWhatsAppParser extends AbstractParser { protected List title = new ArrayList(); + protected List type = new ArrayList(); protected List username = new ArrayList(); protected List userphone = new ArrayList(); protected List useraccount = new ArrayList(); protected List usernotes = new ArrayList(); - protected List participants = new ArrayList(); + protected List> participants = new ArrayList<>(); + protected List> admins = new ArrayList<>(); protected List messagefrom = new ArrayList(); protected List messagebody = new ArrayList(); protected List messageto = new ArrayList(); @@ -53,6 +56,8 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, throws IOException, SAXException, TikaException { if (metadata.get(TikaCoreProperties.TITLE) != null) title.add(metadata.get(TikaCoreProperties.TITLE)); + if (metadata.get(ExtraProperties.CONVERSATION_TYPE) != null) + type.add(metadata.get(ExtraProperties.CONVERSATION_TYPE)); if (metadata.get(ExtraProperties.USER_NAME) != null) username.add(metadata.get(ExtraProperties.USER_NAME)); if (metadata.get(ExtraProperties.USER_PHONE) != null) @@ -61,16 +66,18 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, useraccount.add(metadata.get(ExtraProperties.USER_ACCOUNT)); if (metadata.get(ExtraProperties.USER_NOTES) != null) usernotes.add(metadata.get(ExtraProperties.USER_NOTES)); - if (metadata.get(ExtraProperties.PARTICIPANTS) != null) - participants.add(metadata.get(ExtraProperties.PARTICIPANTS)); - if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM) != null) - messagefrom.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM)); + if (metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS) != null) { + participants.add(Arrays.asList(metadata.getValues(ExtraProperties.CONVERSATION_PARTICIPANTS))); + admins.add(Arrays.asList(metadata.getValues(ExtraProperties.CONVERSATION_ADMINS))); + } + if (metadata.get(ExtraProperties.COMMUNICATION_FROM) != null) + messagefrom.add(metadata.get(ExtraProperties.COMMUNICATION_FROM)); + if (metadata.get(ExtraProperties.COMMUNICATION_TO) != null) + messageto.add(metadata.get(ExtraProperties.COMMUNICATION_TO)); if (metadata.get(ExtraProperties.MESSAGE_BODY) != null) messagebody.add(metadata.get(ExtraProperties.MESSAGE_BODY)); - if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_TO) != null) - messageto.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_TO)); - if (metadata.get(ExtraProperties.MESSAGE_DATE) != null) - messagedate.add(metadata.get(ExtraProperties.MESSAGE_DATE)); + if (metadata.get(ExtraProperties.COMMUNICATION_DATE) != null) + messagedate.add(metadata.get(ExtraProperties.COMMUNICATION_DATE)); } } -} +} \ No newline at end of file diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/WhatsAppParserTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/WhatsAppParserTest.java index fe7354d907..6a7e27cf8f 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/WhatsAppParserTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/WhatsAppParserTest.java @@ -1,7 +1,11 @@ package iped.parsers.whatsapp; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; + import java.io.IOException; import java.io.InputStream; +import java.util.Collections; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; @@ -14,12 +18,19 @@ import org.xml.sax.SAXException; import iped.parsers.standard.StandardParser; +import iped.parsers.util.ConversationConstants; public class WhatsAppParserTest extends AbstractPkgTest { + boolean waDbParsed = false; + @Test public void testWhatsAppParserAndroid() throws IOException, SAXException, TikaException { + if (!waDbParsed) { + testWhatsAppParserWADBAndroid(); + } + String testFile = "test-files/test_whatsAppMsgStore.db"; ParseContext whatsappContext = getContext(testFile); WhatsAppParser parser = new WhatsAppParser(); @@ -43,6 +54,11 @@ public void testWhatsAppParserAndroid() throws IOException, SAXException, TikaEx assertEquals(4, whatsapptracker.messageto.size()); assertEquals(74, whatsapptracker.messagedate.size()); + assertEquals(11, Collections.frequency(whatsapptracker.type, ConversationConstants.TYPE_ONEONONE)); + assertEquals(18, Collections.frequency(whatsapptracker.type, ConversationConstants.TYPE_GROUP)); + assertEquals(0, Collections.frequency(whatsapptracker.type, ConversationConstants.TYPE_BROADCAST)); + assertEquals(0, Collections.frequency(whatsapptracker.type, ConversationConstants.TYPE_UNKNOWN)); + assertEquals("WhatsApp Chat - Nickerida - 556183125151", whatsapptracker.title.get(0)); assertEquals("WhatsApp Chat - Nickerida - 556183125151_message_0", whatsapptracker.title.get(1)); assertEquals("WhatsApp Chat - Nickerida - 556183125151_message_1", whatsapptracker.title.get(2)); @@ -51,18 +67,57 @@ public void testWhatsAppParserAndroid() throws IOException, SAXException, TikaEx assertEquals("WhatsApp Group - Lar - 556185747642-1461964508_message_1", whatsapptracker.title.get(61)); assertEquals("WhatsApp Group - Lar - 556185747642-1461964508_message_2", whatsapptracker.title.get(62)); - assertEquals("unknownAccount", whatsapptracker.participants.get(0)); - assertEquals("unknownAccount", whatsapptracker.participants.get(1)); - assertEquals("unknownAccount", whatsapptracker.participants.get(2)); - assertEquals("unknownAccount", whatsapptracker.participants.get(3)); - assertEquals("unknownAccount", whatsapptracker.participants.get(4)); - assertEquals("unknownAccount", whatsapptracker.participants.get(17)); - assertEquals("unknownAccount", whatsapptracker.participants.get(18)); + // Test OneOnOne Chats + + assertEquals(ConversationConstants.TYPE_ONEONONE, whatsapptracker.type.get(0)); + assertEquals(ConversationConstants.TYPE_ONEONONE, whatsapptracker.type.get(19)); + assertEquals(ConversationConstants.TYPE_ONEONONE, whatsapptracker.type.get(20)); + assertEquals(ConversationConstants.TYPE_ONEONONE, whatsapptracker.type.get(21)); + + assertEquals(2, whatsapptracker.participants.get(0).size()); + assertEquals(2, whatsapptracker.participants.get(19).size()); + assertEquals(2, whatsapptracker.participants.get(20).size()); + assertEquals(2, whatsapptracker.participants.get(21).size()); + + assertThat(whatsapptracker.participants.get(0), hasItems("unknownAccount", "Nickerida (+556183125151)")); + assertThat(whatsapptracker.participants.get(19), hasItems("unknownAccount", "Hotdog412 (+556181704627)")); + assertThat(whatsapptracker.participants.get(20), hasItems("unknownAccount", "Nwi Fibra Ótica (+556133223200)")); + assertThat(whatsapptracker.participants.get(21), hasItems("unknownAccount", "Xavier (+556135952111)")); + + assertEquals(0, whatsapptracker.admins.get(0).size()); + assertEquals(0, whatsapptracker.admins.get(19).size()); + assertEquals(0, whatsapptracker.admins.get(20).size()); + assertEquals(0, whatsapptracker.admins.get(21).size()); + + // Test Group Chats + + assertEquals(ConversationConstants.TYPE_GROUP, whatsapptracker.type.get(1)); + assertEquals(ConversationConstants.TYPE_GROUP, whatsapptracker.type.get(2)); + assertEquals(ConversationConstants.TYPE_GROUP, whatsapptracker.type.get(3)); + assertEquals(ConversationConstants.TYPE_GROUP, whatsapptracker.type.get(4)); + + assertEquals(23, whatsapptracker.participants.get(1).size()); + assertEquals(21, whatsapptracker.participants.get(2).size()); + assertEquals(5, whatsapptracker.participants.get(3).size()); + assertEquals(2, whatsapptracker.participants.get(4).size()); + + assertThat(whatsapptracker.participants.get(1), hasItems("unknownAccount")); + assertThat(whatsapptracker.participants.get(2), hasItems("unknownAccount")); + assertThat(whatsapptracker.participants.get(3), hasItems("unknownAccount")); + assertThat(whatsapptracker.participants.get(4), hasItems("unknownAccount")); + + assertEquals(21, whatsapptracker.admins.get(1).size()); + assertEquals(19, whatsapptracker.admins.get(2).size()); + assertEquals(1, whatsapptracker.admins.get(3).size()); + assertEquals(0, whatsapptracker.admins.get(4).size()); + + assertThat(whatsapptracker.admins.get(1), hasItems("+556192644086")); + assertThat(whatsapptracker.admins.get(2), hasItems("Pedro Gonzaga (+556199351995)")); assertEquals("unknownAccount", whatsapptracker.messagefrom.get(0)); assertEquals("unknownAccount", whatsapptracker.messagefrom.get(1)); assertEquals("unknownAccount", whatsapptracker.messagefrom.get(2)); - assertEquals("Nickerida (556183125151@s.whatsapp.net)", whatsapptracker.messagefrom.get(3)); + assertEquals("Nickerida (+556183125151)", whatsapptracker.messagefrom.get(3)); assertEquals("This is a test for the IPED Whatsapp Parser.", whatsapptracker.messagebody.get(0)); assertEquals("! MESSAGES_NOW_ENCRYPTED", whatsapptracker.messagebody.get(1)); @@ -72,9 +127,9 @@ public void testWhatsAppParserAndroid() throws IOException, SAXException, TikaEx assertEquals("! USER_ADDED_TO_GROUP", whatsapptracker.messagebody.get(42)); assertEquals("! MESSAGES_NOW_ENCRYPTED", whatsapptracker.messagebody.get(43)); - assertEquals("Nickerida (556183125151@s.whatsapp.net)", whatsapptracker.messageto.get(0)); - assertEquals("Nickerida (556183125151@s.whatsapp.net)", whatsapptracker.messageto.get(1)); - assertEquals("Nickerida (556183125151@s.whatsapp.net)", whatsapptracker.messageto.get(2)); + assertEquals("Nickerida (+556183125151)", whatsapptracker.messageto.get(0)); + assertEquals("Nickerida (+556183125151)", whatsapptracker.messageto.get(1)); + assertEquals("Nickerida (+556183125151)", whatsapptracker.messageto.get(2)); assertEquals("unknownAccount", whatsapptracker.messageto.get(3)); assertEquals("2021-06-14T18:55:54Z", whatsapptracker.messagedate.get(0)); @@ -122,12 +177,15 @@ public void testWhatsAppParserWADBAndroid() throws IOException, SAXException, Ti try (InputStream stream = getStream(testFile)) { parser.parse(stream, handler, metadata, whatsappContext); + waDbParsed = true; + assertEquals(384, whatsapptracker.title.size()); assertEquals(384, whatsapptracker.username.size()); assertEquals(365, whatsapptracker.userphone.size()); assertEquals(384, whatsapptracker.useraccount.size()); assertEquals(166, whatsapptracker.usernotes.size()); assertEquals(0, whatsapptracker.participants.size()); + assertEquals(0, whatsapptracker.admins.size()); assertEquals(0, whatsapptracker.messagefrom.size()); assertEquals(0, whatsapptracker.messagebody.size()); assertEquals(0, whatsapptracker.messageto.size()); @@ -213,4 +271,4 @@ public void _testWhatsAppParserUserXMLAndroid() throws IOException, SAXException } } -} +} \ No newline at end of file From 6c46ae8226175385d7e40c50365545a736c1e87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 12:55:43 -0300 Subject: [PATCH 10/25] #2686: Remove unused code --- .../java/iped/parsers/ufed/handler/InstantMessageHandler.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java index 002281c058..65a40ef3a7 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java @@ -167,10 +167,6 @@ protected void fillMetadata(String prefix, Metadata metadata) { }); }); - if (model.isSystemMessage()) { - - } - if (model.getPosition() != null) { metadata.set(Geographic.LATITUDE, model.getPosition().getLatitude()); metadata.set(Geographic.LONGITUDE, model.getPosition().getLongitude()); From 3eeb9f29eba5352c71d43c5578f408a29557da97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 12:57:15 -0300 Subject: [PATCH 11/25] #2686: Use all identified phone numbers of contact/account --- .../iped/parsers/ufed/reference/ReferencedAccountable.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/reference/ReferencedAccountable.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/reference/ReferencedAccountable.java index f0543c3339..104850c699 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/reference/ReferencedAccountable.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/reference/ReferencedAccountable.java @@ -1,7 +1,7 @@ package iped.parsers.ufed.reference; import static iped.parsers.ufed.util.UfedUtils.readUfedMetadata; - +import static iped.parsers.ufed.util.UfedUtils.readUfedMetadataArray; import iped.data.IItemReader; public class ReferencedAccountable extends AbstractReferencedItem { @@ -15,7 +15,7 @@ public String getName() { } public String getPhoneNumber() { - return readUfedMetadata(item, "PhoneNumber"); + return String.join("|", readUfedMetadataArray(item, "PhoneNumber")); } public String getUsername() { From 1895e978b237006bf7ab80014f84b00e3d0737e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 12:58:14 -0300 Subject: [PATCH 12/25] #2686: Safer WhatsApp party phone number calc --- .../src/main/java/iped/parsers/ufed/handler/PartyHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java index 196847e3b5..a53b6181ee 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java @@ -49,8 +49,8 @@ protected void fillMetadata(String prefix, Metadata metadata) { model.getReferencedContact().ifPresentOrElse(ref -> { String id = StringUtils.firstNonBlank(model.getIdentifier(), ref.getUserID()); - String phoneNumber = StringUtils.containsIgnoreCase(source, Chat.SOURCE_WHATSAPP) - ? StringUtils.substringBeforeLast(id, WAContact.waSuffix) + String phoneNumber = StringUtils.containsIgnoreCase(source, Chat.SOURCE_WHATSAPP) && StringUtils.endsWith(id, WAContact.waSuffix) + ? StringUtils.removeEnd(id, WAContact.waSuffix) : ref.getPhoneNumber(); metadata.add(prefix + CONVERSATION_SUFFIX_ID, id); metadata.add(prefix + CONVERSATION_SUFFIX_NAME, StringUtils.firstNonBlank(model.getName(), ref.getName())); From 0eaa9c823501221205b274a3848692a08cd99234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 13:04:11 -0300 Subject: [PATCH 13/25] #2686: Fix when WhatsApp phone number is equals name --- .../java/iped/parsers/chat/WhatsAppPartyStringBuilder.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java index c7b37b253c..21f993c5f4 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java @@ -15,9 +15,6 @@ public String build() { String phoneFromUserID = null; if (StringUtils.endsWithAny(userId, WAContact.waSuffix, WAContact.waStatusSuffix)) { phoneFromUserID = StringUtils.substringBeforeLast(userId, "@"); - if (StringUtils.isNotBlank(phoneFromUserID)) { - phoneFromUserID = StringUtils.prependIfMissing(phoneFromUserID, PHONE_PREFIX); - } } String phoneStr = StringUtils.firstNonBlank(phoneFromUserID, phoneNumber); From 6ce252365e310bf8511040ce9558bbe63e493567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 13:22:38 -0300 Subject: [PATCH 14/25] #2686: prefer Telegram chat names from decoded data in order to avoid lower case names --- .../src/main/java/iped/parsers/telegram/Extractor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java index 346df45f03..51073bf3b3 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java @@ -35,6 +35,7 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,7 +183,8 @@ protected List extractChatList() throws Exception { searchAvatarFileName(cont, androidDecoder.getPhotoData()); - ChatGroup group = new ChatGroup(chatId, cont, chatName); + String chatNameToUse = StringUtils.firstNonBlank((String) androidDecoder.getAlltMetadata().get("title"), chatName); + ChatGroup group = new ChatGroup(chatId, cont, chatNameToUse); Map m = androidDecoder.getAlltMetadata(); if ("true".equals(m.get("broadcast"))) { From 8473b0852959296f8c61b887beb4a33e93e713bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 13:23:08 -0300 Subject: [PATCH 15/25] #2686: use positive numbers for Telegram group IDs --- .../main/java/iped/parsers/telegram/Extractor.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java index 51073bf3b3..51a0f50186 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java @@ -116,7 +116,7 @@ protected Contact extractUserAccountIOS() throws SQLException { private void addParticipants(Connection conn, ChatGroup cg) throws SQLException { try (PreparedStatement stmt = conn.prepareStatement(MEMBERS_CHATS_SQL)) { - stmt.setLong(1, cg.getId()); + stmt.setLong(1, -cg.getId()); ResultSet rs = stmt.executeQuery(); while (rs.next()) { cg.addMember(rs.getLong("uid")); @@ -127,7 +127,7 @@ private void addParticipants(Connection conn, ChatGroup cg) throws SQLException private void addAdmins(Connection conn, ChatGroup cg) throws SQLException { if (SQLite3DBParser.containsTable("channel_admins_v3", conn)) { try (PreparedStatement stmt = conn.prepareStatement(SELECT_CHANNEL_ADMINS)) { - stmt.setLong(1, -cg.getId()); + stmt.setLong(1, cg.getId()); ResultSet rs = stmt.executeQuery(); while (rs.next()) { cg.addAdmin(rs.getLong("uid")); @@ -142,7 +142,7 @@ private int getParticipantsCount(Connection conn, ChatGroup cg) throws SQLExcept String column = "participants_count"; if (SQLite3DBParser.containsTable(table, conn) && SQLite3DBParser.checkIfColumnExists(conn, table, column)) { try (PreparedStatement stmt = conn.prepareStatement(SELECT_CHATS_SETTINGS)) { - stmt.setLong(1, -cg.getId()); + stmt.setLong(1, cg.getId()); ResultSet rs = stmt.executeQuery(); if (rs.next()) { count = rs.getInt("participants_count"); @@ -250,7 +250,7 @@ protected ArrayList extractMessages(Chat chat) throws Exception { ArrayList msgs = new ArrayList(); String SQL = getAndroidExtractMessagesSQL(); try (PreparedStatement stmt = conn.prepareStatement(SQL)) { - stmt.setLong(1, chat.getId()); + stmt.setLong(1, chat.getId() * (chat.isGroupOrChannel() ? -1 : 1)); ResultSet rs = stmt.executeQuery(); ChatGroup cg = null; if (chat.isGroupOrChannel()) { @@ -351,7 +351,7 @@ protected ArrayList extractMessagesIOS(Chat chat) throws SQLException { if (conn != null) { try (PreparedStatement stmt = conn.prepareStatement(EXTRACT_MESSAGES_SQL_IOS)) { ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(chat.getId()); + buffer.putLong(chat.getId() * (chat.isGroupOrChannel() ? -1 : 1)); stmt.setBytes(1, buffer.array()); ResultSet rs = stmt.executeQuery(); if (rs != null) { @@ -634,7 +634,7 @@ private String getAndroidExtractMessagesSQL() throws SQLException { private static final String EXTRACT_USERACCOUNT_SQL_IOS = "SELECT t0.value FROM T0 where key=2"; - private static final String CHATS_SQL_V2 = "SELECT -c.uid as chatId,null as chatName,null as chatData, c.name as groupName, c.data as groupData ,d.date" + private static final String CHATS_SQL_V2 = "SELECT c.uid as chatId,null as chatName,null as chatData, c.name as groupName, c.data as groupData ,d.date" + " from chats c LEFT join dialogs d on c.uid=-d.did " + " UNION " + " SELECT d.did as chatId,u.name as chatName,u.data as chatData, null as groupName, null as groupData, d.date" From 59958b52bfac2b98036c1ea24f95d5f6a2afa1dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 13:24:33 -0300 Subject: [PATCH 16/25] #2686: don't consider participant with invalid uid in Telegram groups --- .../src/main/java/iped/parsers/telegram/Extractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java index 51a0f50186..99bf72ac3c 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java @@ -641,11 +641,11 @@ private String getAndroidExtractMessagesSQL() throws SQLException { + " from dialogs d LEFT join users u on u.uid=d.did WHERE d.did NOT IN (SELECT -uid FROM chats) " + " order by date desc"; - private static final String MEMBERS_CHATS_SQL = "SELECT * from channel_users_v2 where did=?"; + private static final String MEMBERS_CHATS_SQL = "SELECT * from channel_users_v2 where did=? AND uid > 0"; private static final String SELECT_CHATS_SETTINGS = "SELECT participants_count FROM chat_settings_v2 WHERE uid=?"; - private static final String SELECT_CHANNEL_ADMINS = "SELECT uid FROM channel_admins_v3 WHERE did=?"; + private static final String SELECT_CHANNEL_ADMINS = "SELECT uid FROM channel_admins_v3 WHERE did=? AND uid > 0"; private static final String CHATS_SQL_IOS = "select substr(key,16,8) as chatid, false as deleted from t9 " + "UNION SELECT substr(t7.key,1,8) as chatid, true as deleted from t7 " From 4c17cfbba1e84a1c0111828640466318df3ae0f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 13:27:49 -0300 Subject: [PATCH 17/25] #2686: don't add the group/channel itself as member --- .../src/main/java/iped/parsers/telegram/Extractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java index 99bf72ac3c..f60798e5a3 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java @@ -269,7 +269,7 @@ protected ArrayList extractMessages(Chat chat) throws Exception { } setFrom(message, chat); - if (cg != null && message.getFrom().getId() != 0) { + if (cg != null && message.getFrom().getId() != 0 && message.getFrom().getId() != cg.getId()) { cg.addMember(message.getFrom().getId()); } @@ -376,7 +376,7 @@ protected ArrayList extractMessagesIOS(Chat chat) throws SQLException { } } - if (cg != null && message.getFrom().getId() != 0) { + if (cg != null && message.getFrom().getId() != 0 && message.getFrom().getId() != cg.getId()) { cg.addMember(message.getFrom().getId()); } From 748875be21c5357b60e84836ad90b1298dd55e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 13:29:57 -0300 Subject: [PATCH 18/25] #2686: get userAccount earlier in Telegram internal parser --- .../src/main/java/iped/parsers/telegram/Extractor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java index f60798e5a3..4e3029bb0e 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java @@ -504,6 +504,10 @@ protected void extractContacts() throws Exception { e.printStackTrace(); } } + + if (Boolean.parseBoolean((String) androidDecoder.getAlltMetadata().get("self"))) { + userAccount = c; + } } } } From 5704682bc59f8ffe0ea246004d229385497aa369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 13:51:43 -0300 Subject: [PATCH 19/25] #2686: only set valid mediaHashes in Telegram Messages --- .../main/java/iped/parsers/telegram/Message.java | 7 +++++-- .../iped/parsers/telegram/ReportGenerator.java | 2 +- .../java/iped/parsers/telegram/TelegramParser.java | 5 +++-- .../src/main/java/iped/parsers/telegram/Util.java | 14 -------------- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java index 2873fdce88..cc29a015e7 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java @@ -28,6 +28,7 @@ import dpf.ap.gpinf.interfacetelegram.PhotoData; import iped.data.IItemReader; import iped.parsers.util.ChildPornHashLookup; +import iped.parsers.util.HashUtils; public class Message implements MessageInterface { private long id; @@ -70,8 +71,10 @@ public String getMediaHash() { } public void setMediaHash(String mediaHash) { - this.mediaHash = mediaHash; - childPornSets = ChildPornHashLookup.lookupHashAndMerge(mediaHash, childPornSets); + if (HashUtils.isValidHash(mediaHash)) { + this.mediaHash = mediaHash; + childPornSets = ChildPornHashLookup.lookupHashAndMerge(mediaHash, childPornSets); + } } public String getMediaFile() { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/ReportGenerator.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/ReportGenerator.java index 2efe13e10d..ec174f3855 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/ReportGenerator.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/ReportGenerator.java @@ -164,7 +164,7 @@ public byte[] generateNextChatHtml(Chat c) { private TagHtml getThumbTag(Message m, String classnotfound) { byte thumb[] = m.getThumb(); - if (searcher != null && thumb == null && m.getMediaHash() != null && !m.getMediaHash().isBlank()) { + if (searcher != null && thumb == null && m.getMediaHash() != null) { if (!m.getMediaHash().equalsIgnoreCase(emptyMD5)) { List result = iped.parsers.util.Util.getItems("md5:" + m.getMediaHash(), searcher); if (result != null && !result.isEmpty()) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java index b67b873e77..fcef961e0f 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java @@ -54,6 +54,7 @@ import iped.data.IItemReader; import iped.parsers.sqlite.SQLite3DBParser; import iped.parsers.standard.StandardParser; +import iped.parsers.util.HashUtils; import iped.parsers.util.ItemInfo; import iped.parsers.util.PhoneParsingConfig; import iped.properties.BasicProps; @@ -118,7 +119,7 @@ public void setMinChatSplitSize(int minChatSplitSize) { private void storeLinkedHashes(List messages, Metadata metadata) { for (Message m : messages) { - if (Util.isValidHash(m.getMediaHash())) { + if (HashUtils.isValidHash(m.getMediaHash())) { metadata.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + m.getMediaHash()); //$NON-NLS-1$ if (m.isFromMe()) metadata.add(ExtraProperties.SHARED_HASHES, m.getMediaHash()); @@ -291,7 +292,7 @@ private void extractMessages(String chatName, List messages, Contact ac if (m.getMediaSize() != 0) { meta.set("mediaSize", Long.toString(m.getMediaSize())); } - if (Util.isValidHash(m.getMediaHash())) { + if (HashUtils.isValidHash(m.getMediaHash())) { meta.set(StandardParser.INDEXER_CONTENT_TYPE, TELEGRAM_ATTACHMENT.toString()); meta.set(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + m.getMediaHash()); //$NON-NLS-1$ if (!m.getChildPornSets().isEmpty()) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Util.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Util.java index 96151ef4c8..44433a161e 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Util.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Util.java @@ -20,25 +20,11 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.HashSet; -import java.util.Set; import iped.utils.IOUtil; public class Util { - private static final Set zeroLengthHashes = new HashSet(); - static { - // Hashes of empty input (byte[0]), see issue #2157. - zeroLengthHashes.add("d41d8cd98f00b204e9800998ecf8427e"); // MD5 - zeroLengthHashes.add("da39a3ee5e6b4b0d3255bfef95601890afd80709"); // SHA-1 - zeroLengthHashes.add("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); // SHA-256 - } - - public static boolean isValidHash(String hash) { - return hash != null && !hash.isBlank() && !zeroLengthHashes.contains(hash.toLowerCase()); - } - public static void invertByteArray(byte[] array, int start, int len) { for (int i = 0; i < len / 2; i++) { byte aux = array[start + i]; From 3deddd86fe27b2ad0b1e20a24ce34cabb36c9ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 13:57:54 -0300 Subject: [PATCH 20/25] #2686: Use TelegramPartyStringBuilder in Telegram contact toString() --- .../java/iped/parsers/telegram/Contact.java | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java index 0b93cdfffc..9f98563baf 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java @@ -23,6 +23,7 @@ import dpf.ap.gpinf.interfacetelegram.ContactInterface; import dpf.ap.gpinf.interfacetelegram.DecoderTelegramInterface; import dpf.ap.gpinf.interfacetelegram.PhotoData; +import iped.parsers.chat.TelegramPartyStringBuilder; public class Contact implements ContactInterface { private long id; @@ -171,27 +172,16 @@ public boolean isGroupOrChannel() { @Override public String toString() { - StringBuilder sb = new StringBuilder(); - String name = getFullname(); - if (name != null) { - sb.append(name.trim()); - } - String number = getPhone(); - if (number != null && number.length() > 0) { - if (sb.length() > 0) { - sb.append(' '); - } - sb.append("(phone: ").append(number).append(')'); - } else if (getId() > 0) { - if (sb.length() > 0) { - sb.append(' '); - } - sb.append("(ID:").append(getId()).append(')'); - } - if (sb.length() == 0) { - sb.append("[unknown]"); + String str = new TelegramPartyStringBuilder() + .withName(getFullname()) + .withUserId(Long.toString(getId())) + .withPhoneNumber(getPhone()) + .withUsername(getUsername()) + .build(); + if (str.isEmpty()) { + str = "[unknown]"; } - return sb.toString(); + return str; } } From 34659c14ea594d11807af561d05c13e5ee3b6d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Sun, 2 Nov 2025 20:38:41 -0300 Subject: [PATCH 21/25] #2686: Add "Conversation" and "Communication" metadata in TelegramParser --- .../java/iped/parsers/telegram/Contact.java | 2 +- .../iped/parsers/telegram/TelegramParser.java | 86 ++++++++++++++----- .../parsers/telegram/AbstractPkgTest.java | 15 +++- .../parsers/telegram/TelegramParserTest.java | 35 ++++---- 4 files changed, 96 insertions(+), 42 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java index 9f98563baf..241848f1b7 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java @@ -173,8 +173,8 @@ public boolean isGroupOrChannel() { @Override public String toString() { String str = new TelegramPartyStringBuilder() - .withName(getFullname()) .withUserId(Long.toString(getId())) + .withName(getFullname()) .withPhoneNumber(getPhone()) .withUsername(getUsername()) .build(); diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java index fcef961e0f..47ff0654df 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java @@ -54,7 +54,7 @@ import iped.data.IItemReader; import iped.parsers.sqlite.SQLite3DBParser; import iped.parsers.standard.StandardParser; -import iped.parsers.util.HashUtils; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.parsers.util.PhoneParsingConfig; import iped.properties.BasicProps; @@ -119,7 +119,7 @@ public void setMinChatSplitSize(int minChatSplitSize) { private void storeLinkedHashes(List messages, Metadata metadata) { for (Message m : messages) { - if (HashUtils.isValidHash(m.getMediaHash())) { + if (m.getMediaHash() != null) { metadata.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + m.getMediaHash()); //$NON-NLS-1$ if (m.isFromMe()) metadata.add(ExtraProperties.SHARED_HASHES, m.getMediaHash()); @@ -186,6 +186,7 @@ private void generateChat(Chat c, Contact account, Extractor e, IItemSearcher se int frag = 0; int firstMsg = 0; byte[] bytes; + int messagesCount = (int) c.getMessages().stream().filter(m -> m.getType() == null).count(); ReportGenerator r = new ReportGenerator(searcher); r.setMinChatSplitSize(this.minChatSplitSize); while ((bytes = r.generateNextChatHtml(c)) != null) { @@ -204,19 +205,37 @@ private void generateChat(Chat c, Contact account, Extractor e, IItemSearcher se if (c.isGroupOrChannel()) { ChatGroup cg = (ChatGroup) c; - for (long id : cg.getMembers()) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, e.getContact(id).toString()); - } for (long id : cg.getAdmins()) { - chatMetadata.add(ExtraProperties.COMMUNICATION_PREFIX + "ChannelAdmins", - e.getContact(id).toString()); + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_ADMINS, id); + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_PARTICIPANTS, id); + if (id == e.getUserAccount().getId()) { + chatMetadata.set(ExtraProperties.CONVERSATION_IS_OWNER_ADMIN, true); + } } - int participantsCount = cg.getParticipantsCount(); - if (participantsCount > 0) { - chatMetadata.add(ExtraProperties.PARTICIPANTS + "Count", String.valueOf(participantsCount)); + for (long id : cg.getMembers()) { + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_PARTICIPANTS, id); } + } else { + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_PARTICIPANTS, c.getId()); + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_PARTICIPANTS, e.getUserAccount().getId()); + } + + if (c.isChannel()) { + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_BROADCAST); + } else if (c.isGroup()) { + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_GROUP); + } else { + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_ONEONONE); } + // Communication:Account + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_ACCOUNT, e.getUserAccount().getId()); + + // Communication:{ID, Name, , MessagesCount} + chatMetadata.set(ExtraProperties.CONVERSATION_ID, Long.toString(c.getId())); + chatMetadata.set(ExtraProperties.CONVERSATION_NAME, c.getC().getFullname()); + chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, messagesCount); + List msgSubset = c.getMessages().subList(firstMsg, nextMsg); if (extractMessages && !msgSubset.isEmpty()) { @@ -235,8 +254,24 @@ private void generateChat(Chat c, Contact account, Extractor e, IItemSearcher se } } + private void addPartyFields(Metadata meta, Extractor e, String field, long contactId) { + Contact contact = e.getContact(contactId); + addPartyFields(meta, field, contact); + } + + private void addPartyFields(Metadata meta, String field, Contact c) { + if (c == null) { + return; + } + meta.add(field, c.toString()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, Long.toString(c.getId())); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, c.getFullname()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_PHONE, c.getPhone()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_USERNAME, c.getUsername()); + } + private String getChatNamePrefix(Chat c) { - String title = "Telegram_"; + String title = "Telegram "; if (c.isChannel()) { title += "Channel"; } else if (c.isGroup()) { @@ -244,7 +279,7 @@ private String getChatNamePrefix(Chat c) { } else { title += "Chat"; } - title += "_" + c.getName(); + title += " - " + c.getC().toString(); return title; } @@ -264,24 +299,31 @@ private void extractMessages(String chatName, List messages, Contact ac if (m.getLatitude() != null && m.getLongitude() != null) { meta.set(ExtraProperties.LOCATIONS, m.getLatitude() + ";" + m.getLongitude()); } - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, m.getFrom().toString()); + + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, m.getFrom()); + if (m.getChat().isGroupOrChannel()) { ChatGroup groupChat = (ChatGroup) m.getChat(); String to = groupChat.isGroup() ? "Group " : "Channel "; - to += groupChat.getName() + " (id:" + groupChat.getId() + ")"; - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to); - meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true"); - } - if (meta.get(org.apache.tika.metadata.Message.MESSAGE_TO) == null) { + to += groupChat.getName() + " (ID:" + groupChat.getId() + ")"; + meta.add(ExtraProperties.COMMUNICATION_TO, to); + meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); + } else { if (m.getToId() != 0) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, e.getContact(m.getToId()).toString()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, e.getContact(m.getToId())); } else if (m.isFromMe()) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, m.getChat().getC().toString()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, m.getChat().getC()); } else if (account != null) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, account.toString()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, account); } } + if (m.isFromMe()) { + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + } else { + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + } + meta.set(ExtraProperties.MESSAGE_BODY, m.getData()); meta.set("mediaName", m.getMediaName()); @@ -292,7 +334,7 @@ private void extractMessages(String chatName, List messages, Contact ac if (m.getMediaSize() != 0) { meta.set("mediaSize", Long.toString(m.getMediaSize())); } - if (HashUtils.isValidHash(m.getMediaHash())) { + if (m.getMediaHash() != null) { meta.set(StandardParser.INDEXER_CONTENT_TYPE, TELEGRAM_ATTACHMENT.toString()); meta.set(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + m.getMediaHash()); //$NON-NLS-1$ if (!m.getChildPornSets().isEmpty()) { diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java index f00a4d642f..6456c0ce03 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java @@ -19,6 +19,7 @@ import org.xml.sax.SAXException; import iped.data.IItemReader; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.properties.ExtraProperties; import iped.search.IItemSearcher; @@ -80,7 +81,8 @@ protected static class EmbeddedTelegramParser extends AbstractParser { protected List userphone = new ArrayList(); protected List useraccount = new ArrayList(); protected List usernotes = new ArrayList(); - protected List participants = new ArrayList(); + protected List groupParticipants = new ArrayList(); + protected List privateParticipants = new ArrayList(); protected List messagefrom = new ArrayList(); protected List messagebody = new ArrayList(); protected List messageto = new ArrayList(); @@ -102,8 +104,13 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, useraccount.add(metadata.get(ExtraProperties.USER_ACCOUNT)); if (metadata.get(ExtraProperties.USER_NOTES) != null) usernotes.add(metadata.get(ExtraProperties.USER_NOTES)); - if (metadata.get(ExtraProperties.PARTICIPANTS) != null) - participants.add(metadata.get(ExtraProperties.PARTICIPANTS)); + if (metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS) != null) + if (ConversationConstants.TYPE_GROUP.equals(metadata.get(ExtraProperties.CONVERSATION_TYPE)) + || ConversationConstants.TYPE_BROADCAST.equals(metadata.get(ExtraProperties.CONVERSATION_TYPE))) { + groupParticipants.add(metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS)); + } else { + privateParticipants.add(metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS)); + } if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM) != null) messagefrom.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM)); if (metadata.get(ExtraProperties.MESSAGE_BODY) != null) @@ -139,4 +146,4 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, useraccount.add(metadata.get(ExtraProperties.USER_ACCOUNT)); } } -} +} \ No newline at end of file diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java index f0417a0b01..361052326d 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java @@ -36,7 +36,8 @@ public void testTelegramParser() throws IOException, SAXException, TikaException assertEquals(236, telegramtracker.userphone.size()); assertEquals(261, telegramtracker.useraccount.size()); assertEquals(118, telegramtracker.usernotes.size()); - assertEquals(32, telegramtracker.participants.size()); + assertEquals(32, telegramtracker.groupParticipants.size()); + assertEquals(69, telegramtracker.privateParticipants.size()); assertEquals(151, telegramtracker.messagefrom.size()); assertEquals(151, telegramtracker.messagebody.size()); assertEquals(151, telegramtracker.messageto.size()); @@ -46,10 +47,10 @@ public void testTelegramParser() throws IOException, SAXException, TikaException assertEquals("Karol Braz", telegramtracker.title.get(1)); assertEquals("Budi", telegramtracker.title.get(2)); assertEquals("Nickerida", telegramtracker.title.get(3)); - assertEquals("Telegram_Chat_Marcoscachos", telegramtracker.title.get(505)); - assertEquals("Telegram_Chat_Marcoscachos_message_0", telegramtracker.title.get(506)); - assertEquals("Telegram_Group_mixirica e noronhe-se", telegramtracker.title.get(507)); - assertEquals("Telegram_Group_mixirica e noronhe-se_message_0", telegramtracker.title.get(508)); + assertEquals("Telegram Chat - Marcoscachos (@Marquito93 | +5561983468805)", telegramtracker.title.get(505)); + assertEquals("Telegram Chat - Marcoscachos (@Marquito93 | +5561983468805)_message_0", telegramtracker.title.get(506)); + assertEquals("Telegram Group - Mixirica e noronhe-se (ID:136232058)", telegramtracker.title.get(507)); + assertEquals("Telegram Group - Mixirica e noronhe-se (ID:136232058)_message_0", telegramtracker.title.get(508)); assertEquals("Tiago", telegramtracker.username.get(0)); assertEquals("Karol Braz", telegramtracker.username.get(1)); @@ -67,17 +68,21 @@ public void testTelegramParser() throws IOException, SAXException, TikaException assertEquals("RafaelCampos", telegramtracker.usernotes.get(1)); assertEquals("gif", telegramtracker.usernotes.get(3)); - assertEquals("Bruno Chaves (phone: 33667514279)", telegramtracker.participants.get(0)); - assertEquals("Nake Douglas (phone: 5561982616052)", telegramtracker.participants.get(1)); - assertEquals("Guilherme Andreúce (phone: 5561986143035)", telegramtracker.participants.get(2)); + assertEquals("Bruno Chaves (+33667514279)", telegramtracker.groupParticipants.get(0)); + assertEquals("Nake Douglas (+5561982616052)", telegramtracker.groupParticipants.get(1)); + assertEquals("Guilherme Andreúce (@guileb | +5561986143035)", telegramtracker.groupParticipants.get(2)); - assertEquals("Telegram (phone: 42777)", telegramtracker.messagefrom.get(0)); - assertEquals("Nickerida (phone: 5561983125151)", telegramtracker.messagefrom.get(1)); - assertEquals("Guilherme Andreúce (phone: 5561986143035)", telegramtracker.messagefrom.get(3)); + assertEquals("Telegram (+42777)", telegramtracker.privateParticipants.get(0)); + assertEquals("Nickerida (+5561983125151)", telegramtracker.privateParticipants.get(1)); + assertEquals("Yan Victor (@YanVictor | +5561995328012)", telegramtracker.privateParticipants.get(2)); - assertEquals("Telegram (phone: 42777)", telegramtracker.messageto.get(0)); - assertEquals("Nickerida (phone: 5561983125151)", telegramtracker.messageto.get(1)); - assertEquals("Group mixirica e noronhe-se (id:-136232058)", telegramtracker.messageto.get(150)); + assertEquals("Telegram (+42777)", telegramtracker.messagefrom.get(0)); + assertEquals("Nickerida (+5561983125151)", telegramtracker.messagefrom.get(1)); + assertEquals("Guilherme Andreúce (@guileb | +5561986143035)", telegramtracker.messagefrom.get(3)); + + assertEquals("Telegram (+42777)", telegramtracker.messageto.get(0)); + assertEquals("Nickerida (+5561983125151)", telegramtracker.messageto.get(1)); + assertEquals("Group Mixirica e noronhe-se (ID:136232058)", telegramtracker.messageto.get(150)); assertTrue(telegramtracker.messagebody.get(0).contains( "Código de login: 73632. Não envie esse código para ninguém, nem mesmo que eles digam que são do Telegram!")); @@ -117,4 +122,4 @@ public void testTelegramParserAndroidAcc() throws IOException, SAXException, Tik } } -} +} \ No newline at end of file From 395a21f606c7be1cc0f070127f81cd38ee8ad841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Mon, 3 Nov 2025 00:53:58 -0300 Subject: [PATCH 22/25] #2686: Fix Telegram unit tests --- .../test/java/iped/parsers/telegram/AbstractPkgTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java index 6456c0ce03..15031962a8 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java @@ -111,12 +111,12 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, } else { privateParticipants.add(metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS)); } - if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM) != null) - messagefrom.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM)); + if (metadata.get(ExtraProperties.COMMUNICATION_FROM) != null) + messagefrom.add(metadata.get(ExtraProperties.COMMUNICATION_FROM)); if (metadata.get(ExtraProperties.MESSAGE_BODY) != null) messagebody.add(metadata.get(ExtraProperties.MESSAGE_BODY)); - if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_TO) != null) - messageto.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_TO)); + if (metadata.get(ExtraProperties.COMMUNICATION_TO) != null) + messageto.add(metadata.get(ExtraProperties.COMMUNICATION_TO)); if (metadata.get(ExtraProperties.MESSAGE_DATE) != null) messagedate.add(metadata.get(ExtraProperties.MESSAGE_DATE)); } From 15c39e18202ab987a21ba2d7dfa2720881f30872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 7 Nov 2025 11:06:05 -0300 Subject: [PATCH 23/25] #2686: Add new chat metadata to DiscordParser --- .../iped/parsers/discord/DiscordParser.java | 117 +++++++++++++----- 1 file changed, 85 insertions(+), 32 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/DiscordParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/DiscordParser.java index d3d1d04791..a8d9d5969c 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/DiscordParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/DiscordParser.java @@ -7,12 +7,13 @@ import java.io.InputStream; import java.util.Arrays; import java.util.Base64; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.TreeMap; import javax.imageio.ImageIO; @@ -21,6 +22,7 @@ import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; @@ -38,12 +40,14 @@ import iped.data.IItemReader; import iped.parsers.browsers.chrome.CacheIndexParser; +import iped.parsers.chat.PartyStringBuilderFactory; import iped.parsers.discord.cache.Index; import iped.parsers.discord.json.DiscordAttachment; import iped.parsers.discord.json.DiscordAuthor; import iped.parsers.discord.json.DiscordRoot; import iped.parsers.discord.json.DiscordSticker; import iped.parsers.standard.StandardParser; +import iped.parsers.util.ConversationConstants; import iped.properties.BasicProps; import iped.properties.ExtraProperties; import iped.search.IItemSearcher; @@ -52,16 +56,18 @@ import iped.utils.ImageUtil; /*** - * + * * @author PCF Campanini * @author PCF Patrick Dalla Bernardina * */ public class DiscordParser extends AbstractParser { - // TODO + private static final long serialVersionUID = 2785303004503782295L; + + private static final Logger logger = LoggerFactory.getLogger(CacheIndexParser.class); + - private static final long serialVersionUID = 1L; // public static final MediaType DISCORD_INDEX_MIME_TYPE = // MediaType.application("application/x-discord-index"); // public static final MediaType CHROME_INDEX_MIME_TYPE = @@ -80,7 +86,7 @@ public class DiscordParser extends AbstractParser { private static Logger LOGGER = LoggerFactory.getLogger(Index.class); - private static final Set SUPPORTED_TYPES = new HashSet(Arrays.asList(MediaType.parse(CHAT_MIME_TYPE))); + private static final Set SUPPORTED_TYPES = new HashSet<>(Arrays.asList(MediaType.parse(CHAT_MIME_TYPE))); private static final String ME_URL = "https://discord.com/api/v9/users/@me"; static { @@ -108,12 +114,17 @@ public Set getSupportedTypes(ParseContext context) { @Override public void parse(InputStream indexFile, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { - String contentType = metadata.get(Metadata.CONTENT_TYPE); + try { + String contentType = metadata.get(HttpHeaders.CONTENT_TYPE); if (contentType.equals(CHAT_MIME_TYPE)) { parseDiscord(indexFile, handler, metadata, context); } else { parseCacheIndex(indexFile, handler, metadata, context); } + } catch (Exception e) { + logger.warn("Discord error", e); + throw e; + } } // this code may be removed when backward parsersconfig.xml compatibility were @@ -138,7 +149,7 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata IItemReader item = context.get(IItemReader.class); String commonQuery = BasicProps.EVIDENCE_UUID + ":" + item.getDataSource().getUUID() + " AND " + BasicProps.PARENTID + ":" + item.getParentId() + " AND NOT " + BasicProps.LENGTH + ":0 AND " - + CacheIndexParser.IS_CACHE_INDEX_ENTRY.replace(":", "\\:") + ":true"; + + searcher.escapeQuery(CacheIndexParser.IS_CACHE_INDEX_ENTRY) + ":true"; try (InputStream is = TikaInputStream.get(indexFile, new TemporaryResources())) { @@ -204,11 +215,32 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata chatmetadata.set(StandardParser.INDEXER_CONTENT_TYPE, CHAT_MIME_TYPE_HTML); chatmetadata.set(ExtraProperties.ITEM_VIRTUAL_ID, Integer.toString(chatVirtualId)); + // Type + chatmetadata.set(ExtraProperties.CONVERSATION_NAME, chatName); + chatmetadata.set(ExtraProperties.CONVERSATION_TYPE, discordRoot.get(0).getType() + ""); + chatmetadata.set(ExtraProperties.CONVERSATION_ID, discordRoot.get(0).getId()); + + // Account + addPartyFields(chatmetadata, ExtraProperties.CONVERSATION_ACCOUNT, me); + + // Participants + TreeMap participantsMap = new TreeMap<>(); + for (DiscordRoot d : discordRoot) { + if (d.getAuthor() != null) { + participantsMap.putIfAbsent(d.getAuthor().getFullUsername(), d.getAuthor()); + } + } + Collection participants = participantsMap.values(); + for (DiscordAuthor participant : participants) { + addPartyFields(chatmetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, participant); + } + for (DiscordRoot dr : discordRoot) { for (DiscordSticker sticker : dr.getStickers()) { try { - List stickerItems = searcher - .search(commonQuery + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"" + sticker.getId() + ".json\"" + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"discord\""); + List stickerItems = searcher.search(commonQuery + " AND " + + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"" + sticker.getId() + ".json\"" + " AND " + + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"discord\""); for (IItemReader stickerItem : stickerItems) { sticker.setMediaHash(stickerItem.getHash()); } @@ -253,16 +285,32 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata byte[] relatorio = new DiscordHTMLReport(me).convertToHTML(discordRoot, searcher); extractor.parseEmbedded(new ByteArrayInputStream(relatorio), handler, chatmetadata, false); - extractMessages(chatName, discordRoot, handler, extractor, chatVirtualId); + extractMessages(chatName, discordRoot, me, participants, handler, extractor, chatVirtualId); } } + private void addPartyFields(Metadata meta, String field, DiscordAuthor author) { + if (author == null) { + return; + } + meta.add(field, PartyStringBuilderFactory.getBuilder("Discord") + .withUserId(author.getId()) + .withName(author.getName()) + .withPhoneNumber(author.getPhone()) + .withUsername(author.getUsername()) + .build()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, author.getId()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, author.getName()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_PHONE, author.getPhone()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_USERNAME, author.getFullUsername()); + } + private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery, ObjectMapper mapper, HashMap avatarCache, ContentHandler handler, EmbeddedDocumentExtractor extractor) { DiscordAuthor me = null; try { // find me info - List mes = searcher.search(commonQuery + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"" + ME_URL + "\""); + List mes = searcher.search(commonQuery + " AND " + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"" + ME_URL + "\""); for (IItemReader mei : mes) { if (mei.getName().equals("@me")) { try (InputStream is2 = mei.getBufferedInputStream()) { @@ -278,7 +326,7 @@ private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery, memeta.set(ExtraProperties.USER_ACCOUNT_TYPE, "Discord"); memeta.set(ExtraProperties.USER_EMAIL, me.getEmail()); // memeta.set(ExtraProperties.USER_NOTES, me.getBio()); - memeta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); + memeta.set(ExtraProperties.DECODED_DATA, Boolean.toString(true)); if (me.getAvatar() != null) { byte[] meavatar = avatarCache.get(me.getAvatar()); @@ -298,51 +346,56 @@ private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery, return me; } - private void extractMessages(String chatName, List discordRoot, ContentHandler handler, EmbeddedDocumentExtractor extractor, int chatVirtualId) throws SAXException, IOException { + private void extractMessages(String chatName, List discordRoot, DiscordAuthor me, Collection participants, ContentHandler handler, EmbeddedDocumentExtractor extractor, int chatVirtualId) throws SAXException, IOException { int msgCount = 0; - // Checking Participants - LinkedHashSet participants = new LinkedHashSet(); - - for (DiscordRoot d : discordRoot) { - participants.add(d.getAuthor().getFullUsername()); - } - // Set metadata for (DiscordRoot d : discordRoot) { Metadata meta = new Metadata(); meta.set(TikaCoreProperties.TITLE, chatName + "_message_" + msgCount++); meta.set(StandardParser.INDEXER_CONTENT_TYPE, MSG_MIME_TYPE); - meta.set(ExtraProperties.MESSAGE_DATE, d.getTimestamp()); + meta.set(ExtraProperties.COMMUNICATION_DATE, d.getTimestamp()); meta.set(ExtraProperties.MESSAGE_BODY, d.getMessageContent()); - meta.set(ExtraProperties.USER_NAME, d.getAuthor().getFullUsername()); - meta.add(ExtraProperties.PARTICIPANTS, participants.toString()); meta.set(ExtraProperties.PARENT_VIRTUAL_ID, Integer.toString(chatVirtualId)); meta.set(ExtraProperties.PARENT_VIEW_POSITION, String.valueOf(d.getId())); - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, d.getAuthor().getFullUsername()); - meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); + meta.set(ExtraProperties.DECODED_DATA, Boolean.toString(true)); + + // From + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, d.getAuthor()); // Add "Message-TO" field. // In cases where only one participant sends messages, it is not possible to // determine the participants as only the participant list of the calls are cached. if (participants.size() <= 2) { - for (String participant : participants) { - if (!participant.equals(d.getAuthor().getFullUsername())) { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, participant); + for (DiscordAuthor participant : participants) { + if (!participant.getFullUsername().equals(d.getAuthor().getFullUsername())) { + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, participant); } } } else { - meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, chatName); - meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true"); + meta.set(ExtraProperties.COMMUNICATION_TO, chatName); + meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); + } + + // Direction + Boolean isFromMe = null; + if (me != null) { + isFromMe = me.getFullUsername().equals(d.getAuthor().getFullUsername()); + if (isFromMe) { + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + } else { + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + } } for (DiscordAttachment da : d.getAttachments()) { if (da.getMediaHash() != null) { meta.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + da.getMediaHash()); meta.set(StandardParser.INDEXER_CONTENT_TYPE, ATTACH_MIME_TYPE); - // if (m.isFromMe()) - // meta.add(ExtraProperties.SHARED_HASHES, m.getMediaHash()); + if (isFromMe != null && isFromMe) { + meta.add(ExtraProperties.SHARED_HASHES, da.getMediaHash()); + } } } From 916ea5a0a9767b8e5d30c75833870230033b1ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 24 Jul 2026 01:38:48 -0300 Subject: [PATCH 24/25] #2686: support Communication and Coversations metadata in ThreemaParser and SkypeParser --- .../java/iped/parsers/skype/SkypeParser.java | 59 +++++++++++++++++-- .../iped/parsers/threema/ThreemaParser.java | 57 +++++++++++++----- 2 files changed, 97 insertions(+), 19 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java index 66c00d0be2..8c79eaed40 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.StringUtils; import org.apache.tika.config.Field; import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; @@ -30,6 +31,7 @@ import iped.parsers.sqlite.SQLite3Parser; import iped.parsers.standard.StandardParser; import iped.parsers.util.ChildPornHashLookup; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.parsers.util.Messages; import iped.properties.BasicProps; @@ -174,6 +176,8 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, } /* adiciona a conversação */ + boolean isGroup = conv.getParticipantes() != null && conv.getParticipantes().size() > 1; + Metadata chatMetadata = new Metadata(); chatMetadata.set(StandardParser.INDEXER_CONTENT_TYPE, CONVERSATION_MIME_TYPE); chatMetadata.set(HttpHeaders.CONTENT_TYPE, CONVERSATION_MIME_TYPE); @@ -184,6 +188,25 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, chatMetadata.set(BasicProps.HASCHILD, Boolean.TRUE.toString()); chatMetadata.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); + // Conversation:Account + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_ACCOUNT, contactMap, account.getSkypeName()); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, contactMap, account.getSkypeName()); + + // Conversation:Participants + if (conv.getParticipantes() != null) { + for (String participant : conv.getParticipantes()) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, contactMap, participant); + } + } + + // Conversation:{ID, Name, Type, MessagesCount} + chatMetadata.set(ExtraProperties.CONVERSATION_ID, conv.getId()); + chatMetadata.set(ExtraProperties.CONVERSATION_NAME, + StringUtils.firstNonBlank(conv.getDisplayName(), conv.getChatName(), conv.getTitle())); + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, + isGroup ? ConversationConstants.TYPE_GROUP : ConversationConstants.TYPE_ONEONONE); + chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, conv.getMessages().size()); + storeSharedHashes(conv, chatMetadata); if (extractor.shouldParseEmbedded(chatMetadata)) { @@ -205,8 +228,17 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, meta.set(ExtraProperties.PARENT_VIEW_POSITION, String.valueOf(sm.getId())); meta.set(ExtraProperties.USER_ACCOUNT_TYPE, SKYPE); meta.set(ExtraProperties.MESSAGE_DATE, sm.getData()); - meta.set(Metadata.MESSAGE_FROM, formatSkypeName(contactMap, sm.getAutor())); - meta.set(Metadata.MESSAGE_TO, formatSkypeName(contactMap, sm.getDestino())); + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, sm.isFromMe() + ? ConversationConstants.DIRECTION_OUTGOING : ConversationConstants.DIRECTION_INCOMING); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, contactMap, sm.getAutor()); + if (sm.getDestino() != null) { + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, contactMap, sm.getDestino()); + } else if (isGroup) { + meta.set(ExtraProperties.COMMUNICATION_TO, conv.getTitle()); + } + if (isGroup) { + meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); + } meta.set(ExtraProperties.MESSAGE_BODY, sm.getConteudo()); meta.set("messageStatus", String.valueOf(sm.getChatMessageStatus())); meta.set("sendingStatus", String.valueOf(sm.getSendingStatus())); @@ -251,11 +283,15 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, tMetadata.set(TikaCoreProperties.MODIFIED, t.getFinish()); tMetadata.set(ExtraProperties.MESSAGE_DATE, t.getStart()); tMetadata.set(ExtraProperties.MESSAGE_BODY, name); - tMetadata.set(Metadata.MESSAGE_FROM, formatSkypeName(contactMap, t.getFrom())); + tMetadata.set(ExtraProperties.COMMUNICATION_DIRECTION, + t.getFrom() != null && t.getFrom().equals(sqlite.getSkypeName()) + ? ConversationConstants.DIRECTION_OUTGOING : ConversationConstants.DIRECTION_INCOMING); + addPartyFields(tMetadata, ExtraProperties.COMMUNICATION_FROM, contactMap, t.getFrom()); if (t.getType() == 3 && t.getConversation().getParticipantes() != null && t.getConversation().getParticipantes().size() > 1) { - tMetadata.set(Metadata.MESSAGE_TO, t.getConversation().getTitle()); + tMetadata.set(ExtraProperties.COMMUNICATION_TO, t.getConversation().getTitle()); + tMetadata.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); } else { - tMetadata.set(Metadata.MESSAGE_TO, formatSkypeName(contactMap, t.getTo())); + addPartyFields(tMetadata, ExtraProperties.COMMUNICATION_TO, contactMap, t.getTo()); } if (searcher != null) { @@ -333,6 +369,19 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, } + private void addPartyFields(Metadata meta, String field, Map contactMap, String skypeName) { + if (StringUtils.isBlank(skypeName)) { + return; + } + meta.add(field, formatSkypeName(contactMap, skypeName)); + SkypeUser user = contactMap.get(skypeName); + String id = (user != null && StringUtils.isNotBlank(user.getSkypeName())) ? user.getSkypeName() : skypeName; + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, id); + if (user != null && StringUtils.isNotBlank(user.getBestName())) { + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, user.getBestName()); + } + } + private String formatSkypeName(Map contactMap, String skypeName) { if (skypeName == null) return null; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java index 09a24361aa..94bc635247 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.tika.config.Field; import org.apache.tika.exception.TikaException; @@ -55,6 +56,7 @@ import iped.parsers.sqlite.SQLite3Parser; import iped.parsers.standard.StandardParser; import iped.parsers.threema.Message.MessageType; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.properties.BasicProps; import iped.properties.ExtraProperties; @@ -277,21 +279,31 @@ private void createReport(List chatList, IItemSearcher searcher, ContentHa if (extractMessages && !msgSubset.isEmpty()) { chatMetadata.set(BasicProps.HASCHILD, Boolean.TRUE.toString()); } - if (account != null) { - String local = account.getFullId(); - chatMetadata.add(ExtraProperties.PARTICIPANTS, local); + + // Conversation:Account + if (account != null && !account.isUnknown()) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_ACCOUNT, account); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, account); } + + // Conversation:{Participants, Type} if (c.isGroupChat()) { for (ThreemaContact member : c.getParticipants()) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, member.getFullId()); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, member); } - chatMetadata.add(ExtraProperties.GROUP_ID, "Conversation_" + c.getId()); + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_GROUP); } else { if (c.getContact() != null) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, c.getContact().getFullId()); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, c.getContact()); } + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_ONEONONE); } + // Conversation:{ID, Name, MessagesCount} + chatMetadata.set(ExtraProperties.CONVERSATION_ID, Long.toString(c.getId())); + chatMetadata.set(ExtraProperties.CONVERSATION_NAME, c.getTitle()); + chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, c.getMessages().size()); + ByteArrayInputStream chatStream = new ByteArrayInputStream(bytes); extractor.parseEmbedded(chatStream, handler, chatMetadata, false); bytes = nextBytes; @@ -335,10 +347,26 @@ private void fillGroupRecipients(Metadata meta, Chat c) { to += " " + c.getSubject().strip(); } to += " (id:" + c.getId() + ")"; - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to); + meta.add(ExtraProperties.COMMUNICATION_TO, to); meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true"); } + private void addPartyFields(Metadata meta, String field, ThreemaContact contact) { + if (contact == null) { + return; + } + String fullId = contact.getFullId(); + if (StringUtils.isNotBlank(fullId)) { + meta.add(field, fullId); + } + if (contact instanceof ThreemaAccount && ((ThreemaAccount) contact).isUnknown()) { + return; + } + if (StringUtils.isNotBlank(contact.getId())) { + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, contact.getId()); + } + } + private void extractMessages(String chatName, Chat c, List messages, ThreemaAccount account, int parentVirtualId, ContentHandler handler, EmbeddedDocumentExtractor extractor) throws SAXException, IOException { int msgCount = 0; for (Message m : messages) { @@ -353,22 +381,23 @@ private void extractMessages(String chatName, Chat c, List messages, Th meta.set(TikaCoreProperties.CREATED, m.getTimeStamp()); meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); - String local = account.getFullId(); - String remote = m.getRemoteResource(); - if (m.isFromMe()) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, local); + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, account); if (c.isGroupChat()) { fillGroupRecipients(meta, c); } else { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, remote); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, c.getContact()); } } else { - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, remote); + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); if (c.isGroupChat()) { + // for group messages the sender is stored as a display string in the message + meta.set(ExtraProperties.COMMUNICATION_FROM, m.getRemoteResource()); fillGroupRecipients(meta, c); } else { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, local); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, c.getContact()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, account); } } From dfb4b56df2f30b3ce5298466f3a353ed5df16dcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Fri, 24 Jul 2026 04:39:03 -0300 Subject: [PATCH 25/25] #2686: fixes after merging --- .../java/iped/parsers/telegram/Extractor.java | 13 +++--- .../iped/parsers/telegram/TelegramParser.java | 9 ++-- .../parsers/whatsapp/ExtractorAndroid.java | 3 +- .../parsers/telegram/TelegramParserTest.java | 41 +++++++++++-------- 4 files changed, 36 insertions(+), 30 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java index 5729689185..826f372b7e 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java @@ -118,7 +118,7 @@ protected Contact extractUserAccountIOS() throws SQLException { private void addParticipants(Connection conn, ChatGroup cg) throws SQLException { try (PreparedStatement stmt = conn.prepareStatement(MEMBERS_CHATS_SQL)) { - stmt.setLong(1, -cg.getId()); + stmt.setLong(1, cg.getId()); ResultSet rs = stmt.executeQuery(); while (rs.next()) { cg.addMember(rs.getLong("uid")); @@ -129,7 +129,7 @@ private void addParticipants(Connection conn, ChatGroup cg) throws SQLException private void addAdmins(Connection conn, ChatGroup cg) throws SQLException { if (SQLite3DBParser.containsTable("channel_admins_v3", conn)) { try (PreparedStatement stmt = conn.prepareStatement(SELECT_CHANNEL_ADMINS)) { - stmt.setLong(1, cg.getId()); + stmt.setLong(1, -cg.getId()); ResultSet rs = stmt.executeQuery(); while (rs.next()) { cg.addAdmin(rs.getLong("uid")); @@ -144,7 +144,7 @@ private int getParticipantsCount(Connection conn, ChatGroup cg) throws SQLExcept String column = "participants_count"; if (SQLite3DBParser.containsTable(table, conn) && SQLite3DBParser.checkIfColumnExists(conn, table, column)) { try (PreparedStatement stmt = conn.prepareStatement(SELECT_CHATS_SETTINGS)) { - stmt.setLong(1, cg.getId()); + stmt.setLong(1, -cg.getId()); ResultSet rs = stmt.executeQuery(); if (rs.next()) { count = rs.getInt("participants_count"); @@ -344,7 +344,6 @@ protected Chat extractMessages() throws Exception { logger.debug("Message with mid {} has media data", mid); } - if (cg != null && message.getFrom().getId() != 0 && message.getFrom().getId() != cg.getId()) { androidDecoder.setDecoderData(data, DecoderTelegramInterface.MESSAGE); androidDecoder.getMessageData(message); @@ -354,7 +353,7 @@ protected Chat extractMessages() throws Exception { } setFrom(message, chat); - if (cg != null && message.getFrom().getId() != 0) { + if (cg != null && message.getFrom().getId() != 0 && message.getFrom().getId() != cg.getId()) { cg.addMember(message.getFrom().getId()); } @@ -447,7 +446,7 @@ protected ArrayList extractMessagesIOS(Chat chat) throws SQLE if (conn != null) { try (PreparedStatement stmt = conn.prepareStatement(EXTRACT_MESSAGES_SQL_IOS)) { ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(chat.getId() * (chat.isGroupOrChannel() ? -1 : 1)); + buffer.putLong(chat.getId()); stmt.setBytes(1, buffer.array()); ResultSet rs = stmt.executeQuery(); if (rs != null) { @@ -745,7 +744,7 @@ private String getAndroidExtractMessagesSQL() throws SQLException { private static final String EXTRACT_USERACCOUNT_SQL_IOS = "SELECT t0.value FROM T0 where key=2"; - private static final String CHATS_SQL_V2 = "SELECT c.uid as chatId,null as chatName,null as chatData, c.name as groupName, c.data as groupData ,d.date" + private static final String CHATS_SQL_V2 = "SELECT -c.uid as chatId,null as chatName,null as chatData, c.name as groupName, c.data as groupData ,d.date" + " from chats c LEFT join dialogs d on c.uid=-d.did " + " UNION " + " SELECT d.did as chatId,u.name as chatName,u.data as chatData, null as groupName, null as groupData, d.date" diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java index 18e38d4f43..ef9295d98e 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java @@ -55,6 +55,7 @@ import iped.parsers.sqlite.SQLite3DBParser; import iped.parsers.standard.StandardParser; import iped.parsers.util.ConversationConstants; +import iped.parsers.util.HashUtils; import iped.parsers.util.ItemInfo; import iped.parsers.util.PhoneParsingConfig; import iped.properties.BasicProps; @@ -120,7 +121,7 @@ public void setMinChatSplitSize(int minChatSplitSize) { private void storeLinkedHashes(List messages, Metadata metadata) { for (MessageMultiMedia mmm : messages) { for (Message m : mmm.getMessages()) { - if (Util.isValidHash(m.getMediaHash())) { + if (HashUtils.isValidHash(m.getMediaHash())) { metadata.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + m.getMediaHash()); //$NON-NLS-1$ if (m.isFromMe()) metadata.add(ExtraProperties.SHARED_HASHES, m.getMediaHash()); @@ -240,7 +241,7 @@ private void generateChat(Chat c, Contact account, Extractor e, IItemSearcher se addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_ACCOUNT, e.getUserAccount().getId()); // Communication:{ID, Name, , MessagesCount} - chatMetadata.set(ExtraProperties.CONVERSATION_ID, Long.toString(c.getId())); + chatMetadata.set(ExtraProperties.CONVERSATION_ID, Long.toString(c.isGroupOrChannel() ? -c.getId() :c.getId())); chatMetadata.set(ExtraProperties.CONVERSATION_NAME, c.getC().getFullname()); chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, messagesCount); @@ -316,7 +317,7 @@ private void extractMessages(String chatName, List messages, if (m.getChat().isGroupOrChannel()) { ChatGroup groupChat = (ChatGroup) m.getChat(); String to = groupChat.isGroup() ? "Group " : "Channel "; - to += groupChat.getName() + " (ID:" + groupChat.getId() + ")"; + to += groupChat.getName() + " (ID:" + (-groupChat.getId()) + ")"; meta.add(ExtraProperties.COMMUNICATION_TO, to); meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); } else { @@ -346,7 +347,7 @@ private void extractMessages(String chatName, List messages, if (media.getMediaSize() != 0) { meta.set("mediaSize", Long.toString(media.getMediaSize())); } - if (Util.isValidHash(media.getMediaHash())) { + if (HashUtils.isValidHash(media.getMediaHash())) { meta.add(StandardParser.INDEXER_CONTENT_TYPE, TELEGRAM_ATTACHMENT.toString()); meta.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + media.getMediaHash()); //$NON-NLS-1$ if (!media.getChildPornSets().isEmpty()) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java index 9439281a63..b1ad267ce1 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java @@ -152,7 +152,8 @@ protected List extractChatList() throws WAExtractorException { hasChatView = databaseHasChatView(conn); hasThumbTable = SQLite3DBParser.containsTable("message_thumbnails", conn); //$NON-NLS-1$ hasGroupParticipantsTable = SQLite3DBParser.containsTable("group_participants", conn); //$NON-NLS-1$ - hasGroupParticipantUserTable = SQLite3DBParser.containsTable("group_participant_user", conn); //$NON-NLS-1$ hasEditVersionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "edit_version"); //$NON-NLS-1$ //$NON-NLS-2$ + hasGroupParticipantUserTable = SQLite3DBParser.containsTable("group_participant_user", conn); //$NON-NLS-1$ + hasEditVersionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "edit_version"); //$NON-NLS-1$ //$NON-NLS-2$ hasMediaCaptionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "media_caption"); //$NON-NLS-1$ //$NON-NLS-2$ hasMediaDurationCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "media_duration"); //$NON-NLS-1$ //$NON-NLS-2$ hasForwardedCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "forwarded"); //$NON-NLS-1$ //$NON-NLS-2$ diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java index 251f9279e2..382db985dd 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java @@ -1,5 +1,8 @@ package iped.parsers.telegram; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; + import java.io.IOException; import java.io.InputStream; @@ -47,10 +50,10 @@ public void testTelegramParser() throws IOException, SAXException, TikaException assertEquals("Karol Braz", telegramtracker.title.get(1)); assertEquals("Budi", telegramtracker.title.get(2)); assertEquals("Nickerida", telegramtracker.title.get(3)); - assertEquals("Telegram_Chat_Marilê", telegramtracker.title.get(505)); - assertEquals("Telegram_Chat_Marilê_message_0", telegramtracker.title.get(506)); - assertEquals("Telegram_Chat_Rafael CiC", telegramtracker.title.get(507)); - assertEquals("Telegram_Chat_Rafael CiC_message_0", telegramtracker.title.get(508)); + assertEquals("Telegram Chat - Marilê (+5561981993740)", telegramtracker.title.get(505)); + assertEquals("Telegram Chat - Marilê (+5561981993740)_message_0", telegramtracker.title.get(506)); + assertEquals("Telegram Chat - Rafael CiC (+5562991688700)", telegramtracker.title.get(507)); + assertEquals("Telegram Chat - Rafael CiC (+5562991688700)_message_0", telegramtracker.title.get(508)); assertEquals("Tiago", telegramtracker.username.get(0)); assertEquals("Karol Braz", telegramtracker.username.get(1)); @@ -68,20 +71,22 @@ public void testTelegramParser() throws IOException, SAXException, TikaException assertEquals("RafaelCampos", telegramtracker.usernotes.get(1)); assertEquals("gif", telegramtracker.usernotes.get(3)); - assertEquals("Bruno Chaves (+33667514279)", telegramtracker.groupParticipants.get(0)); - assertEquals("Nake Douglas (+5561982616052)", telegramtracker.groupParticipants.get(1)); - assertEquals("Guilherme Andreúce (@guileb | +5561986143035)", telegramtracker.groupParticipants.get(2)); - assertEquals("Telegram (+42777)", telegramtracker.privateParticipants.get(0)); - assertEquals("Nickerida (+5561983125151)", telegramtracker.privateParticipants.get(1)); - assertEquals("Yan Victor (@YanVictor | +5561995328012)", telegramtracker.privateParticipants.get(2)); - - assertEquals("Telegram (+42777)", telegramtracker.messagefrom.get(0)); - assertEquals("Nickerida (+5561983125151)", telegramtracker.messagefrom.get(1)); - assertEquals("Guilherme Andreúce (@guileb | +5561986143035)", telegramtracker.messagefrom.get(3)); - - assertEquals("Telegram (+42777)", telegramtracker.messageto.get(0)); - assertEquals("Nickerida (+5561983125151)", telegramtracker.messageto.get(1)); - assertEquals("Group Mixirica e noronhe-se (ID:136232058)", telegramtracker.messageto.get(150)); + assertThat(telegramtracker.groupParticipants, hasItems( + "Bruno Chaves (+33667514279)", + "Nake Douglas (+5561982616052)", + "Guilherme Andreúce (@guileb | +5561986143035)")); + assertThat(telegramtracker.privateParticipants, hasItems( + "Telegram (+42777)", + "Nickerida (+5561983125151)", + "Yan Victor (@YanVictor | +5561995328012)")); + + assertEquals("Vayu (+5561985757632)", telegramtracker.messagefrom.get(0)); + assertEquals("Felix (@Very_Hungery_Boye | +5561982943741)", telegramtracker.messagefrom.get(1)); + assertEquals("Caio Brandão (@CaioBrandao)", telegramtracker.messagefrom.get(3)); + + assertEquals("Group Constelação Familiar e Barras de Access (ID:1459548546)", telegramtracker.messageto.get(0)); + assertEquals("Group Le Chad et les Chadettes (ID:1391969250)", telegramtracker.messageto.get(1)); + assertEquals("Rafael CiC (+5562991688700)", telegramtracker.messageto.get(150)); assertTrue(telegramtracker.messagebody.get(1).contains("KKKKK")); assertTrue(telegramtracker.messagebody.get(2).contains("Parece outro carro até"));