diff --git a/iped-api/src/main/java/iped/properties/ExtraProperties.java b/iped-api/src/main/java/iped/properties/ExtraProperties.java index 252e840e6b..670c9ced0b 100644 --- a/iped-api/src/main/java/iped/properties/ExtraProperties.java +++ b/iped-api/src/main/java/iped/properties/ExtraProperties.java @@ -57,6 +57,8 @@ public class ExtraProperties { public static final Property CONVERSATION_MESSAGES_COUNT = Property.internalInteger(CONVERSATION_PREFIX + "messagesCount"); + public static final Property CONVERSATION_IS_EMPTY = Property.internalBoolean(CONVERSATION_PREFIX + "isEmpty"); + public static final String CONVERSATION_SUFFIX_ID = ":id"; public static final String CONVERSATION_SUFFIX_NAME = ":name"; public static final String CONVERSATION_SUFFIX_PHONE = ":phoneNumber"; @@ -78,7 +80,9 @@ public class ExtraProperties { public static final String GROUP_ID = "GroupID"; - public static final String IS_GROUP_MESSAGE = "isGroupMessage"; + public static final String IS_GROUP_MESSAGE = COMMUNICATION_PREFIX + "isGroupMessage"; + public static final Property COMMUNICATION_SYSTEM_MESSAGE = Property.internalBoolean(COMMUNICATION_PREFIX + "isSystemMessage"); + public static final String MESSAGE_BODY = MESSAGE_PREFIX + "Body"; //$NON-NLS-1$ diff --git a/iped-engine/src/main/java/iped/engine/task/index/IndexItem.java b/iped-engine/src/main/java/iped/engine/task/index/IndexItem.java index dfeab6f552..2258f5f567 100644 --- a/iped-engine/src/main/java/iped/engine/task/index/IndexItem.java +++ b/iped-engine/src/main/java/iped/engine/task/index/IndexItem.java @@ -82,7 +82,6 @@ import iped.engine.preview.PreviewConstants; import iped.engine.preview.PreviewInputStreamFactory; import iped.engine.sleuthkit.SleuthkitInputStreamFactory; -import iped.engine.task.ImageThumbTask; import iped.engine.task.MinIOTask.MinIOInputInputStreamFactory; import iped.engine.task.ThumbTask; import iped.engine.task.similarity.ImageSimilarityTask; @@ -126,8 +125,12 @@ public class IndexItem extends BasicProps { public static final char EVENT_IDX_SEPARATOR2 = ','; public static final String EVENT_SEPARATOR = " | "; + private static final String COUNT_SUFFIX = ":count"; + static HashSet ignoredMetadata = new HashSet(); + static HashSet countMetadata = new HashSet<>(); + private static Map inputStreamFactories = new HashMap<>(); private static Map> typesMap = MetadataUtil.getMetadataTypes(); @@ -158,6 +161,9 @@ public class IndexItem extends BasicProps { // ocrCharCount is already copied to an extra attribute ignoredMetadata.add(OCRParser.OCR_CHAR_COUNT); + countMetadata.add(ExtraProperties.CONVERSATION_PARTICIPANTS); + countMetadata.add(ExtraProperties.CONVERSATION_ADMINS); + BasicProps.SET.add(ID_IN_SOURCE); BasicProps.SET.add(SOURCE_PATH); BasicProps.SET.add(SOURCE_DECODER); @@ -711,6 +717,12 @@ private static void addMetadataToDoc(Document doc, Metadata metadata, Set SUPPORTED_TYPES = new HashSet(Arrays.asList(MediaType.parse(CHAT_MIME_TYPE))); + private static final Set SUPPORTED_TYPES = new HashSet<>(Arrays.asList(MediaType.parse(CHAT_MIME_TYPE))); private static final String ME_URL = "https://discord.com/api/v9/users/@me"; static { @@ -111,12 +117,17 @@ public Set getSupportedTypes(ParseContext context) { @Override public void parse(InputStream indexFile, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { - String contentType = metadata.get(Metadata.CONTENT_TYPE); + try { + String contentType = metadata.get(HttpHeaders.CONTENT_TYPE); if (contentType.equals(CHAT_MIME_TYPE)) { parseDiscord(indexFile, handler, metadata, context); } else { parseCacheIndex(indexFile, handler, metadata, context); } + } catch (Exception e) { + logger.warn("Discord error", e); + throw e; + } } // this code may be removed when backward parsersconfig.xml compatibility were @@ -141,7 +152,7 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata IItemReader item = context.get(IItemReader.class); String commonQuery = BasicProps.EVIDENCE_UUID + ":" + item.getDataSource().getUUID() + " AND " + BasicProps.PARENTID + ":" + item.getParentId() + " AND NOT " + BasicProps.LENGTH + ":0 AND " - + CacheIndexParser.IS_CACHE_INDEX_ENTRY.replace(":", "\\:") + ":true"; + + searcher.escapeQuery(CacheIndexParser.IS_CACHE_INDEX_ENTRY) + ":true"; try (InputStream is = TikaInputStream.get(indexFile, new TemporaryResources())) { @@ -238,11 +249,32 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata chatmetadata.set(StandardParser.INDEXER_CONTENT_TYPE, CHAT_MIME_TYPE_HTML); chatmetadata.set(ExtraProperties.ITEM_VIRTUAL_ID, Integer.toString(chatVirtualId)); + // Type + chatmetadata.set(ExtraProperties.CONVERSATION_NAME, chatName); + chatmetadata.set(ExtraProperties.CONVERSATION_TYPE, discordRoot.get(0).getType() + ""); + chatmetadata.set(ExtraProperties.CONVERSATION_ID, discordRoot.get(0).getId()); + + // Account + addPartyFields(chatmetadata, ExtraProperties.CONVERSATION_ACCOUNT, me); + + // Participants + TreeMap participantsMap = new TreeMap<>(); + for (DiscordRoot d : discordRoot) { + if (d.getAuthor() != null) { + participantsMap.putIfAbsent(d.getAuthor().getFullUsername(), d.getAuthor()); + } + } + Collection participants = participantsMap.values(); + for (DiscordAuthor participant : participants) { + addPartyFields(chatmetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, participant); + } + for (DiscordRoot dr : discordRoot) { for (DiscordSticker sticker : dr.getStickers()) { try { - List stickerItems = searcher - .search(commonQuery + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"" + sticker.getId() + ".json\"" + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"discord\""); + List stickerItems = searcher.search(commonQuery + " AND " + + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"" + sticker.getId() + ".json\"" + " AND " + + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"discord\""); for (IItemReader stickerItem : stickerItems) { sticker.setMediaHash(stickerItem.getHash()); } @@ -287,16 +319,32 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata byte[] relatorio = new DiscordHTMLReport(me).convertToHTML(discordRoot, searcher); extractor.parseEmbedded(new ByteArrayInputStream(relatorio), handler, chatmetadata, false); - extractMessages(chatName, discordRoot, handler, extractor, chatVirtualId); + extractMessages(chatName, discordRoot, me, participants, handler, extractor, chatVirtualId); } } + private void addPartyFields(Metadata meta, String field, DiscordAuthor author) { + if (author == null) { + return; + } + meta.add(field, PartyStringBuilderFactory.getBuilder("Discord") + .withUserId(author.getId()) + .withName(author.getName()) + .withPhoneNumber(author.getPhone()) + .withUsername(author.getUsername()) + .build()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, author.getId()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, author.getName()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_PHONE, author.getPhone()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_USERNAME, author.getFullUsername()); + } + private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery, ObjectMapper mapper, HashMap avatarCache, ContentHandler handler, EmbeddedDocumentExtractor extractor) { DiscordAuthor me = null; try { // find me info - List mes = searcher.search(commonQuery + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"" + ME_URL + "\""); + List mes = searcher.search(commonQuery + " AND " + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"" + ME_URL + "\""); for (IItemReader mei : mes) { if (mei.getName().equals("@me")) { try (InputStream is2 = getDecompressedStream(mei.getBufferedInputStream())) { @@ -312,7 +360,7 @@ private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery, memeta.set(ExtraProperties.USER_ACCOUNT_TYPE, "Discord"); memeta.set(ExtraProperties.USER_EMAIL, me.getEmail()); // memeta.set(ExtraProperties.USER_NOTES, me.getBio()); - memeta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); + memeta.set(ExtraProperties.DECODED_DATA, Boolean.toString(true)); if (me.getAvatar() != null) { byte[] meavatar = avatarCache.get(me.getAvatar()); @@ -334,51 +382,56 @@ private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery, return me; } - private void extractMessages(String chatName, List discordRoot, ContentHandler handler, EmbeddedDocumentExtractor extractor, int chatVirtualId) throws SAXException, IOException { + private void extractMessages(String chatName, List discordRoot, DiscordAuthor me, Collection participants, ContentHandler handler, EmbeddedDocumentExtractor extractor, int chatVirtualId) throws SAXException, IOException { int msgCount = 0; - // Checking Participants - LinkedHashSet participants = new LinkedHashSet(); - - for (DiscordRoot d : discordRoot) { - participants.add(d.getAuthor().getFullUsername()); - } - // Set metadata for (DiscordRoot d : discordRoot) { Metadata meta = new Metadata(); meta.set(TikaCoreProperties.TITLE, chatName + "_message_" + msgCount++); meta.set(StandardParser.INDEXER_CONTENT_TYPE, MSG_MIME_TYPE); - meta.set(ExtraProperties.MESSAGE_DATE, d.getTimestamp()); + meta.set(ExtraProperties.COMMUNICATION_DATE, d.getTimestamp()); meta.set(ExtraProperties.MESSAGE_BODY, d.getMessageContent()); - meta.set(ExtraProperties.USER_NAME, d.getAuthor().getFullUsername()); - meta.add(ExtraProperties.PARTICIPANTS, participants.toString()); meta.set(ExtraProperties.PARENT_VIRTUAL_ID, Integer.toString(chatVirtualId)); meta.set(ExtraProperties.PARENT_VIEW_POSITION, String.valueOf(d.getId())); - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, d.getAuthor().getFullUsername()); - meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); + meta.set(ExtraProperties.DECODED_DATA, Boolean.toString(true)); + + // From + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, d.getAuthor()); // Add "Message-TO" field. // In cases where only one participant sends messages, it is not possible to // determine the participants as only the participant list of the calls are cached. if (participants.size() <= 2) { - for (String participant : participants) { - if (!participant.equals(d.getAuthor().getFullUsername())) { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, participant); + for (DiscordAuthor participant : participants) { + if (!participant.getFullUsername().equals(d.getAuthor().getFullUsername())) { + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, participant); } } } else { - meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, chatName); - meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true"); + meta.set(ExtraProperties.COMMUNICATION_TO, chatName); + meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); + } + + // Direction + Boolean isFromMe = null; + if (me != null) { + isFromMe = me.getFullUsername().equals(d.getAuthor().getFullUsername()); + if (isFromMe) { + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + } else { + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + } } for (DiscordAttachment da : d.getAttachments()) { if (da.getMediaHash() != null) { meta.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + da.getMediaHash()); meta.set(StandardParser.INDEXER_CONTENT_TYPE, ATTACH_MIME_TYPE); - // if (m.isFromMe()) - // meta.add(ExtraProperties.SHARED_HASHES, m.getMediaHash()); + if (isFromMe != null && isFromMe) { + meta.add(ExtraProperties.SHARED_HASHES, da.getMediaHash()); + } } } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java index 66c00d0be2..8c79eaed40 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.StringUtils; import org.apache.tika.config.Field; import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; @@ -30,6 +31,7 @@ import iped.parsers.sqlite.SQLite3Parser; import iped.parsers.standard.StandardParser; import iped.parsers.util.ChildPornHashLookup; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.parsers.util.Messages; import iped.properties.BasicProps; @@ -174,6 +176,8 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, } /* adiciona a conversação */ + boolean isGroup = conv.getParticipantes() != null && conv.getParticipantes().size() > 1; + Metadata chatMetadata = new Metadata(); chatMetadata.set(StandardParser.INDEXER_CONTENT_TYPE, CONVERSATION_MIME_TYPE); chatMetadata.set(HttpHeaders.CONTENT_TYPE, CONVERSATION_MIME_TYPE); @@ -184,6 +188,25 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, chatMetadata.set(BasicProps.HASCHILD, Boolean.TRUE.toString()); chatMetadata.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); + // Conversation:Account + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_ACCOUNT, contactMap, account.getSkypeName()); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, contactMap, account.getSkypeName()); + + // Conversation:Participants + if (conv.getParticipantes() != null) { + for (String participant : conv.getParticipantes()) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, contactMap, participant); + } + } + + // Conversation:{ID, Name, Type, MessagesCount} + chatMetadata.set(ExtraProperties.CONVERSATION_ID, conv.getId()); + chatMetadata.set(ExtraProperties.CONVERSATION_NAME, + StringUtils.firstNonBlank(conv.getDisplayName(), conv.getChatName(), conv.getTitle())); + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, + isGroup ? ConversationConstants.TYPE_GROUP : ConversationConstants.TYPE_ONEONONE); + chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, conv.getMessages().size()); + storeSharedHashes(conv, chatMetadata); if (extractor.shouldParseEmbedded(chatMetadata)) { @@ -205,8 +228,17 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, meta.set(ExtraProperties.PARENT_VIEW_POSITION, String.valueOf(sm.getId())); meta.set(ExtraProperties.USER_ACCOUNT_TYPE, SKYPE); meta.set(ExtraProperties.MESSAGE_DATE, sm.getData()); - meta.set(Metadata.MESSAGE_FROM, formatSkypeName(contactMap, sm.getAutor())); - meta.set(Metadata.MESSAGE_TO, formatSkypeName(contactMap, sm.getDestino())); + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, sm.isFromMe() + ? ConversationConstants.DIRECTION_OUTGOING : ConversationConstants.DIRECTION_INCOMING); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, contactMap, sm.getAutor()); + if (sm.getDestino() != null) { + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, contactMap, sm.getDestino()); + } else if (isGroup) { + meta.set(ExtraProperties.COMMUNICATION_TO, conv.getTitle()); + } + if (isGroup) { + meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); + } meta.set(ExtraProperties.MESSAGE_BODY, sm.getConteudo()); meta.set("messageStatus", String.valueOf(sm.getChatMessageStatus())); meta.set("sendingStatus", String.valueOf(sm.getSendingStatus())); @@ -251,11 +283,15 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, tMetadata.set(TikaCoreProperties.MODIFIED, t.getFinish()); tMetadata.set(ExtraProperties.MESSAGE_DATE, t.getStart()); tMetadata.set(ExtraProperties.MESSAGE_BODY, name); - tMetadata.set(Metadata.MESSAGE_FROM, formatSkypeName(contactMap, t.getFrom())); + tMetadata.set(ExtraProperties.COMMUNICATION_DIRECTION, + t.getFrom() != null && t.getFrom().equals(sqlite.getSkypeName()) + ? ConversationConstants.DIRECTION_OUTGOING : ConversationConstants.DIRECTION_INCOMING); + addPartyFields(tMetadata, ExtraProperties.COMMUNICATION_FROM, contactMap, t.getFrom()); if (t.getType() == 3 && t.getConversation().getParticipantes() != null && t.getConversation().getParticipantes().size() > 1) { - tMetadata.set(Metadata.MESSAGE_TO, t.getConversation().getTitle()); + tMetadata.set(ExtraProperties.COMMUNICATION_TO, t.getConversation().getTitle()); + tMetadata.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); } else { - tMetadata.set(Metadata.MESSAGE_TO, formatSkypeName(contactMap, t.getTo())); + addPartyFields(tMetadata, ExtraProperties.COMMUNICATION_TO, contactMap, t.getTo()); } if (searcher != null) { @@ -333,6 +369,19 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, } + private void addPartyFields(Metadata meta, String field, Map contactMap, String skypeName) { + if (StringUtils.isBlank(skypeName)) { + return; + } + meta.add(field, formatSkypeName(contactMap, skypeName)); + SkypeUser user = contactMap.get(skypeName); + String id = (user != null && StringUtils.isNotBlank(user.getSkypeName())) ? user.getSkypeName() : skypeName; + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, id); + if (user != null && StringUtils.isNotBlank(user.getBestName())) { + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, user.getBestName()); + } + } + private String formatSkypeName(Map contactMap, String skypeName) { if (skypeName == null) return null; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java index 0b93cdfffc..241848f1b7 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Contact.java @@ -23,6 +23,7 @@ import dpf.ap.gpinf.interfacetelegram.ContactInterface; import dpf.ap.gpinf.interfacetelegram.DecoderTelegramInterface; import dpf.ap.gpinf.interfacetelegram.PhotoData; +import iped.parsers.chat.TelegramPartyStringBuilder; public class Contact implements ContactInterface { private long id; @@ -171,27 +172,16 @@ public boolean isGroupOrChannel() { @Override public String toString() { - StringBuilder sb = new StringBuilder(); - String name = getFullname(); - if (name != null) { - sb.append(name.trim()); - } - String number = getPhone(); - if (number != null && number.length() > 0) { - if (sb.length() > 0) { - sb.append(' '); - } - sb.append("(phone: ").append(number).append(')'); - } else if (getId() > 0) { - if (sb.length() > 0) { - sb.append(' '); - } - sb.append("(ID:").append(getId()).append(')'); - } - if (sb.length() == 0) { - sb.append("[unknown]"); + String str = new TelegramPartyStringBuilder() + .withUserId(Long.toString(getId())) + .withName(getFullname()) + .withPhoneNumber(getPhone()) + .withUsername(getUsername()) + .build(); + if (str.isEmpty()) { + str = "[unknown]"; } - return sb.toString(); + return str; } } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java index 73d96bfb23..826f372b7e 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java @@ -35,6 +35,7 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,7 +185,8 @@ protected List extractChatList() throws Exception { searchAvatarFileName(cont, androidDecoder.getPhotoData()); - ChatGroup group = new ChatGroup(chatId, cont, chatName); + String chatNameToUse = StringUtils.firstNonBlank((String) androidDecoder.getAlltMetadata().get("title"), chatName); + ChatGroup group = new ChatGroup(chatId, cont, chatNameToUse); Map m = androidDecoder.getAlltMetadata(); if ("true".equals(m.get("broadcast"))) { @@ -351,7 +353,7 @@ protected Chat extractMessages() throws Exception { } setFrom(message, chat); - if (cg != null && message.getFrom().getId() != 0) { + if (cg != null && message.getFrom().getId() != 0 && message.getFrom().getId() != cg.getId()) { cg.addMember(message.getFrom().getId()); } @@ -469,7 +471,7 @@ protected ArrayList extractMessagesIOS(Chat chat) throws SQLE } } - if (cg != null && message.getFrom().getId() != 0) { + if (cg != null && message.getFrom().getId() != 0 && message.getFrom().getId() != cg.getId()) { cg.addMember(message.getFrom().getId()); } @@ -602,6 +604,10 @@ protected void extractContacts() throws Exception { e.printStackTrace(); } } + + if (Boolean.parseBoolean((String) androidDecoder.getAlltMetadata().get("self"))) { + userAccount = c; + } } } } @@ -745,11 +751,11 @@ private String getAndroidExtractMessagesSQL() throws SQLException { + " from dialogs d LEFT join users u on u.uid=d.did WHERE d.did NOT IN (SELECT -uid FROM chats) " + " order by date desc"; - private static final String MEMBERS_CHATS_SQL = "SELECT * from channel_users_v2 where did=?"; + private static final String MEMBERS_CHATS_SQL = "SELECT * from channel_users_v2 where did=? AND uid > 0"; private static final String SELECT_CHATS_SETTINGS = "SELECT participants_count FROM chat_settings_v2 WHERE uid=?"; - private static final String SELECT_CHANNEL_ADMINS = "SELECT uid FROM channel_admins_v3 WHERE did=?"; + private static final String SELECT_CHANNEL_ADMINS = "SELECT uid FROM channel_admins_v3 WHERE did=? AND uid > 0"; private static final String CHATS_SQL_IOS = "select substr(key,16,8) as chatid, false as deleted from t9 " + "UNION SELECT substr(t7.key,1,8) as chatid, true as deleted from t7 " diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java index 2f4b5a956f..1f9928c1ce 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java @@ -28,6 +28,7 @@ import dpf.ap.gpinf.interfacetelegram.PhotoData; import iped.data.IItemReader; import iped.parsers.util.ChildPornHashLookup; +import iped.parsers.util.HashUtils; public class Message implements MessageInterface { private long id; @@ -72,8 +73,10 @@ public String getMediaHash() { } public void setMediaHash(String mediaHash) { - this.mediaHash = mediaHash; - childPornSets = ChildPornHashLookup.lookupHashAndMerge(mediaHash, childPornSets); + if (HashUtils.isValidHash(mediaHash)) { + this.mediaHash = mediaHash; + childPornSets = ChildPornHashLookup.lookupHashAndMerge(mediaHash, childPornSets); + } } public String getMediaFile() { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/ReportGenerator.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/ReportGenerator.java index 69900f8548..c628928418 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/ReportGenerator.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/ReportGenerator.java @@ -169,7 +169,7 @@ public byte[] generateNextChatHtml(Chat c) { private TagHtml getThumbTag(Message m, String classnotfound) { byte thumb[] = m.getThumb(); - if (searcher != null && thumb == null && m.getMediaHash() != null && !m.getMediaHash().isBlank()) { + if (searcher != null && thumb == null && m.getMediaHash() != null) { if (!m.getMediaHash().equalsIgnoreCase(emptyMD5)) { List result = iped.parsers.util.Util.getItems("md5:" + m.getMediaHash(), searcher); if (result != null && !result.isEmpty()) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java index 944b02f49d..ef9295d98e 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java @@ -54,6 +54,8 @@ import iped.data.IItemReader; import iped.parsers.sqlite.SQLite3DBParser; import iped.parsers.standard.StandardParser; +import iped.parsers.util.ConversationConstants; +import iped.parsers.util.HashUtils; import iped.parsers.util.ItemInfo; import iped.parsers.util.PhoneParsingConfig; import iped.properties.BasicProps; @@ -119,7 +121,7 @@ public void setMinChatSplitSize(int minChatSplitSize) { private void storeLinkedHashes(List messages, Metadata metadata) { for (MessageMultiMedia mmm : messages) { for (Message m : mmm.getMessages()) { - if (Util.isValidHash(m.getMediaHash())) { + if (HashUtils.isValidHash(m.getMediaHash())) { metadata.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + m.getMediaHash()); //$NON-NLS-1$ if (m.isFromMe()) metadata.add(ExtraProperties.SHARED_HASHES, m.getMediaHash()); @@ -193,6 +195,7 @@ private void generateChat(Chat c, Contact account, Extractor e, IItemSearcher se int frag = 0; int firstMsg = 0; byte[] bytes; + int messagesCount = (int) c.getMessages().stream().filter(m -> m.getType() == null).count(); ReportGenerator r = new ReportGenerator(searcher); r.setMinChatSplitSize(this.minChatSplitSize); while ((bytes = r.generateNextChatHtml(c)) != null) { @@ -211,19 +214,37 @@ private void generateChat(Chat c, Contact account, Extractor e, IItemSearcher se if (c.isGroupOrChannel()) { ChatGroup cg = (ChatGroup) c; - for (long id : cg.getMembers()) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, e.getContact(id).toString()); - } for (long id : cg.getAdmins()) { - chatMetadata.add(ExtraProperties.COMMUNICATION_PREFIX + "ChannelAdmins", - e.getContact(id).toString()); + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_ADMINS, id); + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_PARTICIPANTS, id); + if (id == e.getUserAccount().getId()) { + chatMetadata.set(ExtraProperties.CONVERSATION_IS_OWNER_ADMIN, true); + } } - int participantsCount = cg.getParticipantsCount(); - if (participantsCount > 0) { - chatMetadata.add(ExtraProperties.PARTICIPANTS + "Count", String.valueOf(participantsCount)); + for (long id : cg.getMembers()) { + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_PARTICIPANTS, id); } + } else { + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_PARTICIPANTS, c.getId()); + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_PARTICIPANTS, e.getUserAccount().getId()); } + if (c.isChannel()) { + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_BROADCAST); + } else if (c.isGroup()) { + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_GROUP); + } else { + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_ONEONONE); + } + + // Communication:Account + addPartyFields(chatMetadata, e, ExtraProperties.CONVERSATION_ACCOUNT, e.getUserAccount().getId()); + + // Communication:{ID, Name, , MessagesCount} + chatMetadata.set(ExtraProperties.CONVERSATION_ID, Long.toString(c.isGroupOrChannel() ? -c.getId() :c.getId())); + chatMetadata.set(ExtraProperties.CONVERSATION_NAME, c.getC().getFullname()); + chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, messagesCount); + List msgSubset = c.getMessages().subList(firstMsg, nextMsg); if (extractMessages && !msgSubset.isEmpty()) { @@ -242,8 +263,24 @@ private void generateChat(Chat c, Contact account, Extractor e, IItemSearcher se } } + private void addPartyFields(Metadata meta, Extractor e, String field, long contactId) { + Contact contact = e.getContact(contactId); + addPartyFields(meta, field, contact); + } + + private void addPartyFields(Metadata meta, String field, Contact c) { + if (c == null) { + return; + } + meta.add(field, c.toString()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, Long.toString(c.getId())); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, c.getFullname()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_PHONE, c.getPhone()); + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_USERNAME, c.getUsername()); + } + private String getChatNamePrefix(Chat c) { - String title = "Telegram_"; + String title = "Telegram "; if (c.isChannel()) { title += "Channel"; } else if (c.isGroup()) { @@ -251,7 +288,7 @@ private String getChatNamePrefix(Chat c) { } else { title += "Chat"; } - title += "_" + c.getName(); + title += " - " + c.getC().toString(); return title; } @@ -275,24 +312,30 @@ private void extractMessages(String chatName, List messages, for (String location : m.getLocations()) { meta.add(ExtraProperties.LOCATIONS, location); } - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, m.getFrom().toString()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, m.getFrom()); + if (m.getChat().isGroupOrChannel()) { ChatGroup groupChat = (ChatGroup) m.getChat(); String to = groupChat.isGroup() ? "Group " : "Channel "; - to += groupChat.getName() + " (id:" + groupChat.getId() + ")"; - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to); - meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true"); - } - if (meta.get(org.apache.tika.metadata.Message.MESSAGE_TO) == null) { + to += groupChat.getName() + " (ID:" + (-groupChat.getId()) + ")"; + meta.add(ExtraProperties.COMMUNICATION_TO, to); + meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); + } else { if (m.getToId() != 0) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, e.getContact(m.getToId()).toString()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, e.getContact(m.getToId())); } else if (m.isFromMe()) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, m.getChat().getC().toString()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, m.getChat().getC()); } else if (account != null) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, account.toString()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, account); } } + if (m.isFromMe()) { + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + } else { + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + } + meta.set(ExtraProperties.MESSAGE_BODY, m.getData()); for (Message media : m.getMessages()) { @@ -304,7 +347,7 @@ private void extractMessages(String chatName, List messages, if (media.getMediaSize() != 0) { meta.set("mediaSize", Long.toString(media.getMediaSize())); } - if (Util.isValidHash(media.getMediaHash())) { + if (HashUtils.isValidHash(media.getMediaHash())) { meta.add(StandardParser.INDEXER_CONTENT_TYPE, TELEGRAM_ATTACHMENT.toString()); meta.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + media.getMediaHash()); //$NON-NLS-1$ if (!media.getChildPornSets().isEmpty()) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Util.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Util.java index 96151ef4c8..44433a161e 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Util.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Util.java @@ -20,25 +20,11 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.HashSet; -import java.util.Set; import iped.utils.IOUtil; public class Util { - private static final Set zeroLengthHashes = new HashSet(); - static { - // Hashes of empty input (byte[0]), see issue #2157. - zeroLengthHashes.add("d41d8cd98f00b204e9800998ecf8427e"); // MD5 - zeroLengthHashes.add("da39a3ee5e6b4b0d3255bfef95601890afd80709"); // SHA-1 - zeroLengthHashes.add("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); // SHA-256 - } - - public static boolean isValidHash(String hash) { - return hash != null && !hash.isBlank() && !zeroLengthHashes.contains(hash.toLowerCase()); - } - public static void invertByteArray(byte[] array, int start, int len) { for (int i = 0; i < len / 2; i++) { byte aux = array[start + i]; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java index 09a24361aa..94bc635247 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.tika.config.Field; import org.apache.tika.exception.TikaException; @@ -55,6 +56,7 @@ import iped.parsers.sqlite.SQLite3Parser; import iped.parsers.standard.StandardParser; import iped.parsers.threema.Message.MessageType; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.properties.BasicProps; import iped.properties.ExtraProperties; @@ -277,21 +279,31 @@ private void createReport(List chatList, IItemSearcher searcher, ContentHa if (extractMessages && !msgSubset.isEmpty()) { chatMetadata.set(BasicProps.HASCHILD, Boolean.TRUE.toString()); } - if (account != null) { - String local = account.getFullId(); - chatMetadata.add(ExtraProperties.PARTICIPANTS, local); + + // Conversation:Account + if (account != null && !account.isUnknown()) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_ACCOUNT, account); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, account); } + + // Conversation:{Participants, Type} if (c.isGroupChat()) { for (ThreemaContact member : c.getParticipants()) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, member.getFullId()); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, member); } - chatMetadata.add(ExtraProperties.GROUP_ID, "Conversation_" + c.getId()); + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_GROUP); } else { if (c.getContact() != null) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, c.getContact().getFullId()); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, c.getContact()); } + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_ONEONONE); } + // Conversation:{ID, Name, MessagesCount} + chatMetadata.set(ExtraProperties.CONVERSATION_ID, Long.toString(c.getId())); + chatMetadata.set(ExtraProperties.CONVERSATION_NAME, c.getTitle()); + chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, c.getMessages().size()); + ByteArrayInputStream chatStream = new ByteArrayInputStream(bytes); extractor.parseEmbedded(chatStream, handler, chatMetadata, false); bytes = nextBytes; @@ -335,10 +347,26 @@ private void fillGroupRecipients(Metadata meta, Chat c) { to += " " + c.getSubject().strip(); } to += " (id:" + c.getId() + ")"; - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to); + meta.add(ExtraProperties.COMMUNICATION_TO, to); meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true"); } + private void addPartyFields(Metadata meta, String field, ThreemaContact contact) { + if (contact == null) { + return; + } + String fullId = contact.getFullId(); + if (StringUtils.isNotBlank(fullId)) { + meta.add(field, fullId); + } + if (contact instanceof ThreemaAccount && ((ThreemaAccount) contact).isUnknown()) { + return; + } + if (StringUtils.isNotBlank(contact.getId())) { + meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, contact.getId()); + } + } + private void extractMessages(String chatName, Chat c, List messages, ThreemaAccount account, int parentVirtualId, ContentHandler handler, EmbeddedDocumentExtractor extractor) throws SAXException, IOException { int msgCount = 0; for (Message m : messages) { @@ -353,22 +381,23 @@ private void extractMessages(String chatName, Chat c, List messages, Th meta.set(TikaCoreProperties.CREATED, m.getTimeStamp()); meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); - String local = account.getFullId(); - String remote = m.getRemoteResource(); - if (m.isFromMe()) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, local); + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, account); if (c.isGroupChat()) { fillGroupRecipients(meta, c); } else { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, remote); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, c.getContact()); } } else { - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, remote); + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); if (c.isGroupChat()) { + // for group messages the sender is stored as a display string in the message + meta.set(ExtraProperties.COMMUNICATION_FROM, m.getRemoteResource()); fillGroupRecipients(meta, c); } else { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, local); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, c.getContact()); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, account); } } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/ChatHandler.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/ChatHandler.java index 929f47101a..ec90201af2 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/ChatHandler.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/ChatHandler.java @@ -29,6 +29,7 @@ import iped.parsers.ufed.model.Chat; import iped.parsers.ufed.model.ContactPhoto; import iped.parsers.ufed.model.Party; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.Messages; import iped.parsers.whatsapp.WAContact; import iped.properties.MediaTypes; @@ -81,9 +82,6 @@ public void fillMetadata(String prefix, Metadata metadata) { model.getParticipants().forEach(p -> { new PartyHandler(p, model.getSource()).fillMetadata(CONVERSATION_PARTICIPANTS, metadata); }); - if (!model.getParticipants().isEmpty()) { - metadata.add(CONVERSATION_PARTICIPANTS + ":count", Integer.toString(model.getParticipants().size())); - } // Chat Owner Participant model.getPhoneOwnerParticipant().ifPresent(p -> { @@ -94,10 +92,6 @@ public void fillMetadata(String prefix, Metadata metadata) { model.getParticipants().stream().filter(Party::isGroupAdmin).forEach(p -> { new PartyHandler(p, model.getSource()).fillMetadata(CONVERSATION_ADMINS, metadata); }); - long groupAdminsCount = model.getParticipants().stream().filter(Party::isGroupAdmin).count(); - if (groupAdminsCount > 0) { - metadata.set(CONVERSATION_ADMINS + ":count", Long.toString(groupAdminsCount)); - } // Chat Owner is Group Admin if (model.getParticipants().stream().filter(Party::isGroupAdmin).filter(Party::isPhoneOwner).findAny().isPresent()) { @@ -165,15 +159,15 @@ else if (StringUtils.isNotBlank(model.getServiceIdentifier())) if (StringUtils.isNotBlank(chatType)) { switch (chatType) { - case Chat.TYPE_ONEONONE: + case ConversationConstants.TYPE_ONEONONE: sb.append("Chat").append(' '); break; - case Chat.TYPE_GROUP: + case ConversationConstants.TYPE_GROUP: sb.append(Messages.getString("UfedChatParser.Group")).append(' '); break; - case Chat.TYPE_BROADCAST: + case ConversationConstants.TYPE_BROADCAST: if (model.getParticipants().size() == 1 && StringUtils.containsAnyIgnoreCase(source, Chat.SOURCE_TELEGRAM, Chat.SOURCE_WHATSAPP)) { sb.append(Messages.getString("UfedChatParser.Status")); } else if (Chat.SOURCE_TELEGRAM.equalsIgnoreCase(source)) { @@ -184,7 +178,7 @@ else if (StringUtils.isNotBlank(model.getServiceIdentifier())) sb.append(' '); break; - case Chat.TYPE_UNKNOWN: + case ConversationConstants.TYPE_UNKNOWN: break; default: diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java index e3956ccb07..65a40ef3a7 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/InstantMessageHandler.java @@ -34,6 +34,7 @@ import iped.parsers.ufed.model.ReplyMessageData; import iped.parsers.util.ConversationConstants; import iped.properties.BasicProps; +import iped.properties.ExtraProperties; import iped.search.IItemSearcher; /** @@ -64,62 +65,68 @@ protected void fillMetadata(String prefix, Metadata metadata) { } if (!model.getAttachments().isEmpty()) { - metadata.set(MESSAGE_ATTACHMENT_COUNT, Integer.toString(model.getAttachments().size())); + metadata.set(MESSAGE_ATTACHMENT_COUNT, model.getAttachments().size()); } - // Message -> Direction - model.getFrom().ifPresentOrElse(from -> { - if (from.isPhoneOwner()) { - metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); - } else { - metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); - } - }, () -> { - if (model.getTo().stream().filter(Party::isPhoneOwner).findAny().isPresent()) { - metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); - } - }); + if (model.isSystemMessage()) { + metadata.set(ExtraProperties.COMMUNICATION_SYSTEM_MESSAGE, true); - // Message -> From - model.getFrom().ifPresent(from -> { - new PartyHandler(from, model.getSource()).fillMetadata(COMMUNICATION_FROM, metadata); - }); + } else { - // Message -> To - if (model.getTo().size() == 1) { - new PartyHandler(model.getTo().get(0), model.getSource()).fillMetadata(COMMUNICATION_TO, metadata); + // Message -> Direction + model.getFrom().ifPresentOrElse(from -> { + if (from.isPhoneOwner()) { + metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + } else { + metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + } + }, () -> { + if (model.getTo().stream().filter(Party::isPhoneOwner).findAny().isPresent()) { + metadata.set(COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + } + }); - } else if (model.getChat() != null) { + // Message -> From + model.getFrom().ifPresent(from -> { + new PartyHandler(from, model.getSource()).fillMetadata(COMMUNICATION_FROM, metadata); + }); - // in case "To" is NOT set in InstantMessage, try to... - List otherParticipants; - if (model.getFrom().isPresent()) { + // Message -> To + if (model.getTo().size() == 1) { + new PartyHandler(model.getTo().get(0), model.getSource()).fillMetadata(COMMUNICATION_TO, metadata); - // ...get other participants (without "from") - otherParticipants = model.getChat().getParticipants().stream() // - .filter(p -> !p.equals(model.getFrom().get())) // - .collect(Collectors.toList()); // - } else { - // ...get other participants - otherParticipants = model.getChat().getParticipants(); - } + } else if (model.getChat() != null) { - if (otherParticipants.size() == 1) { - new PartyHandler(otherParticipants.get(0), model.getSource()).fillMetadata(COMMUNICATION_TO, metadata); + // in case "To" is NOT set in InstantMessage, try to... + List otherParticipants; + if (model.getFrom().isPresent()) { - } else if (otherParticipants.size() > 2) { + // ...get other participants (without "from") + otherParticipants = model.getChat().getParticipants().stream() // + .filter(p -> !p.equals(model.getFrom().get())) // + .collect(Collectors.toList()); // + } else { + // ...get other participants + otherParticipants = model.getChat().getParticipants(); + } - // there is more than one "To", so use chat information - metadata.add(COMMUNICATION_TO, new ChatHandler(model.getChat()).getTitle(false, false)); + if (otherParticipants.size() == 1) { + new PartyHandler(otherParticipants.get(0), model.getSource()).fillMetadata(COMMUNICATION_TO, metadata); - String chatId = model.getChat().getFieldId(); - if (StringUtils.isNotBlank(chatId)) { - metadata.add(COMMUNICATION_TO + ":id", chatId); - } + } else if (otherParticipants.size() > 2) { + + // there is more than one "To", so use chat information + metadata.add(COMMUNICATION_TO, new ChatHandler(model.getChat()).getTitle(false, false)); + + String chatId = model.getChat().getFieldId(); + if (StringUtils.isNotBlank(chatId)) { + metadata.add(COMMUNICATION_TO + ":id", chatId); + } - String chatName = model.getChat().getName(); - if (StringUtils.isNotBlank(chatName)) { - metadata.add(COMMUNICATION_TO + ":name", chatName); + String chatName = model.getChat().getName(); + if (StringUtils.isNotBlank(chatName)) { + metadata.add(COMMUNICATION_TO + ":name", chatName); + } } } } @@ -160,10 +167,6 @@ protected void fillMetadata(String prefix, Metadata metadata) { }); }); - if (model.isSystemMessage()) { - metadata.set(UFED_META_PREFIX + "isSystemMessage", Boolean.toString(true)); - } - if (model.getPosition() != null) { metadata.set(Geographic.LATITUDE, model.getPosition().getLatitude()); metadata.set(Geographic.LONGITUDE, model.getPosition().getLongitude()); diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java index 9b9803a658..a53b6181ee 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/handler/PartyHandler.java @@ -18,7 +18,9 @@ import iped.data.IItemReader; import iped.parsers.chat.PartyStringBuilderFactory; +import iped.parsers.ufed.model.Chat; import iped.parsers.ufed.model.Party; +import iped.parsers.whatsapp.WAContact; import iped.properties.ExtraProperties; import iped.properties.MediaTypes; import iped.search.IItemSearcher; @@ -46,9 +48,13 @@ protected void fillMetadata(String prefix, Metadata metadata) { metadata.add(prefix, getTitle()); model.getReferencedContact().ifPresentOrElse(ref -> { - metadata.add(prefix + CONVERSATION_SUFFIX_ID, StringUtils.firstNonBlank(model.getIdentifier(), ref.getUserID())); + String id = StringUtils.firstNonBlank(model.getIdentifier(), ref.getUserID()); + String phoneNumber = StringUtils.containsIgnoreCase(source, Chat.SOURCE_WHATSAPP) && StringUtils.endsWith(id, WAContact.waSuffix) + ? StringUtils.removeEnd(id, WAContact.waSuffix) + : ref.getPhoneNumber(); + metadata.add(prefix + CONVERSATION_SUFFIX_ID, id); metadata.add(prefix + CONVERSATION_SUFFIX_NAME, StringUtils.firstNonBlank(model.getName(), ref.getName())); - metadata.add(prefix + CONVERSATION_SUFFIX_PHONE, ref.getPhoneNumber()); + metadata.add(prefix + CONVERSATION_SUFFIX_PHONE, phoneNumber); metadata.add(prefix + CONVERSATION_SUFFIX_USERNAME, ref.getUsername()); }, () -> { metadata.add(prefix + CONVERSATION_SUFFIX_ID, model.getIdentifier()); diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/model/Chat.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/model/Chat.java index a383cdbf5e..5ebde73b60 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/model/Chat.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/model/Chat.java @@ -20,11 +20,6 @@ public class Chat extends BaseModel { private static final long serialVersionUID = -568791121571161053L; - public static final String TYPE_ONEONONE = "OneOnOne"; - public static final String TYPE_GROUP = "Group"; - public static final String TYPE_BROADCAST = "Broadcast"; - public static final String TYPE_UNKNOWN = "Unknown"; - public static final String SOURCE_WHATSAPP = "WhatsApp"; public static final String SOURCE_WHATSAPP_BUSINESS = "WhatsApp Business"; public static final String SOURCE_TELEGRAM = "Telegram"; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/reference/ReferencedAccountable.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/reference/ReferencedAccountable.java index f0543c3339..104850c699 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/reference/ReferencedAccountable.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/ufed/reference/ReferencedAccountable.java @@ -1,7 +1,7 @@ package iped.parsers.ufed.reference; import static iped.parsers.ufed.util.UfedUtils.readUfedMetadata; - +import static iped.parsers.ufed.util.UfedUtils.readUfedMetadataArray; import iped.data.IItemReader; public class ReferencedAccountable extends AbstractReferencedItem { @@ -15,7 +15,7 @@ public String getName() { } public String getPhoneNumber() { - return readUfedMetadata(item, "PhoneNumber"); + return String.join("|", readUfedMetadataArray(item, "PhoneNumber")); } public String getUsername() { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ConversationConstants.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ConversationConstants.java index 9b6e24df5b..972f2f7cfa 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ConversationConstants.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ConversationConstants.java @@ -5,4 +5,8 @@ public class ConversationConstants { public static final String DIRECTION_INCOMING = "Incoming"; public static final String DIRECTION_OUTGOING = "Outgoing"; + public static final String TYPE_ONEONONE = "OneOnOne"; + public static final String TYPE_GROUP = "Group"; + public static final String TYPE_BROADCAST = "Broadcast"; + public static final String TYPE_UNKNOWN = "Unknown"; } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Chat.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Chat.java index ebce23ab69..e6dbc0bc8d 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Chat.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Chat.java @@ -24,6 +24,10 @@ public class Chat { private String recoveredFrom; private Set groupMembers = new HashSet<>(); + private Set groupAdmins = new HashSet<>(); + + private WAAccount account; + private boolean isOwnerAdmin; public Chat(WAContact remote) { this.remote = remote; @@ -160,8 +164,24 @@ public Set getGroupMembers() { return groupMembers; } - public void setGroupMembers(Set groupmembers) { - this.groupMembers = groupmembers; + public Set getGroupAdmins() { + return groupAdmins; + } + + public WAAccount getAccount() { + return account; + } + + public void setAccount(WAAccount account) { + this.account = account; + } + + public boolean isOwnerAdmin() { + return isOwnerAdmin; + } + + public void setOwnerAdmin(boolean isOwnerAdmin) { + this.isOwnerAdmin = isOwnerAdmin; } public void setDeleted(boolean isDeleted) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Extractor.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Extractor.java index a52e12315f..c34c2bc370 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Extractor.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Extractor.java @@ -9,6 +9,7 @@ import java.util.List; public abstract class Extractor { + protected final File databaseFile; protected List chatList; protected final WAContactsDirectory contacts; @@ -43,17 +44,25 @@ public File getDatabaseFile() { protected abstract Connection getConnection() throws SQLException; - protected void setGroupMembers(Chat c, Connection conn, String SELECT_GROUP_MEMBERS) throws SQLException { + protected void setGroupMembers(Chat c, Connection conn, String selectGroupMembersQuery) throws SQLException { // adds all contacts that sent at least one message for (Message m : c.getMessages()) { - if (m.getRemoteResource() != null) + if (m.getRemoteResource() != null) { + if (c.isGroupOrChannelChat() && c.getRemote() != null && m.getRemoteResource().equals(c.getRemote().getFullId())) { + continue; // don't add the group itself as a member + } c.getGroupMembers().add(contacts.getContact(m.getRemoteResource())); + } } - if (SELECT_GROUP_MEMBERS == null) { + + // add account as member + c.getGroupMembers().add(contacts.getContact(account.getFullId())); + + if (selectGroupMembersQuery == null) { return; } // adds all contacts which is a member of the group now - try (PreparedStatement stmt = conn.prepareStatement(SELECT_GROUP_MEMBERS)) { + try (PreparedStatement stmt = conn.prepareStatement(selectGroupMembersQuery)) { stmt.setString(1, c.getRemote().getFullId()); try (ResultSet rs = stmt.executeQuery()) { @@ -61,6 +70,9 @@ protected void setGroupMembers(Chat c, Connection conn, String SELECT_GROUP_MEMB String memberId = rs.getString("member"); if (memberId != null && !memberId.trim().isEmpty()) { c.getGroupMembers().add(contacts.getContact(memberId)); + if (rs.getBoolean("admin")) { + c.getGroupAdmins().add(contacts.getContact(memberId)); + } } } } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java index 7f61acc52f..b1ad267ce1 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java @@ -78,7 +78,8 @@ public abstract class ExtractorAndroid extends Extractor { private boolean hasMediaCaptionCol = false; private boolean hasSubjectCol = false; private boolean hasMediaDurationCol = false; - private boolean hasGroupParticiantsTable = true; + private boolean hasGroupParticipantsTable = true; + private boolean hasGroupParticipantUserTable = true; private boolean hasForwardedCol = false; private boolean hasQuoteCol = false; private SQLException parsingException = null; @@ -150,7 +151,8 @@ protected List extractChatList() throws WAExtractorException { hasSortTimestamp = databaseHasSortTimestamp(conn); hasChatView = databaseHasChatView(conn); hasThumbTable = SQLite3DBParser.containsTable("message_thumbnails", conn); //$NON-NLS-1$ - hasGroupParticiantsTable = SQLite3DBParser.containsTable("group_participants", conn); //$NON-NLS-1$ + hasGroupParticipantsTable = SQLite3DBParser.containsTable("group_participants", conn); //$NON-NLS-1$ + hasGroupParticipantUserTable = SQLite3DBParser.containsTable("group_participant_user", conn); //$NON-NLS-1$ hasEditVersionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "edit_version"); //$NON-NLS-1$ //$NON-NLS-2$ hasMediaCaptionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "media_caption"); //$NON-NLS-1$ //$NON-NLS-2$ hasMediaDurationCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "media_duration"); //$NON-NLS-1$ //$NON-NLS-2$ @@ -218,7 +220,9 @@ protected List extractChatList() throws WAExtractorException { undeletedMessagesTable, hasThumbTable, hasEditVersionCol, firstTry)); if (c.isGroupChat()) { try { - setGroupMembers(c, conn, hasGroupParticiantsTable ? SELECT_GROUP_MEMBERS : null); + String query = hasGroupParticipantUserTable ? SELECT_GROUP_MEMBERS_2 + : hasGroupParticipantsTable ? SELECT_GROUP_MEMBERS : null; + setGroupMembers(c, conn, query); } catch (SQLException ex) { if (firstTry || !isSqliteCorruptException(ex)) { throw ex; @@ -779,7 +783,9 @@ private static String getMessagesQueryQuote(boolean hasThumbTable, boolean hasEd + "WHERE remoteId=? and status!=-1 ORDER BY timestamp"; //$NON-NLS-1$ // to address a field must use ` instead of ' - private static final String SELECT_GROUP_MEMBERS = "select gjid as 'group', jid as member FROM group_participants where `group`=?"; //$NON-NLS-1$ + private static final String SELECT_GROUP_MEMBERS = "select gjid as 'group', jid as member, admin FROM group_participants where `group`=?"; + static final String SELECT_GROUP_MEMBERS_2 = "select g._id as group_id, g.raw_string as group_name, u._id as user_id, u.raw_string as member, gp.rank > 0 as admin " + + "FROM group_participant_user gp inner join jid g on g._id=gp.group_jid_row_id inner join jid u on u._id=gp.user_jid_row_id where u.server='s.whatsapp.net' and u.type=0 and group_name=?"; private static final String SELECT_QUOTES_NO_THUMBS_TABLE = "SELECT mq._id AS id, mq.key_remote_jid " + "as remoteId, mq.remote_resource AS remoteResource, mq.status, mq.data, " diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java index d2bf3b0098..d84b31fcd4 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java @@ -165,7 +165,7 @@ protected List extractChatList() throws WAExtractorException { for (Chat c : list) { Message.sort(c.getMessages()); if (c.isGroupChat()) { - setGroupMembers(c, conn, SELECT_GROUP_MEMBERS); + setGroupMembers(c, conn, ExtractorAndroid.SELECT_GROUP_MEMBERS_2); } } @@ -1279,8 +1279,5 @@ private static String getSelectBlockedQuery(Connection conn) throws SQLException + " left join chat chat1 on chat1.jid_row_id = log.jid_row_id" + " left join chat chat2 on chat2.jid_row_id = log.group_jid_row_id"; - private static final String SELECT_GROUP_MEMBERS = "select g._id as group_id, g.raw_string as group_name, u._id as user_id, u.raw_string as member " - + "FROM group_participant_user gp inner join jid g on g._id=gp.group_jid_row_id inner join jid u on u._id=gp.user_jid_row_id where u.server='s.whatsapp.net' and u.type=0 and group_name=?"; //$NON-NLS-1$ - private static final String SELECT_JID_MAP = "SELECT a.raw_string AS lid, b.raw_string AS jid FROM jid_map map, jid a, jid b WHERE map.lid_row_id = a._id AND map.jid_row_id = b._id"; } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java index a108387e9f..6fd2db2afa 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java @@ -1855,7 +1855,7 @@ public void extractPushNames() { private static final String SELECT_PUSH_NAMES = "select ZJID as jid, ZPUSHNAME as pushname from ZWAPROFILEPUSHNAME"; - private static final String SELECT_GROUP_MEMBERS = "select CS.ZCONTACTJID as `group`, ZMEMBERJID as member from ZWAGROUPMEMBER GM " + private static final String SELECT_GROUP_MEMBERS = "select CS.ZCONTACTJID as `group`, ZMEMBERJID as member, ZISADMIN as admin from ZWAGROUPMEMBER GM " + "inner join ZWACHATSESSION CS on GM.ZCHATSESSION=CS.Z_PK where `group`=?"; private static final String SELECT_MESSAGES_USER = "SELECT ZWAMESSAGE.Z_PK AS id, ZCHATSESSION " //$NON-NLS-1$ diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java index 83c2d38379..fd40b0d04d 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java @@ -19,6 +19,7 @@ import iped.data.IItemReader; import iped.parsers.util.ChildPornHashLookup; +import iped.parsers.util.HashUtils; /** * @author Fabio Melo Pfeifer @@ -229,15 +230,15 @@ public String getMediaHash() { } public void setMediaHash(String mediaHash, boolean encoded) { - if (mediaHash == null || mediaHash.isEmpty()) - return; - if (encoded) { - byte[] hash = Base64.getDecoder().decode(mediaHash); - this.mediaHash = new String(Hex.encodeHex(hash, false)); - } else { - this.mediaHash = mediaHash; + if (HashUtils.isValidHash(mediaHash)) { + if (encoded) { + byte[] hash = Base64.getDecoder().decode(mediaHash); + this.mediaHash = new String(Hex.encodeHex(hash, false)); + } else { + this.mediaHash = mediaHash; + } + childPornSets = ChildPornHashLookup.lookupHashAndMerge(this.mediaHash, childPornSets); } - childPornSets = ChildPornHashLookup.lookupHashAndMerge(this.mediaHash, childPornSets); } public byte[] getThumbData() { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java index eef4ab33ff..a599331956 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java @@ -1268,7 +1268,7 @@ private String formatLocation(Message message) { private void printQuote(PrintWriter out, Message message, WAContactsDirectory contactsDirectory, WAAccount account) { - String quoteClass = message.isFromMe() ? "quoteTo" : "quoteFrom"; + String quoteClass = "quoteBlock " + (message.isFromMe() ? "quoteTo" : "quoteFrom"); Message messageQuote = message.getMessageQuote(); if (messageQuote != null) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAAccount.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAAccount.java index 47f751992f..66d7d54d12 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAAccount.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAAccount.java @@ -58,15 +58,16 @@ public static WAAccount getFromAndroidXml(InputStream is) throws SAXException, I } else { expr = xpath.compile("/map/string[@name=\"registration_jid\"]"); value = (String) expr.evaluate(doc, XPathConstants.STRING); - if (StringUtils.isBlank(value)) { - expr = xpath.compile("/map/string[@name=\"ph\"]"); - value = (String) expr.evaluate(doc, XPathConstants.STRING); - } if (StringUtils.isNotBlank(value)) { - if (!value.endsWith(waSuffix)) { - value += waSuffix; - } id = value; + } else { + expr = xpath.compile("/map/string[@name=\"cc\"]"); + String cc = (String) expr.evaluate(doc, XPathConstants.STRING); + expr = xpath.compile("/map/string[@name=\"ph\"]"); + String ph = (String) expr.evaluate(doc, XPathConstants.STRING); + if (StringUtils.isNoneBlank(cc, ph)) { + id = cc + ph + waSuffix; + } } } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAContact.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAContact.java index d1fa9d4521..f35cfa20a5 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAContact.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WAContact.java @@ -1,5 +1,7 @@ package iped.parsers.whatsapp; +import org.apache.commons.lang3.StringUtils; + public class WAContact { public static final String waSuffix = "@s.whatsapp.net"; @@ -98,16 +100,7 @@ public void setNickName(String nickName) { } public String getName() { - if (displayName != null && !displayName.isBlank()) - return displayName; - else if (waName != null && !waName.isBlank()) - return waName; - else if (givenName != null && !givenName.isBlank()) - return givenName; - else if (nickName != null && !nickName.isBlank()) - return nickName; - else - return id; + return StringUtils.firstNonBlank(displayName, waName, givenName, nickName, id); } @@ -169,6 +162,12 @@ public int hashCode() { return this.getFullId().hashCode(); } + @Override + public String toString() { + // for debug + return getName() + " (" + getFullId() + ")"; + } + public boolean matches(WAContact o) { if (id != null && o.id != null && id.equals(o.id)) { return true; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java index ec4976e045..c10018b427 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java @@ -49,6 +49,7 @@ import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.tika.config.Field; import org.apache.tika.exception.TikaException; @@ -70,10 +71,12 @@ import iped.data.IItem; import iped.data.IItemReader; import iped.io.SeekableInputStream; +import iped.parsers.chat.WhatsAppPartyStringBuilder; import iped.parsers.plist.detector.PListDetector; import iped.parsers.sqlite.SQLite3DBParser; import iped.parsers.sqlite.SQLite3Parser; import iped.parsers.standard.StandardParser; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.parsers.util.PhoneParsingConfig; import iped.parsers.vcard.VCardParser; @@ -131,6 +134,8 @@ public class WhatsAppParser extends SQLite3DBParser { public static final String DOWNLOAD_MEDIA_FILES_PROP = "downloadWhatsAppMediaProp"; + public static final String UNKNOWN_ACCOUNT = "unknownAccount"; + // TODO: Once #2286 is merged, use some property to identify WhatsApp Status chats/messages // private static final String STATUS_PROP = ExtraProperties.COMMUNICATION_PREFIX + "isStatus"; @@ -303,6 +308,7 @@ private void createReport(List chatList, IItemSearcher searcher, WAContact searchMediaFilesForMessagesInBatches(c.getMessages(), searcher, handler, extractor, dbPath, context, null); int frag = 0; int firstMsg = 0; + int messagesCount = (int) c.getMessages().stream().filter(m -> !m.isSystemMessage()).count(); ReportGenerator reportGenerator = new ReportGenerator(); reportGenerator.setMinChatSplitSize(this.minChatSplitSize); StringBuilder histFrag = new StringBuilder(); @@ -339,23 +345,36 @@ private void createReport(List chatList, IItemSearcher searcher, WAContact if (extractMessages && msgSubset.size() > 0) { chatMetadata.set(BasicProps.HASCHILD, Boolean.TRUE.toString()); } - if (account != null) { - String local = formatContact(account, cache); - chatMetadata.add(ExtraProperties.PARTICIPANTS, local); - } - if (c.isGroupChat()) { + if (c.isGroupOrChannelChat()) { for (WAContact member : c.getGroupMembers()) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, formatContact(member, cache)); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, member); } // string formatted as {creator's phone number}-{creation time}@g.us - chatMetadata.add(ExtraProperties.GROUP_ID, c.getRemote().getFullId()); + for (WAContact admin : c.getGroupAdmins()) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_ADMINS, admin); + } + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, c.isGroupChat() ? ConversationConstants.TYPE_GROUP : ConversationConstants.TYPE_BROADCAST); } else { if (c.getRemote() != null) { - chatMetadata.add(ExtraProperties.PARTICIPANTS, formatContact(c.getRemote(), cache)); + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, c.getRemote()); + } + if (account != null) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, account); } + chatMetadata.set(ExtraProperties.CONVERSATION_TYPE, ConversationConstants.TYPE_ONEONONE); } - if (c.isBroadcast()) { - // TODO: chatMetadata.set(STATUS_PROP, Boolean.TRUE.toString()); + + // Communication:{ID, Name, Account, MessagesCount, isEmpty} + chatMetadata.set(ExtraProperties.CONVERSATION_ID, c.getRemote() != null ? c.getRemote().getFullId() : c.getPrintId()); + chatMetadata.set(ExtraProperties.CONVERSATION_NAME, c.getRemote() != null ? c.getRemote().getName() : ""); + if (account != null) { + addPartyFields(chatMetadata, ExtraProperties.CONVERSATION_ACCOUNT, account); + } + chatMetadata.set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, messagesCount); + chatMetadata.set(ExtraProperties.CONVERSATION_IS_EMPTY, (messagesCount == 0)); + + if (c.isOwnerAdmin()) { + chatMetadata.set(ExtraProperties.CONVERSATION_IS_OWNER_ADMIN, true); } // Set created and modified dates based on the first and last messages dates @@ -366,16 +385,6 @@ private void createReport(List chatList, IItemSearcher searcher, WAContact chatMetadata.set(TikaCoreProperties.MODIFIED, last.getTimeStamp()); } - // "isEmpty" = the chat is empty or contains only system messages - boolean isEmpty = true; - for (Message m : msgSubset) { - if (!m.isSystemMessage()) { - isEmpty = false; - break; - } - } - chatMetadata.set(ExtraProperties.COMMUNICATION_PREFIX + "isEmpty", Boolean.valueOf(isEmpty).toString()); - ByteArrayInputStream chatStream = new ByteArrayInputStream(bytes); extractor.parseEmbedded(chatStream, handler, chatMetadata, false); bytes = nextBytes; @@ -391,6 +400,29 @@ private void createReport(List chatList, IItemSearcher searcher, WAContact } + private void addPartyFields(Metadata chatMetadata, String field, WAContact member) { + if (member == null) { + return; + } + String phoneNumber = null; + if (StringUtils.endsWithAny(member.getId(), WAContact.waSuffix, WAContact.waStatusSuffix)) { + phoneNumber = StringUtils.substringAfterLast(member.getId(), "@"); + } + chatMetadata.add(field, new WhatsAppPartyStringBuilder() + .withName(member.getName()) + .withUserId(member.getFullId()) + .withPhoneNumber(phoneNumber) + .build()); + + if (member instanceof WAAccount && ((WAAccount) member).isUnknown()) { + return; + } + + chatMetadata.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, member.getFullId()); + chatMetadata.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, member.getName()); + chatMetadata.add(field + ExtraProperties.CONVERSATION_SUFFIX_PHONE, phoneNumber); + } + private void parseWhatsAppMessages(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context, ExtractorFactory extFactory) throws IOException, SAXException, TikaException { @@ -417,6 +449,12 @@ private void parseWhatsAppMessages(InputStream stream, ContentHandler handler, M Extractor waExtractor = extFactory.createMessageExtractor(filePath, tempDbFile, contacts, account, recoverDeletedRecords); List chatList = waExtractor.getChatList(); + for (Chat chat : chatList) { + chat.setAccount(account); + if (chat.getGroupAdmins().stream().map(WAContact::getFullId).anyMatch(account.getFullId()::equals)) { + chat.setOwnerAdmin(true); + } + } createReport(chatList, searcher, contacts, handler, extractor, account, tis.getFile(), context); } catch (Exception e) { @@ -917,7 +955,7 @@ private String getInternationalPhone(String id) { } private WAAccount getUserAccount(IItemSearcher searcher, String dbPath, boolean isAndroid) { - WAAccount account = new WAAccount("unknownAccount"); + WAAccount account = new WAAccount(UNKNOWN_ACCOUNT); account.setUnknown(true); if (searcher != null) { StringBuilder query = new StringBuilder(); @@ -1000,41 +1038,6 @@ private static boolean isWABusiness(String path) { return path.contains(".w4b") || path.contains("WhatsApp Business"); } - private String formatContact(WAContact contact, Map cache) { - String result = cache.get(contact.getId()); - if (result == null) { - result = contact.getName(); - String number = contact.getFullId(); - if (number.endsWith(WAContact.lidSuffix)) { - number = contact.getId(); - } - if (number.endsWith(WAContact.waSuffix)) { - number = number.substring(0, number.length() - WAContact.waSuffix.length()); - } - result = result == null ? "" : result.trim(); - number = number == null ? "" : number.trim(); - if (!number.isEmpty()) { - if (result.isEmpty()) { - result = number; - } else if (!number.equals(result)) { - result += " (" + number + ")"; - } - } - cache.put(contact.getId(), result); - } - return result; - } - - private void fillGroupRecipients(Metadata meta, Chat c) { - String to = c.isChannelChat() ? "Channel " : "Group "; - if (c.getSubject() != null) { - to += c.getSubject().strip(); - } - to += " (id:" + c.getId() + ")"; - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to); - meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true"); - } - private void extractMessages(String chatName, Chat c, List messages, WAAccount account, WAContactsDirectory contacts, int parentVirtualId, ContentHandler handler, EmbeddedDocumentExtractor extractor, Map cache) throws SAXException, IOException { @@ -1047,37 +1050,29 @@ private void extractMessages(String chatName, Chat c, List messages, WA meta.set(ExtraProperties.PARENT_VIRTUAL_ID, Integer.toString(parentVirtualId)); meta.set(ExtraProperties.PARENT_VIEW_POSITION, String.valueOf(m.getUniqueId())); meta.set(ExtraProperties.USER_ACCOUNT_TYPE, WHATSAPP); - meta.set(ExtraProperties.MESSAGE_DATE, m.getTimeStamp()); + meta.set(ExtraProperties.COMMUNICATION_DATE, m.getTimeStamp()); meta.set(TikaCoreProperties.CREATED, m.getTimeStamp()); meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); if (m.isDeleted()) { meta.set(ExtraProperties.DELETED, Boolean.toString(true)); } - if (!m.isSystemMessage()) { - String local = formatContact(account, cache); + if (m.isSystemMessage()) { + meta.set(ExtraProperties.COMMUNICATION_SYSTEM_MESSAGE, true); + } else { String remote = m.getRemoteResource(); - if (remote != null) { - WAContact contact = contacts.getContact(remote); - remote = contact == null ? remote : formatContact(contact, cache); - } + WAContact contact = remote != null ? contacts.getContact(remote) : null; if (m.isFromMe()) { - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, local); - if (c.isGroupOrChannelChat()) { - fillGroupRecipients(meta, c); - } else { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, remote); - } + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, account); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, c.isGroupOrChannelChat() ? c.getRemote() : contact); } else { - meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, remote); - if (c.isGroupOrChannelChat()) { - fillGroupRecipients(meta, c); - } else { - meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, local); - } + meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING); + addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, contact); + addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, c.isGroupOrChannelChat() ? c.getRemote() : account); } - if (c.isBroadcast()) { - // TODO: meta.set(STATUS_PROP, Boolean.TRUE.toString()); + if (c.isGroupOrChannelChat()) { + meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true)); } } meta.set(ExtraProperties.MESSAGE_BODY, m.getData()); diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java index f00a4d642f..15031962a8 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/AbstractPkgTest.java @@ -19,6 +19,7 @@ import org.xml.sax.SAXException; import iped.data.IItemReader; +import iped.parsers.util.ConversationConstants; import iped.parsers.util.ItemInfo; import iped.properties.ExtraProperties; import iped.search.IItemSearcher; @@ -80,7 +81,8 @@ protected static class EmbeddedTelegramParser extends AbstractParser { protected List userphone = new ArrayList(); protected List useraccount = new ArrayList(); protected List usernotes = new ArrayList(); - protected List participants = new ArrayList(); + protected List groupParticipants = new ArrayList(); + protected List privateParticipants = new ArrayList(); protected List messagefrom = new ArrayList(); protected List messagebody = new ArrayList(); protected List messageto = new ArrayList(); @@ -102,14 +104,19 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, useraccount.add(metadata.get(ExtraProperties.USER_ACCOUNT)); if (metadata.get(ExtraProperties.USER_NOTES) != null) usernotes.add(metadata.get(ExtraProperties.USER_NOTES)); - if (metadata.get(ExtraProperties.PARTICIPANTS) != null) - participants.add(metadata.get(ExtraProperties.PARTICIPANTS)); - if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM) != null) - messagefrom.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM)); + if (metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS) != null) + if (ConversationConstants.TYPE_GROUP.equals(metadata.get(ExtraProperties.CONVERSATION_TYPE)) + || ConversationConstants.TYPE_BROADCAST.equals(metadata.get(ExtraProperties.CONVERSATION_TYPE))) { + groupParticipants.add(metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS)); + } else { + privateParticipants.add(metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS)); + } + if (metadata.get(ExtraProperties.COMMUNICATION_FROM) != null) + messagefrom.add(metadata.get(ExtraProperties.COMMUNICATION_FROM)); if (metadata.get(ExtraProperties.MESSAGE_BODY) != null) messagebody.add(metadata.get(ExtraProperties.MESSAGE_BODY)); - if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_TO) != null) - messageto.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_TO)); + if (metadata.get(ExtraProperties.COMMUNICATION_TO) != null) + messageto.add(metadata.get(ExtraProperties.COMMUNICATION_TO)); if (metadata.get(ExtraProperties.MESSAGE_DATE) != null) messagedate.add(metadata.get(ExtraProperties.MESSAGE_DATE)); } @@ -139,4 +146,4 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, useraccount.add(metadata.get(ExtraProperties.USER_ACCOUNT)); } } -} +} \ No newline at end of file diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java index d9c8fa1f41..382db985dd 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java @@ -1,5 +1,8 @@ package iped.parsers.telegram; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; + import java.io.IOException; import java.io.InputStream; @@ -36,7 +39,8 @@ public void testTelegramParser() throws IOException, SAXException, TikaException assertEquals(236, telegramtracker.userphone.size()); assertEquals(261, telegramtracker.useraccount.size()); assertEquals(118, telegramtracker.usernotes.size()); - assertEquals(32, telegramtracker.participants.size()); + assertEquals(32, telegramtracker.groupParticipants.size()); + assertEquals(69, telegramtracker.privateParticipants.size()); assertEquals(151, telegramtracker.messagefrom.size()); assertEquals(151, telegramtracker.messagebody.size()); assertEquals(151, telegramtracker.messageto.size()); @@ -46,10 +50,10 @@ public void testTelegramParser() throws IOException, SAXException, TikaException assertEquals("Karol Braz", telegramtracker.title.get(1)); assertEquals("Budi", telegramtracker.title.get(2)); assertEquals("Nickerida", telegramtracker.title.get(3)); - assertEquals("Telegram_Chat_Marilê", telegramtracker.title.get(505)); - assertEquals("Telegram_Chat_Marilê_message_0", telegramtracker.title.get(506)); - assertEquals("Telegram_Chat_Rafael CiC", telegramtracker.title.get(507)); - assertEquals("Telegram_Chat_Rafael CiC_message_0", telegramtracker.title.get(508)); + assertEquals("Telegram Chat - Marilê (+5561981993740)", telegramtracker.title.get(505)); + assertEquals("Telegram Chat - Marilê (+5561981993740)_message_0", telegramtracker.title.get(506)); + assertEquals("Telegram Chat - Rafael CiC (+5562991688700)", telegramtracker.title.get(507)); + assertEquals("Telegram Chat - Rafael CiC (+5562991688700)_message_0", telegramtracker.title.get(508)); assertEquals("Tiago", telegramtracker.username.get(0)); assertEquals("Karol Braz", telegramtracker.username.get(1)); @@ -67,19 +71,22 @@ public void testTelegramParser() throws IOException, SAXException, TikaException assertEquals("RafaelCampos", telegramtracker.usernotes.get(1)); assertEquals("gif", telegramtracker.usernotes.get(3)); - assertEquals("Vayu (phone: 5561985757632)", telegramtracker.participants.get(0)); - assertEquals("Felix (phone: 5561982943741)", telegramtracker.participants.get(1)); - assertEquals("Nake Douglas (phone: 5561982616052)", telegramtracker.participants.get(2)); - - assertEquals("Vayu (phone: 5561985757632)", telegramtracker.messagefrom.get(0)); - assertEquals("Felix (phone: 5561982943741)", telegramtracker.messagefrom.get(1)); - assertEquals("Caio Brandão (ID:445292655)", telegramtracker.messagefrom.get(3)); - - assertEquals("Group constelação familiar e barras de access (id:-1459548546)", - telegramtracker.messageto.get(0)); - assertEquals("Group le chad et les chadettes (id:-1391969250)", telegramtracker.messageto.get(1)); - assertEquals("Rafael CiC (phone: 5562991688700)", telegramtracker.messageto.get(150)); - + assertThat(telegramtracker.groupParticipants, hasItems( + "Bruno Chaves (+33667514279)", + "Nake Douglas (+5561982616052)", + "Guilherme Andreúce (@guileb | +5561986143035)")); + assertThat(telegramtracker.privateParticipants, hasItems( + "Telegram (+42777)", + "Nickerida (+5561983125151)", + "Yan Victor (@YanVictor | +5561995328012)")); + + assertEquals("Vayu (+5561985757632)", telegramtracker.messagefrom.get(0)); + assertEquals("Felix (@Very_Hungery_Boye | +5561982943741)", telegramtracker.messagefrom.get(1)); + assertEquals("Caio Brandão (@CaioBrandao)", telegramtracker.messagefrom.get(3)); + + assertEquals("Group Constelação Familiar e Barras de Access (ID:1459548546)", telegramtracker.messageto.get(0)); + assertEquals("Group Le Chad et les Chadettes (ID:1391969250)", telegramtracker.messageto.get(1)); + assertEquals("Rafael CiC (+5562991688700)", telegramtracker.messageto.get(150)); assertTrue(telegramtracker.messagebody.get(1).contains("KKKKK")); assertTrue(telegramtracker.messagebody.get(2).contains("Parece outro carro até")); diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/AbstractPkgTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/AbstractPkgTest.java index b9823509fd..a27db67da2 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/AbstractPkgTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/AbstractPkgTest.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -35,11 +36,13 @@ protected ParseContext getContext(String resource) throws IOException { @SuppressWarnings("serial") protected static class EmbeddedWhatsAppParser extends AbstractParser { protected List title = new ArrayList(); + protected List type = new ArrayList(); protected List username = new ArrayList(); protected List userphone = new ArrayList(); protected List useraccount = new ArrayList(); protected List usernotes = new ArrayList(); - protected List participants = new ArrayList(); + protected List> participants = new ArrayList<>(); + protected List> admins = new ArrayList<>(); protected List messagefrom = new ArrayList(); protected List messagebody = new ArrayList(); protected List messageto = new ArrayList(); @@ -53,6 +56,8 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, throws IOException, SAXException, TikaException { if (metadata.get(TikaCoreProperties.TITLE) != null) title.add(metadata.get(TikaCoreProperties.TITLE)); + if (metadata.get(ExtraProperties.CONVERSATION_TYPE) != null) + type.add(metadata.get(ExtraProperties.CONVERSATION_TYPE)); if (metadata.get(ExtraProperties.USER_NAME) != null) username.add(metadata.get(ExtraProperties.USER_NAME)); if (metadata.get(ExtraProperties.USER_PHONE) != null) @@ -61,16 +66,18 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, useraccount.add(metadata.get(ExtraProperties.USER_ACCOUNT)); if (metadata.get(ExtraProperties.USER_NOTES) != null) usernotes.add(metadata.get(ExtraProperties.USER_NOTES)); - if (metadata.get(ExtraProperties.PARTICIPANTS) != null) - participants.add(metadata.get(ExtraProperties.PARTICIPANTS)); - if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM) != null) - messagefrom.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_FROM)); + if (metadata.get(ExtraProperties.CONVERSATION_PARTICIPANTS) != null) { + participants.add(Arrays.asList(metadata.getValues(ExtraProperties.CONVERSATION_PARTICIPANTS))); + admins.add(Arrays.asList(metadata.getValues(ExtraProperties.CONVERSATION_ADMINS))); + } + if (metadata.get(ExtraProperties.COMMUNICATION_FROM) != null) + messagefrom.add(metadata.get(ExtraProperties.COMMUNICATION_FROM)); + if (metadata.get(ExtraProperties.COMMUNICATION_TO) != null) + messageto.add(metadata.get(ExtraProperties.COMMUNICATION_TO)); if (metadata.get(ExtraProperties.MESSAGE_BODY) != null) messagebody.add(metadata.get(ExtraProperties.MESSAGE_BODY)); - if (metadata.get(org.apache.tika.metadata.Message.MESSAGE_TO) != null) - messageto.add(metadata.get(org.apache.tika.metadata.Message.MESSAGE_TO)); - if (metadata.get(ExtraProperties.MESSAGE_DATE) != null) - messagedate.add(metadata.get(ExtraProperties.MESSAGE_DATE)); + if (metadata.get(ExtraProperties.COMMUNICATION_DATE) != null) + messagedate.add(metadata.get(ExtraProperties.COMMUNICATION_DATE)); } } -} +} \ No newline at end of file diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/WhatsAppParserTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/WhatsAppParserTest.java index d42fec2735..cb7ef33b26 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/WhatsAppParserTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/whatsapp/WhatsAppParserTest.java @@ -1,7 +1,11 @@ package iped.parsers.whatsapp; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; + import java.io.IOException; import java.io.InputStream; +import java.util.Collections; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; @@ -14,12 +18,19 @@ import org.xml.sax.SAXException; import iped.parsers.standard.StandardParser; +import iped.parsers.util.ConversationConstants; public class WhatsAppParserTest extends AbstractPkgTest { + boolean waDbParsed = false; + @Test public void testWhatsAppParserAndroid() throws IOException, SAXException, TikaException { + if (!waDbParsed) { + testWhatsAppParserWADBAndroid(); + } + String testFile = "test-files/test_whatsAppMsgStore.db"; ParseContext whatsappContext = getContext(testFile); WhatsAppParser parser = new WhatsAppParser(); @@ -43,6 +54,11 @@ public void testWhatsAppParserAndroid() throws IOException, SAXException, TikaEx assertEquals(4, whatsapptracker.messageto.size()); assertEquals(74, whatsapptracker.messagedate.size()); + assertEquals(11, Collections.frequency(whatsapptracker.type, ConversationConstants.TYPE_ONEONONE)); + assertEquals(18, Collections.frequency(whatsapptracker.type, ConversationConstants.TYPE_GROUP)); + assertEquals(0, Collections.frequency(whatsapptracker.type, ConversationConstants.TYPE_BROADCAST)); + assertEquals(0, Collections.frequency(whatsapptracker.type, ConversationConstants.TYPE_UNKNOWN)); + assertEquals("WhatsApp Chat - Nickerida - 556183125151", whatsapptracker.title.get(0)); assertEquals("WhatsApp Chat - Nickerida - 556183125151_message_0", whatsapptracker.title.get(1)); assertEquals("WhatsApp Chat - Nickerida - 556183125151_message_1", whatsapptracker.title.get(2)); @@ -51,18 +67,57 @@ public void testWhatsAppParserAndroid() throws IOException, SAXException, TikaEx assertEquals("WhatsApp Group - Lar - 556185747642-1461964508_message_1", whatsapptracker.title.get(61)); assertEquals("WhatsApp Group - Lar - 556185747642-1461964508_message_2", whatsapptracker.title.get(62)); - assertEquals("unknownAccount", whatsapptracker.participants.get(0)); - assertEquals("unknownAccount", whatsapptracker.participants.get(1)); - assertEquals("unknownAccount", whatsapptracker.participants.get(2)); - assertEquals("unknownAccount", whatsapptracker.participants.get(3)); - assertEquals("unknownAccount", whatsapptracker.participants.get(4)); - assertEquals("unknownAccount", whatsapptracker.participants.get(17)); - assertEquals("unknownAccount", whatsapptracker.participants.get(18)); + // Test OneOnOne Chats + + assertEquals(ConversationConstants.TYPE_ONEONONE, whatsapptracker.type.get(0)); + assertEquals(ConversationConstants.TYPE_ONEONONE, whatsapptracker.type.get(19)); + assertEquals(ConversationConstants.TYPE_ONEONONE, whatsapptracker.type.get(20)); + assertEquals(ConversationConstants.TYPE_ONEONONE, whatsapptracker.type.get(21)); + + assertEquals(2, whatsapptracker.participants.get(0).size()); + assertEquals(2, whatsapptracker.participants.get(19).size()); + assertEquals(2, whatsapptracker.participants.get(20).size()); + assertEquals(2, whatsapptracker.participants.get(21).size()); + + assertThat(whatsapptracker.participants.get(0), hasItems("unknownAccount", "Nickerida (+556183125151)")); + assertThat(whatsapptracker.participants.get(19), hasItems("unknownAccount", "Hotdog412 (+556181704627)")); + assertThat(whatsapptracker.participants.get(20), hasItems("unknownAccount", "Nwi Fibra Ótica (+556133223200)")); + assertThat(whatsapptracker.participants.get(21), hasItems("unknownAccount", "Xavier (+556135952111)")); + + assertEquals(0, whatsapptracker.admins.get(0).size()); + assertEquals(0, whatsapptracker.admins.get(19).size()); + assertEquals(0, whatsapptracker.admins.get(20).size()); + assertEquals(0, whatsapptracker.admins.get(21).size()); + + // Test Group Chats + + assertEquals(ConversationConstants.TYPE_GROUP, whatsapptracker.type.get(1)); + assertEquals(ConversationConstants.TYPE_GROUP, whatsapptracker.type.get(2)); + assertEquals(ConversationConstants.TYPE_GROUP, whatsapptracker.type.get(3)); + assertEquals(ConversationConstants.TYPE_GROUP, whatsapptracker.type.get(4)); + + assertEquals(23, whatsapptracker.participants.get(1).size()); + assertEquals(21, whatsapptracker.participants.get(2).size()); + assertEquals(5, whatsapptracker.participants.get(3).size()); + assertEquals(2, whatsapptracker.participants.get(4).size()); + + assertThat(whatsapptracker.participants.get(1), hasItems("unknownAccount")); + assertThat(whatsapptracker.participants.get(2), hasItems("unknownAccount")); + assertThat(whatsapptracker.participants.get(3), hasItems("unknownAccount")); + assertThat(whatsapptracker.participants.get(4), hasItems("unknownAccount")); + + assertEquals(21, whatsapptracker.admins.get(1).size()); + assertEquals(19, whatsapptracker.admins.get(2).size()); + assertEquals(1, whatsapptracker.admins.get(3).size()); + assertEquals(0, whatsapptracker.admins.get(4).size()); + + assertThat(whatsapptracker.admins.get(1), hasItems("+556192644086")); + assertThat(whatsapptracker.admins.get(2), hasItems("Pedro Gonzaga (+556199351995)")); assertEquals("unknownAccount", whatsapptracker.messagefrom.get(0)); assertEquals("unknownAccount", whatsapptracker.messagefrom.get(1)); assertEquals("unknownAccount", whatsapptracker.messagefrom.get(2)); - assertEquals("Nickerida (556183125151)", whatsapptracker.messagefrom.get(3)); + assertEquals("Nickerida (+556183125151)", whatsapptracker.messagefrom.get(3)); assertEquals("This is a test for the IPED Whatsapp Parser.", whatsapptracker.messagebody.get(0)); assertEquals("! MESSAGES_NOW_ENCRYPTED", whatsapptracker.messagebody.get(1)); @@ -72,9 +127,9 @@ public void testWhatsAppParserAndroid() throws IOException, SAXException, TikaEx assertEquals("! USER_ADDED_TO_GROUP", whatsapptracker.messagebody.get(42)); assertEquals("! MESSAGES_NOW_ENCRYPTED", whatsapptracker.messagebody.get(43)); - assertEquals("Nickerida (556183125151)", whatsapptracker.messageto.get(0)); - assertEquals("Nickerida (556183125151)", whatsapptracker.messageto.get(1)); - assertEquals("Nickerida (556183125151)", whatsapptracker.messageto.get(2)); + assertEquals("Nickerida (+556183125151)", whatsapptracker.messageto.get(0)); + assertEquals("Nickerida (+556183125151)", whatsapptracker.messageto.get(1)); + assertEquals("Nickerida (+556183125151)", whatsapptracker.messageto.get(2)); assertEquals("unknownAccount", whatsapptracker.messageto.get(3)); assertEquals("2021-06-14T18:55:54Z", whatsapptracker.messagedate.get(0)); @@ -122,12 +177,15 @@ public void testWhatsAppParserWADBAndroid() throws IOException, SAXException, Ti try (InputStream stream = getStream(testFile)) { parser.parse(stream, handler, metadata, whatsappContext); + waDbParsed = true; + assertEquals(384, whatsapptracker.title.size()); assertEquals(384, whatsapptracker.username.size()); assertEquals(365, whatsapptracker.userphone.size()); assertEquals(384, whatsapptracker.useraccount.size()); assertEquals(166, whatsapptracker.usernotes.size()); assertEquals(0, whatsapptracker.participants.size()); + assertEquals(0, whatsapptracker.admins.size()); assertEquals(0, whatsapptracker.messagefrom.size()); assertEquals(0, whatsapptracker.messagebody.size()); assertEquals(0, whatsapptracker.messageto.size());