Add "Conversation" and "Communication" metadata to internal parsers

iped-api/src/main/java/iped/properties/ExtraProperties.java

@@ -57,6 +57,8 @@ public class ExtraProperties {
 
     public static final Property CONVERSATION_MESSAGES_COUNT = Property.internalInteger(CONVERSATION_PREFIX + "messagesCount");
 
+    public static final Property CONVERSATION_IS_EMPTY = Property.internalBoolean(CONVERSATION_PREFIX + "isEmpty");
+
     public static final String CONVERSATION_SUFFIX_ID = ":id";
     public static final String CONVERSATION_SUFFIX_NAME = ":name";
     public static final String CONVERSATION_SUFFIX_PHONE = ":phoneNumber";
@@ -78,7 +80,9 @@ public class ExtraProperties {
 
     public static final String GROUP_ID = "GroupID";
 
-    public static final String IS_GROUP_MESSAGE = "isGroupMessage";
+    public static final String IS_GROUP_MESSAGE = COMMUNICATION_PREFIX + "isGroupMessage";
+    public static final Property COMMUNICATION_SYSTEM_MESSAGE = Property.internalBoolean(COMMUNICATION_PREFIX + "isSystemMessage");
+
 
     public static final String MESSAGE_BODY = MESSAGE_PREFIX + "Body"; //$NON-NLS-1$
 

iped-engine/src/main/java/iped/engine/task/index/IndexItem.java

@@ -82,7 +82,6 @@
 import iped.engine.preview.PreviewConstants;
 import iped.engine.preview.PreviewInputStreamFactory;
 import iped.engine.sleuthkit.SleuthkitInputStreamFactory;
-import iped.engine.task.ImageThumbTask;
 import iped.engine.task.MinIOTask.MinIOInputInputStreamFactory;
 import iped.engine.task.ThumbTask;
 import iped.engine.task.similarity.ImageSimilarityTask;
@@ -126,8 +125,12 @@ public class IndexItem extends BasicProps {
     public static final char EVENT_IDX_SEPARATOR2 = ',';
     public static final String EVENT_SEPARATOR = " | ";
 
+    private static final String COUNT_SUFFIX = ":count";
+
     static HashSet<String> ignoredMetadata = new HashSet<String>();
 
+    static HashSet<String> countMetadata = new HashSet<>();
+
     private static Map<String, SeekableInputStreamFactory> inputStreamFactories = new HashMap<>();
 
     private static Map<String, Class<?>> typesMap = MetadataUtil.getMetadataTypes();
@@ -158,6 +161,9 @@ public class IndexItem extends BasicProps {
         // ocrCharCount is already copied to an extra attribute
         ignoredMetadata.add(OCRParser.OCR_CHAR_COUNT);
 
+        countMetadata.add(ExtraProperties.CONVERSATION_PARTICIPANTS);
+        countMetadata.add(ExtraProperties.CONVERSATION_ADMINS);
+
         BasicProps.SET.add(ID_IN_SOURCE);
         BasicProps.SET.add(SOURCE_PATH);
         BasicProps.SET.add(SOURCE_DECODER);
@@ -711,6 +717,12 @@ private static void addMetadataToDoc(Document doc, Metadata metadata, Set<TimeSt
                     addMetadataKeyToDoc(doc, key, val, isMultiValued, mimetype, timeEventSet);
             }
 
+            if (countMetadata.contains(key)) {
+                String newKey = key + COUNT_SUFFIX;
+                MetadataUtil.setMetadataType(newKey, Integer.class);
+                doc.removeFields(newKey);
+                addExtraAttributeToDoc(doc, newKey, metadata.getValues(key).length, false, timeEventSet);
+            }
         }
     }
 

iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/chat/WhatsAppPartyStringBuilder.java

@@ -2,23 +2,33 @@
 
 import org.apache.commons.lang3.StringUtils;
 
+import iped.parsers.whatsapp.WAContact;
+import iped.parsers.whatsapp.WhatsAppParser;
+
 public class WhatsAppPartyStringBuilder extends PartyStringBuilder {
 
     private static final String PHONE_PREFIX = "+";
 
     @Override
     public String build() {
 
-        String phoneFromUserID = StringUtils.substringBefore(userId, '@');
-        if (StringUtils.isNotBlank(phoneFromUserID)) {
-            phoneFromUserID = StringUtils.prependIfMissing(phoneFromUserID, PHONE_PREFIX);
+        String phoneFromUserID = null;
+        if (StringUtils.endsWithAny(userId, WAContact.waSuffix, WAContact.waStatusSuffix)) {
+            phoneFromUserID = StringUtils.substringBeforeLast(userId, "@");
         }
+
         String phoneStr = StringUtils.firstNonBlank(phoneFromUserID, phoneNumber);
+        if (StringUtils.isNotBlank(phoneStr)) {
+            phoneStr = StringUtils.prependIfMissing(phoneStr, PHONE_PREFIX);
+        }
 
         if (StringUtils.isNotBlank(name)) {
-            if (StringUtils.isBlank(phoneStr) || StringUtils.equalsAny(name, phoneNumber, phoneFromUserID)) {
+            if (StringUtils.isBlank(phoneStr) || name.equals(WhatsAppParser.UNKNOWN_ACCOUNT)) {
                 return name;
             }
+            if (StringUtils.equalsAny(name, phoneNumber, phoneFromUserID)) {
+                return StringUtils.prependIfMissing(name, PHONE_PREFIX);
+            }
             return name + " (" + phoneStr + ")";
         } else if (StringUtils.isNotBlank(phoneStr)) {
             return phoneStr;

iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/DiscordParser.java

@@ -9,12 +9,13 @@
 import java.io.InputStream;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.TreeMap;
 
 import javax.imageio.ImageIO;
 
@@ -23,6 +24,7 @@
 import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor;
 import org.apache.tika.io.TemporaryResources;
 import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.mime.MediaType;
@@ -41,12 +43,14 @@
 
 import iped.data.IItemReader;
 import iped.parsers.browsers.chrome.CacheIndexParser;
+import iped.parsers.chat.PartyStringBuilderFactory;
 import iped.parsers.discord.cache.Index;
 import iped.parsers.discord.json.DiscordAttachment;
 import iped.parsers.discord.json.DiscordAuthor;
 import iped.parsers.discord.json.DiscordRoot;
 import iped.parsers.discord.json.DiscordSticker;
 import iped.parsers.standard.StandardParser;
+import iped.parsers.util.ConversationConstants;
 import iped.properties.BasicProps;
 import iped.properties.ExtraProperties;
 import iped.search.IItemSearcher;
@@ -55,16 +59,18 @@
 import iped.utils.ImageUtil;
 
 /***
- * 
+ *
  * @author PCF Campanini
  * @author PCF Patrick Dalla Bernardina
  *
  */
 public class DiscordParser extends AbstractParser {
 
-    // TODO
+    private static final long serialVersionUID = 2785303004503782295L;
+
+    private static final Logger logger = LoggerFactory.getLogger(CacheIndexParser.class);
+
 
-    private static final long serialVersionUID = 1L;
     // public static final MediaType DISCORD_INDEX_MIME_TYPE =
     // MediaType.application("application/x-discord-index");
     // public static final MediaType CHROME_INDEX_MIME_TYPE =
@@ -83,7 +89,7 @@ public class DiscordParser extends AbstractParser {
 
     private static Logger LOGGER = LoggerFactory.getLogger(Index.class);
 
-    private static final Set<MediaType> SUPPORTED_TYPES = new HashSet<MediaType>(Arrays.asList(MediaType.parse(CHAT_MIME_TYPE)));
+    private static final Set<MediaType> SUPPORTED_TYPES = new HashSet<>(Arrays.asList(MediaType.parse(CHAT_MIME_TYPE)));
     private static final String ME_URL = "https://discord.com/api/v9/users/@me";
 
     static {
@@ -111,12 +117,17 @@ public Set<MediaType> getSupportedTypes(ParseContext context) {
 
     @Override
     public void parse(InputStream indexFile, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException {
-        String contentType = metadata.get(Metadata.CONTENT_TYPE);
+        try {
+        String contentType = metadata.get(HttpHeaders.CONTENT_TYPE);
         if (contentType.equals(CHAT_MIME_TYPE)) {
             parseDiscord(indexFile, handler, metadata, context);
         } else {
             parseCacheIndex(indexFile, handler, metadata, context);
         }
+        } catch (Exception e) {
+            logger.warn("Discord error", e);
+            throw e;
+        }
     }
 
     // this code may be removed when backward parsersconfig.xml compatibility were
@@ -141,7 +152,7 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata
         IItemReader item = context.get(IItemReader.class);
 
         String commonQuery = BasicProps.EVIDENCE_UUID + ":" + item.getDataSource().getUUID() + " AND " + BasicProps.PARENTID + ":" + item.getParentId() + " AND NOT " + BasicProps.LENGTH + ":0 AND "
-                + CacheIndexParser.IS_CACHE_INDEX_ENTRY.replace(":", "\\:") + ":true";
+                + searcher.escapeQuery(CacheIndexParser.IS_CACHE_INDEX_ENTRY) + ":true";
 
         try (InputStream is = TikaInputStream.get(indexFile, new TemporaryResources())) {
 
@@ -238,11 +249,32 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata
             chatmetadata.set(StandardParser.INDEXER_CONTENT_TYPE, CHAT_MIME_TYPE_HTML);
             chatmetadata.set(ExtraProperties.ITEM_VIRTUAL_ID, Integer.toString(chatVirtualId));
 
+            // Type
+            chatmetadata.set(ExtraProperties.CONVERSATION_NAME, chatName);
+            chatmetadata.set(ExtraProperties.CONVERSATION_TYPE, discordRoot.get(0).getType() + "");
+            chatmetadata.set(ExtraProperties.CONVERSATION_ID, discordRoot.get(0).getId());
+
+            // Account
+            addPartyFields(chatmetadata, ExtraProperties.CONVERSATION_ACCOUNT, me);
+
+            // Participants
+            TreeMap<String, DiscordAuthor> participantsMap = new TreeMap<>();
+            for (DiscordRoot d : discordRoot) {
+                if (d.getAuthor() != null) {
+                    participantsMap.putIfAbsent(d.getAuthor().getFullUsername(), d.getAuthor());
+                }
+            }
+            Collection<DiscordAuthor> participants = participantsMap.values();
+            for (DiscordAuthor participant : participants) {
+                addPartyFields(chatmetadata, ExtraProperties.CONVERSATION_PARTICIPANTS, participant);
+            }
+
             for (DiscordRoot dr : discordRoot) {
                 for (DiscordSticker sticker : dr.getStickers()) {
                     try {
-                        List<IItemReader> stickerItems = searcher
-                                .search(commonQuery + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"" + sticker.getId() + ".json\"" + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"discord\"");
+                        List<IItemReader> stickerItems = searcher.search(commonQuery + " AND " 
+                                + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"" + sticker.getId() + ".json\"" + " AND " 
+                                + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"discord\"");
                         for (IItemReader stickerItem : stickerItems) {
                             sticker.setMediaHash(stickerItem.getHash());
                         }
@@ -287,16 +319,32 @@ public void parseDiscord(InputStream indexFile, ContentHandler handler, Metadata
             byte[] relatorio = new DiscordHTMLReport(me).convertToHTML(discordRoot, searcher);
             extractor.parseEmbedded(new ByteArrayInputStream(relatorio), handler, chatmetadata, false);
 
-            extractMessages(chatName, discordRoot, handler, extractor, chatVirtualId);
+            extractMessages(chatName, discordRoot, me, participants, handler, extractor, chatVirtualId);
         }
     }
 
+    private void addPartyFields(Metadata meta, String field, DiscordAuthor author) {
+        if (author == null) {
+            return;
+        }
+        meta.add(field,  PartyStringBuilderFactory.getBuilder("Discord")
+                .withUserId(author.getId())
+                .withName(author.getName())
+                .withPhoneNumber(author.getPhone())
+                .withUsername(author.getUsername())
+                .build());
+        meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_ID, author.getId());
+        meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_NAME, author.getName());
+        meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_PHONE, author.getPhone());
+        meta.add(field + ExtraProperties.CONVERSATION_SUFFIX_USERNAME, author.getFullUsername());
+    }
+
     private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery, ObjectMapper mapper, HashMap<String, byte[]> avatarCache, ContentHandler handler, EmbeddedDocumentExtractor extractor) {
         DiscordAuthor me = null;
 
         try {
             // find me info
-            List<IItemReader> mes = searcher.search(commonQuery + " AND " + CacheIndexParser.CACHE_URL.replace(":", "\\:") + ":\"" + ME_URL + "\"");
+            List<IItemReader> mes = searcher.search(commonQuery + " AND " + searcher.escapeQuery(CacheIndexParser.CACHE_URL) + ":\"" + ME_URL + "\"");
             for (IItemReader mei : mes) {
                 if (mei.getName().equals("@me")) {
                     try (InputStream is2 = getDecompressedStream(mei.getBufferedInputStream())) {
@@ -312,7 +360,7 @@ private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery,
                             memeta.set(ExtraProperties.USER_ACCOUNT_TYPE, "Discord");
                             memeta.set(ExtraProperties.USER_EMAIL, me.getEmail());
                             // memeta.set(ExtraProperties.USER_NOTES, me.getBio());
-                            memeta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString());
+                            memeta.set(ExtraProperties.DECODED_DATA, Boolean.toString(true));
 
                             if (me.getAvatar() != null) {
                                 byte[] meavatar = avatarCache.get(me.getAvatar());
@@ -334,51 +382,56 @@ private DiscordAuthor extractAccount(IItemSearcher searcher, String commonQuery,
         return me;
     }
 
-    private void extractMessages(String chatName, List<DiscordRoot> discordRoot, ContentHandler handler, EmbeddedDocumentExtractor extractor, int chatVirtualId) throws SAXException, IOException {
+    private void extractMessages(String chatName, List<DiscordRoot> discordRoot, DiscordAuthor me, Collection<DiscordAuthor> participants, ContentHandler handler, EmbeddedDocumentExtractor extractor, int chatVirtualId) throws SAXException, IOException {
         int msgCount = 0;
 
-        // Checking Participants
-        LinkedHashSet<String> participants = new LinkedHashSet<String>();
-
-        for (DiscordRoot d : discordRoot) {
-            participants.add(d.getAuthor().getFullUsername());
-        }
-
         // Set metadata
         for (DiscordRoot d : discordRoot) {
 
             Metadata meta = new Metadata();
             meta.set(TikaCoreProperties.TITLE, chatName + "_message_" + msgCount++);
             meta.set(StandardParser.INDEXER_CONTENT_TYPE, MSG_MIME_TYPE);
-            meta.set(ExtraProperties.MESSAGE_DATE, d.getTimestamp());
+            meta.set(ExtraProperties.COMMUNICATION_DATE, d.getTimestamp());
             meta.set(ExtraProperties.MESSAGE_BODY, d.getMessageContent());
-            meta.set(ExtraProperties.USER_NAME, d.getAuthor().getFullUsername());
-            meta.add(ExtraProperties.PARTICIPANTS, participants.toString());
             meta.set(ExtraProperties.PARENT_VIRTUAL_ID, Integer.toString(chatVirtualId));
             meta.set(ExtraProperties.PARENT_VIEW_POSITION, String.valueOf(d.getId()));
-            meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, d.getAuthor().getFullUsername());
-            meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString());
+            meta.set(ExtraProperties.DECODED_DATA, Boolean.toString(true));
+
+            // From
+            addPartyFields(meta, ExtraProperties.COMMUNICATION_FROM, d.getAuthor());
 
             // Add "Message-TO" field.
             // In cases where only one participant sends messages, it is not possible to
             // determine the participants as only the participant list of the calls are cached.
             if (participants.size() <= 2) {
-                for (String participant : participants) {
-                    if (!participant.equals(d.getAuthor().getFullUsername())) {
-                        meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, participant);
+                for (DiscordAuthor participant : participants) {
+                    if (!participant.getFullUsername().equals(d.getAuthor().getFullUsername())) {
+                        addPartyFields(meta, ExtraProperties.COMMUNICATION_TO, participant);
                     }
                 }
             } else {
-                meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, chatName);
-                meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
+                meta.set(ExtraProperties.COMMUNICATION_TO, chatName);
+                meta.set(ExtraProperties.IS_GROUP_MESSAGE, Boolean.toString(true));
+            }
+
+            // Direction
+            Boolean isFromMe = null;
+            if (me != null) {
+                isFromMe = me.getFullUsername().equals(d.getAuthor().getFullUsername());
+                if (isFromMe) {
+                    meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_OUTGOING);
+                } else {
+                    meta.set(ExtraProperties.COMMUNICATION_DIRECTION, ConversationConstants.DIRECTION_INCOMING);
+                }
             }
 
             for (DiscordAttachment da : d.getAttachments()) {
                 if (da.getMediaHash() != null) {
                     meta.add(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + da.getMediaHash());
                     meta.set(StandardParser.INDEXER_CONTENT_TYPE, ATTACH_MIME_TYPE);
-                    // if (m.isFromMe())
-                    // meta.add(ExtraProperties.SHARED_HASHES, m.getMediaHash());
+                    if (isFromMe != null && isFromMe) {
+                         meta.add(ExtraProperties.SHARED_HASHES, da.getMediaHash());
+                    }
                 }
             }