diff --git a/ThirdParty.txt b/ThirdParty.txt index 64d1e7c110..3cc6288eb0 100644 --- a/ThirdParty.txt +++ b/ThirdParty.txt @@ -182,4 +182,13 @@ JUnRAR Lottie-player - Site: https://github.com/LottieFiles/lottie-player/blob/master/LICENSE -- License: licenses/LOTTIE-PLAYER.txt \ No newline at end of file +- License: licenses/LOTTIE-PLAYER.txt + +------------------------------------------ + +ALeapp +- Site: https://github.com/abrignoni/ALEAPP +- Project: ALEAPP (Android Logs Events And Protobuf Parser) +- Project contributors: https://github.com/abrignoni/ALEAPP/graphs/contributors?from=18%2F04%2F2026 +- License: MIT + diff --git a/iped-app/pom.xml b/iped-app/pom.xml index 2afa6ce15f..c5790747c2 100644 --- a/iped-app/pom.xml +++ b/iped-app/pom.xml @@ -13,6 +13,7 @@ ${project.parent.build.directory}/release/iped-${project.version} ${resources.dir}/../.. + v2026.1.0 @@ -61,30 +62,56 @@ findbugs-maven-plugin 3.0.0 - - com.googlecode.maven-download-plugin - download-maven-plugin - 1.6.8 - - - download-regripper - - wget - - - - - https://github.com/sepinf-inc/RegRipper3.0/archive/refs/tags/3.0-p3.zip - true - - - RegRipper[^/]+/ - ./ - - - ${tools.dir}/regripper - - + + io.github.download-maven-plugin + download-maven-plugin + 2.1.0 + + + download-regripper + + wget + + + https://github.com/sepinf-inc/RegRipper3.0/archive/refs/tags/3.0-p3.zip + true + + + ^RegRipper[^/]+/(.*)$ + $1 + + + ${tools.dir}/regripper + + + + download-aleapp + + wget + + + https://github.com/abrignoni/ALEAPP/archive/refs/tags/${aleapp.version}.zip + true + + + ^ALEAPP[^/]+/(.*)$ + $1 + + + + */admin/ + */assets/ + */tests/ + **/*.md + **/*.lcasedata + **/*.alprofile + */aleappGUI.py + + ${tools.dir}/aleapp + + + + org.apache.maven.plugins maven-dependency-plugin @@ -854,8 +881,6 @@ 2.18.1 true - true - true diff --git a/iped-app/resources/config/IPEDConfig.txt b/iped-app/resources/config/IPEDConfig.txt index b385a8632f..6208710cd4 100644 --- a/iped-app/resources/config/IPEDConfig.txt +++ b/iped-app/resources/config/IPEDConfig.txt @@ -1,4 +1,4 @@ -######################################################################## +######################################################################## # Processing Settings ######################################################################## @@ -171,3 +171,8 @@ enableHTMLReport = true # It performs classification of image and video files. # Advanced settings can be modified in file "conf/RemoteImageClassifierConfig.txt" enableRemoteImageClassifier = false + +# Enables ALEAPP integration. +# You can configure a custom path to the ALEAPP installation folder in conf/ALeappConfig.txt. +enableAleapp = true + diff --git a/iped-app/resources/config/conf/ALeappConfig.txt b/iped-app/resources/config/conf/ALeappConfig.txt new file mode 100644 index 0000000000..d87931f9b7 --- /dev/null +++ b/iped-app/resources/config/conf/ALeappConfig.txt @@ -0,0 +1,30 @@ +# ----------------------------------------------------------------------------- +# ALEAPP Integration Settings +# ----------------------------------------------------------------------------- + +# Path to the main ALEAPP installation directory. +# If this property is commented out or left empty, IPED will search for ALEAPP +# in the default directory: /tools/aleapp +# Example for Windows: aleapFolder=C:\Forensic Tools\aleapp +# Example for Linux: aleapFolder=/home/user/aleapp +# aleapFolder= + +# ----------------------------------------------------------------------------- + +# Comma-separated list of specific ALEAPP plugins to run. +# Use this to create an inclusion list, forcing IPED to run ONLY the specified plugins. +# If left empty (default behavior), IPED will run all available ALEAPP plugins +# except for those defined in the 'excludePlugins' property below. +# Example: includePlugins=firefox,teams,googleCalendar +includePlugins= + +# ----------------------------------------------------------------------------- + +# Comma-separated list of ALEAPP plugins to specifically exclude from processing. +# This is useful for disabling plugins that are redundant with native IPED parsers, +# are irrelevant to the case, or produce undesired output. +# +# The default exclusions are listed below with justifications: +# - walStrings: Excluded because IPED natively parses strings from unparsed files. +# - WhatsApp: Excluded because IPED uses its own robust, internal parser for WhatsApp artifacts. +excludePlugins=walStrings,WhatsApp diff --git a/iped-app/resources/config/conf/CategoriesConfig.json b/iped-app/resources/config/conf/CategoriesConfig.json index da48d64b3f..9f786f5bf6 100644 --- a/iped-app/resources/config/conf/CategoriesConfig.json +++ b/iped-app/resources/config/conf/CategoriesConfig.json @@ -19,7 +19,7 @@ ]}, {"name": "Emails and Mailboxes", "categories":[ {"name": "Mailboxes", "mimes": ["application/vnd.ms-outlook-pst", "application/outlook-dbx", "application/mbox", "application/x-incredimail", "application/vnd.lotus-notes", "application/x-win10-mail-db"]}, - {"name": "Emails", "mimes": ["message/rfc822", "message/x-emlx", "application/vnd.ms-outlook", "message/outlook-pst", "application/x-ufed-html-mails", "application/x-ufed-email", "message/x-win10-mail-msg"]}, + {"name": "Emails", "mimes": ["message/rfc822", "message/x-emlx", "application/vnd.ms-outlook", "message/outlook-pst", "application/x-ufed-html-mails", "application/x-ufed-email", "message/x-win10-mail-msg", "application/x-aleapp-email"]}, {"name": "Appointments", "mimes": ["application/x-win10-mail-appointment"]}, {"name": "Win10 Mail Info", "mimes": ["application/x-win10-mail-store"]}, {"name": "Email Attachments"} @@ -80,31 +80,31 @@ {"name": "GDrive File Entries", "mimes": ["application/x-gdrive-cloud-graph-registry", "application/x-gdrive-snapshot-registry"]} ]}, {"name": "Databases", "mimes": ["application/x-edb", "application/x-edb-table", "application/irpf", "application/x-msaccess", "application/x-dbf", "application/vnd.oasis.opendocument.database", "application/x-sqlite3", "application/x-mysql-db", "application/x-berkeley-db", "application/x-mssql-data", "application/x-database-table"]}, - {"name": "Compressed Archives", "mimes": ["application/x-tika-ooxml", "application/zlib", "application/applefile", "application/vnd.ms-tnef", "application/zip", "application/x-rar-compressed", "application/x-tar", "application/gzip", "application/x-gzip", "application/x-xz", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed", "application/x-arj", "application/x-gtar", "application/x-archive", "application/x-cpio", "application/x-tika-unix-dump", "application/x-snappy-framed", "application/x-snappy", "application/x-snappy-raw", "application/x-compress", "application/x-java-pack200", "application/x-lzma", "application/x-lz4", "application/x-lz4-block", "application/x-brotli", "application/zstd", "application/deflate64", "image/x-emf-compressed", "application/x-lzfse"]}, - {"name": "Contacts", "mimes": ["text/x-vcard", "application/x-vcard-html", "application/windows-adress-book", "application/outlook-contact", "application/x-livecontacts", "application/x-livecontacts-table", "contact/x-skype-contact", "application/x-whatsapp-wadb", "application/x-whatsapp-contactsv2", "contact/x-whatsapp-contact", "application/x-ufed-html-contacts", "application/x-ufed-contact", "contact/x-telegram-contact", "application/x-ios-addressbook-db", "application/x-win10-mail-contact"]}, + {"name": "Compressed Archives", "mimes": ["application/x-tika-ooxml", "application/zlib", "application/applefile", "application/vnd.ms-tnef", "application/zip", "application/x-rar-compressed", "application/x-tar", "application/gzip", "application/x-gzip", "application/x-xz", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed", "application/x-arj", "application/x-gtar", "application/x-archive", "application/x-cpio", "application/x-tika-unix-dump", "application/x-snappy-framed", "application/x-snappy", "application/x-snappy-raw", "application/x-compress", "application/x-java-pack200", "application/x-lzma", "application/x-lz4", "application/x-lz4-block", "application/x-brotli", "application/zstd", "application/deflate64", "image/x-emf-compressed", "application/x-lzfse", "application/x-android-backup"]}, + {"name": "Contacts", "mimes": ["text/x-vcard", "application/x-vcard-html", "application/windows-adress-book", "application/outlook-contact", "application/x-livecontacts", "application/x-livecontacts-table", "contact/x-skype-contact", "application/x-whatsapp-wadb", "application/x-whatsapp-contactsv2", "contact/x-whatsapp-contact", "application/x-ufed-html-contacts", "application/x-ufed-contact", "contact/x-telegram-contact", "application/x-ios-addressbook-db", "application/x-win10-mail-contact", "application/x-aleapp-facebook-contact"]}, {"name": "Chats", "categories":[ - {"name": "WhatsApp", "mimes":["application/x-whatsapp-db", "application/x-whatsapp-chatstorage", "application/x-whatsapp-chat","application/x-ufed-chat-preview-whatsapp"]}, - {"name": "Threema", "mimes":["application/x-threema-chat", "application/x-threema-chatstorage", "application/x-ufed-chat-preview-threema"]}, + {"name": "WhatsApp", "mimes":["application/x-whatsapp-db", "application/x-whatsapp-chatstorage", "application/x-whatsapp-chat","application/x-ufed-chat-preview-whatsapp", "application/x-aleapp-chat-preview-whatsapp"]}, + {"name": "Threema", "mimes":["application/x-threema-chat", "application/x-threema-chatstorage", "application/x-ufed-chat-preview-threema", "application/x-aleapp-chat-preview-threema"]}, {"name": "Messenger", "mimes":["application/messenger-plus", "application/messenger-chat"]}, - {"name": "Skype", "mimes":["application/sqlite-skype", "application/skype", "application/x-skype-conversation","application/x-ufed-chat-preview-skype"]}, - {"name": "Telegram", "mimes":["application/x-telegram-chat","application/x-ufed-chat-preview-telegram"]}, + {"name": "Skype", "mimes":["application/sqlite-skype", "application/skype", "application/x-skype-conversation","application/x-ufed-chat-preview-skype", "application/x-aleapp-chat-preview-skype"]}, + {"name": "Telegram", "mimes":["application/x-telegram-chat","application/x-ufed-chat-preview-telegram", "application/x-aleapp-chat-preview-telegram"]}, {"name": "Jabber", "mimes":["application/x-jabber-chat"]}, - {"name": "Facebook", "mimes":["application/x-ufed-chat-preview-facebook"]}, - {"name": "Instagram", "mimes":["application/x-ufed-chat-preview-instagram"]}, - {"name": "Signal", "mimes":["application/x-ufed-chat-preview-signal"]}, - {"name": "Snapchat", "mimes":["application/x-ufed-chat-preview-snapchat"]}, - {"name": "TikTok", "mimes":["application/x-ufed-chat-preview-tiktok"]}, - {"name": "Viber", "mimes":["application/x-ufed-chat-preview-viber"]}, - {"name": "Discord", "mimes":["application/x-discord-chat","application/x-ufed-chat-preview-discord"]}, - {"name": "Others Chats", "mimes":["application/x-ufed-html-chats", "application/x-ufed-chats-txt", "application/x-ufed-chat", "application/x-ufed-chat-preview"]} + {"name": "Facebook", "mimes":["application/x-ufed-chat-preview-facebook", "application/x-aleapp-chat-preview-facebook"]}, + {"name": "Instagram", "mimes":["application/x-ufed-chat-preview-instagram", "application/x-aleapp-chat-preview-instagram"]}, + {"name": "Signal", "mimes":["application/x-ufed-chat-preview-signal", "application/x-aleapp-chat-preview-signal"]}, + {"name": "Snapchat", "mimes":["application/x-ufed-chat-preview-snapchat", "application/x-aleapp-chat-preview-snapchat"]}, + {"name": "TikTok", "mimes":["application/x-ufed-chat-preview-tiktok", "application/x-aleapp-chat-preview-tiktok"]}, + {"name": "Viber", "mimes":["application/x-ufed-chat-preview-viber", "application/x-aleapp-chat-preview-viber"]}, + {"name": "Discord", "mimes":["application/x-discord-chat","application/x-ufed-chat-preview-discord", "application/x-aleapp-chat-preview-discord"]}, + {"name": "Badoo", "mimes":["application/x-aleapp-badoo-chat"]}, + {"name": "Others Chats", "mimes":["application/x-ufed-html-chats", "application/x-ufed-chats-txt", "application/x-ufed-chat", "application/x-ufed-chat-preview", "application/x-aleapp-chat-preview"]} ]}, {"name": "Open Financial Exchange", "mimes": ["application/x-ofx-v1","application/x-ofx-v2","application/x-ofc"]}, {"name": "USN Journal", "mimes": ["application/x-usnjournal-$J", "application/x-usnjournal-report-html", "application/x-usnjournal-report-csv", "application/x-usnjournal-registry"]}, - {"name": "Programs and Libraries", + {"name": "Programs and Libraries", "mimes": ["application/java-archive", "application/x-dosexec", "application/x-msdownload", "application/x-bat", "application/vnd.ms-cab-compressed", "application/x-font-ttf", "application/pkcs7-signature", "application/vnd.ms-htmlhelp", "application/java-vm", "application/vnd.ms-pki.seccat", "application/x-ms-installer", "application/x-ufed-html-apps", "application/x-ufed-installedapplication"], "categories":[{"name": "Android Applications", "mimes": ["application/vnd.android.package-archive"]}] - }, - + }, {"name": "Unallocated", "mimes": ["application/x-unallocated"]}, {"name": "File Slacks", "mimes": ["application/x-fileslack"]}, {"name": "Plain Texts", "mimes": ["text"], "categories":[ @@ -162,12 +162,13 @@ {"name": "Extraction Summary", "mimes": ["application/x-ufed-html-summary"]}, {"name": "Calls", "categories":[ {"name": "Discord Calls", "mimes":["call/x-discord-call"]}, - {"name": "Facebook Calls", "mimes":[]}, + {"name": "Facebook Calls", "mimes":["application/aleapp-facebook-call", "application/aleapp-facebookapp-call", "application/aleapp-facebookmessenger-call"]}, + {"name": "Google Duo Calls", "mimes":["application/aleapp-googleduo-call"]}, {"name": "Phone Calls", "mimes":[]}, {"name": "Signal Calls", "mimes":[]}, {"name": "Telegram Calls", "mimes":["call/x-telegram-call"]}, {"name": "Threema Calls", "mimes":["call/x-threema-call"]}, - {"name": "WhatsApp Calls", "mimes":["call/x-whatsapp-call"]}, + {"name": "WhatsApp Calls", "mimes":["call/x-whatsapp-call", "application/x-aleapp-whatsapp-call"]}, {"name": "Other Calls", "mimes":["application/x-ufed-html-calls", "application/x-ufed-call", "application/x-ios-calllog-db", "application/x-ios8-calllog-db"]} ]}, {"name": "SMS Messages", "mimes": ["application/x-ufed-html-sms", "application/x-ufed-sms", "application/x-ios-sms-db"]}, @@ -179,31 +180,35 @@ {"name": "Shared Contacts"} ]}, {"name": "Bluetooth Devices", "mimes": ["application/x-ufed-html-bluetooth", "application/x-ufed-bluetoothdevice"]}, - {"name": "SIM Data", "mimes": ["application/x-ufed-html-simdata", "application/x-ufed-simdata"]}, - {"name": "Calendar", "mimes": ["application/x-ufed-html-calendar", "application/x-ufed-calendarentry", "application/x-ios-calendar-db"]}, + {"name": "SIM Data", "mimes": ["application/x-ufed-html-simdata", "application/x-ufed-simdata", "application/x-aleapp-siminfo"]}, + {"name": "Calendar", "mimes": ["application/x-ufed-html-calendar", "application/x-ufed-calendarentry", "application/x-ios-calendar-db", "application/aleapp-calendar-calendars"]}, {"name": "Logs", "mimes": ["application/x-ufed-html-logs", "application/x-ufed-logentry"]}, - {"name": "User Accounts", "mimes": ["application/x-ufed-html-users", "application/x-ufed-useraccount", "application/x-ufed-user", "contact/x-skype-account", "application/x-whatsapp-account", "application/x-telegram-account", "application/x-gdrive-account-info", "application/x-whatsapp-user-xml"]}, - {"name": "Searches", "mimes": ["application/x-ufed-html-searches", "application/x-ufed-searcheditem"]}, + {"name": "User Accounts", "mimes": ["application/x-ufed-html-users", "application/x-ufed-useraccount", "application/x-ufed-user", "contact/x-skype-account", "application/x-whatsapp-account", "application/x-telegram-account", "application/x-gdrive-account-info", "application/x-whatsapp-user-xml", "application/aleapp-accountdata", "application/aleapp-gmail-active", "application/aleapp-chrome-logindata", "application/x-aleapp-account", "application/x-aleapp-facebook-account"]}, + {"name": "Searches", "mimes": ["application/x-ufed-html-searches", "application/x-ufed-searcheditem", "application/aleapp-googleplaysearches"]}, {"name": "Notes", "mimes": ["application/x-ufed-html-notes", "application/x-ufed-note", "application/x-ios-oldnotes-db", "application/x-ios-notes-db"]}, - {"name": "Wireless Networks", "mimes": ["application/x-ufed-html-wifi", "application/x-ufed-wirelessnetwork"]}, - {"name": "Notifications", "mimes": ["application/x-ufed-html-notifications", "application/x-ufed-notification"]}, - {"name": "Locations", "mimes": ["application/x-ufed-html-locations", "application/x-ufed-location", "application/x-ios-locations-db", "application/x-apple-location"]}, - {"name": "Cookies", "mimes": ["application/x-ufed-html-cookies", "application/x-ufed-cookie"]}, + {"name": "Wireless Networks", "mimes": ["application/x-ufed-html-wifi", "application/x-ufed-wirelessnetwork","application/aleapp-wi-fiprofiles"]}, + {"name": "Notifications", "mimes": ["application/x-ufed-html-notifications", "application/x-ufed-notification", "application/x-aleapp-notification"]}, + {"name": "Locations", "mimes": ["application/x-ufed-html-locations", "application/x-ufed-location", "application/x-ios-locations-db", "application/x-apple-location", "application/aleapp-googlemapssearches-1", "application/aleapp-googlesearchhistorymaps", "application/aleapp-googlemapslasttrip" ]}, + {"name": "Cookies", "mimes": ["application/x-ufed-html-cookies", "application/x-ufed-cookie", "application/aleapp-chrome-cookies", "application/x-aleapp-cookies"]}, {"name": "Configuration", "mimes": ["application/x-ufed-html-configurations"]}, - {"name": "Passwords", "mimes": ["application/x-ufed-html-passwords", "application/x-ufed-password"]}, + {"name": "Passwords", "mimes": ["application/x-ufed-html-passwords", "application/x-ufed-password", "application/x-aleapp-account-authtoken"]}, {"name": "Autofill", "mimes": ["application/x-ufed-html-autofill", "application/x-ufed-autofill"]}, {"name": "Cell Towers", "mimes": ["application/x-ufed-html-celltowers", "application/x-ufed-celltower"]}, - {"name": "Power Events", "mimes": ["application/x-ufed-html-energyevents", "application/x-ufed-poweringevent"]}, + {"name": "Power Events", "mimes": ["application/x-ufed-html-energyevents", "application/x-ufed-poweringevent", "application/aleapp-shutdowncheckpoints", "application/aleapp-lastboottime"]}, {"name": "User Dictionaries", "mimes": ["application/x-ufed-html-userdict", "application/x-ufed-dictionaryword"]}, {"name": "IP Connections", "mimes": ["application/x-ufed-html-ips", "application/x-ufed-ipconnection"]}, {"name": "Recordings", "mimes": ["application/x-ufed-html-recordings", "application/x-ufed-recording", "application/x-ios-voicemail-db"]}, {"name": "Mobile Cards", "mimes": ["application/x-ufed-mobilecard"]}, {"name": "Applications Usage", "mimes": ["application/x-ufed-appsusagelog", "application/x-ufed-applicationusage"]}, - {"name": "Device Information", "mimes": ["application/x-ufed-deviceinfo", "application/x-ufed-deviceinfoentry"]}, + {"name": "Device Information", "mimes": ["application/x-ufed-deviceinfo", "application/x-ufed-deviceinfoentry", "application/x-aleapp-deviceinfo"]}, {"name": "Activities Sensor", "mimes": ["application/x-ufed-activitysensordata", "application/x-ufed-activitysensordatameasurement", "application/x-ufed-activitysensordatasample"]}, {"name": "Credit Cards", "mimes": ["application/x-ufed-creditcard"]}, - {"name": "Device Connectivity", "mimes": ["application/x-ufed-deviceconnectivity"]}, - {"name": "Device Events", "mimes": ["application/x-ufed-deviceevent"]}, + {"name": "Device Connectivity", "mimes": ["application/x-ufed-deviceconnectivity", "application/aleapp-adbhosts"]}, + {"name": "Device Events", "mimes": ["application/x-ufed-deviceevent", "application/aleapp-factoryreset", "application/aleapp-appops.xml"]}, + {"name": "Privacy Dashboard", "mimes": ["application/aleapp-privacydashboard"]}, + {"name": "GDrive File Entries", "mimes": ["application/x-aleapp-gdrive-file-entry"]}, + {"name": "App Roles", "mimes": ["application/x-aleapp-app-role"]}, + {"name": "Update Information", "mimes": ["application/x-aleapp-update-info"]}, {"name": "File Downloads", "mimes": ["application/x-ufed-filedownload"]}, {"name": "File Uploads", "mimes": ["application/x-ufed-fileupload"]}, {"name": "Financial Accounts", "mimes": ["application/x-ufed-financialaccount"]}, diff --git a/iped-app/resources/config/conf/CustomSignatures.xml b/iped-app/resources/config/conf/CustomSignatures.xml index 5a504fd03c..fdfca6f353 100644 --- a/iped-app/resources/config/conf/CustomSignatures.xml +++ b/iped-app/resources/config/conf/CustomSignatures.xml @@ -195,6 +195,10 @@ + + + + @@ -1476,6 +1480,12 @@ + + + + + + @@ -1703,4 +1713,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/iped-app/resources/config/conf/ParserConfig.xml b/iped-app/resources/config/conf/ParserConfig.xml index 43f2ab18a7..841758ecf1 100644 --- a/iped-app/resources/config/conf/ParserConfig.xml +++ b/iped-app/resources/config/conf/ParserConfig.xml @@ -272,6 +272,7 @@ + image/svg+xml diff --git a/iped-app/resources/config/conf/TaskInstaller.xml b/iped-app/resources/config/conf/TaskInstaller.xml index a17823f825..a4151f923f 100644 --- a/iped-app/resources/config/conf/TaskInstaller.xml +++ b/iped-app/resources/config/conf/TaskInstaller.xml @@ -54,6 +54,9 @@ + + + diff --git a/iped-app/resources/config/conf/metadataTypes.txt b/iped-app/resources/config/conf/metadataTypes.txt index 3422bd1b20..fd2af4e66e 100644 --- a/iped-app/resources/config/conf/metadataTypes.txt +++ b/iped-app/resources/config/conf/metadataTypes.txt @@ -1738,6 +1738,35 @@ accountType = java.lang.String apple-mobile-web-app-capable = java.lang.String apple-mobile-web-app-status-bar-style = java.lang.String array = java.lang.Integer +aleapp:Added Date = java.util.Date +aleapp:Created Date = java.util.Date +aleapp:Created Time = java.util.Date +aleapp:Created = java.util.Date +aleapp:Creation Time = java.util.Date +aleapp:Date Created = java.util.Date +aleapp:Date Last Used = java.util.Date +aleapp:Date sent = java.util.Date +aleapp:Date = java.util.Date +aleapp:End Time = java.util.Date +aleapp:Expiration Date = java.util.Date +aleapp:First Download = java.util.Date +aleapp:Install Time = java.util.Date +aleapp:Last Access Date = java.util.Date +aleapp:Last Access Time = java.util.Date +aleapp:Last Connected Timestamp = java.util.Date +aleapp:Last Updated Timestamp = java.util.Date +aleapp:Last Updated = java.util.Date +aleapp:Latest = java.util.Date +aleapp:Modified/Downloaded Timestamp = java.util.Date +aleapp:Modified = java.util.Date +aleapp:Purchase Time = java.util.Date +aleapp:Start Time = java.util.Date +aleapp:Timestamp Added = java.util.Date +aleapp:Timestamp Modified = java.util.Date +aleapp:Timestamp = java.util.Date +aleapp:Update Time = java.util.Date +aleapp:Value = java.util.Date +aleapp:ft = java.util.Date audio:bits = java.lang.Integer audio:channels = java.lang.Integer audio:dc:creator = java.lang.String @@ -4643,3 +4672,7 @@ CarvedOffset = java.lang.Long mediaSize = java.lang.Long mediaMime = java.lang.String mediaName = java.lang.String +zip:comment = java.lang.String +zip:compressedRatio = java.lang.Double +zip:compressedSize = java.lang.Integer +zip:unixMode = java.lang.String diff --git a/iped-app/resources/config/profiles/triage/IPEDConfig.txt b/iped-app/resources/config/profiles/triage/IPEDConfig.txt index 461cdf3352..381b804ecd 100644 --- a/iped-app/resources/config/profiles/triage/IPEDConfig.txt +++ b/iped-app/resources/config/profiles/triage/IPEDConfig.txt @@ -33,3 +33,4 @@ enableSearchHardwareWallets = true enableVideoThumbs = false enableDocThumbs = false enableHTMLReport = false +enableAleapp = false diff --git a/iped-app/resources/scripts/tasks/RefineCategoryTask.js b/iped-app/resources/scripts/tasks/RefineCategoryTask.js index 826081e5b7..ff7716955f 100644 --- a/iped-app/resources/scripts/tasks/RefineCategoryTask.js +++ b/iped-app/resources/scripts/tasks/RefineCategoryTask.js @@ -202,6 +202,20 @@ function process(e){ } } + if (categorias.contains("Other files") && mime.startsWith("application/x-aleapp-")) { + if (mime.endsWith("call")) { + e.setCategory("Other Calls"); + } else if (mime.endsWith("-chat") || mime.endsWith("-conversation")) { + e.setCategory("Others Chats"); + } else if (mime.endsWith("-message")) { + e.setCategory("Instant Messages"); + } else if (mime.endsWith("-contact")) { + e.setCategory("Contacts"); + } else if (mime.endsWith("-autofill")) { + e.setCategory("Autofill"); + } + } + // Usually, conditions that overwrite the category (using setCategory()) // should go before the ones that add other categories (using addCategory()). diff --git a/iped-app/src/main/java/iped/app/ui/IconManager.java b/iped-app/src/main/java/iped/app/ui/IconManager.java index c34abfc921..c0c604e4ff 100644 --- a/iped-app/src/main/java/iped/app/ui/IconManager.java +++ b/iped-app/src/main/java/iped/app/ui/IconManager.java @@ -65,6 +65,10 @@ public class IconManager { private static final String folderOpenedKey = "folder-opened"; private static final String folderClosedKey = "folder-closed"; + + private static final String reportFolderOpenedKey = "report-folder-opened"; + private static final String reportFolderClosedKey = "report-folder-closed"; + private static final String diskKey = "drive"; private static final String fileKey = "file"; @@ -121,6 +125,14 @@ public static Icon getFolderIcon(boolean isOpened) { return getTreeIcon(isOpened ? folderOpenedKey : folderClosedKey); } + public static Icon getReportFolderIcon() { + return getReportFolderIcon(false); + } + + public static Icon getReportFolderIcon(boolean isOpened) { + return getTreeIcon(isOpened ? reportFolderOpenedKey : reportFolderClosedKey); + } + public static Icon getFolderIconGallery() { return getFolderIconGallery(false); } @@ -143,7 +155,11 @@ public static Icon getFileIcon(String mimeType, String extension, Icon defaultIc private static Icon getFileIcon(String mimeType, String extension, Map mimeIconMap, Map extIconMap, Icon defaultIcon) { if (mimeType != null && !mimeType.isBlank()) { - Icon icon = mimeIconMap.get(mimeType.strip()); + String mime = mimeType.strip(); + Icon icon = mimeIconMap.get(mime); + if (icon == null && mime.startsWith(ALEAPP_MIME_PREFIX)) { + icon = getAleappIcon(mime, mimeIconMap); + } if (icon != null) { return icon; } @@ -157,6 +173,49 @@ private static Icon getFileIcon(String mimeType, String extension, Map mimeIconMap) { + for (String app : ALEAPP_CHAT_APPS) { + if (mimeType.contains(app)) { + Icon icon = mimeIconMap.get(ALEAPP_MIME_PREFIX + "chat-preview-" + app); + if (icon != null) { + return icon; + } + } + } + if (mimeType.endsWith("-chat") || mimeType.endsWith("-message") || mimeType.endsWith("-conversation")) { + return mimeIconMap.get(ALEAPP_MIME_PREFIX + "chat-preview"); + } + return null; + } + public static Icon getCategoryIcon(String category) { if (category != null && !category.isBlank()) { return catIconMap.getOrDefault(category.strip(), defaultCategoryIcon); @@ -279,6 +338,12 @@ private static Map initMimeToIconMap(int size) { mimeIconMap.put("application/x-windows-registry-report", icon); } + icon = availableIconsMap.get("chat"); + if (icon != null) { + mimeIconMap.put("application/x-ufed-chat-preview", icon); + mimeIconMap.put("application/x-aleapp-chat-preview", icon); + } + icon = availableIconsMap.get("whatsapp"); if (icon != null) { mimeIconMap.put("application/x-whatsapp-db", icon); @@ -286,6 +351,7 @@ private static Map initMimeToIconMap(int size) { mimeIconMap.put("application/x-whatsapp-chatstorage", icon); mimeIconMap.put("application/x-whatsapp-chat", icon); mimeIconMap.put("application/x-ufed-chat-preview-whatsapp", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-whatsapp", icon); } icon = availableIconsMap.get("threema"); @@ -294,36 +360,43 @@ private static Map initMimeToIconMap(int size) { mimeIconMap.put("application/x-threema-user-plist", icon); mimeIconMap.put("application/x-threema-chatstorage", icon); mimeIconMap.put("application/x-ufed-chat-preview-threema", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-threema", icon); } icon = availableIconsMap.get("facebook"); if (icon != null) { mimeIconMap.put("application/x-ufed-chat-preview-facebook", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-facebook", icon); } icon = availableIconsMap.get("signal"); if (icon != null) { mimeIconMap.put("application/x-ufed-chat-preview-signal", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-signal", icon); } icon = availableIconsMap.get("snapchat"); if (icon != null) { mimeIconMap.put("application/x-ufed-chat-preview-snapchat", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-snapchat", icon); } icon = availableIconsMap.get("tiktok"); if (icon != null) { mimeIconMap.put("application/x-ufed-chat-preview-tiktok", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-tiktok", icon); } icon = availableIconsMap.get("viber"); if (icon != null) { mimeIconMap.put("application/x-ufed-chat-preview-viber", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-viber", icon); } icon = availableIconsMap.get("instagram"); if (icon != null) { mimeIconMap.put("application/x-ufed-chat-preview-instagram", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-instagram", icon); } icon = availableIconsMap.get("skype"); @@ -332,6 +405,7 @@ private static Map initMimeToIconMap(int size) { mimeIconMap.put("application/skype", icon); mimeIconMap.put("application/x-skype-conversation", icon); mimeIconMap.put("application/x-ufed-chat-preview-skype", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-skype", icon); } icon = availableIconsMap.get("telegram"); @@ -339,6 +413,7 @@ private static Map initMimeToIconMap(int size) { mimeIconMap.put("application/x-telegram-chat", icon); mimeIconMap.put("application/x-telegram-db", icon); mimeIconMap.put("application/x-ufed-chat-preview-telegram", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-telegram", icon); } icon = availableIconsMap.get("apple-config"); @@ -485,7 +560,6 @@ private static Map initMimeToIconMap(int size) { icon = availableIconsMap.get("message"); if (icon != null) { mimeIconMap.put("application/x-ufed-instantmessage", icon); - mimeIconMap.put("application/x-ufed-chat-preview", icon); mimeIconMap.put("message/x-chat-message", icon); mimeIconMap.put("message/x-discord-message", icon); } @@ -666,6 +740,7 @@ private static Map initMimeToIconMap(int size) { mimeIconMap.put("application/x-discord-index", icon); mimeIconMap.put("application/x-discord-chat", icon); mimeIconMap.put("application/x-ufed-chat-preview-discord", icon); + mimeIconMap.put("application/x-aleapp-chat-preview-discord", icon); } icon = availableIconsMap.get("discord-attachment"); diff --git a/iped-app/src/main/java/iped/app/ui/ReferencedByTableModel.java b/iped-app/src/main/java/iped/app/ui/ReferencedByTableModel.java index 4b854ff0ac..81141c2e73 100644 --- a/iped-app/src/main/java/iped/app/ui/ReferencedByTableModel.java +++ b/iped-app/src/main/java/iped/app/ui/ReferencedByTableModel.java @@ -109,6 +109,18 @@ public Query createQuery(Document doc) { } } + // globalId + String globalId = doc.get(ExtraProperties.GLOBAL_ID); + if (StringUtils.isNotBlank(globalId)) { + String globalIdQuery = ExtraProperties.LINKED_ITEMS + ":" + ExtraProperties.GLOBAL_ID // + + " AND " + ExtraProperties.LINKED_ITEMS + ":" + globalId; + try { + queryBuilder.add(b.getQuery(globalIdQuery), Occur.SHOULD); + } catch (ParseException | QueryNodeException e) { + e.printStackTrace(); + } + } + // jumpList:ids String[] appIds = doc.getValues(JumpListTask.JUMPLIST_PROGRAM_APP_IDS); for (String appId: appIds) { diff --git a/iped-app/src/main/java/iped/app/ui/TableCellRenderer.java b/iped-app/src/main/java/iped/app/ui/TableCellRenderer.java index d7729d37bc..d4bcfa2e5e 100644 --- a/iped-app/src/main/java/iped/app/ui/TableCellRenderer.java +++ b/iped-app/src/main/java/iped/app/ui/TableCellRenderer.java @@ -36,6 +36,8 @@ import iped.data.IMultiBookmarks; import iped.engine.task.index.IndexItem; import iped.localization.LocalizedProperties; +import iped.properties.BasicProps; +import iped.properties.ExtraProperties; public class TableCellRenderer extends DefaultTableCellRenderer { @@ -89,9 +91,20 @@ public Component getTableCellRendererComponent(JTable table, Object value, boole if (Boolean.valueOf(doc.get(IndexItem.ISDIR))) { icon = IconManager.getFolderIcon(); } else { + boolean isDecodedReport = Boolean.parseBoolean(doc.get(ExtraProperties.DECODED_DATA)) + && Boolean.parseBoolean(doc.get(BasicProps.HASCHILD)); String type = doc.get(IndexItem.TYPE); String contentType = doc.get(IndexItem.CONTENTTYPE); - icon = Boolean.valueOf(doc.get(IndexItem.ISROOT)) ? IconManager.getFileIcon(contentType, type, IconManager.getDiskIcon()) : IconManager.getFileIcon(contentType, type); + // a specific mime icon (e.g. an app chat preview) takes precedence over the + // generic decoded report folder icon, which is used only as a fallback + Icon mimeIcon = IconManager.getMimeIcon(contentType); + if (mimeIcon != null) { + icon = mimeIcon; + } else if (isDecodedReport) { + icon = IconManager.getReportFolderIcon(false); + } else { + icon = Boolean.valueOf(doc.get(IndexItem.ISROOT)) ? IconManager.getFileIcon(contentType, type, IconManager.getDiskIcon()) : IconManager.getFileIcon(contentType, type); + } } } catch (IOException e) { } diff --git a/iped-app/src/main/java/iped/app/ui/TreeCellRenderer.java b/iped-app/src/main/java/iped/app/ui/TreeCellRenderer.java index 67422056de..0e6df37fbe 100644 --- a/iped-app/src/main/java/iped/app/ui/TreeCellRenderer.java +++ b/iped-app/src/main/java/iped/app/ui/TreeCellRenderer.java @@ -29,6 +29,7 @@ import iped.app.ui.TreeViewModel.Node; import iped.engine.task.index.IndexItem; import iped.properties.BasicProps; +import iped.properties.ExtraProperties; public class TreeCellRenderer extends DefaultTreeCellRenderer { @@ -42,6 +43,9 @@ public Component getTreeCellRendererComponent(JTree tree, Object value, boolean boolean isDir = Boolean.valueOf(node.getDoc().get(IndexItem.ISDIR)) || node.docId == -1; super.getTreeCellRendererComponent(tree, value, selected, expanded, !isDir, row, hasFocus); + boolean isDecodedReport = Boolean.parseBoolean(node.getDoc().get(ExtraProperties.DECODED_DATA)) + && Boolean.parseBoolean(node.getDoc().get(BasicProps.HASCHILD)); + if (row == 0) { setIcon(rootIcon); } else if (isDir) { @@ -50,8 +54,17 @@ public Component getTreeCellRendererComponent(JTree tree, Object value, boolean Document doc = node.getDoc(); String type = doc.get(BasicProps.TYPE); String contentType = doc.get(BasicProps.CONTENTTYPE); - Icon icon = Boolean.valueOf(doc.get(IndexItem.ISROOT)) ? IconManager.getFileIcon(contentType, type, IconManager.getDiskIcon()) : IconManager.getFileIcon(contentType, type); - setIcon(icon); + // a specific mime icon (e.g. an app chat preview) takes precedence over the + // generic decoded report folder icon, which is used only as a fallback + Icon mimeIcon = IconManager.getMimeIcon(contentType); + if (mimeIcon != null) { + setIcon(mimeIcon); + } else if (isDecodedReport) { + setIcon(IconManager.getReportFolderIcon(expanded)); + } else { + Icon icon = Boolean.valueOf(doc.get(IndexItem.ISROOT)) ? IconManager.getFileIcon(contentType, type, IconManager.getDiskIcon()) : IconManager.getFileIcon(contentType, type); + setIcon(icon); + } } return this; diff --git a/iped-app/src/main/resources/iped/app/ui/mime/chat.png b/iped-app/src/main/resources/iped/app/ui/mime/chat.png new file mode 100644 index 0000000000..dc3819cb80 Binary files /dev/null and b/iped-app/src/main/resources/iped/app/ui/mime/chat.png differ diff --git a/iped-app/src/main/resources/iped/app/ui/tree/report-folder-closed.png b/iped-app/src/main/resources/iped/app/ui/tree/report-folder-closed.png new file mode 100644 index 0000000000..225e325dc2 Binary files /dev/null and b/iped-app/src/main/resources/iped/app/ui/tree/report-folder-closed.png differ diff --git a/iped-app/src/main/resources/iped/app/ui/tree/report-folder-opened.png b/iped-app/src/main/resources/iped/app/ui/tree/report-folder-opened.png new file mode 100644 index 0000000000..b067127c0f Binary files /dev/null and b/iped-app/src/main/resources/iped/app/ui/tree/report-folder-opened.png differ diff --git a/iped-engine/src/main/java/iped/engine/config/ALeappConfig.java b/iped-engine/src/main/java/iped/engine/config/ALeappConfig.java new file mode 100644 index 0000000000..c58470bf79 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/config/ALeappConfig.java @@ -0,0 +1,94 @@ +package iped.engine.config; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + +import iped.utils.UTF8Properties; + +public class ALeappConfig extends AbstractTaskPropertiesConfig { + + private static final long serialVersionUID = -6174870703965268807L; + + private static final String CONFIG_FILE = "ALeappConfig.txt"; + + private File aleappFolder; + + ArrayList excludedPlugins = new ArrayList<>(); + ArrayList includedPlugins = new ArrayList<>(); + + @Override + public String getTaskEnableProperty() { + return "enableAleapp"; + } + + @Override + public String getTaskConfigFileName() { + return CONFIG_FILE; + } + + @Override + void processProperties(UTF8Properties properties) { + + String aleapFolderValue = properties.getProperty("aleapFolder"); + if (StringUtils.isNotBlank(aleapFolderValue)) { + Path aleappPath = Paths.get(aleapFolderValue.trim()); + if (aleappPath.isAbsolute()) { + aleappFolder = aleappPath.toFile(); + } else { + aleappFolder = new File(Configuration.getInstance().appRoot, aleappPath.toString()); + } + } else { + aleappFolder = new File(Configuration.getInstance().appRoot, "tools/aleapp"); + } + if (!aleappFolder.exists() || !aleappFolder.isDirectory()) { + throw new IllegalArgumentException("Invalid Aleapp folder: " + aleappFolder); + } + + String excludePlugins = properties.getProperty("excludePlugins"); + if (StringUtils.isNotBlank(excludePlugins)) { + Arrays.stream(StringUtils.split(excludePlugins, ',')) + .map(StringUtils::strip) + .collect(Collectors.toCollection(() -> excludedPlugins)); + } + + String includePlugins = properties.getProperty("includePlugins"); + if (StringUtils.isNotBlank(includePlugins)) { + Arrays.stream(StringUtils.split(includePlugins, ',')) + .map(StringUtils::strip) + .collect(Collectors.toCollection(() -> includedPlugins)); + } + } + + public File getAleappFolder() { + return aleappFolder; + } + + public ArrayList getExcludedPlugins() { + return excludedPlugins; + } + + public ArrayList getIncludedPlugins() { + return includedPlugins; + } + + public boolean isPluginIncluded(String moduleName) { + + boolean include = true; + + if (!includedPlugins.isEmpty()) { + include &= includedPlugins.contains(moduleName); + } + + if (!excludedPlugins.isEmpty()) { + include &= !excludedPlugins.contains(moduleName); + } + + return include; + } +} diff --git a/iped-engine/src/main/java/iped/engine/core/QueuesProcessingOrder.java b/iped-engine/src/main/java/iped/engine/core/QueuesProcessingOrder.java index 5c8716016c..52d0b32372 100644 --- a/iped-engine/src/main/java/iped/engine/core/QueuesProcessingOrder.java +++ b/iped-engine/src/main/java/iped/engine/core/QueuesProcessingOrder.java @@ -9,6 +9,7 @@ import org.apache.tika.mime.MediaType; import org.apache.tika.mime.MediaTypeRegistry; +import iped.engine.task.leapp.AleappTask; import iped.parsers.ares.AresParser; import iped.parsers.bittorrent.BitTorrentResumeDatEntryParser; import iped.parsers.bittorrent.BitTorrentResumeDatParser; @@ -116,6 +117,10 @@ private static Map installTypesToPostProcess() { mediaTypes.put(RFC822Parser.RFC822_PARTIAL0_MIME, 2); mediaTypes.put(RFC822Parser.RFC822_PARTIAL1_MIME, 2); + mediaTypes.put(AleappTask.ALEAPP_CASE_MEDIATYPE, 5); + mediaTypes.put(AleappTask.ALEAPP_CATEGORY_MEDIATYPE, 6); + mediaTypes.put(AleappTask.ALEAPP_DEVICE_INFO_MEDIATYPE, 6); + return mediaTypes; } diff --git a/iped-engine/src/main/java/iped/engine/datasource/ItemProducer.java b/iped-engine/src/main/java/iped/engine/datasource/ItemProducer.java index 0092e3c599..c6f307eeab 100644 --- a/iped-engine/src/main/java/iped/engine/datasource/ItemProducer.java +++ b/iped-engine/src/main/java/iped/engine/datasource/ItemProducer.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -70,10 +71,14 @@ public ItemProducer(Manager manager, ICaseData caseData, boolean listOnly, List< private void installDataSourceReaders() throws Exception { - Class[] readerList = new Class[] { SleuthkitReader.class, - IPEDReader.class, UfedXmlReader.class, AD1DataSourceReader.class, - FolderTreeReader.class // deve ser o último - }; + List> readerList = Arrays.asList( + SleuthkitReader.class, + IPEDReader.class, + UfedXmlReader.class, + AD1DataSourceReader.class, + ZipFileReader.class, + FolderTreeReader.class // must be the last one + ); for (Class srcReader : readerList) { Constructor constr = srcReader.getConstructor(ICaseData.class, File.class, diff --git a/iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java b/iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java index fdbc5cf8df..15bf0ebe55 100644 --- a/iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java +++ b/iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java @@ -122,6 +122,9 @@ public class UfedXmlReader extends DataSourceReader { public static final String UFED_NATIVE_SCENE_CLASSIFICATION = ExtraProperties.UFED_META_PREFIX + "Native Scene Classification"; public static final String MSISDN_PROP = "MSISDN"; + /** Name of the virtual folder holding the decoded models, a direct child of the ufdr root item. */ + public static final String DECODED_DATA_FOLDER_NAME = "_DecodedData"; + private static final String EMPTY_EXTRACTION_STR = "-"; private static final String FILE_ID_ATTR = ExtraProperties.UFED_META_PREFIX + "file_id"; //$NON-NLS-1$ @@ -392,7 +395,7 @@ private void addVirtualDecodedFolder() throws InterruptedException { return; decodedFolder = new Item(); - decodedFolder.setName("_DecodedData"); //$NON-NLS-1$ + decodedFolder.setName(DECODED_DATA_FOLDER_NAME); decodedFolder.setParent(rootItem); decodedFolder.setIdInDataSource(""); decodedFolder.setPath(rootItem.getPath() + "/" + decodedFolder.getName()); //$NON-NLS-1$ diff --git a/iped-engine/src/main/java/iped/engine/datasource/ZipFileReader.java b/iped-engine/src/main/java/iped/engine/datasource/ZipFileReader.java new file mode 100644 index 0000000000..85107807f6 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/datasource/ZipFileReader.java @@ -0,0 +1,263 @@ +package iped.engine.datasource; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import iped.data.ICaseData; +import iped.data.IItem; +import iped.engine.core.Manager; +import iped.engine.data.DataSource; +import iped.engine.data.Item; +import iped.engine.io.ZIPInputStreamFactory; +import iped.properties.ExtraProperties; +import iped.utils.SeekableInputStreamFactory; + +/** + * DataSourceReader that reads items directly from ZIP files without expanding + * them in the file system. The archive entries are enumerated and the content + * of each entry is read on demand through a {@link SeekableInputStreamFactory}. + * + * Analogous to {@link FolderTreeReader}, which walks real folders from the OS. + */ +public class ZipFileReader extends DataSourceReader { + + private static final Logger logger = LoggerFactory.getLogger(ZipFileReader.class); + + private static final String ZIP_METADATA_PREFIX = "zip:"; + + private static final Set SUPPORTED_EXTENSIONS = Set.of("zip"); + private static final Set SUPPORTED_TYPES = Set.of(ArchiveStreamFactory.ZIP); + + private String evidenceName; + private SeekableInputStreamFactory inputStreamFactory; + + /** Maps normalized entry path -> directory item already created */ + private final Map dirItems = new HashMap<>(); + + private IItem rootItem; + + public ZipFileReader(ICaseData caseData, File output, boolean listOnly) { + super(caseData, output, listOnly); + } + + @Override + public boolean isSupported(File datasource) { + + if (!datasource.isFile()) { + return false; + } + + // detect by extension + String ext = FilenameUtils.getExtension(datasource.getName()).toLowerCase(); + if (SUPPORTED_EXTENSIONS.contains(ext)) { + return true; + } + + // detection by file signature + try (InputStream is = new FileInputStream(datasource)) { + String detectedType = ArchiveStreamFactory.detect(is); + if (SUPPORTED_TYPES.contains(detectedType)) { + return true; + } + } catch (ArchiveException | IOException e) { + } + + return false; + } + + @Override + public void read(File file) throws Exception { + read(file, null); + } + + @Override + public void read(File file, Item parent) throws Exception { + + evidenceName = getEvidenceName(file); + if (evidenceName == null) { + evidenceName = file.getName(); + } + + inputStreamFactory = new ZIPInputStreamFactory(file.toPath()); + + if (parent == null) { + dataSource = new DataSource(file); + dataSource.setName(evidenceName); + } + + if (!listOnly) { + // creates a root item representing the archive file itself + BasicFileAttributes attributes = Files.readAttributes(file.toPath(), BasicFileAttributes.class); + rootItem = createItem(evidenceName, evidenceName, true, null, attributes.creationTime(), attributes.lastModifiedTime(), attributes.lastAccessTime()); + rootItem.setExtraAttribute(ExtraProperties.DATASOURCE_READER, this.getClass().getSimpleName()); + rootItem.setIdInDataSource(""); + if (parent != null) { + rootItem.setParent(parent); + } else { + rootItem.setRoot(true); + } + addToQueue(rootItem); + } else { + caseData.incDiscoveredEvidences(1); + } + + readZipEntries(file); + } + + /** + * Reads ZIP entries using ZipFile, which parses the central directory at the + * end of the file. This supports data descriptors, gets correct sizes for all + * entries and allows random access to entry contents. + */ + private void readZipEntries(File file) throws IOException, InterruptedException { + try (ZipFile zip = ZipFile.builder().setFile(file).get()) { + Enumeration entries = zip.getEntriesInPhysicalOrder(); + while (entries.hasMoreElements()) { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + ZipArchiveEntry entry = entries.nextElement(); + processEntry(entry); + } + } + } + + private void processEntry(ZipArchiveEntry entry) + throws InterruptedException { + + String path = normalize(entry.getName()); + if (path.isEmpty() || path.contains("../")) { + // protection against malicious (zip slip) or invalid entries + logger.warn("Entry ignored (invalid path): {} in {}", entry.getName(), dataSource); + return; + } + + if (listOnly) { + caseData.incDiscoveredEvidences(1); + if (entry.getSize() > 0) { + caseData.incDiscoveredVolume(entry.getSize()); + } + return; + } + + IItem parent = getOrCreateParent(path); + + String name = path.substring(path.lastIndexOf('/') + 1); + String evidencePath = (parent != null ? parent.getPath() : evidenceName) + '/' + name; + + IItem item = createItem(name, evidencePath, entry.isDirectory(), entry.getSize(), entry.getCreationTime(), entry.getLastModifiedTime(), entry.getLastAccessTime()); + addMetadata(item, entry); + item.setIdInDataSource(path); + + + boolean isDir = entry.isDirectory(); + if (parent != null) { + item.setParent(parent); + } + + addToQueue(item); + + if (isDir) { + dirItems.put(path, item); + } + } + + private void addMetadata(IItem item, ZipArchiveEntry entry) { + item.getMetadata().set(ZIP_METADATA_PREFIX + "name", entry.getName()); + item.getMetadata().set(ZIP_METADATA_PREFIX + "comment", entry.getComment()); + item.getMetadata().set(ZIP_METADATA_PREFIX + "compressedSize", Long.toString(entry.getCompressedSize())); + if (entry.getSize() > 0) { + item.getMetadata().set(ZIP_METADATA_PREFIX + "compressedRatio", Double.toString((double) entry.getCompressedSize() / entry.getSize())); + } + item.getMetadata().set(ZIP_METADATA_PREFIX + "unixMode", Long.toOctalString(entry.getUnixMode())); + } + + /** + * Ensures the existence of the parent directory chain of an entry. Many + * archives do not have explicit entries for directories, or the entries come + * out of order, so missing directories are created here. + */ + private IItem getOrCreateParent(String path) throws InterruptedException { + int idx = path.lastIndexOf('/'); + if (idx == -1) { + return rootItem; + } + String parentPath = path.substring(0, idx); + IItem parent = dirItems.get(parentPath); + if (parent != null) { + return parent; + } + IItem grandParent = getOrCreateParent(parentPath); + String name = parentPath.substring(parentPath.lastIndexOf('/') + 1); + String evidencePath = (grandParent != null ? grandParent.getPath() : evidenceName) + '/' + name; + + parent = createItem(name, evidencePath, true, null, null, null, null); + parent.setIdInDataSource(parentPath); + if (grandParent != null) { + parent.setParent(grandParent); + if (grandParent == rootItem) { + parent.setExtraAttribute(ExtraProperties.DATASOURCE_READER, this.getClass().getSimpleName()); + } + } + caseData.incDiscoveredEvidences(1); + addToQueue(parent); + dirItems.put(parentPath, parent); + return parent; + } + + private IItem createItem(String name, String evidencePath, boolean isDir, Long size, FileTime creationTime, FileTime modificationTime, FileTime accessTime) { + IItem item = new Item(); + item.setDataSource(dataSource); + item.setInputStreamFactory(inputStreamFactory); + item.setName(name); + item.setPath(evidencePath); + item.setIsDir(isDir); + if (!isDir) { + item.setLength(size); + } + if (creationTime != null) { + item.setCreationDate(new Date(creationTime.toMillis())); + } + if (modificationTime != null) { + item.setModificationDate(new Date(modificationTime.toMillis())); + } + if (accessTime != null) { + item.setAccessDate(new Date(accessTime.toMillis())); + } + return item; + } + + private void addToQueue(IItem item) throws InterruptedException { + Manager.getInstance().addItemToQueue(item); + } + + public static String normalize(String entryName) { + // 1. Standardize to Unix slashes + // 2. Strip ALL leading and trailing slashes instantly + return StringUtils.strip(entryName.replace('\\', '/'), "/"); + } + + @Override + public void close() throws IOException { + dirItems.clear(); + } +} \ No newline at end of file diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/AleappMediaTypeResolver.java b/iped-engine/src/main/java/iped/engine/task/leapp/AleappMediaTypeResolver.java new file mode 100644 index 0000000000..4375609830 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/AleappMediaTypeResolver.java @@ -0,0 +1,131 @@ +package iped.engine.task.leapp; + +import org.apache.commons.lang3.StringUtils; +import org.apache.tika.mime.MediaType; + +/** + * Gives each ALEAPP plugin row subitem its media type. + * + *

+ * The media type is how these items get a category: they are not categorized by code, only by the mime → category + * map in CategoriesConfig.json. + * + *

+ * The type is built in one of two ways: + *

    + *
  • {@link #pluginSpecificSubtype}: a few plugins get a fixed type so all their rows share one category (e.g. every + * FCMQueued* plugin → {@code x-aleapp-notification}); + *
  • {@link #genericSubtype}: any other plugin gets a type built from its artifact name. + *
+ * + *

+ * A category that depends on a cell value (e.g. "settingsSecure Name = bluetooth_address") cannot be set here: the type + * is decided once per plugin run, not per row. + */ +public final class AleappMediaTypeResolver { + + private AleappMediaTypeResolver() { + } + + /** + * Returns the media type for a row: the plugin-specific type when there is one, otherwise the generic type. + * + * @param moduleName the plugin module (python file stem, e.g. "chromeCookies") + * @param pluginName the artifact key (e.g. "get_fb_user_id") + * @param artifactName the artifact display name (e.g. "Gmail - App Emails") + */ + public static MediaType resolveMediaType(String moduleName, String pluginName, String artifactName) { + String subtype = pluginSpecificSubtype(moduleName, pluginName, artifactName); + if (subtype == null) { + subtype = genericSubtype(pluginName, artifactName); + } + return MediaType.application(AleappTask.ALEAPP_APPLICATION_PREFIX + subtype); + } + + /** + * Stable subtypes for plugins whose forensic category is driven by identity (module or artifact), so that + * CategoriesConfig.json can map the whole group with a single mime. Returns null when no specific rule applies. + */ + private static String pluginSpecificSubtype(String moduleName, String pluginName, String artifactName) { + + if (moduleName.startsWith("FCMQueued")) { + return "notification"; + } + if (moduleName.equals("accounts_de")) { + return "account"; + } + if (moduleName.equals("accounts_ce")) { + // accounts_ce.py declares two artifacts: "Accounts_ce" and "Authentication tokens" + return "Authentication tokens".equals(artifactName) ? "account-authtoken" : "account"; + } + if (moduleName.equals("siminfo")) { + return "siminfo"; + } + if (moduleName.equals("Cello")) { + return "gdrive-file-entry"; + } + if (moduleName.equals("roles")) { + return "app-role"; + } + if (moduleName.equals("frosting")) { + return "update-info"; + } + if (moduleName.equals("gmailEmails") && "Gmail - App Emails".equals(artifactName)) { + return "email"; + } + if (moduleName.equals("FacebookMessenger")) { + // artifact keys: get_fb_*_contacts, get_fb_user_id, get_fb_*_chats (chats are chat previews) + if (StringUtils.containsIgnoreCase(pluginName, "contacts")) { + return "facebook-contact"; + } + if (StringUtils.containsIgnoreCase(pluginName, "user_id")) { + return "facebook-account"; + } + } + return null; + } + + /** + * Generic per-artifact subtype. A module can register many plugins, so the plugin name (often prefixed with "get_", + * which is stripped) plus hints from the artifact name ("Call", "Chat", ...) are used to build a specific subtype. + */ + private static String genericSubtype(String pluginName, String artifactName) { + + String mimePluginName = pluginName.toLowerCase().replace(".", ""); + mimePluginName = StringUtils.removeStart(mimePluginName, "get_"); + + // Facebook plugins share generic plugin names: the artifact name prefix (before "- ") + // is more specific, so use it instead + if (StringUtils.containsIgnoreCase(mimePluginName, "facebook")) { + mimePluginName = StringUtils.substringBefore(artifactName, "- ").toLowerCase(); + } + + // Chrome plugins are named per artifact already, so the artifact name alone is used + // (mimePluginName is intentionally ignored in this branch) + if (StringUtils.containsIgnoreCase(pluginName, "chrome")) { + return artifactNameToType(artifactName); + } else if (StringUtils.containsIgnoreCase(artifactName, "Call")) { + return mimePluginName + "-call"; + } else if (StringUtils.containsIgnoreCase(artifactName, "Chat")) { + return mimePluginName + "-chat"; + } else if (StringUtils.containsIgnoreCase(artifactName, "Message")) { + return mimePluginName + "-message"; + } else if (StringUtils.containsAnyIgnoreCase(artifactName, "Activity", "Activities")) { + return mimePluginName + "-activity"; + } else if (StringUtils.containsIgnoreCase(artifactName, "Contact")) { + return mimePluginName + "-contact"; + } else if (StringUtils.containsIgnoreCase(artifactName, "Conversation")) { + return mimePluginName + "-conversation"; + } else if (StringUtils.containsIgnoreCase(artifactName, "Autofill")) { + return mimePluginName + "-autofill"; + } else { + return artifactNameToType(artifactName); + } + } + + private static String artifactNameToType(String artifactName) { + String type = StringUtils.substringBefore(artifactName, " ("); + type = type.replace(" - ", "-").replace(" ", "-").replace("--", "-"); + return type; + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/AleappTask.java b/iped-engine/src/main/java/iped/engine/task/leapp/AleappTask.java new file mode 100644 index 0000000000..32a5cf8741 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/AleappTask.java @@ -0,0 +1,601 @@ +package iped.engine.task.leapp; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.file.PathUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.tika.mime.MediaType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import iped.configuration.Configurable; +import iped.data.IItem; +import iped.data.IItemReader; +import iped.datasource.IDataSource; +import iped.engine.config.ALeappConfig; +import iped.engine.config.ConfigurationManager; +import iped.engine.core.Worker.ProcessTime; +import iped.engine.data.Item; +import iped.engine.datasource.UfedXmlReader; +import iped.engine.task.AbstractTask; +import iped.engine.task.ExportFileTask; +import iped.engine.task.leapp.conversation.ConversationViewSpec; +import iped.parsers.android.backup.AndroidBackupParser; +import iped.parsers.python.PythonParser; +import iped.properties.BasicProps; +import iped.properties.ExtraProperties; +import iped.search.IItemSearcher; +import jep.Jep; +import jep.python.PyObject; + +public class AleappTask extends AbstractTask { + + protected static final Logger logger = LoggerFactory.getLogger(AleappTask.class); + + public static final String ALEAPP_APPLICATION_PREFIX = "x-aleapp-"; + + public static final MediaType ALEAPP_CASE_MEDIATYPE = MediaType.application(ALEAPP_APPLICATION_PREFIX + "case"); + public static final MediaType ALEAPP_CATEGORY_MEDIATYPE = MediaType.application(ALEAPP_APPLICATION_PREFIX + "category"); + public static final MediaType ALEAPP_PLUGIN_RESULTS_MEDIATYPE = MediaType.application(ALEAPP_APPLICATION_PREFIX + "plugin-results"); + public static final MediaType ALEAPP_ACTIVITY_MEDIATYPE = MediaType.application(ALEAPP_APPLICATION_PREFIX + "activity"); + public static final MediaType ALEAPP_DEVICE_INFO_MEDIATYPE = MediaType.application(ALEAPP_APPLICATION_PREFIX + "deviceinfo"); + + public static final String ALEAPP_PLUGIN_CATEGORY_KEY = "aleapp_category"; + + // Temp attribute (survives the ProcessTime.LATER queue) carrying the "aleapp:

" metadata keys of a row + // subitem whose values are device file paths. Resolving each path needs an index search, too costly to run per + // row inside the plugin call, so it is deferred: the subitem comes back through process() and each recorded path + // is resolved to its case item and added to LINKED_ITEMS (see processDeferredFilePathLinks). + public static final String ALEAPP_METADATA_PATHS = "aleapp_metadata_paths"; + + private static final String DEVICE_INFO_HTML = "DeviceInfo.html"; + + private static final String CASE_EVIDENCE_NAME = "ALEAPP_Results"; + + private static final String ZIP_EXT = "zip"; + private static final String UFDR_EXT = "ufdr"; + + private static final Set DUMP_ROOT_FOLDER_NAMES = Set.of("Dump", "backup"); + + // top level folders of an android file system, used to tell if ALEAPP should run over a extraction (see isAndroidFileSystem) + private static final Set ANDROID_ROOT_FOLDER_NAMES = Set.of("data", "system"); + + // data_views is not copied to metadata: it is consumed by ConversationViewSpec + // to build chat-preview items (see LavaInsertSqliteDataInterceptor) + private static final Set ARTIFACT_INFO_KEYS_TO_IGNORE = Set + .of("function", "paths", "requirements", "output_types", "notes", "sample_data", "artifact_icon", + ConversationViewSpec.DATA_VIEWS_KEY); + + public static final String ALEAPP_METADATA_PREFIX = "aleapp:"; + public static final String ALEAPP_PLUGIN_METADATA_PREFIX = ALEAPP_METADATA_PREFIX + "plugin:"; + public static final String ALEAPP_EXTRACTION_TYPE_META = ALEAPP_METADATA_PREFIX + "extractionType"; + public static final String ALEAPP_PLUGIN_KEYNAME_META = ALEAPP_PLUGIN_METADATA_PREFIX + "key"; + + private static final String EXTRACTION_TYPE_ANDROID_BACKUP = "android-backup"; + private static final String EXTRACTION_TYPE_ZIP = "zip"; + private static final String EXTRACTION_TYPE_UFDR = "ufdr"; + private static final String EXTRACTION_TYPE_DUMP = "dump"; + + /** + * Guards all access to the shared ilapfuncs.identifiers dict: device_info() writes (serialized by + * IlapfuncsDeviceInfoInterceptor) and the write_device_info() iteration in processDeviceInfoEvidence(). + */ + public static final Object DEVICE_INFO_LOCK = new Object(); + + private volatile boolean initialized = false; + private static volatile boolean interceptorsInstalled = false; + + private Map selectedPlugins; + private ALeappConfig config; + + private Path outputFolder; + private String outputFolderBase; + + private Path exportFilesFolder; + + // The per-worker-thread Jep interpreter shared with PythonParser/PythonTask + // (see PythonParser.getJep()). Jep is thread-confined, but every AleappTask + // instance belongs to a single worker and process()/finish() always run on + // that worker's thread, so no dedicated Python thread is needed. The + // interpreter's lifecycle is owned by PythonParser: never close it here. + private Jep jep; + + public AleappTask() { + } + + @Override + public List> getConfigurables() { + return Arrays.asList(new ALeappConfig()); + } + + @Override + public boolean isEnabled() { + return config.isEnabled(); + } + + @Override + public void init(ConfigurationManager configurationManager) throws Exception { + + config = (ALeappConfig) configurationManager.findObject(ALeappConfig.class); + + outputFolder = Files.createTempDirectory("aleapp-output-"); + exportFilesFolder = outputFolder.resolve("data"); + } + + public void initialize() throws Exception { + if (!initialized) { + synchronized (this) { + if (!initialized) { + doSetup(); + initialized = true; + } + } + } + } + + private void doSetup() throws Exception { + + // reuses the worker thread's interpreter, shared with PythonParser/PythonTask. + // NOTE: the interpreter namespace is shared with the python parsers/tasks + // running on this thread, so every global this integration creates is either + // prefixed with _iped_leapp_ or deleted right after use. + jep = PythonParser.getJep(); + if (jep == null) { + logger.error("Python environment not available, ALeapp task disabled."); + return; + } + + jep.exec("import sys"); + jep.exec("sys.path.append('" + config.getAleappFolder().getCanonicalPath() + "')"); + + // SharedInterpreter instances all share the same Python module state, so interceptors are + // installed only ONCE globally, even though each worker + // thread has its own Jep instance + synchronized (AleappTask.class) { + if (!interceptorsInstalled) { + LeappInterceptors interceptors = new LeappInterceptors(); + interceptors.install(jep); + interceptorsInstalled = true; + } + } + + // load all available plugins + // (mimics https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/aleapp.py#L181) + jep.exec("import scripts.plugin_loader"); + jep.exec("_iped_leapp_plugins = list(scripts.plugin_loader.PluginLoader().plugins)"); + + @SuppressWarnings("unchecked") + List availablePlugins = (List) jep.getValue("_iped_leapp_plugins"); + + selectedPlugins = availablePlugins + .stream() + .map(PluginSpec::new) + .filter(plugin -> config.isPluginIncluded(plugin.getModuleName())) + .collect(Collectors.toMap(PluginSpec::getName, Function.identity())); + + // the PyObjects held by PluginSpec keep their own references: the temp global can be removed + jep.exec("del _iped_leapp_plugins"); + + jep.exec("import scripts.ilapfuncs"); + jep.exec("import scripts.context"); + + // mimics https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/aleapp.py#L307 + jep.exec("_iped_leapp_out_params = scripts.ilapfuncs.OutputParameters('" + outputFolder.toString() + "', 'ALEAPP_Reports')"); + outputFolderBase = jep.getValue("_iped_leapp_out_params.output_folder_base", String.class); + + // the Context keeps the reference: the temp global can be removed + jep.exec("scripts.context.Context.set_output_params(_iped_leapp_out_params)"); + jep.exec("del _iped_leapp_out_params"); + } + + @Override + public void process(IItem item) throws Exception { + + initialize(); + + if (jep == null) { + // python environment not available (see doSetup) + return; + } + + if (isExtractionRoot(item)) { + processExtractionRoot(item); + } else if (isCaseEvidence(item)) { + processCaseEvidence(item); + } else if (isCategoryEvidence(item)) { + processCategoryEvidence(item); + } else if (isPluginEvidence(item)) { + processPluginEvidence(item); + } else if (isDeviceInfoEvidence(item)) { + processDeviceInfoEvidence(item); + } else if (isDeferredFilePathLinks(item)) { + processDeferredFilePathLinks(item); + } + } + + private boolean isExtractionRoot(IItem evidence) { + + if (AndroidBackupParser.SUPPORTED_TYPES.contains(evidence.getMediaType())) { + return true; + } + + // in a ufdr the device tree does not hang from the ufdr item itself, but from its file system + // item, so the latter is the extraction root (see isUfdrFileSystem) + if (isUfdrFileSystem(evidence)) { + return true; + } + + String realName = evidence.getName(); + if (evidence.isRoot()) { + // if evidence is root, its realname can be changed via -dname parameter, so we + // need to get it from other source. + realName = evidence.getDataSource().getSourceFile().getName(); + } + + return DUMP_ROOT_FOLDER_NAMES.contains(realName); + } + + /** + * Checks if the item is a file system item of a ufdr: the virtual folder UfedXmlReader creates for the "fs" + * attribute of report.xml entries (e.g. "extraction-001.ufdr/EXTRACTION_FFS.zip"). The device paths + * hang from it, so it is the extraction root, and not the ufdr item. + */ + private boolean isUfdrFileSystem(IItem evidence) { + + // file system items are always virtual folders, direct children of the ufdr item + if (evidence.isRoot() || !evidence.isDir() + || UfedXmlReader.DECODED_DATA_FOLDER_NAME.equals(evidence.getName())) { + return false; + } + + String path = evidence.getPath(); + String parentPath = StringUtils.substringBeforeLast(path, "/"); + if (StringUtils.isEmpty(parentPath) || parentPath.equals(path)) { + return false; + } + + // a ufdr processed as a subitem of another evidence keeps its file name in the path + if (UFDR_EXT.equalsIgnoreCase(FilenameUtils.getExtension(parentPath))) { + return true; + } + + // a ufdr processed as a case root may have been renamed via -dname, so check the data source + // file instead, requiring a direct child of the root (whose path has no separator) + IDataSource dataSource = evidence.getDataSource(); + File sourceFile = dataSource != null ? dataSource.getSourceFile() : null; + + return sourceFile != null && !StringUtils.contains(parentPath, '/') + && UFDR_EXT.equalsIgnoreCase(FilenameUtils.getExtension(sourceFile.getName())); + } + + private boolean isCaseEvidence(IItem evidence) { + return ALEAPP_CASE_MEDIATYPE.equals(evidence.getMediaType()); + } + + private boolean isCategoryEvidence(IItem evidence) { + return ALEAPP_CATEGORY_MEDIATYPE.equals(evidence.getMediaType()); + } + + private boolean isPluginEvidence(IItem evidence) { + return ALEAPP_PLUGIN_RESULTS_MEDIATYPE.equals(evidence.getMediaType()); + } + + private boolean isDeviceInfoEvidence(IItem evidence) { + return ALEAPP_DEVICE_INFO_MEDIATYPE.equals(evidence.getMediaType()); + } + + private boolean isDeferredFilePathLinks(IItem evidence) { + return evidence.getTempAttribute(ALEAPP_METADATA_PATHS) != null; + } + + private void processExtractionRoot(IItem rootEvidence) { + + if (selectedPlugins.isEmpty()) { + return; + } + + // creates a subitem to represent the ALeapp report + Item caseEvidence = (Item) rootEvidence.createChildItem(); + caseEvidence.setMediaType(ALEAPP_CASE_MEDIATYPE); + + String name = CASE_EVIDENCE_NAME; + caseEvidence.setName(name); + caseEvidence.setPath(rootEvidence.getPath() + "/" + name); + caseEvidence.setIdInDataSource(""); + caseEvidence.setHasChildren(true); + caseEvidence.setExtraAttribute(BasicProps.TREENODE, true); + caseEvidence.setExtraAttribute(ExtraProperties.DECODED_DATA, true); + + String extractionType; + if (AndroidBackupParser.SUPPORTED_TYPES.contains(rootEvidence.getMediaType())) { + extractionType = EXTRACTION_TYPE_ANDROID_BACKUP; + } else if (isUfdrFileSystem(rootEvidence)) { + extractionType = EXTRACTION_TYPE_UFDR; + } else { + String realExt = rootEvidence.getExt(); + if (rootEvidence.isRoot()) { + String realName = rootEvidence.getDataSource().getSourceFile().getName(); + realExt = FilenameUtils.getExtension(realName); + } + extractionType = ZIP_EXT.equalsIgnoreCase(realExt) ? EXTRACTION_TYPE_ZIP : EXTRACTION_TYPE_DUMP; + } + caseEvidence.getMetadata().set(ALEAPP_EXTRACTION_TYPE_META, extractionType); + + worker.processNewItem(caseEvidence, ProcessTime.LATER); + } + + private void processCaseEvidence(IItem caseEvidence) throws Exception { + + if (!isSupportedAndroidExtraction(caseEvidence)) { + caseEvidence.setToIgnore(true); + return; + } + + Map categoryItems = new HashMap<>(); + + // creates one subitem for each plugin execution + // (PluginSpec getters call PyObject.getAttr on this worker's interpreter) + for (PluginSpec plugin : selectedPlugins.values()) { + + Item categoryItem = categoryItems.computeIfAbsent(plugin.getCategory(), name -> { + Item categoryEvidence = (Item) caseEvidence.createChildItem(); + categoryEvidence.setMediaType(ALEAPP_CATEGORY_MEDIATYPE); + + categoryEvidence.setName(name); + categoryEvidence.setExtension(""); + categoryEvidence.setPath(caseEvidence.getPath() + "/" + name); + categoryEvidence.setIdInDataSource(""); + categoryEvidence.setExtraAttribute(ExtraProperties.DECODED_DATA, true); + + worker.processNewItem(categoryEvidence, ProcessTime.LATER); + + return categoryEvidence; + }); + + Item pluginEvidence = (Item) categoryItem.createChildItem(); + pluginEvidence.setMediaType(ALEAPP_PLUGIN_RESULTS_MEDIATYPE); + pluginEvidence.setTempAttribute(ALEAPP_PLUGIN_CATEGORY_KEY, categoryItem); + + String name = StringUtils.firstNonBlank((String) plugin.getArtifactInfo().get("name"), plugin.getName()); + pluginEvidence.setName(name); + pluginEvidence.setExtension(""); + pluginEvidence.setPath(categoryItem.getPath() + "/" + name); + pluginEvidence.setIdInDataSource(""); + pluginEvidence.setExtraAttribute(ExtraProperties.DECODED_DATA, true); + + pluginEvidence.getMetadata().set(ALEAPP_PLUGIN_KEYNAME_META, plugin.getName()); + pluginEvidence.getMetadata().set(ALEAPP_PLUGIN_METADATA_PREFIX + "moduleName", plugin.getModuleName()); + for (Entry entry : plugin.getArtifactInfo().entrySet()) { + if (ARTIFACT_INFO_KEYS_TO_IGNORE.contains(entry.getKey())) { + continue; + } + pluginEvidence.getMetadata().set(ALEAPP_PLUGIN_METADATA_PREFIX + entry.getKey(), entry.getValue().toString()); + } + + worker.processNewItem(pluginEvidence, ProcessTime.LATER); + } + + // creates subitem to hold device info collected + Item deviceInfoEvidence = (Item) caseEvidence.createChildItem(); + deviceInfoEvidence.setName(DEVICE_INFO_HTML); + deviceInfoEvidence.setMediaType(ALEAPP_DEVICE_INFO_MEDIATYPE); + deviceInfoEvidence.setPath(caseEvidence.getPath() + "/" + DEVICE_INFO_HTML); + deviceInfoEvidence.setIdInDataSource(""); + worker.processNewItem(deviceInfoEvidence, ProcessTime.LATER); + } + + /** + * Checks if the extraction root really holds an Android extraction the plugins can decode, applying the checks each + * extraction type needs. + */ + private boolean isSupportedAndroidExtraction(IItem caseEvidence) { + + String extractionType = caseEvidence.getMetadata().get(ALEAPP_EXTRACTION_TYPE_META); + + switch (extractionType) { + + case EXTRACTION_TYPE_ANDROID_BACKUP: + // expanded as a tar, holding a file system of its own ("apps/", "shared/0"...) + return true; + + case EXTRACTION_TYPE_UFDR: + return isAndroidFileSystem(caseEvidence); + + case EXTRACTION_TYPE_DUMP: + case EXTRACTION_TYPE_ZIP: + return isAndroidFileSystem(caseEvidence) && isInsideRealDump(caseEvidence); + + default: + throw new IllegalStateException("Unexpected extraction type: " + extractionType); + } + } + + /** + * Checks if the case evidence really corresponds to an Android dump, by looking for a folder every Android device + * has ("/data/data/com.android.vending"). Avoids running plugins over unrelated zips/folders that just happen to + * match the extraction root naming rules. + */ + private boolean isInsideRealDump(IItem caseEvidence) { + + String checkFolder = "/data/data/com.android.vending"; + + IItemSearcher searcher = (IItemSearcher) caseData.getCaseObject(IItemSearcher.class.getName()); + + return LeappUtils.findItemByPath(searcher, getExtractionRootPath(caseEvidence), checkFolder) != null; + } + + /** + * Checks if the extraction root holds an Android file system, by looking for at least one of its top level folders + * ({@link #ANDROID_ROOT_FOLDER_NAMES}). + * A ufdr can hold an iOS, KeyChain or KeyStore extractions, whose file system have no plugin can decode. + */ + @SuppressWarnings("resource") + private boolean isAndroidFileSystem(IItem caseEvidence) { + + String rootPath = getExtractionRootPath(caseEvidence); + + IItemSearcher searcher = (IItemSearcher) caseData.getCaseObject(IItemSearcher.class.getName()); + + // anyMatch short-circuits: the next folder is only searched if the previous one was not found + return ANDROID_ROOT_FOLDER_NAMES.stream() + .anyMatch(folder -> LeappUtils.findItemByPath(searcher, rootPath, "/" + folder) != null); + } + + /** The in-case path of the extraction root, i.e. the path prefix shared by every item of this extraction. */ + private String getExtractionRootPath(IItem aleappEvidence) { + return StringUtils.substringBeforeLast(aleappEvidence.getPath(), "/" + CASE_EVIDENCE_NAME); + } + + private void processCategoryEvidence(IItem categoryEvidence) throws Exception { + if (!categoryEvidence.hasChildren()) { + logger.info("Ignoring Aleapp category {}: no children", categoryEvidence.getName()); + categoryEvidence.setToIgnore(true); + } + } + + private void processPluginEvidence(IItem pluginEvidence) throws Exception { + + String pluginName = pluginEvidence.getMetadata().get(ALEAPP_PLUGIN_KEYNAME_META); + PluginSpec plugin = selectedPlugins.get(pluginName); + if (plugin == null) { + throw new IllegalStateException("Plugin should have been found: " + pluginName); + } + + // look for the files the plugin needs + // (mimics https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/aleapp.py#L386) + IItemSearcher searcher = (IItemSearcher) caseData.getCaseObject(IItemSearcher.class.getName()); + FileSeeker seeker = new FileSeeker(getExtractionRootPath(pluginEvidence), exportFilesFolder, searcher); + + try { + + List filesFound = seeker.searchItems(plugin.getSearchGlobs()); + + if (filesFound.isEmpty()) { + logger.warn("Ignoring Aleapp {} plugin: no files found", pluginName); + pluginEvidence.setToIgnore(true); + return; + } + + LeappContext.create(seeker, worker, jep, plugin, pluginEvidence, filesFound); + + try { + // mimics https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/aleapp.py#L409 + Path categoryFolder = Paths.get(outputFolderBase, "_HTML", plugin.getCategory()); + Files.createDirectories(categoryFolder); + + // export all files found (and -wal/-journal companions of sqlite items) + ArrayList filesFoundStringList = seeker.exportItems(filesFound); + + // call the plugin method + // plugin.method(files_found, report_folder, seeker, wrap_text) + // + // https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/aleapp.py#L418 + plugin.getMethod().call(filesFoundStringList, categoryFolder.toString(), seeker, false); + + } catch (Exception e) { + logger.error("Aleapp {} plugin ended prematurely: {}", pluginName, ExceptionUtils.getMessage(e)); + logger.warn(pluginName, e); + } + + if (!pluginEvidence.hasChildren()) { + logger.warn("Ignoring Aleapp {} plugin: no children", pluginName); + pluginEvidence.setToIgnore(true); + } + } finally { + try { + // clears this thread's Python Context state, mirroring aleapp.py's per-artifact + // Context.clear() (state is thread-local, see context_thread_local_patch.py) + jep.exec("import scripts.context"); + jep.exec("scripts.context.Context.clear()"); + } catch (Exception e) { + logger.warn("Failed to clear Python Context after {} plugin", pluginName, e); + } + seeker.cleanup(); + LeappContext.clear(); + } + } + + private void processDeviceInfoEvidence(IItem deviceInfoEvidence) throws Exception { + + // https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/aleapp.py#L432 + jep.exec("import scripts.ilapfuncs"); + + Path deviceInfoPath = Files.createTempFile("screen_output_file_path_devinfo", ".html"); + try { + // OutputParameters attributes are class-level and identifiers is a module + // global, both shared across all workers: serialize the path switch + write + // against concurrent device_info() calls (see IlapfuncsDeviceInfoInterceptor) + synchronized (DEVICE_INFO_LOCK) { + jep.exec("scripts.ilapfuncs.OutputParameters.screen_output_file_path_devinfo = '" + deviceInfoPath.toString() + "'"); + jep.exec("scripts.ilapfuncs.write_device_info()"); + } + + byte[] deviceInfoBytes = Files.readAllBytes(deviceInfoPath); + + if (deviceInfoBytes.length > 0) { + ExportFileTask.getLastInstance().insertIntoStorage(deviceInfoEvidence, deviceInfoBytes, deviceInfoBytes.length); + } else { + deviceInfoEvidence.setToIgnore(true); + } + } finally { + Files.deleteIfExists(deviceInfoPath); + } + } + + /** + * Resolves the deferred file-path links of a plugin row subitem. The interceptor recorded the "aleapp:<header>" + * metadata keys whose values are device file paths in the {@link #ALEAPP_METADATA_PATHS} temp attribute; here each + * value is resolved to its case item (via {@link LeappUtils#findItemByPath}) and added to LINKED_ITEMS. Doing it here + * (instead of inside the plugin call) spreads the one-search-per-path cost across the worker pool. + */ + private void processDeferredFilePathLinks(IItem item) { + + @SuppressWarnings("unchecked") + List metadataKeys = (List) item.getTempAttribute(ALEAPP_METADATA_PATHS); + + IItemSearcher searcher = (IItemSearcher) caseData.getCaseObject(IItemSearcher.class.getName()); + String pathRoot = getExtractionRootPath(item); + + for (String metadataKey : metadataKeys) { + String pathValue = StringUtils.trimToEmpty(item.getMetadata().get(metadataKey)); + if (pathValue.isEmpty() || pathValue.equals("/")) { + continue; + } + IItemReader found = LeappUtils.findItemByPath(searcher, pathRoot, pathValue); + if (found != null) { + item.getMetadata().add(ExtraProperties.LINKED_ITEMS, + ExtraProperties.GLOBAL_ID + ":" + found.getExtraAttribute(ExtraProperties.GLOBAL_ID)); + } + } + } + + @Override + public void finish() throws Exception { + // the Jep interpreter is shared with PythonParser/PythonTask and owned by them: do NOT close it here + jep = null; + + try { + if (outputFolder != null && Files.exists(outputFolder)) { + PathUtils.deleteDirectory(outputFolder); + } + } catch (Exception e) { + logger.warn("Failed to delete outputFolder: {}", outputFolder, e); + } + outputFolder = null; + exportFilesFolder = null; + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/CallInterceptor.java b/iped-engine/src/main/java/iped/engine/task/leapp/CallInterceptor.java new file mode 100644 index 0000000000..1c9f2c731e --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/CallInterceptor.java @@ -0,0 +1,151 @@ +package iped.engine.task.leapp; + +import java.util.Arrays; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jep.Jep; +import jep.PyMethod; +import jep.python.PyCallable; + +/** + * A Java class that can be used as a Python function. + * + * It intercepts calls to Python function and then calls the original function. + */ +public class CallInterceptor { + + protected static final Logger logger = LoggerFactory.getLogger(CallInterceptor.class); + + private String pythonModule; + private String pythonFunction; + private boolean isClassMethod = false; + + private PyCallable originalCall; + + public CallInterceptor(String pythonModule, String pythonFunction, boolean isClassMethod) { + this.pythonModule = pythonModule; + this.pythonFunction = pythonFunction; + this.isClassMethod = isClassMethod; + } + + public CallInterceptor(String pythonModule, String pythonFunction) { + this(pythonModule, pythonFunction, false); + } + + public void install(Jep jep) { + + if (StringUtils.isNotBlank(pythonModule)) { + jep.exec("import " + pythonModule); + } + + // "_iped_leapp_interceptor" is only a temporary handoff variable used during + // install: both branches below capture this interceptor immediately, so the + // global is deleted at the end. The name is prefixed because the interpreter + // namespace is shared with PythonParser/PythonTask scripts. + jep.set("_iped_leapp_interceptor", this); + + if (isClassMethod) { + + String clazz = StringUtils.substringBeforeLast(pythonFunction, "."); + String method = StringUtils.substringAfterLast(pythonFunction, "."); + + originalCall = jep.getValue("getattr(" + clazz + ", \"" + method + "\")", PyCallable.class); + if (originalCall == null) { + throw new IllegalStateException("Original call is null for: " + pythonFunction); + } + + // The keyword-only default "_interceptor=..." captures the interceptor at def + // time. Referencing the global directly in the body would be resolved at CALL + // time (late binding), breaking once the temp global is deleted below. + jep.exec("def _iped_leapp_interceptor_method(self, *args, _interceptor=_iped_leapp_interceptor, **kwargs):" + + " return _interceptor.call(self, *args, **kwargs)"); + + // the class attribute keeps the function reference: the temp global can go + jep.exec("setattr(" + clazz + ", \"" + method + "\", _iped_leapp_interceptor_method)"); + jep.exec("del _iped_leapp_interceptor_method"); + + } else { + + originalCall = jep.getValue(pythonFunction, PyCallable.class); + if (originalCall == null) { + throw new IllegalStateException("Original call is null for: " + pythonFunction); + } + + // "_iped_leapp_interceptor.call" is evaluated NOW, at exec time: the module + // attribute ends up holding a bound callable, independent of the global name. + jep.exec(pythonFunction + " = _iped_leapp_interceptor.call"); + } + + jep.exec("del _iped_leapp_interceptor"); + } + + // The three call() overloads below exist because Jep dispatches by the Python + // call shape (positional-only, kwargs-only, or both), selected via @PyMethod. + // Subclasses overriding call(Object[], Map) must repeat the @PyMethod + // annotation, otherwise Jep dispatch breaks. + + @PyMethod(varargs = true, kwargs = true) + public Object call(Object[] args, Map kwargs) throws Exception { + + if (logger.isDebugEnabled()) { + logger.debug("JAVA INTERCEPTOR: ---- 001 ----"); + logger.debug("JAVA INTERCEPTOR: call: " + pythonFunction); + logger.debug("JAVA INTERCEPTOR: varargs: " + Arrays.toString(args)); + logger.debug("JAVA INTERCEPTOR: kwargs: " + kwargs); + } + + handleArgs(args, kwargs); + + return this.originalCall.call(args, kwargs); + } + + @PyMethod(varargs = true, kwargs = false) + public Object call(Object... args) throws Exception { + return call(args, null); + } + + @PyMethod(varargs = false, kwargs = true) + public Object call(Object str, Map kwargs) throws Exception { + Object[] args = new Object[] { str }; + return call(args, kwargs); + } + + protected void handleArgs(Object[] args, Map kwargs) throws Exception { + } + + protected Object getArgumentValue(String key, int index, Object[] args, Map kwargs) { + if (key != null && kwargs != null && kwargs.containsKey(key)) { + return kwargs.get(key); + } + + if (index >= 0 && args != null && args.length > index) { + return args[index]; + } + + return null; + } + + protected void setArgumentValue(String key, int index, Object value, Object[] args, Map kwargs) { + + if (logger.isDebugEnabled()) { + logger.debug(String.format("Setting value: [%s,%d] <= %s", key, index, value)); + } + + if (key != null && kwargs != null && kwargs.containsKey(key)) { + kwargs.put(key, value); + return; + } + + if (index >= 0 && args != null && args.length > index) { + args[index] = value; + return; + } + + throw new IllegalArgumentException(String.format("Invalid key or index: [%s,%d] <= %s %s", key, index, Arrays.toString(args), kwargs)); + } + +} \ No newline at end of file diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/FileSeeker.java b/iped-engine/src/main/java/iped/engine/task/leapp/FileSeeker.java new file mode 100644 index 0000000000..6799587b64 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/FileSeeker.java @@ -0,0 +1,270 @@ +package iped.engine.task.leapp; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.io.file.PathUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import iped.data.IItemReader; +import iped.properties.BasicProps; +import iped.search.IItemSearcher; +import iped.utils.DateUtil; +import iped.utils.IOUtil; + +public class FileSeeker { + + protected static final Logger logger = LoggerFactory.getLogger(FileSeeker.class); + + private String pathRoot; + private IItemSearcher searcher; + private Path exportFolder; + + private Map exportedFiles = new HashMap<>(); + + // Public snake_case fields: LEAPP plugins access these directly on the Python + // side as attributes of the "seeker" object (e.g. seeker.data_folder), so the + // names MUST match FileSeekerBase attributes. Do not rename. + public String data_folder; + public HashMap file_infos = new HashMap<>(); + + public static class FileInfo { + public String source_path; + // epoch SECONDS, mirroring Python's FileInfo which stores os.stat() + // st_ctime/st_mtime floats (0 when unknown, like FileSeekerTar does). + // Plugins compare/convert these as numbers, so java.util.Date must not + // be used here. + public double creation_date; + public double modification_date; + + public FileInfo(String source_path, Date creationDate, Date modificationDate) { + this.source_path = source_path; + this.creation_date = creationDate == null ? 0 : creationDate.getTime() / 1000.0; + this.modification_date = modificationDate == null ? 0 : modificationDate.getTime() / 1000.0; + } + } + + public FileSeeker(String pathRoot, Path exportFolder, IItemSearcher searcher) { + this.pathRoot = pathRoot; + this.searcher = searcher; + this.exportFolder = exportFolder.toAbsolutePath().normalize(); + this.data_folder = this.exportFolder.toString(); + } + + /** + * Python-facing API, mirroring ALEAPP's FileSeekerBase: + * + * def search(self, filepattern, return_on_first_hit=False): '''Returns a list of paths for files/folders that + * matched''' + * + * Unlike ALEAPP seekers, the matched files live inside the IPED case, so each hit is exported to the local export + * folder and the local paths are returned. For sqlite hits, -wal and -journal companion files are exported too, so + * the databases can be opened consistently. + * + * NOTE: Jep cannot map Python keyword arguments onto Java methods, so plugins must call seeker.search(pattern, True) + * positionally; the kwarg form return_on_first_hit=True is not supported. + * + * https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/scripts/search_files.py#L57 + */ + public List search(String filepattern) throws IOException { + return search(filepattern, false); + } + + public List search(String filepattern, boolean returnOnFirstHit) throws IOException { + + List items = searchItems(List.of(filepattern)); + + if (returnOnFirstHit && !items.isEmpty()) { + // mimics ALEAPP semantics: a list containing only the first hit + items = items.subList(0, 1); + } + + return exportItems(items); + } + + /** + * Exports the given case items to the local export folder and returns their local paths, in the same order. For + * sqlite items, -wal and -journal companion files are exported too, so the databases can be opened consistently. + */ + public ArrayList exportItems(List items) throws IOException { + + ArrayList paths = new ArrayList<>(); + + // ids of the items themselves: avoids exporting a -wal/-journal companion a + // second time when the plugin's search pattern also matched it directly + Set itemIds = new HashSet<>(); + for (IItemReader item : items) { + itemIds.add(item.getId()); + } + + for (IItemReader item : items) { + + paths.add(exportItemToFile(item).toString()); + + if ("sqlite".equals(item.getType())) { + for (IItemReader walOrJournal : getJournalAndWalFiles(item)) { + if (!itemIds.contains(walOrJournal.getId())) { + exportItemToFile(walOrJournal); + } + } + } + } + + return paths; + } + + /** + * Java-facing search used by the task itself: returns the matched case items without exporting them. + */ + public List searchItems(List globPatterns) { + + String query = "(" + + globPatterns.stream() + .map(glob -> LeappUtils.buildFileSearchQuery(pathRoot, glob)) + .collect(Collectors.joining(") OR (")) + + ")"; + + logger.debug("query=[{}], patterns=[{}]", query, globPatterns); + + // The Lucene query is a high-recall pre-filter and may return extra hits (see + // LeappUtils.globToLuceneQuery), so re-check each hit strictly: it must start + // with the evidence root path and fnmatch at least one of the glob patterns. + List hits = searcher + .search(query) // + .stream() // + .filter(item -> item.getPath().startsWith(pathRoot)) // + .filter(item -> !item.getPath().substring(pathRoot.length()).contains(">>")) // + .filter(item -> { + for (String glob : globPatterns) { + if (LeappUtils.matchesGlob(item, glob)) { + return true; + } + } + return false; + }) + .collect(Collectors.toList()); + + // Two or more items can share the same path (e.g. an active file plus a carved + // copy or one recovered from deletion), which would collide in the export + // folder. Keep only the most significant item per path: active > deleted > + // carved; ties keep the first hit. + Map bestItemPerPath = new LinkedHashMap<>(); + for (IItemReader item : hits) { + bestItemPerPath.merge(item.getPath(), item, + (current, candidate) -> significanceRank(candidate) < significanceRank(current) ? candidate : current); + } + + return new ArrayList<>(bestItemPerPath.values()); + } + + /** + * Lower is better: an active file is preferred over one recovered from deletion, which is preferred over a carved one + */ + private static int significanceRank(IItemReader item) { + if (item.isCarved()) { + return 2; + } + if (item.isDeleted()) { + return 1; + } + return 0; + } + + public Path exportItemToFile(IItemReader item) throws IOException { + + Path filePath; + + if (IOUtil.hasFile(item)) { + + filePath = IOUtil.getFile(item).toPath(); + + } else { + + // we are sure that the item path starts with the evidence root path + // because searchItems() already filtered it, so this substring is safe + String fileRelativePathStr = item.getPath().substring(pathRoot.length() + 1); + filePath = exportFolder.resolve(fileRelativePathStr); + + // create the folder for the item + Files.createDirectories(filePath.getParent()); + + if (item.isDir()) { + Files.createDirectories(filePath); + } else { + if (!Files.exists(filePath) || item.getLength() == null || Files.size(filePath) != item.getLength()) { + + // copy the item to the folder + try (InputStream is = item.getBufferedInputStream()) { + Files.copy(is, filePath, StandardCopyOption.REPLACE_EXISTING); + } + } + } + + DateUtil.updatePathTimes(filePath, item); + } + + filePath = filePath.toAbsolutePath().normalize(); + + exportedFiles.put(filePath.toString(), item); + + // keep file_infos in sync: some plugins read seeker.file_infos[path] to recover + // the original source path and timestamps of an exported file + file_infos.put(filePath.toString(), new FileInfo(item.getPath(), item.getCreationDate(), item.getModDate())); + + return filePath; + } + + public List getJournalAndWalFiles(IItemReader item) { + if (!"sqlite".equals(item.getType())) { + return Collections.emptyList(); + } + + String basePath = item.getPath(); + String walPath = basePath + "-wal"; + String journalPath = basePath + "-journal"; + + return searcher.search(BasicProps.PATH + ":(\"" + walPath + "\" \"" + journalPath + "\")") + .stream() + .filter(i -> i.getPath().equals(walPath) || i.getPath().equals(journalPath)) + .collect(Collectors.toList()); + } + + // https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/scripts/search_files.py#L61 + public void cleanup() { + try { + if (Files.exists(exportFolder)) { + PathUtils.deleteDirectory(exportFolder); + } + } catch (IOException e) { + logger.warn("Failed to delete export folder: {}", exportFolder, e); + } + exportedFiles.clear(); + file_infos.clear(); + } + + public Map getExportedFiles() { + return exportedFiles; + } + + public String getPathRoot() { + return pathRoot; + } + + public IItemSearcher getSearcher() { + return searcher; + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/LeappContext.java b/iped-engine/src/main/java/iped/engine/task/leapp/LeappContext.java new file mode 100644 index 0000000000..840272d2b2 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/LeappContext.java @@ -0,0 +1,69 @@ +package iped.engine.task.leapp; + +import java.util.List; + +import iped.data.IItem; +import iped.data.IItemReader; +import iped.engine.core.Worker; +import jep.Jep; + +public class LeappContext { + + private FileSeeker fileSeeker; + private Worker worker; + private PluginSpec plugin; + private Jep jep; + private IItem pluginItem; + + private List foundFiles; + + private static final ThreadLocal threadLocal = new ThreadLocal<>(); + + private LeappContext(FileSeeker seeker, Worker worker, Jep jep, PluginSpec plugin, IItem pluginItem, List foundFiles) { + this.fileSeeker = seeker; + this.worker = worker; + this.plugin = plugin; + this.jep = jep; + this.foundFiles = foundFiles; + this.pluginItem = pluginItem; + } + + public static LeappContext create(FileSeeker seeker, Worker worker, Jep jep, PluginSpec plugin, IItem pluginItem, List foundFiles) { + LeappContext context = new LeappContext(seeker, worker, jep, plugin, pluginItem, foundFiles); + threadLocal.set(context); + return context; + } + + public static void clear() { + threadLocal.remove(); + } + + public static LeappContext get() { + return threadLocal.get(); + } + + public FileSeeker getFileSeeker() { + return fileSeeker; + } + + public Worker getWorker() { + return worker; + } + + public PluginSpec getPlugin() { + return plugin; + } + + public Jep getJep() { + return jep; + } + + public IItem getPluginItem() { + return pluginItem; + } + + public List getFoundFiles() { + return foundFiles; + } + +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/LeappInterceptors.java b/iped-engine/src/main/java/iped/engine/task/leapp/LeappInterceptors.java new file mode 100644 index 0000000000..2a1a24ff6e --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/LeappInterceptors.java @@ -0,0 +1,160 @@ +package iped.engine.task.leapp; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import iped.engine.task.leapp.interceptors.IlapfuncsDeviceInfoInterceptor; +import iped.engine.task.leapp.interceptors.IlapfuncsLogfuncInterceptor; +import iped.engine.task.leapp.interceptors.LavaInsertSqliteDataInterceptor; +import jep.Jep; + +public class LeappInterceptors { + + protected static final Logger logger = LoggerFactory.getLogger(LeappInterceptors.class); + + private List interceptors = new ArrayList<>(); + + public LeappInterceptors() { + interceptors.add(new IlapfuncsLogfuncInterceptor()); + // results are captured from lava_insert_sqlite_data (NOT tsv/timeline/html): + // it is the only output call receiving raw typed headers and raw values + interceptors.add(new LavaInsertSqliteDataInterceptor()); + // serializes concurrent writes to the shared ilapfuncs.identifiers dict + interceptors.add(new IlapfuncsDeviceInfoInterceptor()); + } + + public void install(Jep jep) { + + disableFunctions(jep); + + // patches LEAPP's Context so its per-plugin-run state becomes thread-local, + // isolating concurrent plugin runs on different IPED workers sharing the same + // Python interpreter (full rationale in the resource file) + execPythonResource(jep, "context_thread_local_patch.py"); + + for (CallInterceptor interceptor : interceptors) { + interceptor.install(jep); + } + } + + private void execPythonResource(Jep jep, String resourceName) { + String script; + try (InputStream is = getClass().getResourceAsStream(resourceName)) { + script = new String(is.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException("Failed to load resource: " + resourceName, e); + } + jep.exec(script); + } + + public void disableFunctions(Jep jep) { + + disableLavaFuncs(jep); + + // tsv, timeline and kml outputs are not used: results are captured from + // lava_insert_sqlite_data, which receives the raw typed data + disablePythonFunction(jep, "scripts.ilapfuncs", "scripts.ilapfuncs.tsv"); + disablePythonFunction(jep, "scripts.ilapfuncs", "scripts.ilapfuncs.timeline"); + disablePythonFunction(jep, "scripts.ilapfuncs", "scripts.ilapfuncs.kmlgen"); + disablePythonFunction(jep, "scripts.ilapfuncs", "scripts.ilapfuncs.set_media_references"); + + // avoid to use backslash as path separator + disablePythonFunction(jep, "scripts.ilapfuncs", "scripts.ilapfuncs.is_platform_windows", "False"); + + // html output is not used + disablePythonClass(jep, "scripts.artifact_report", "scripts.artifact_report.ArtifactHtmlReport"); + + patchMediaFunctions(jep); + } + + /** + * Media handling: plugins register media files through check_in_media, whose return value ends up in the + * data_list cells of 'media' typed columns. + * + * The original implementation copies files and registers them in the LAVA media database (disabled here). It is + * replaced by a version that simply returns the extraction path of the media file (resolved against the + * thread-local Context), which LavaInsertSqliteDataInterceptor then maps back to the original case item and adds + * to the subitem's linkedItems. + * + * get_data_list_with_media is also patched: artifact_processor calls it whenever a 'media' column exists (BEFORE + * checking output types) to build the html/tsv views, and it would crash against the disabled + * lava_get_full_media_info stub. Since html/tsv outputs are disabled, it just passes data through. + */ + private void patchMediaFunctions(Jep jep) { + jep.exec("import scripts.ilapfuncs"); + jep.exec("import scripts.context"); + + // the keyword-only default "_context=..." captures the Context class at def + // time, so no extra global is left behind after the temp names are deleted + jep.exec("def _iped_leapp_check_in_media(file_path, name='', *args, _context=scripts.context.Context, **kwargs):" + + " return _context.get_source_file_path(file_path)"); + jep.exec("scripts.ilapfuncs.check_in_media = _iped_leapp_check_in_media"); + jep.exec("del _iped_leapp_check_in_media"); + + // embedded media has no corresponding case item to link: keep it disabled + disablePythonFunction(jep, "scripts.ilapfuncs", "scripts.ilapfuncs.check_in_embedded_media"); + + jep.exec("scripts.ilapfuncs.get_data_list_with_media = lambda media_header_info, data_list: (data_list, data_list)"); + } + + private void disableLavaFuncs(Jep jep) { + + // Important!! ---> lavafuncs MUST be disabled before import ilapfuncs + + // disable lava init/finalize (not called by IPED explicitly) + // https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/aleapp.py#L312 + disablePythonFunction(jep, "scripts.lavafuncs", "scripts.lavafuncs.initialize_lava"); + disablePythonFunction(jep, "scripts.lavafuncs", "scripts.lavafuncs.lava_finalize_output"); + + // artifact_processor unpacks lava_process_artifact's return: keep the shape. + // LavaInsertSqliteDataInterceptor does not need these values, the types come + // from the data_headers tuples + disablePythonFunction(jep, "scripts.lavafuncs", "scripts.lavafuncs.lava_process_artifact", "[None, None, None]"); + + // disabled here (BEFORE scripts.ilapfuncs is imported, so its from-import + // captures the no-op) as a safety net for plugins importing it directly from + // lavafuncs; the binding actually used by artifact_processor is + // scripts.ilapfuncs.lava_insert_sqlite_data, which is replaced later by + // LavaInsertSqliteDataInterceptor + disablePythonFunction(jep, "scripts.lavafuncs", "scripts.lavafuncs.lava_insert_sqlite_data"); + + // used in ilapfuncs.artifact_processor function (get_data_list_with_media) + // https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/scripts/ilapfuncs.py#L402 + disablePythonFunction(jep, "scripts.lavafuncs", "scripts.lavafuncs.lava_get_full_media_info", "['', '', '', '', '', '', '', '']"); + + // used by the original check_in_media/check_in_embedded_media (replaced in + // patchMediaFunctions, so these are just a safety net) + disablePythonFunction(jep, "scripts.lavafuncs", "scripts.lavafuncs.lava_insert_sqlite_media_item"); + disablePythonFunction(jep, "scripts.lavafuncs", "scripts.lavafuncs.lava_insert_sqlite_media_references"); + disablePythonFunction(jep, "scripts.lavafuncs", "scripts.lavafuncs.lava_get_media_references"); + } + + private void disablePythonFunction(Jep jep, String module, String function, String returnValue) { + jep.exec("import " + module); + jep.exec(function + " = lambda *args, **kwargs: " + returnValue); + } + + private void disablePythonFunction(Jep jep, String module, String function) { + disablePythonFunction(jep, module, function, "None"); + } + + private void disablePythonClass(Jep jep, String module, String clazz) { + jep.exec("import " + module); + jep.exec("for name in dir(" + clazz + "):\n" + + " # We only want to replace public methods, not special ones like __init__\n" + + " if not name.startswith('__'):\n" + + " attr = getattr(" + clazz + ", name)\n" + + " \n" + + " # Check if the attribute is a callable method\n" + + " if callable(attr):\n" + + " # Replace the method with a function that does nothing\n" + + " setattr(" + clazz + ", name, lambda *args, **kwargs: None)"); + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/LeappUtils.java b/iped-engine/src/main/java/iped/engine/task/leapp/LeappUtils.java new file mode 100644 index 0000000000..ef16fe97d1 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/LeappUtils.java @@ -0,0 +1,258 @@ +package iped.engine.task.leapp; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + +import iped.data.IItemReader; +import iped.properties.BasicProps; +import iped.search.IItemSearcher; + +/** + * A utility class containing helper methods for ALEAPP. + */ +public final class LeappUtils { + + // Cache to avoid recompiling the same glob patterns into Regex repeatedly + private static final ConcurrentHashMap PATTERN_CACHE = new ConcurrentHashMap<>(); + + private LeappUtils() { + } + + /** + * Builds the complete Lucene query used by FileSeeker: the glob-derived clauses from + * {@link #globToLuceneQuery(String)}, anchored to the evidence root path and excluding carved file fragments. + */ + public static String buildFileSearchQuery(String basePath, String globPattern) { + StringBuilder query = new StringBuilder(); + query.append(BasicProps.PATH).append(":\"").append(basePath).append("\""); + + String globQuery = globToLuceneQuery(globPattern); + if (StringUtils.isNotBlank(globQuery)) { + query.append(" && (").append(globQuery).append(")"); + } + + query.append(" && -fileFragment:true"); + + return query.toString(); + } + + /** + * Converts a glob pattern to a Lucene query combining exact path logic and a robust name pre-filter. + * + * IMPORTANT ARCHITECTURAL NOTE: This method acts as a high-recall "pre-filter". Because Lucene tokenizes text and this + * method strips certain wildcards/punctuation to maximize performance, the generated Lucene query WILL often match MORE + * files than the original glob. + * + * To guarantee 100% precision without missing any results, the hits returned by this Lucene query MUST be checked + * again in memory using a strict Java PathMatcher or Regex (e.g., via AleappUtils.matchesGlob, as FileSeeker.search + * does). + * + * @param globPattern The glob pattern to convert. + * @return A Lucene query string. + */ + public static String globToLuceneQuery(String globPattern) { + if (StringUtils.isBlank(globPattern)) { + return ""; + } + + // Normalize separators and isolate path from filename + String normalizedPath = globPattern.replace("\\", "/"); + int lastSlash = normalizedPath.lastIndexOf('/'); + + String parentPathString = lastSlash >= 0 ? normalizedPath.substring(0, lastSlash) : ""; + String fileName = lastSlash >= 0 ? normalizedPath.substring(lastSlash + 1) : normalizedPath; + + // 1. Build the exact path query logic provided + String pathQuery = ""; + if (StringUtils.isNotBlank(parentPathString)) { + String[] cleanedPaths = StringUtils.split(parentPathString, '*'); + pathQuery = Arrays + .stream(cleanedPaths) + .map(term -> StringUtils.strip(term, "/")) + .filter(StringUtils::isNotBlank) + .map(term -> String.format("%s:\"%s\"", BasicProps.PATH, term)) + .collect(Collectors.joining(" && ")); + } + + // 2. Handle pure catch-all wildcards + if (StringUtils.equalsAny(fileName, "*", "*.*", "**", "")) { + return pathQuery; + } + + // 3. Build the Name Query + String nameQuery = ""; + + // If there are absolutely no wildcards in the filename, we can safely quote it for an exact phrase match + if (!fileName.contains("*") && !fileName.contains("?")) { + nameQuery = String.format("%s:\"%s\"", BasicProps.NAME, fileName); + } else { + // Contains wildcards: Split by ALL punctuation (including hyphens) to ensure robust boolean queries. + // Hyphens unquoted in Lucene act as the NOT operator (e.g. test-profile -> test AND NOT profile), + // so we must split them into separate required terms to maximize recall. + String normalizedName = fileName.replaceAll("[^a-zA-Z0-9*?]+", " "); + String[] tokens = normalizedName.trim().split("\\s+"); + List queryParts = new ArrayList<>(); + + for (String token : tokens) { + if (StringUtils.isBlank(token)) { + continue; + } + + // Compress duplicate wildcards + String term = token.replaceAll("\\*+", "*"); + term = term.replaceAll("\\?+", "?"); + + // Skip standalone wildcards to avoid unoptimized match-all name clauses (e.g., from splitting "*.") + if (term.equals("*") || term.equals("?")) { + continue; + } + + queryParts.add(BasicProps.NAME + ":" + term); + } + nameQuery = String.join(" && ", queryParts); + } + + // 4. Combine pathQuery and nameQuery + if (StringUtils.isNoneBlank(pathQuery, nameQuery)) { + return String.format("%s && %s", pathQuery, nameQuery); + } else if (StringUtils.isNotBlank(nameQuery)) { + return nameQuery; + } else { + return pathQuery; + } + } + + /** + * Replicates the exact behavior of Python's fnmatch logic. Evaluates a file path against a glob pattern to guarantee + * 100% precision. Used as the strict in-memory re-check for hits returned by the high-recall Lucene pre-filter built + * by {@link #globToLuceneQuery(String)}. + * + * See: https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/scripts/search_files.py#L111 + */ + public static boolean matchesGlob(IItemReader item, String globPattern) { + if (globPattern == null || item == null) { + return false; + } + + // 1. Fetch or compile the fnmatch-equivalent regex pattern + Pattern pat = PATTERN_CACHE.computeIfAbsent(globPattern, LeappUtils::compileFnmatchPattern); + + // 2. Normalize the path to use forward slashes (cross-platform safety) + String pathToCheck = item.getPath().replace('\\', '/'); + + // 3. Match against the full string exactly like Python + return pat.matcher(pathToCheck).matches(); + } + + /** + * Maps a shell-style glob pattern to a Java Regex exactly like Python's fnmatch. Unlike Java's PathMatcher, this treats + * '*' as matching across directory boundaries. + */ + private static Pattern compileFnmatchPattern(String glob) { + StringBuilder sb = new StringBuilder("(?i)^"); // (?i) for case-insensitive, ^ for start + for (int i = 0; i < glob.length(); i++) { + char c = glob.charAt(i); + switch (c) { + case '*': + sb.append(".*"); // fnmatch treats * as .* (crosses slashes) + break; + case '?': + sb.append("."); // fnmatch treats ? as . + break; + // Escape all standard regex metacharacters + case '.': + case '+': + case '^': + case '$': + case '\\': + case '|': + case '{': + case '}': + case '(': + case ')': + case '[': + case ']': + sb.append('\\').append(c); + break; + default: + sb.append(c); + break; + } + } + sb.append("$"); // $ for end of string + return Pattern.compile(sb.toString()); + } + + // Android exposes the same storage under several symbolic links / bind mounts, so the file path an app + // records often differs from where the file physically lives in the extraction tree. Each entry maps an + // app-visible prefix (key) to the underlying extraction location (value). More specific keys come first. + private static final Map SYMLINK_PREFIXES = new LinkedHashMap<>(); + static { + SYMLINK_PREFIXES.put("/storage/emulated", "/data/media"); + SYMLINK_PREFIXES.put("/storage/self/primary", "/data/media/0"); + SYMLINK_PREFIXES.put("/mnt/sdcard", "/data/media/0"); + SYMLINK_PREFIXES.put("/mnt/user/0/primary", "/data/media/0"); + SYMLINK_PREFIXES.put("/sdcard", "/data/media/0"); + SYMLINK_PREFIXES.put("/data/user/0", "/data/data"); + } + + /** + * Locates the case item that corresponds to a device file path stored in plugin data (e.g. the "File Path" column of + * chromeOfflinePages). The item path is expected to be {@code pathRoot + pathValue}; since Android exposes the same + * storage under several symbolic links, the known {@link #SYMLINK_PREFIXES} variants of the value are tried too. The + * item whose in-case path exactly equals one of the candidate paths is returned, or {@code null} when none matches. + * + * @param searcher the case searcher + * @param pathRoot the evidence root path (the in-case prefix of every item under this extraction) + * @param pathValue the device file path stored in the plugin data cell + * @return the corresponding case item, or {@code null} when nothing matches + */ + public static IItemReader findItemByPath(IItemSearcher searcher, String pathRoot, String pathValue) { + if (searcher == null || StringUtils.isBlank(pathValue)) { + return null; + } + + String path = pathValue.trim(); + + // first try the path exactly as stored by the app + IItemReader item = searchItemByPath(searcher, pathRoot + path); + if (item != null) { + return item; + } + + // then try the known Android storage symlink resolutions, one at a time + for (Map.Entry symlink : SYMLINK_PREFIXES.entrySet()) { + String prefix = symlink.getKey(); + if (path.equals(prefix) || path.startsWith(prefix + "/")) { + String resolved = symlink.getValue() + path.substring(prefix.length()); + item = searchItemByPath(searcher, pathRoot + resolved); + if (item != null) { + return item; + } + } + } + + return null; + } + + /** Returns the case item whose path exactly equals {@code itemPath}, or {@code null} when there is none. */ + private static IItemReader searchItemByPath(IItemSearcher searcher, String itemPath) { + // the path query matches whole terms in any position, so it can hit the whole subtree of itemPath: + // iterating lazily loads only the items up to the exact one, instead of all the hits at once + for (IItemReader item : searcher.searchIterable(BasicProps.PATH + ":\"" + itemPath + "\"")) { + if (itemPath.equals(item.getPath())) { + return item; + } + } + return null; + } + +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/PluginSpec.java b/iped-engine/src/main/java/iped/engine/task/leapp/PluginSpec.java new file mode 100644 index 0000000000..52be8920cc --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/PluginSpec.java @@ -0,0 +1,76 @@ +package iped.engine.task.leapp; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import jep.python.PyCallable; +import jep.python.PyObject; + +/** + * A Java representation of a Python PluginSpec object that retrieves attributes on-demand (lazily) when getter methods + * are called. + */ +public class PluginSpec { + + private final PyObject pluginObject; + + private List searchGlobs; + + public PluginSpec(PyObject pluginObject) { + this.pluginObject = pluginObject; + } + + // --- Getters that call getAttr() on-demand --- + + public String getName() { + return this.pluginObject.getAttr("name", String.class); + } + + public String getModuleName() { + return this.pluginObject.getAttr("module_name", String.class); + } + + public String getCategory() { + return this.pluginObject.getAttr("category", String.class); + } + + /** + * Returns the plugin's file search patterns. Despite ALEAPP calling them "regexes" in places, these are shell-style + * glob patterns (fnmatch semantics). + * + * mimics https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/aleapp.py#L386 + */ + @SuppressWarnings("unchecked") + public List getSearchGlobs() { + + if (searchGlobs == null) { + List list = new ArrayList<>(); + Object search = this.pluginObject.getAttr("search"); + if (search instanceof String) { + list.add((String) search); + } else if (search instanceof Collection) { + list.addAll((Collection) search); + } else if (search != null) { + throw new IllegalArgumentException("Invalid plugin search: " + search.getClass() + " > " + search); + } + searchGlobs = list; + } + return searchGlobs; + } + + public PyCallable getMethod() { + return this.pluginObject.getAttr("method", PyCallable.class); + } + + @SuppressWarnings("unchecked") + public Map getArtifactInfo() { + return this.pluginObject.getAttr("artifact_info", Map.class); + } + + @Override + public String toString() { + return pluginObject.toString(); + } +} \ No newline at end of file diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/conversation/Conversation.java b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/Conversation.java new file mode 100644 index 0000000000..fd93fbde68 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/Conversation.java @@ -0,0 +1,72 @@ +package iped.engine.task.leapp.conversation; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.lang3.StringUtils; + +/** + * A conversation assembled from the data_list rows sharing the same discriminator column value. + */ +public class Conversation { + + /** Value of the conversationDiscriminatorColumn (e.g. a Discord channel id). */ + private final String id; + + /** Value of the conversationLabelColumn, when declared: a human-friendly conversation name. */ + private final String label; + + /** Artifact (display) name of the plugin that produced the rows. */ + private final String artifactName; + + private final List messages = new ArrayList<>(); + + public Conversation(String id, String label, String artifactName) { + this.id = id; + this.label = label; + this.artifactName = artifactName; + } + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public String getArtifactName() { + return artifactName; + } + + public String getTitle() { + return StringUtils.firstNonBlank(label, id, "?"); + } + + public List getMessages() { + return messages; + } + + /** Distinct message senders, in first-seen order: the best participant information LEAPP rows can provide. */ + public Set getParticipants() { + Set participants = new LinkedHashSet<>(); + for (ConversationMessage message : messages) { + if (StringUtils.isNotBlank(message.getSender())) { + participants.add(message.getSender()); + } + } + return participants; + } + + /** + * Sorts messages chronologically (stable: rows without a parseable timestamp keep their original relative order, + * before dated ones, like a null date sorts first elsewhere in IPED). + */ + public void sortMessages() { + messages.sort(Comparator.comparing(ConversationMessage::getTimestamp, + Comparator.nullsFirst(Comparator.naturalOrder()))); + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationCreator.java b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationCreator.java new file mode 100644 index 0000000000..f09f6fc74c --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationCreator.java @@ -0,0 +1,211 @@ +package iped.engine.task.leapp.conversation; + +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.StringUtils; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; + +import iped.data.IItem; +import iped.engine.core.Worker.ProcessTime; +import iped.engine.data.Item; +import iped.engine.task.ExportFileTask; +import iped.engine.task.leapp.AleappTask; +import iped.engine.task.leapp.LeappContext; +import iped.parsers.util.ConversationConstants; +import iped.properties.ExtraProperties; +import iped.utils.DateUtil; + +/** + * Turns the {@link Conversation}s of one plugin run into case items: for each conversation (or part of it, when the + * HTML is split by size) one chat-preview item is created as child of the plugin evidence, its HTML rendered by + * {@link ConversationHtmlReportGenerator} and stored via {@link ExportFileTask#insertIntoStorage}; the row subitems of + * the messages are then created as children of the part item they were rendered into, mirroring the UFED chat + * structure (chat preview → message subitems). + */ +public class ConversationCreator { + + public static final MediaType ALEAPP_CONVERSATION_MEDIATYPE = MediaType + .application(AleappTask.ALEAPP_APPLICATION_PREFIX + "chat-preview"); + + /** + * App-specific chat-preview media types, mirroring UfedChatParser: they carry the source app in the mime so + * IconManager and CategoriesConfig can give conversations the proper app icon/category. All of them MUST be + * registered in CustomSignatures.xml as sub-class-of x-aleapp-chat-preview, whose parent x-preview-with-links + * routes them to HtmlLinkViewer (enabling the app.open/app.check javascript bridge). + */ + private static final Map APP_PREVIEW_MEDIATYPES = Map.ofEntries( // + Map.entry("whatsapp", previewType("whatsapp")), // + Map.entry("telegram", previewType("telegram")), // + Map.entry("skype", previewType("skype")), // + Map.entry("facebook", previewType("facebook")), // + Map.entry("instagram", previewType("instagram")), // + Map.entry("signal", previewType("signal")), // + Map.entry("snapchat", previewType("snapchat")), // + Map.entry("threema", previewType("threema")), // + Map.entry("tiktok", previewType("tiktok")), // + Map.entry("viber", previewType("viber")), // + Map.entry("discord", previewType("discord"))); + + private static MediaType previewType(String app) { + return MediaType.application(AleappTask.ALEAPP_APPLICATION_PREFIX + "chat-preview-" + app); + } + + /** Same default used by UfedChatParser/WhatsAppParser: bigger chats are split into multiple HTML parts. */ + private static final int MIN_CHAT_SPLIT_SIZE = 6000000; + + /** + * Creates the subitem of one data_list row. Implemented by LavaInsertSqliteDataInterceptor, which owns the + * row-to-metadata mapping: this class only decides WHERE in the item tree the subitem goes. + */ + public interface MessageItemFactory { + Item create(IItem parent, int rowIndex, int subitemId); + } + + private final LeappContext context; + private final ConversationViewSpec view; + private final MessageItemFactory messageItemFactory; + + public ConversationCreator(LeappContext context, ConversationViewSpec view, MessageItemFactory messageItemFactory) { + this.context = context; + this.view = view; + this.messageItemFactory = messageItemFactory; + } + + public void createConversations(List conversations, AtomicInteger subitemIdSeq) throws Exception { + for (Conversation conversation : conversations) { + createConversation(conversation, subitemIdSeq); + } + } + + private void createConversation(Conversation conversation, AtomicInteger subitemIdSeq) throws Exception { + + conversation.sortMessages(); + + ConversationHtmlReportGenerator generator = new ConversationHtmlReportGenerator(conversation, MIN_CHAT_SPLIT_SIZE); + + byte[] bytes = generator.generateNextChatHtml(); + int frag = 0; + int firstMsg = 0; + + while (bytes != null) { + int nextMsg = generator.getNextMsgNum(); + byte[] nextBytes = generator.generateNextChatHtml(); + + String name = conversation.getArtifactName() + " - " + conversation.getTitle(); + if (frag > 0 || nextBytes != null) { + // fragment naming mirroring UfedChatParser/WhatsAppParser + name += "_" + (++frag); + } + + List partMessages = conversation.getMessages().subList(firstMsg, nextMsg); + + Item convItem = (Item) context.getPluginItem().createChildItem(); + convItem.setMediaType(resolvePreviewMediaType()); + convItem.setName(name); + convItem.setExtension(""); + convItem.setPath(context.getPluginItem().getPath() + "/" + name); + convItem.setIdInDataSource(""); + convItem.setSubItem(true); + convItem.setSubitemId(subitemIdSeq.getAndIncrement()); + convItem.setExtraAttribute(ExtraProperties.DECODED_DATA, true); + convItem.setHasChildren(!partMessages.isEmpty()); + + // standard cross-parser "Conversation:" metadata, as set by other chat + // parsers (e.g. iped.parsers.ufed.handler.ChatHandler) + convItem.getMetadata().set(ExtraProperties.CONVERSATION_ID, conversation.getId()); + convItem.getMetadata().set(ExtraProperties.CONVERSATION_NAME, conversation.getTitle()); + convItem.getMetadata().set(ExtraProperties.CONVERSATION_MESSAGES_COUNT, partMessages.size()); + Set participants = conversation.getParticipants(); + for (String participant : participants) { + convItem.getMetadata().add(ExtraProperties.CONVERSATION_PARTICIPANTS, participant); + } + if (!participants.isEmpty()) { + convItem.getMetadata().add(ExtraProperties.CONVERSATION_PARTICIPANTS + ":count", + Integer.toString(participants.size())); + } + + ExportFileTask.getLastInstance().insertIntoStorage(convItem, bytes, bytes.length); + context.getWorker().processNewItem(convItem, ProcessTime.LATER); + + for (ConversationMessage message : partMessages) { + Item msgItem = messageItemFactory.create(convItem, message.getRowIndex(), subitemIdSeq.getAndIncrement()); + setCommunicationMetadata(msgItem, message, conversation); + context.getWorker().processNewItem(msgItem, ProcessTime.LATER); + } + + firstMsg = nextMsg; + bytes = nextBytes; + } + } + + /** + * Standard cross-parser "Communication:" metadata on a message subitem, as set by other chat parsers (e.g. + * iped.parsers.ufed.handler.InstantMessageHandler). The row cells mapped by column classification + * (Communication:From/To/Date) may already be present: the data view mapping only fills the gaps, so nothing is + * overwritten. + */ + private void setCommunicationMetadata(Item msgItem, ConversationMessage message, Conversation conversation) { + + Metadata metadata = msgItem.getMetadata(); + + if (message.getOutgoing() != null) { + metadata.set(ExtraProperties.COMMUNICATION_DIRECTION, message.isOutgoing() + ? ConversationConstants.DIRECTION_OUTGOING + : ConversationConstants.DIRECTION_INCOMING); + removeAleappMetadata(metadata, view.getDirectionColumn()); + } + + if (metadata.get(ExtraProperties.COMMUNICATION_FROM) == null && StringUtils.isNotBlank(message.getSender())) { + metadata.set(ExtraProperties.COMMUNICATION_FROM, message.getSender()); + removeAleappMetadata(metadata, view.getSenderColumn()); + } + + // no per-row recipient in LEAPP data: the conversation itself is the + // destination, mirroring what InstantMessageHandler does for group chats + if (metadata.get(ExtraProperties.COMMUNICATION_TO) == null) { + metadata.set(ExtraProperties.COMMUNICATION_TO, conversation.getTitle()); + metadata.add(ExtraProperties.COMMUNICATION_TO + ExtraProperties.CONVERSATION_SUFFIX_ID, conversation.getId()); + // the discriminator/label cells became the chat's Conversation:id/Name + // and this message's Communication:To + removeAleappMetadata(metadata, view.getDiscriminatorColumn()); + removeAleappMetadata(metadata, view.getLabelColumn()); + } + + if (metadata.get(ExtraProperties.MESSAGE_DATE) == null && message.getTimestamp() != null) { + metadata.set(ExtraProperties.MESSAGE_DATE, DateUtil.dateToString(message.getTimestamp())); + removeAleappMetadata(metadata, view.getTimeColumn()); + } + + if (metadata.get(ExtraProperties.MESSAGE_BODY) == null && StringUtils.isNotBlank(message.getBody())) { + metadata.set(ExtraProperties.MESSAGE_BODY, message.getBody()); + removeAleappMetadata(metadata, view.getTextColumn()); + } + } + + /** Values promoted to a standard property are not kept duplicated under the "aleapp:" prefix. */ + private static void removeAleappMetadata(Metadata metadata, String column) { + if (column != null) { + metadata.remove(AleappTask.ALEAPP_METADATA_PREFIX + column); + } + } + + /** + * Resolves the app-specific chat-preview media type by looking for a known app keyword in the plugin module name + * (e.g. "discordChats") or artifact name (e.g. "Discord Chats"); falls back to the generic type. + */ + private MediaType resolvePreviewMediaType() { + String hint = (context.getPlugin().getModuleName() + " " + context.getPlugin().getName()) + .toLowerCase(Locale.ROOT); + for (Map.Entry entry : APP_PREVIEW_MEDIATYPES.entrySet()) { + if (hint.contains(entry.getKey())) { + return entry.getValue(); + } + } + return ALEAPP_CONVERSATION_MEDIATYPE; + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationHtmlReportGenerator.java b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationHtmlReportGenerator.java new file mode 100644 index 0000000000..119e1e4045 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationHtmlReportGenerator.java @@ -0,0 +1,348 @@ +package iped.engine.task.leapp.conversation; + +import static j2html.TagCreator.attrs; +import static j2html.TagCreator.b; +import static j2html.TagCreator.br; +import static j2html.TagCreator.div; +import static j2html.TagCreator.img; +import static j2html.TagCreator.table; +import static j2html.TagCreator.td; +import static j2html.TagCreator.tr; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.List; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.tika.metadata.XMPDM; + +import iped.data.IItemReader; +import iped.parsers.util.ChildPornHashLookup; +import iped.parsers.util.Messages; +import iped.parsers.whatsapp.Util; +import iped.properties.BasicProps; +import iped.properties.ExtraProperties; +import iped.utils.EmojiUtil; +import iped.utils.SimpleHTMLEncoder; +import j2html.tags.specialized.DivTag; + +/** + * Renders a LEAPP {@link Conversation} as a chat-style HTML, replicating the features of + * iped.parsers.ufed.ReportGenerator: WhatsApp-like css/js, topbar with the conversation title, date separator lines, + * incoming/outgoing bubbles with sender and time, media attachments with embedded base64 thumbs, audio/video player + * hooks (iped-audio/iped-video + data-src), links/checkboxes to the original case items, audio transcriptions, child + * porn hash set hits, location blocks and splitting of big chats into multiple HTML parts (with continuation markers + * and a loading modal for very long parts). + * + * Features of the UFED report that have no data source in LEAPP rows (reply/quote references, forwarded flags, + * message status ticks, edited/deleted states, shared contacts and contact photos) are intentionally absent. + */ +public class ConversationHtmlReportGenerator { + + private static final int MIN_MESSAGES_TO_SHOW_MODAL = 500; + + private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + private final SimpleDateFormat timeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z"); + + private final Conversation conversation; + private final int minChatSplitSize; + + private boolean firstHtml = true; + private int currentMsg = 0; + + public ConversationHtmlReportGenerator(Conversation conversation, int minChatSplitSize) { + this.conversation = conversation; + this.minChatSplitSize = minChatSplitSize; + } + + /** + * Index (into the conversation messages list) of the first message NOT yet rendered: after each + * {@link #generateNextChatHtml()} call, messages [previous, getNextMsgNum()) were written to that part. + */ + public int getNextMsgNum() { + return currentMsg; + } + + /** + * Generates the next HTML part, or null when all messages were already rendered. Mirrors the iteration contract of + * iped.parsers.ufed.ReportGenerator#generateNextChatHtml(). + */ + public byte[] generateNextChatHtml() throws UnsupportedEncodingException { + + List messages = conversation.getMessages(); + + if ((!firstHtml && currentMsg == 0) || (currentMsg > 0 && currentMsg == messages.size())) { + return null; + } + + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + PrintWriter out = new PrintWriter(new OutputStreamWriter(bout, StandardCharsets.UTF_8)); + + String title = conversation.getTitle(); + boolean printModal = (messages.size() - currentMsg) > MIN_MESSAGES_TO_SHOW_MODAL; + printMessageFileHeader(out, title, printModal); + + if (currentMsg > 0) { + printDateSeparator(out, Messages.getString("WhatsAppReport.ChatContinuation")); + } + + String lastDate = null; + while (currentMsg < messages.size()) { + ConversationMessage m = messages.get(currentMsg); + String thisDate = m.getTimestamp() != null ? dateFormat.format(m.getTimestamp()) + : Messages.getString("ReportGenerator.UnknownDate"); + if (!thisDate.equals(lastDate)) { + printDateSeparator(out, thisDate); + lastDate = thisDate; + } + printMessage(out, m); + + if (currentMsg++ != messages.size() - 1 && bout.size() >= minChatSplitSize) { + printDateSeparator(out, Messages.getString("WhatsAppReport.ChatContinues")); + break; + } + } + + printMessageFileFooter(out); + out.flush(); + + firstHtml = false; + + return EmojiUtil.replaceByImages(bout.toByteArray()); + } + + private static void printDateSeparator(PrintWriter out, String text) { + out.println("
" + text + "
"); + } + + private void printMessage(PrintWriter out, ConversationMessage message) { + + boolean isOutgoing = message.isOutgoing(); + + out.println("
"); + if (isOutgoing) { + out.println("
"); + } else { + out.println("
"); + } + + String name = message.getSender(); + if (StringUtils.isBlank(name)) { + name = isOutgoing ? Messages.getString("WhatsAppReport.Owner") + : Messages.getString("ReportGenerator.Unknown"); + } + out.println("" + format(name) + "
"); + + printMessageContent(out, message); + + out.println(""); + if (message.getTimestamp() != null) { + out.println(timeFormat.format(message.getTimestamp())); + } else if (isNotBlank(message.getRawTime())) { + out.println(format(message.getRawTime())); + } + out.println(""); + + if (isOutgoing) { + out.println("
"); + } else { + out.println("
"); + } + out.println("
"); + } + + private void printMessageContent(PrintWriter out, ConversationMessage message) { + + out.println(formatLocation(message)); + + for (IItemReader mediaItem : message.getMediaItems()) { + printMediaItem(out, mediaItem); + } + + String body = message.getBody(); + if (isNotBlank(body)) { + out.print(format(body)); + out.print("
"); + } + } + + /** + * Renders one media attachment, with the same viewer integrations of the UFED report: hash checkbox + app.open + * link, embedded thumb, iped-audio/iped-video hooks with export/source paths, duration, transcription and child + * porn hash set hits. + */ + private void printMediaItem(PrintWriter out, IItemReader mediaItem) { + + byte[] thumb = mediaItem.getThumb(); + String exportPath = iped.parsers.util.Util.getExportPath(mediaItem); + String source = iped.parsers.util.Util.getSourceFileIfExists(mediaItem).orElse(""); + Float duration = getDuration(mediaItem); + String hash = mediaItem.getHash(); + + boolean startedLink = false; + if (isNotBlank(hash)) { + String query = BasicProps.HASH + ":" + hash; + String href = iped.parsers.util.Util.getReportHref(mediaItem); + out.println(""); + out.println(""); + startedLink = true; + } + + String mediaType = mediaItem.getMediaType() != null ? mediaItem.getMediaType().toString() : ""; + + if (mediaType.startsWith("audio")) { + out.println(Messages.getString("UfedChatParser.AudioMessageTitle") + "
"); + out.println("
"); + if (duration != null && duration > 0) { + out.print(" " + formatDuration(duration) + ""); + } + out.print("
"); + + } else if (mediaType.startsWith("video")) { + out.println(Messages.getString("UfedChatParser.VideoMessageTitle") + "
"); + if (thumb != null) { + out.print("
"); + } else { + out.println("
"); + } + } else if (thumb != null) { + out.print("
"); + } else if (mediaType.startsWith("image")) { + out.println("
"); + } else { + out.println("Attachment:
"); + } + + if (startedLink) { + out.println("
"); + } + + // file name below the media, so the examiner can identify it + String mediaName = mediaItem.getName(); + if (isNotBlank(mediaName)) { + out.println("

" + format(mediaName) + "

"); + } + + // render audio transcription + String transcription = mediaItem.getMetadata().get(ExtraProperties.TRANSCRIPT_ATTR); + if (isNotBlank(transcription)) { + out.print("
"); + out.print(Messages.getString("ReportGenerator.TranscriptionTitle")); + String confidence = mediaItem.getMetadata().get(ExtraProperties.CONFIDENCE_ATTR); + if (confidence != null) { + float score = Float.valueOf(confidence) * 100; + out.print(" [" + (int) score + "%]"); + } + out.println(": "); + out.println(format(transcription)); + out.println("
"); + } + + // render child porn sets + if (isNotBlank(hash)) { + List childPornSets = ChildPornHashLookup.lookupHash(hash); + if (childPornSets != null && !childPornSets.isEmpty()) { + out.print("

" + Messages.getString("WhatsAppReport.FoundInPedoHashDB") + " " + + format(childPornSets.toString()) + "

"); + } + } + } + + private String formatLocation(ConversationMessage message) { + + String lat = message.getLatitude(); + String lon = message.getLongitude(); + + if (StringUtils.isAllBlank(lat, lon)) { + return StringUtils.EMPTY; + } + + DivTag div = div(img(attrs(".location")), b(Messages.getString("UfedChatParser.Location.Title")), br()); + div.with(table(attrs(".contact-table"), // + tr(td(Messages.getString("UfedChatParser.Location.Latitude")), td(lat)), // + tr(td(Messages.getString("UfedChatParser.Location.Longitude")), td(lon))), // + br()); + return div.render(); + } + + private static Float getDuration(IItemReader item) { + String duration = StringUtils.firstNonBlank( + item.getMetadata().get(ExtraProperties.AUDIO_META_PREFIX + XMPDM.DURATION.getName()), + item.getMetadata().get(ExtraProperties.VIDEO_META_PREFIX + XMPDM.DURATION.getName())); + if (duration != null) { + try { + return Float.parseFloat(duration); + } catch (NumberFormatException e) { + } + } + return null; + } + + private static String formatDuration(float duration) { + if (duration == 0) { + return ""; + } + return "(" + String.format("%02d:%02d", (int) duration / 60, (int) duration % 60) + ")"; + } + + private static String format(String text) { + String ret = SimpleHTMLEncoder.htmlEncode(text); + // Keep line breaks present in the content, converting to an HTML
+ ret = ret.replaceAll("\n", "
\n"); + return ret; + } + + private void printMessageFileHeader(PrintWriter out, String title, boolean printModal) { + + out.println("\n" + + "\n" + + "\n" + + "" + format(title) + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + "" + "\n" + + ""); + if (printModal) { + out.println("
\n" + + "
" + Messages.getString("UfedChatParser.Loading") + "
\n" + + "
"); + } + out.println("
\n" + + " " + + "   " + + format(title) + "\n" + + "
\n" + + "
"); + } + + private static void printMessageFileFooter(PrintWriter out) { + out.println("


\n" + + "
\n" + + "
 
\n" + + "\n" + + "\n" + + ""); + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationMessage.java b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationMessage.java new file mode 100644 index 0000000000..1a210fe6dd --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationMessage.java @@ -0,0 +1,102 @@ +package iped.engine.task.leapp.conversation; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import iped.data.IItemReader; +import iped.utils.DateUtil; + +/** + * One message of a LEAPP conversation view: the values of the view-mapped columns of a single data_list row, plus the + * case items of the row's 'media' typed columns. + */ +public class ConversationMessage { + + /** Index of the row in the plugin's data_list, used as a stable per-plugin anchor id. */ + private final int rowIndex; + + private final String sender; + private final String body; + + /** True/false when the view declares a directionColumn; null when the direction is unknown. */ + private final Boolean outgoing; + + private final String rawTime; + private final Date timestamp; + + private final List mediaItems = new ArrayList<>(); + + private String latitude; + private String longitude; + + public ConversationMessage(int rowIndex, String sender, String body, Boolean outgoing, String rawTime) { + this.rowIndex = rowIndex; + this.sender = sender; + this.body = body; + this.outgoing = outgoing; + this.rawTime = rawTime; + this.timestamp = parseTime(rawTime); + } + + /** + * Time cells arrive stringified from Python (str(datetime) produces "2023-05-12 14:33:12+00:00"-like values, other + * plugins emit ISO strings); DateUtil handles both the 'T' and space separated ISO-8601 variants. Parsing failures + * are not fatal: the raw string is still shown in the HTML. + */ + private static Date parseTime(String value) { + if (value == null || value.isBlank()) { + return null; + } + return DateUtil.tryToParseDate(value.trim()); + } + + public int getRowIndex() { + return rowIndex; + } + + public String getAnchorId() { + return "leapp-msg-" + rowIndex; + } + + public String getSender() { + return sender; + } + + public String getBody() { + return body; + } + + public boolean isOutgoing() { + return Boolean.TRUE.equals(outgoing); + } + + public Boolean getOutgoing() { + return outgoing; + } + + public String getRawTime() { + return rawTime; + } + + public Date getTimestamp() { + return timestamp; + } + + public List getMediaItems() { + return mediaItems; + } + + public void setLocation(String latitude, String longitude) { + this.latitude = latitude; + this.longitude = longitude; + } + + public String getLatitude() { + return latitude; + } + + public String getLongitude() { + return longitude; + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationViewSpec.java b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationViewSpec.java new file mode 100644 index 0000000000..84ffa9be4e --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/conversation/ConversationViewSpec.java @@ -0,0 +1,115 @@ +package iped.engine.task.leapp.conversation; + +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +/** + * The "conversation" data view a plugin declares in the "data_views" field of its __artifacts_v2__ block, e.g.: + * + *
+ * "data_views": {
+ *     "conversation": {
+ *         "conversationDiscriminatorColumn": "Channel ID",
+ *         "textColumn": "Content",
+ *         "directionColumn": "Direction",
+ *         "directionSentValue": "Outgoing",
+ *         "timeColumn": "Timestamp",
+ *         "senderColumn": "Username"
+ *     }
+ * }
+ * 
+ * + * In ALEAPP this declaration is only copied into the LAVA JSON by lava_process_artifact (the rendering happens in the + * LAVA viewer app), so there is no need to enable that function: the raw dict is read here straight from the plugin's + * artifact_info, and the column names match the raw data_headers names received by LavaInsertSqliteDataInterceptor. + * + * The backward compatibility rules of lava_process_artifact are mirrored: a "chat" view is upgraded to "conversation" + * and thread*Column keys are remapped to conversation*Column. + * + * https://github.com/abrignoni/ALEAPP/blob/v2026.1.0/scripts/lavafuncs.py#L253 + */ +public class ConversationViewSpec { + + public static final String DATA_VIEWS_KEY = "data_views"; + + private final String discriminatorColumn; + private final String labelColumn; + private final String textColumn; + private final String directionColumn; + private final String directionSentValue; + private final String timeColumn; + private final String senderColumn; + + private ConversationViewSpec(Map params) { + this.discriminatorColumn = str(params, "conversationDiscriminatorColumn", "threadDiscriminatorColumn"); + this.labelColumn = str(params, "conversationLabelColumn", "threadLabelColumn"); + this.textColumn = str(params, "textColumn"); + this.directionColumn = str(params, "directionColumn"); + this.directionSentValue = str(params, "directionSentValue"); + this.timeColumn = str(params, "timeColumn"); + this.senderColumn = str(params, "senderColumn"); + } + + /** + * Returns the conversation view declared in the given artifact_info, or null if there is none usable (a view + * without a discriminator column cannot group rows into conversations). + */ + public static ConversationViewSpec from(Map artifactInfo) { + + Object dataViews = artifactInfo.get(DATA_VIEWS_KEY); + if (!(dataViews instanceof Map)) { + return null; + } + + Object view = ((Map) dataViews).get("conversation"); + if (!(view instanceof Map)) { + // backward compatibility, mirroring lava_process_artifact + view = ((Map) dataViews).get("chat"); + } + if (!(view instanceof Map)) { + return null; + } + + ConversationViewSpec spec = new ConversationViewSpec((Map) view); + return spec.discriminatorColumn != null ? spec : null; + } + + private static String str(Map params, String... keys) { + for (String key : keys) { + Object value = params.get(key); + if (value != null && StringUtils.isNotBlank(value.toString())) { + return value.toString(); + } + } + return null; + } + + public String getDiscriminatorColumn() { + return discriminatorColumn; + } + + public String getLabelColumn() { + return labelColumn; + } + + public String getTextColumn() { + return textColumn; + } + + public String getDirectionColumn() { + return directionColumn; + } + + public String getDirectionSentValue() { + return directionSentValue; + } + + public String getTimeColumn() { + return timeColumn; + } + + public String getSenderColumn() { + return senderColumn; + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/IlapfuncsDeviceInfoInterceptor.java b/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/IlapfuncsDeviceInfoInterceptor.java new file mode 100644 index 0000000000..94b0dd9f3c --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/IlapfuncsDeviceInfoInterceptor.java @@ -0,0 +1,35 @@ +package iped.engine.task.leapp.interceptors; + +import java.util.Map; + +import iped.engine.task.leapp.AleappTask; +import iped.engine.task.leapp.CallInterceptor; +import jep.PyMethod; + +/** + * Intercepts ilapfuncs.device_info only to serialize it: ilapfuncs.identifiers is a shared module-global dict, + * populated by device_info with a read-modify-write sequence, and IPED workers run plugins concurrently, so + * unsynchronized calls could lose updates. + * + * The original Python function is called unchanged, under {@link AleappTask#DEVICE_INFO_LOCK} — the same monitor + * guarding write_device_info() in AleappTask.processDeviceInfoEvidence(), so the dict cannot be mutated while it is + * being iterated for the report. + * + * NOTE: intercepting in Java (instead of wrapping in Python) preserves device_info's caller attribution: it uses + * inspect.stack()[1] to record which plugin function reported the value, and the plugin -> Java -> original call + * round-trip pushes no intermediate Python frame. + */ +public class IlapfuncsDeviceInfoInterceptor extends CallInterceptor { + + public IlapfuncsDeviceInfoInterceptor() { + super("scripts.ilapfuncs", "scripts.ilapfuncs.device_info"); + } + + @Override + @PyMethod(varargs = true, kwargs = true) + public Object call(Object[] args, Map kwargs) throws Exception { + synchronized (AleappTask.DEVICE_INFO_LOCK) { + return super.call(args, kwargs); + } + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/IlapfuncsLogfuncInterceptor.java b/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/IlapfuncsLogfuncInterceptor.java new file mode 100644 index 0000000000..b0ede5cbb2 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/IlapfuncsLogfuncInterceptor.java @@ -0,0 +1,31 @@ +package iped.engine.task.leapp.interceptors; + +import java.util.Map; + +import iped.engine.task.leapp.CallInterceptor; +import iped.engine.task.leapp.LeappContext; +import jep.PyMethod; + +/** + * Replaces the ilapfuncs.logfunc function so ALEAPP plugin log messages are redirected to the IPED log instead of + * ALEAPP's own report log file. + */ +public class IlapfuncsLogfuncInterceptor extends CallInterceptor { + + public IlapfuncsLogfuncInterceptor() { + super("scripts.ilapfuncs", "scripts.ilapfuncs.logfunc"); + } + + @Override + @PyMethod(varargs = true, kwargs = true) + public Object call(Object[] args, Map kwargs) throws Exception { + + String message = (String) getArgumentValue("message", 0, args, kwargs); + String moduleName = LeappContext.get().getPlugin().getModuleName(); + String pluginName = LeappContext.get().getPlugin().getName(); + + logger.info("{} [{}]: {}", moduleName, pluginName, message); + + return null; + } +} diff --git a/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/LavaInsertSqliteDataInterceptor.java b/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/LavaInsertSqliteDataInterceptor.java new file mode 100644 index 0000000000..9a4f2ac7d6 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/task/leapp/interceptors/LavaInsertSqliteDataInterceptor.java @@ -0,0 +1,532 @@ +package iped.engine.task.leapp.interceptors; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.StringUtils; +import org.apache.tika.metadata.Property; +import org.apache.tika.mime.MediaType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import iped.data.IItem; +import iped.data.IItemReader; +import iped.engine.core.Worker.ProcessTime; +import iped.engine.data.Item; +import iped.engine.task.leapp.AleappMediaTypeResolver; +import iped.engine.task.leapp.AleappTask; +import iped.engine.task.leapp.CallInterceptor; +import iped.engine.task.leapp.LeappContext; +import iped.engine.task.leapp.conversation.Conversation; +import iped.engine.task.leapp.conversation.ConversationCreator; +import iped.engine.task.leapp.conversation.ConversationMessage; +import iped.engine.task.leapp.conversation.ConversationViewSpec; +import iped.properties.ExtraProperties; +import jep.PyMethod; + +/** + * Replaces the lava_insert_sqlite_data function. This is the main IPED-LEAPP integration point: instead of letting the + * plugin results be written to the LAVA sqlite database, each data row is turned into an IPED subitem of the current + * plugin evidence. + * + * lava_insert_sqlite_data is intercepted (rather than tsv/timeline/html) because it is the only output call that + * receives the RAW data_headers and data_list: headers keep their (name, type) tuples and values keep their original + * Python types — the other outputs receive stripped headers and stringified values. + * + * NOTE: the interception target is the binding INSIDE scripts.ilapfuncs ("from scripts.lavafuncs import + * lava_insert_sqlite_data" is captured at import time), which is the name artifact_processor actually calls. + */ +public class LavaInsertSqliteDataInterceptor extends CallInterceptor { + + protected static final Logger logger = LoggerFactory.getLogger(LavaInsertSqliteDataInterceptor.class); + + // Fallback classification for UNTYPED (plain string) headers, calibrated against + // ALEAPP v2026.1.0 sources. Date columns vary a lot across plugins, so dates use + // exact names plus a suffix family. Sender, recipient and body use exact matches + // only: mislabeling those is forensically costly (e.g. "Account" is usually the + // device owner, who is the receiver of incoming records, not the sender). + // "Date of Birth"-style personal dates must not become the record's event date, + // so there is no "date " prefix rule — "date *" event columns are listed explicitly. + private static final Set DATE_HEADERS = Set.of( // + "datetime", "date/time", "date", "created", "created at", "updated at", // + "time created", "last updated", "last login", "last modified", "last access", "last accessed", // + "date added", "date created", "date modified", "date sent", "date taken"); + private static final Set FROM_HEADERS = Set.of("sender", "from", "author"); + private static final Set TO_HEADERS = Set.of("recipient", "to", "receiver"); + private static final Set BODY_HEADERS = Set.of("message", "body", "text", "content"); + + // UNTYPED columns whose value is a device file path but which ALEAPP did NOT declare as ('name', 'media'). + // These cells are not rewritten by the media interception (check_in_media), so their value stays as the raw + // device path (e.g. chromeOfflinePages "File Path", WhatsApp "Local Path To Media"). When the seeker did not + // export the file, the fallback below still links it to its original case item via IItemSearcher. + // The suffix rules in isFilePathHeader cover the many "* Path" columns across plugins (Download Path, Source + // File Path, Save Path, Full Path, Original Path, Screenshot Path, Code Path, file_path, *_filepath, ...); this + // set only holds the few file-path column names those rules do not catch. + private static final Set FILE_PATH_HEADERS = Set.of("local path to media"); + + // Per-plugin exceptions: columns whose name matches the file-path rules but do NOT hold a filesystem path in + // that specific plugin, so the file lookup must be skipped. Kept plugin-scoped (keyed by the plugin module name) + // because the same column name can be a real file path elsewhere: "Path" is a cookie/URL path in the cookie + // plugins but a MediaStore file path in emulatedSmeta and a transfer path in Zapya. Module names and column + // names are written exactly as declared in the plugins. + private static final Map> NON_FILE_PATH_COLUMNS = Map.of( // + "chromeCookies", Set.of("Path"), // + "firefoxCookies", Set.of("Path"), // + "OrnetBrowser", Set.of("Path"), // + "FairEmail", Set.of("Return Path"), // + "DuckDuckGo", Set.of("Folder Path"), // + "libretorrentFR", Set.of("Length - Path")); + + // types used in LEAPP data_headers tuples (see lavafuncs.get_sql_type and + // ilapfuncs.get_media_header_info) + private static final String TYPE_DATETIME = "datetime"; + private static final String TYPE_DATE = "date"; + private static final String TYPE_MEDIA = "media"; + + private enum StandardField { + DATE, FROM, TO, BODY, LATITUDE, LONGITUDE, MEDIA, FILE_PATH, NONE + } + + /** A data_headers entry: plain string or (name, type[, style]) tuple. */ + private static class Header { + final String name; + final String type; + + Header(String name, String type) { + this.name = name; + this.type = type; + } + } + + public LavaInsertSqliteDataInterceptor() { + super("scripts.ilapfuncs", "scripts.ilapfuncs.lava_insert_sqlite_data"); + } + + @SuppressWarnings("unchecked") + @Override + @PyMethod(varargs = true, kwargs = true) + public Object call(Object[] args, Map kwargs) throws Exception { + + // lava_insert_sqlite_data(table_name, data, object_columns, headers, column_map) + // table_name/object_columns/column_map come from lava_process_artifact, which is + // disabled and returns None values: only data and headers are used here + List> dataList = (List>) getArgumentValue("data", 1, args, kwargs); + List rawHeaders = (List) getArgumentValue("headers", 3, args, kwargs); + + if (dataList == null || dataList.isEmpty()) { + return null; + } + + // the interceptor is installed globally in the Python interpreter, so the + // thread-local context tells us which plugin run this call belongs to + LeappContext context = LeappContext.get(); + + // set hasChildren, so plugin will not be ignored in AleappTask.processPluginEvidence() + // and category will not be ignored in AleappTask.processCategoryEvidence() + context.getPluginItem().setHasChildren(true); + IItem categoryItem = (IItem) context.getPluginItem().getTempAttribute(AleappTask.ALEAPP_PLUGIN_CATEGORY_KEY); + categoryItem.setHasChildren(true); + + addLinkedItems(context); + + String artifactName = StringUtils.firstNonBlank( + (String) context.getPlugin().getArtifactInfo().get("name"), context.getPlugin().getName()); + String pluginName = context.getPluginItem().getMetadata().get(AleappTask.ALEAPP_PLUGIN_KEYNAME_META); + String pluginModule = context.getPlugin().getModuleName(); + MediaType mediaType = AleappMediaTypeResolver.resolveMediaType(pluginModule, pluginName, artifactName); + + // headers are constant across rows: parse and classify each column once + Header[] headers = new Header[rawHeaders.size()]; + StandardField[] standardFields = new StandardField[rawHeaders.size()]; + for (int i = 0; i < rawHeaders.size(); i++) { + headers[i] = parseHeader(rawHeaders.get(i)); + standardFields[i] = classifyHeader(headers[i]); + // a column named like a file path may still be a non-file path in a specific plugin + // (e.g. the cookie "Path"): drop it back to NONE so no file lookup is attempted + if (standardFields[i] == StandardField.FILE_PATH && isNonFilePathColumn(pluginModule, headers[i].name)) { + standardFields[i] = StandardField.NONE; + } + } + + // a "conversation" data view groups the rows into chats: each conversation + // becomes a chat-preview child item of the plugin evidence (with a UFED-like + // HTML rendering) and the row subitems become children of their conversation + ConversationViewSpec view = ConversationViewSpec.from(context.getPlugin().getArtifactInfo()); + int discriminatorIdx = view == null ? -1 : indexOfColumn(headers, view.getDiscriminatorColumn()); + + AtomicInteger subitemIdSeq = new AtomicInteger(); + + if (discriminatorIdx >= 0) { + createConversations(context, mediaType, artifactName, headers, standardFields, dataList, view, discriminatorIdx, subitemIdSeq); + } else { + for (int index = 0; index < dataList.size(); index++) { + Item subItem = createSubItem(context, context.getPluginItem(), mediaType, artifactName, index, + subitemIdSeq.getAndIncrement(), headers, standardFields, dataList.get(index)); + context.getWorker().processNewItem(subItem, ProcessTime.LATER); + } + } + + return null; + } + + /** + * Groups the data rows by the discriminator column into {@link Conversation}s and delegates item/HTML creation to + * {@link ConversationCreator}. The row subitems are still created by {@link #createSubItem} (via the factory), so + * their metadata is identical to the non-conversation case: only their parent changes. + */ + private void createConversations(LeappContext context, MediaType mediaType, String artifactName, Header[] headers, + StandardField[] standardFields, List> dataList, ConversationViewSpec view, + int discriminatorIdx, AtomicInteger subitemIdSeq) throws Exception { + + int labelIdx = indexOfColumn(headers, view.getLabelColumn()); + int textIdx = indexOfColumn(headers, view.getTextColumn()); + int directionIdx = indexOfColumn(headers, view.getDirectionColumn()); + int timeIdx = indexOfColumn(headers, view.getTimeColumn()); + int senderIdx = indexOfColumn(headers, view.getSenderColumn()); + + Map conversations = new LinkedHashMap<>(); + + for (int index = 0; index < dataList.size(); index++) { + List data = dataList.get(index); + + String conversationId = StringUtils.defaultString(cellString(data, discriminatorIdx)); + String label = cellString(data, labelIdx); + Conversation conversation = conversations.computeIfAbsent(conversationId, + id -> new Conversation(id, label, artifactName)); + + // null (unknown) when the view declares no direction column/value: the + // Communication:Direction metadata is only set when the direction is known + Boolean outgoing = (directionIdx < 0 || view.getDirectionSentValue() == null) ? null + : view.getDirectionSentValue().equalsIgnoreCase(StringUtils.trim(cellString(data, directionIdx))); + + ConversationMessage message = new ConversationMessage(index, cellString(data, senderIdx), + cellString(data, textIdx), outgoing, cellString(data, timeIdx)); + + String lat = null, lon = null; + for (int i = 0; i < headers.length; i++) { + Object value = cellValue(data, i); + if (value == null) { + continue; + } + if (standardFields[i] == StandardField.MEDIA) { + message.getMediaItems().addAll(getMediaCaseItems(context, value)); + } else if (standardFields[i] == StandardField.LATITUDE) { + lat = value.toString(); + } else if (standardFields[i] == StandardField.LONGITUDE) { + lon = value.toString(); + } + } + message.setLocation(lat, lon); + + conversation.getMessages().add(message); + } + + ConversationCreator creator = new ConversationCreator(context, view, + (parent, rowIndex, subitemId) -> createSubItem(context, parent, mediaType, artifactName, rowIndex, + subitemId, headers, standardFields, dataList.get(rowIndex))); + + creator.createConversations(new ArrayList<>(conversations.values()), subitemIdSeq); + } + + /** + * Index of the given data view column in data_headers; -1 when absent. Exact match first, then a trimmed + * case-insensitive fallback, since view declarations are hand-written in the plugins. + */ + private static int indexOfColumn(Header[] headers, String column) { + if (column == null) { + return -1; + } + for (int i = 0; i < headers.length; i++) { + if (column.equals(headers[i].name)) { + return i; + } + } + for (int i = 0; i < headers.length; i++) { + if (column.trim().equalsIgnoreCase(StringUtils.trim(headers[i].name))) { + return i; + } + } + return -1; + } + + private static String cellString(List data, int idx) { + if (idx < 0) { + return null; + } + Object value = cellValue(data, idx); + return value == null ? null : value.toString(); + } + + /** + * Resolves the case items referenced by a 'media' typed cell (one exported path or a list of them). Paths not + * exported by the seeker have no case item counterpart and are skipped. + */ + @SuppressWarnings("unchecked") + private static List getMediaCaseItems(LeappContext context, Object value) { + List mediaPaths = (value instanceof List) ? (List) value : List.of(value); + List items = new ArrayList<>(); + for (Object mediaPath : mediaPaths) { + if (mediaPath == null) { + continue; + } + IItemReader item = context.getFileSeeker().getExportedFiles().get(mediaPath.toString()); + if (item != null) { + items.add(item); + } + } + return items; + } + + /** + * Links the plugin evidence to all files found by the seeker for this plugin. + */ + private void addLinkedItems(LeappContext context) { + Set globalIds = new HashSet<>(); + for (IItemReader foundFile : context.getFoundFiles()) { + globalIds.add((String) foundFile.getExtraAttribute(ExtraProperties.GLOBAL_ID)); + } + String linkedItems = ExtraProperties.GLOBAL_ID + ":(" + String.join(" ", globalIds) + ")"; + context.getPluginItem().getMetadata().add(ExtraProperties.LINKED_ITEMS, linkedItems); + } + + /** + * Creates one subitem for a data row under the given parent (the plugin evidence or, when a conversation view + * exists, the conversation part item), storing each cell as "aleapp:<header>" metadata and mapping typed or + * recognized columns to IPED standard properties. + */ + private Item createSubItem(LeappContext context, IItem parent, MediaType mediaType, String artifactName, int index, + int subitemId, Header[] headers, StandardField[] standardFields, List data) { + + String subItemName = artifactName + "-" + index; + Item subItem = (Item) parent.createChildItem(); + subItem.setMediaType(mediaType); + subItem.setName(subItemName); + subItem.setExtension(""); + subItem.setPath(parent.getPath() + "/" + subItemName); + subItem.setExtraAttribute(ExtraProperties.DECODED_DATA, true); + subItem.setSubItem(true); + subItem.setSubitemId(subitemId); + + // data as metadata + String lat = null, lon = null; + List filePathMetaKeys = null; + for (int i = 0; i < headers.length; i++) { + Object value = cellValue(data, i); + if (value == null) { + continue; + } + + if (standardFields[i] == StandardField.MEDIA) { + addMediaValue(context, subItem, headers[i].name, value); + continue; + } + + String valueStr = value.toString(); + + // cells holding a path exported by the seeker are rewritten back to the + // original in-case path and linked to the original item + boolean linkedToCaseItem = false; + if (context.getFileSeeker().getExportedFiles().containsKey(valueStr)) { + IItemReader valueItem = context.getFileSeeker().getExportedFiles().get(valueStr); + valueStr = StringUtils.removeStart(valueItem.getPath(), context.getFileSeeker().getPathRoot()); + subItem.getMetadata().add(ExtraProperties.LINKED_ITEMS, ExtraProperties.GLOBAL_ID + ":" + valueItem.getExtraAttribute(ExtraProperties.GLOBAL_ID)); + linkedToCaseItem = true; + } + + // fallback for UNTYPED file-path columns (e.g. "File Path"): the value is a device file + // path the seeker did not export, so it is not in getExportedFiles(). Resolving it needs an + // index search per value, too costly to run here for every row: just record the metadata key + // and let AleappTask.process() resolve and link it when this subitem is reprocessed by a worker. + if (!linkedToCaseItem && standardFields[i] == StandardField.FILE_PATH) { + if (filePathMetaKeys == null) { + filePathMetaKeys = new ArrayList<>(); + } + filePathMetaKeys.add("aleapp:" + headers[i].name); + } + + // cells promoted to a standard property (Communication:*, Message-Body) are + // not duplicated as "aleapp:" metadata + boolean promoted = false; + if (standardFields[i] == StandardField.LATITUDE) { + lat = valueStr; + } else if (standardFields[i] == StandardField.LONGITUDE) { + lon = valueStr; + } else if (standardFields[i] != StandardField.NONE) { + promoted = applyStandardField(subItem, standardFields[i], valueStr); + } + if (!promoted) { + subItem.getMetadata().set("aleapp:" + headers[i].name, valueStr); + } + } + setLocationIfValid(subItem, lat, lon); + + // the recorded file-path columns are resolved and linked later, in AleappTask.process() + if (filePathMetaKeys != null) { + subItem.setTempAttribute(AleappTask.ALEAPP_METADATA_PATHS, filePathMetaKeys); + } + + return subItem; + } + + /** + * Handles a cell of a 'media' typed column. With check_in_media patched to return the exported extraction path + * (see LeappInterceptors), the cell holds one exported path or a list of them: link each one to the original case + * item and store its in-case path as metadata. + */ + @SuppressWarnings("unchecked") + private void addMediaValue(LeappContext context, Item subItem, String headerName, Object value) { + + List mediaPaths = (value instanceof List) ? (List) value : List.of(value); + + for (Object mediaPath : mediaPaths) { + if (mediaPath == null) { + continue; + } + String pathStr = mediaPath.toString(); + IItemReader mediaItem = context.getFileSeeker().getExportedFiles().get(pathStr); + if (mediaItem != null) { + pathStr = StringUtils.removeStart(mediaItem.getPath(), context.getFileSeeker().getPathRoot()); + subItem.getMetadata().add(ExtraProperties.LINKED_ITEMS, + ExtraProperties.GLOBAL_ID + ":" + mediaItem.getExtraAttribute(ExtraProperties.GLOBAL_ID)); + } + subItem.getMetadata().add("aleapp:" + headerName, pathStr); + } + } + + private static Object cellValue(List data, int i) { + return (i < data.size()) ? data.get(i) : null; + } + + /** + * data_headers entries are plain strings or (name, type[, style]) tuples — Jep converts tuples to Lists. + */ + private static Header parseHeader(Object rawHeader) { + if (rawHeader instanceof List) { + List tuple = (List) rawHeader; + String name = String.valueOf(tuple.get(0)); + String type = tuple.size() > 1 ? StringUtils.lowerCase(String.valueOf(tuple.get(1)), Locale.ROOT) : null; + return new Header(name, type); + } + return new Header(String.valueOf(rawHeader), null); + } + + private static boolean isDateHeader(String h) { + return DATE_HEADERS.contains(h) || h.contains("timestamp") // + || h.endsWith(" time") || h.endsWith(" date") || h.endsWith("_date"); + } + + private static boolean isFilePathHeader(String h) { + return FILE_PATH_HEADERS.contains(h) || h.equals("path") // + || h.endsWith(" path") || h.endsWith("_path") || h.endsWith("filepath"); + } + + /** + * Whether the given column is a per-plugin exception that looks like a file path by name but is not one in this + * plugin (see {@link #NON_FILE_PATH_COLUMNS}), so the file lookup must be skipped. + */ + private static boolean isNonFilePathColumn(String moduleName, String headerName) { + Set columns = NON_FILE_PATH_COLUMNS.get(moduleName); + return columns != null && columns.contains(headerName); + } + + /** + * Classification is type-driven when the header carries a type; the name-based heuristics are only a fallback for + * untyped headers. + */ + private static StandardField classifyHeader(Header header) { + + if (TYPE_MEDIA.equals(header.type)) { + return StandardField.MEDIA; + } + if (TYPE_DATETIME.equals(header.type) || TYPE_DATE.equals(header.type)) { + return StandardField.DATE; + } + + String h = header.name.toLowerCase(Locale.ROOT).trim(); + if (header.type == null && isDateHeader(h)) { + return StandardField.DATE; + } else if (FROM_HEADERS.contains(h)) { + return StandardField.FROM; + } else if (TO_HEADERS.contains(h)) { + return StandardField.TO; + } else if (BODY_HEADERS.contains(h)) { + return StandardField.BODY; + } else if (h.equals("latitude")) { + return StandardField.LATITUDE; + } else if (h.equals("longitude")) { + return StandardField.LONGITUDE; + } else if (header.type == null && isFilePathHeader(h)) { + return StandardField.FILE_PATH; + } + return StandardField.NONE; + } + + /** Returns true when the value was stored in the standard property. */ + private static boolean applyStandardField(Item item, StandardField field, String value) { + if (StringUtils.isBlank(value)) { + return false; + } + switch (field) { + case DATE: + return setIfAbsent(item, ExtraProperties.MESSAGE_DATE, value); + case FROM: + // Communication:From/To directly (the standard cross-parser properties): + // the Message:From/To -> Communication:From/To rename of + // MetadataUtil.normalizeMetadata only runs inside StandardParser + return setIfAbsent(item, ExtraProperties.COMMUNICATION_FROM, value); + case TO: + return setIfAbsent(item, ExtraProperties.COMMUNICATION_TO, value); + case BODY: + return setIfAbsent(item, ExtraProperties.MESSAGE_BODY, value); + default: + return false; + } + } + + private static void setLocationIfValid(Item item, String lat, String lon) { + if (StringUtils.isBlank(lat) || StringUtils.isBlank(lon)) { + return; + } + // ALeapp data may use comma as decimal separator (locale-formatted + // devices), as MetadataUtil.normalizeGPSMeta also handles + lat = lat.trim().replace(',', '.'); + lon = lon.trim().replace(',', '.'); + try { + double la = Double.parseDouble(lat); + double lo = Double.parseDouble(lon); + if (!Double.isFinite(la) || !Double.isFinite(lo) // + || (la == 0 && lo == 0) || la < -90 || la > 90 || lo < -180 || lo > 180) { + return; + } + // store the original strings like other LOCATIONS writers do: + // Double.toString would emit scientific notation for small values, + // which the KML consumers don't accept + item.getMetadata().set(ExtraProperties.LOCATIONS, lat + ";" + lon); + } catch (NumberFormatException e) { + // non-numeric coordinate values: leave them as aleapp:* metadata only + } + } + + private static boolean setIfAbsent(Item item, Property key, String value) { + if (item.getMetadata().get(key) == null) { + item.getMetadata().set(key, value); + return true; + } + return false; + } + + private static boolean setIfAbsent(Item item, String key, String value) { + if (item.getMetadata().get(key) == null) { + item.getMetadata().set(key, value); + return true; + } + return false; + } + +} diff --git a/iped-engine/src/main/resources/iped/engine/task/leapp/context_thread_local_patch.py b/iped-engine/src/main/resources/iped/engine/task/leapp/context_thread_local_patch.py new file mode 100644 index 0000000000..4259e71f5f --- /dev/null +++ b/iped-engine/src/main/resources/iped/engine/task/leapp/context_thread_local_patch.py @@ -0,0 +1,93 @@ +""" +IPED patch: makes LEAPP's Context state thread-local. + +scripts.context.Context stores per-plugin-run state (files_found, seeker, +report_folder, ...) in CLASS attributes. IPED runs many workers over a single +shared CPython interpreter (Jep SharedInterpreter), so that state would be +shared and concurrent plugin runs would overwrite each other. + +Each IPED worker interacts with Python from its own thread, so thread +identity == worker identity: moving the state to threading.local() isolates +each worker. + +The methods are patched IN PLACE (class identity is preserved), so bindings +created earlier via "from scripts.context import Context" (e.g. inside +ilapfuncs or plugin modules) also see the patched behavior. + +One-time global state (_output_params, _data_folder) is left shared on +purpose. get_source_file_path and get_relative_path need no patch: they only +touch state through the patched methods or through the shared globals. + +Everything is defined inside a single function so the patched methods keep +their dependencies as CLOSURES: after the installer global is deleted below, +no name is left behind in the interpreter namespace (which is shared with +PythonParser/PythonTask scripts). +""" + + +def _iped_leapp_patch_context(): + import threading + from os.path import basename + + from scripts.context import Context + + tls = threading.local() + + per_run_attrs = ('report_folder', 'seeker', 'artifact_info', 'module_name', + 'module_file_path', 'artifact_name', 'files_found') + + def state(): + st = getattr(tls, 'state', None) + if st is None: + st = dict.fromkeys(per_run_attrs) + st['files_found'] = [] + st['filename_lookup_map'] = {} + tls.state = st + return st + + def make_setter(name): + def _set(value): + st = state() + st[name] = value + if name == 'files_found': + # the lookup map is derived from files_found: invalidate it + st['filename_lookup_map'] = {} + return _set + + def make_getter(name): + def _get(): + value = state()[name] + if not value: + # same behavior and message as the original Context getters + raise ValueError('Context not set. This function should be' + + ' called from within an artifact.') + return value + return _get + + def get_filename_lookup_map(): + st = state() + if not st['filename_lookup_map']: + if not st['files_found']: + raise ValueError( + 'Cannot build lookup map: _files_found is not set.') + lookup = {} + for path in st['files_found']: + lookup.setdefault(basename(path), []).append(path) + st['filename_lookup_map'] = lookup + return st['filename_lookup_map'] + + def clear(): + # re-initialized lazily on next access + tls.state = None + + for name in per_run_attrs: + setattr(Context, 'set_' + name, staticmethod(make_setter(name))) + setattr(Context, 'get_' + name, staticmethod(make_getter(name))) + + Context.get_filename_lookup_map = staticmethod(get_filename_lookup_map) + Context._build_lookup_map = staticmethod(get_filename_lookup_map) + Context.clear = staticmethod(clear) + + +_iped_leapp_patch_context() +del _iped_leapp_patch_context diff --git a/iped-engine/src/test/java/iped/engine/task/leapp/AleappMediaTypeResolverTest.java b/iped-engine/src/test/java/iped/engine/task/leapp/AleappMediaTypeResolverTest.java new file mode 100644 index 0000000000..8e9874888e --- /dev/null +++ b/iped-engine/src/test/java/iped/engine/task/leapp/AleappMediaTypeResolverTest.java @@ -0,0 +1,74 @@ +package iped.engine.task.leapp; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class AleappMediaTypeResolverTest { + + private String mime(String module, String pluginKey, String artifactName) { + return AleappMediaTypeResolver.resolveMediaType(module, pluginKey, artifactName).toString(); + } + + // ── plugin-specific subtypes (drive the category via CategoriesConfig.json) ── + + @Test + public void testFcmQueuedPluginsShareNotificationType() { + assertEquals("application/x-aleapp-notification", + mime("FCMQueuedMessagesTwitter", "get_fcm_twitter", "FCM Queued Messages - Twitter")); + assertEquals("application/x-aleapp-notification", + mime("FCMQueuedMessageKik", "get_fcm_kik", "FCM Queued Messages - Kik")); + } + + @Test + public void testAccountsPlugins() { + assertEquals("application/x-aleapp-account", mime("accounts_de", "get_accounts_de", "Accounts_de")); + assertEquals("application/x-aleapp-account", mime("accounts_ce", "get_accounts_ce", "Accounts_ce")); + assertEquals("application/x-aleapp-account-authtoken", + mime("accounts_ce", "get_accounts_ce_authtokens", "Authentication tokens")); + } + + @Test + public void testSingleModulePlugins() { + assertEquals("application/x-aleapp-siminfo", mime("siminfo", "get_siminfo", "Sim Info")); + assertEquals("application/x-aleapp-gdrive-file-entry", mime("Cello", "get_cello", "Cello")); + assertEquals("application/x-aleapp-app-role", mime("roles", "get_roles", "Roles")); + assertEquals("application/x-aleapp-update-info", mime("frosting", "get_frosting", "Frosting")); + } + + @Test + public void testGmailOnlyAppEmailsBecomesEmail() { + assertEquals("application/x-aleapp-email", mime("gmailEmails", "get_gmail_app_emails", "Gmail - App Emails")); + // other gmail artifacts fall back to the generic subtype + assertEquals("application/x-aleapp-gmail-label-details", + mime("gmailEmails", "get_gmail_labels", "Gmail - Label Details")); + } + + @Test + public void testFacebookMessengerContactsAndUser() { + assertEquals("application/x-aleapp-facebook-contact", + mime("FacebookMessenger", "get_fb_msys_contacts", "Facebook Messenger - Contacts")); + assertEquals("application/x-aleapp-facebook-account", + mime("FacebookMessenger", "get_fb_user_id", "Facebook Messenger - User")); + } + + // ── generic derivation (unchanged behaviour) ──────────────────────────────── + + @Test + public void testChromePluginsUseArtifactName() { + assertEquals("application/x-aleapp-cookies", mime("chromeCookies", "get_chromeCookies", "Cookies")); + assertEquals("application/x-aleapp-offline-pages", + mime("chromeOfflinePages", "get_chromeOfflinePages", "Offline Pages")); + } + + @Test + public void testArtifactNameHintSuffixes() { + assertEquals("application/x-aleapp-whatsapp-call", mime("WhatsApp", "get_whatsapp", "WhatsApp Calls")); + assertEquals("application/x-aleapp-someapp-message", mime("someapp", "get_someapp", "App Messages")); + } + + @Test + public void testGenericFallbackUsesArtifactName() { + assertEquals("application/x-aleapp-app-usage", mime("wellbeing", "get_wellbeing", "App Usage")); + } +} diff --git a/iped-engine/src/test/java/iped/engine/task/leapp/LavaInsertSqliteDataInterceptorTest.java b/iped-engine/src/test/java/iped/engine/task/leapp/LavaInsertSqliteDataInterceptorTest.java new file mode 100644 index 0000000000..09496d1e08 --- /dev/null +++ b/iped-engine/src/test/java/iped/engine/task/leapp/LavaInsertSqliteDataInterceptorTest.java @@ -0,0 +1,334 @@ +package iped.engine.task.leapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Test; + +import iped.engine.data.Item; +import iped.engine.task.leapp.interceptors.LavaInsertSqliteDataInterceptor; +import iped.engine.tika.SyncMetadata; +import iped.properties.ExtraProperties; + +public class LavaInsertSqliteDataInterceptorTest { + + private Item newItem() { + Item item = new Item(); + item.setMetadata(new SyncMetadata()); + return item; + } + + private static final Class HEADER; + private static final Class STANDARD_FIELD; + static { + try { + HEADER = Class.forName( + "iped.engine.task.leapp.interceptors.LavaInsertSqliteDataInterceptor$Header"); + STANDARD_FIELD = Class.forName( + "iped.engine.task.leapp.interceptors.LavaInsertSqliteDataInterceptor$StandardField"); + } catch (ClassNotFoundException e) { + throw new ExceptionInInitializerError(e); + } + } + + private Object newHeader(String name, String type) throws Exception { + Constructor c = HEADER.getDeclaredConstructor(String.class, String.class); + c.setAccessible(true); + return c.newInstance(name, type); + } + + private Object invokeClassify(String name, String type) throws Exception { + Method classify = LavaInsertSqliteDataInterceptor.class.getDeclaredMethod("classifyHeader", HEADER); + classify.setAccessible(true); + return classify.invoke(null, newHeader(name, type)); + } + + private void invokeMapStandard(Item item, String name, String type, String value) throws Exception { + Object field = invokeClassify(name, type); + Method apply = LavaInsertSqliteDataInterceptor.class + .getDeclaredMethod("applyStandardField", Item.class, STANDARD_FIELD, String.class); + apply.setAccessible(true); + apply.invoke(null, item, field, value); + } + + private Object invokeCellValue(List data, int i) throws Exception { + Method m = LavaInsertSqliteDataInterceptor.class.getDeclaredMethod("cellValue", List.class, int.class); + m.setAccessible(true); + return m.invoke(null, data, i); + } + + // ── Typed headers (lava (name, type) tuples) ─────────────────────────────── + + @Test + public void testDatetimeTypeMapsToDate() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "Whatever Name", "datetime", "2024-01-15 10:00:00"); + assertEquals("2024-01-15 10:00:00", item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + + @Test + public void testDateTypeMapsToDate() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "Whatever Name", "date", "2024-01-15"); + assertEquals("2024-01-15", item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + + @Test + public void testMediaTypeClassifiesAsMedia() throws Exception { + assertEquals("MEDIA", invokeClassify("Image", "media").toString()); + } + + @Test + public void testTypedNonDateHeaderDoesNotUseDateNameHeuristics() throws Exception { + // typed headers must not fall back to the name heuristics for dates: + // e.g. a TEXT column that happens to end with " time" + Item item = newItem(); + invokeMapStandard(item, "Prayer Time", "str", "value"); + assertNull(item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + + // ── Bounds safety ────────────────────────────────────────────────────────── + + @Test + public void testShortRowYieldsNullForMissingColumns() throws Exception { + // Row has 2 values but headers expect 4 — must not throw IndexOutOfBoundsException + List shortRow = Arrays.asList("2024-01-15T10:00:00Z", "Alice"); + + assertEquals("Alice", invokeCellValue(shortRow, 1)); + assertNull("Missing column must be null, not an exception", invokeCellValue(shortRow, 2)); + assertNull("Missing column must be null, not an exception", invokeCellValue(shortRow, 3)); + } + + @Test + public void testEmptyRowYieldsNull() throws Exception { + assertNull(invokeCellValue(Collections.emptyList(), 0)); + assertNull(invokeCellValue(Collections.emptyList(), 1)); + } + + // ── Untyped headers: name-based fallback ─────────────────────────────────── + + @Test + public void testTimestampMapsToMessageDate() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "Timestamp", null, "2024-01-15T10:00:00Z"); + assertEquals("2024-01-15T10:00:00Z", item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + + @Test + public void testDateOfBirthDoesNotMapToDate() throws Exception { + // A personal date must never become the record's event/timeline date + Item item = newItem(); + invokeMapStandard(item, "Date of Birth", null, "1985-03-10"); + assertNull(item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + + @Test + public void testSenderMapsToCommunicationFrom() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "Sender", null, "Alice"); + assertEquals("Alice", item.getMetadata().get(ExtraProperties.COMMUNICATION_FROM)); + } + + @Test + public void testAmbiguousAccountDoesNotMapToCommunicationFrom() throws Exception { + // "Account" usually denotes the device owner, who is the RECEIVER of + // incoming records — mapping it to COMMUNICATION_FROM would mislabel senders. + Item item = newItem(); + invokeMapStandard(item, "Account", null, "user@example.com"); + assertNull(item.getMetadata().get(ExtraProperties.COMMUNICATION_FROM)); + } + + @Test + public void testDateHeaderVariantsMap() throws Exception { + for (String header : Arrays.asList("Start Time", "Message Timestamp", + "Created Timestamp", "Last Updated Timestamp", "Visit Date", + "Date Added", "End Time")) { + Item item = newItem(); + invokeMapStandard(item, header, null, "2024-01-15 10:00:00"); + assertEquals("header must map to MESSAGE_DATE: " + header, + "2024-01-15 10:00:00", item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + } + + @Test + public void testNonEventDateHeadersDoNotMap() throws Exception { + for (String header : Arrays.asList("Timezone", "Birthdate", "Duration")) { + Item item = newItem(); + invokeMapStandard(item, header, null, "some value"); + assertNull("header must NOT map to MESSAGE_DATE: " + header, + item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + } + + @Test + public void testRecipientMapsToCommunicationTo() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "Recipient", null, "Carol"); + assertEquals("Carol", item.getMetadata().get(ExtraProperties.COMMUNICATION_TO)); + } + + @Test + public void testMessageMapsToBody() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "Message", null, "Hello world"); + assertEquals("Hello world", item.getMetadata().get(ExtraProperties.MESSAGE_BODY)); + } + + @Test + public void testBlankValueDoesNotBlockLaterColumn() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "Timestamp", null, ""); + invokeMapStandard(item, "Date", null, "2024-01-15 10:00:00"); + assertEquals("2024-01-15 10:00:00", item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + + @Test + public void testFirstValueWinsForDuplicateHeaders() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "Timestamp", null, "first"); + invokeMapStandard(item, "Timestamp", null, "second"); + assertEquals("first", item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + + @Test + public void testCaseInsensitiveHeader() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "TIMESTAMP", null, "2024-01-15T10:00:00Z"); + assertEquals("2024-01-15T10:00:00Z", item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + } + + @Test + public void testUnknownHeaderDoesNotMapStandard() throws Exception { + Item item = newItem(); + invokeMapStandard(item, "CustomField", null, "value"); + assertNull(item.getMetadata().get(ExtraProperties.MESSAGE_DATE)); + assertNull(item.getMetadata().get(ExtraProperties.COMMUNICATION_FROM)); + assertNull(item.getMetadata().get(ExtraProperties.COMMUNICATION_TO)); + assertNull(item.getMetadata().get(ExtraProperties.MESSAGE_BODY)); + } + + // ── Untyped file-path columns ────────────────────────────────────────────── + + @Test + public void testFilePathColumnsClassifyAsFilePath() throws Exception { + // representative "* Path" / *filepath / file_path columns across ALEAPP plugins + for (String header : Arrays.asList("File Path", "Local Path To Media", "Download Path", "Full Path", + "Original Path", "Source File Path", "Screenshot Path", "Code Path", "Save Path", "Item Path", + "Target File Path", "Path", "file_path", "Original Filepath", "vault_filepath")) { + assertEquals("header must classify as FILE_PATH: " + header, "FILE_PATH", + invokeClassify(header, null).toString()); + } + } + + private boolean invokeIsNonFilePathColumn(String moduleName, String headerName) throws Exception { + Method m = LavaInsertSqliteDataInterceptor.class + .getDeclaredMethod("isNonFilePathColumn", String.class, String.class); + m.setAccessible(true); + return (boolean) m.invoke(null, moduleName, headerName); + } + + @Test + public void testNonFilePathColumnsAreExcludedPerPlugin() throws Exception { + // known per-plugin exceptions: the name looks like a file path but is not one in that plugin + assertTrue(invokeIsNonFilePathColumn("chromeCookies", "Path")); + assertTrue(invokeIsNonFilePathColumn("firefoxCookies", "Path")); + assertTrue(invokeIsNonFilePathColumn("FairEmail", "Return Path")); + assertTrue(invokeIsNonFilePathColumn("DuckDuckGo", "Folder Path")); + } + + @Test + public void testSameColumnIsFilePathInOtherPlugins() throws Exception { + // "Path" is a real file path in these plugins, so it must NOT be excluded there + assertFalse(invokeIsNonFilePathColumn("emulatedSmeta", "Path")); + assertFalse(invokeIsNonFilePathColumn("Zapya", "path")); + // and on its own the name still classifies as FILE_PATH — the plugin decides the exception + assertEquals("FILE_PATH", invokeClassify("Path", null).toString()); + } + + @Test + public void testTypedColumnIsNotClassifiedAsFilePath() throws Exception { + // a column carrying an explicit type must not fall back to the file-path name heuristic + assertEquals("NONE", invokeClassify("File Path", "str").toString()); + } + + @Test + public void testMediaTypedPathColumnStaysMedia() throws Exception { + // ('Attachment File', 'media') and similar must remain MEDIA, not FILE_PATH + assertEquals("MEDIA", invokeClassify("Attachment File Path", "media").toString()); + } + + @Test + public void testNonPathColumnsAreNotFilePath() throws Exception { + for (String header : Arrays.asList("Title", "URL", "Account", "File Size", "File Name")) { + assertEquals("header must NOT classify as FILE_PATH: " + header, "NONE", + invokeClassify(header, null).toString()); + } + } + + // ── Geolocation mapping ──────────────────────────────────────────────────── + + private void invokeSetLocation(Item item, String lat, String lon) throws Exception { + Method m = LavaInsertSqliteDataInterceptor.class + .getDeclaredMethod("setLocationIfValid", Item.class, String.class, String.class); + m.setAccessible(true); + m.invoke(null, item, lat, lon); + } + + @Test + public void testValidCoordinatesMapToLocations() throws Exception { + Item item = newItem(); + invokeSetLocation(item, "-15.7942", "-47.8822"); + assertEquals("-15.7942;-47.8822", item.getMetadata().get(ExtraProperties.LOCATIONS)); + } + + @Test + public void testOriginalStringIsStoredNotReparsedDouble() throws Exception { + Item item = newItem(); + invokeSetLocation(item, "15.50", "-0.0005"); + assertEquals("15.50;-0.0005", item.getMetadata().get(ExtraProperties.LOCATIONS)); + } + + @Test + public void testCommaDecimalCoordinatesAreNormalized() throws Exception { + Item item = newItem(); + invokeSetLocation(item, "-15,7942", "-47,8822"); + assertEquals("-15.7942;-47.8822", item.getMetadata().get(ExtraProperties.LOCATIONS)); + } + + @Test + public void testZeroZeroCoordinatesDoNotMap() throws Exception { + // 0;0 is the classic "no fix" placeholder — must not plot at Null Island + Item item = newItem(); + invokeSetLocation(item, "0", "0"); + assertNull(item.getMetadata().get(ExtraProperties.LOCATIONS)); + } + + @Test + public void testOutOfRangeCoordinatesDoNotMap() throws Exception { + Item item = newItem(); + invokeSetLocation(item, "91.0", "10.0"); + assertNull(item.getMetadata().get(ExtraProperties.LOCATIONS)); + } + + @Test + public void testNonNumericCoordinatesDoNotMap() throws Exception { + Item item = newItem(); + invokeSetLocation(item, "n/a", "n/a"); + assertNull(item.getMetadata().get(ExtraProperties.LOCATIONS)); + } + + @Test + public void testMissingCoordinateDoesNotMap() throws Exception { + Item item = newItem(); + invokeSetLocation(item, "-15.7942", null); + assertNull(item.getMetadata().get(ExtraProperties.LOCATIONS)); + } +} diff --git a/iped-engine/src/test/java/iped/engine/task/leapp/LeappUtilsTest.java b/iped-engine/src/test/java/iped/engine/task/leapp/LeappUtilsTest.java new file mode 100644 index 0000000000..1d4325af5d --- /dev/null +++ b/iped-engine/src/test/java/iped/engine/task/leapp/LeappUtilsTest.java @@ -0,0 +1,108 @@ +package iped.engine.task.leapp; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class LeappUtilsTest { + + @Test + public void testPatternWithoutPathSeparators() { + assertEquals("name:*com && name:whatsapp*", LeappUtils.globToLuceneQuery("*com.whatsapp*")); + } + + @Test + public void testExactFilenameMatch() { + assertEquals("path:\"com.whatsapp/shared_prefs\" && name:\"com.whatsapp_preferences_light.xml\"", + LeappUtils.globToLuceneQuery("*/com.whatsapp/shared_prefs/com.whatsapp_preferences_light.xml")); + } + + @Test + public void testFilenameWithPrefixAndTrailingWildcard() { + assertEquals("path:\"com.whatsapp/databases\" && name:db*", LeappUtils.globToLuceneQuery("*/com.whatsapp/databases/*.db*")); + } + + @Test + public void testFilenameWithPrefixAndDotWildcard() { + assertEquals("path:\"WhatsApp Videos\" && name:video", LeappUtils.globToLuceneQuery("*/WhatsApp Videos/video.*")); + } + + @Test + public void testFilenameWithExtensionMatch() { + assertEquals("path:\"WhatsApp Audio\" && name:ogg", LeappUtils.globToLuceneQuery("*/WhatsApp Audio/*.ogg")); + } + + @Test + public void testMatchAllFilesWithAnyExtension() { + assertEquals("path:\"WhatsApp Images\"", LeappUtils.globToLuceneQuery("*/WhatsApp Images/*.*")); + } + + @Test + public void testMatchAllFiles() { + assertEquals("path:\"WhatsApp Images\"", LeappUtils.globToLuceneQuery("*/WhatsApp Images/*")); + } + + @Test + public void testFilenameWithInfixMatch() { + assertEquals("path:\"WhatsApp Images\" && name:*file*", LeappUtils.globToLuceneQuery("*/WhatsApp Images/*file*")); + } + + @Test + public void testFilenameWithInternalWildcard() { + assertEquals("path:\"com.runtastic.android/databases\" && name:user && name:db*", + LeappUtils.globToLuceneQuery("*com.runtastic.android/databases/user.db*")); + } + + @Test + public void testRootPathWithExtensionMatch() { + assertEquals("name:log", LeappUtils.globToLuceneQuery("/*.log")); + } + + @Test + public void testPathWithIntermediateWildcardDirectories() { + assertEquals("path:\"system/usagestats\" && name:\"version\"", LeappUtils.globToLuceneQuery("*/system/usagestats/*/version")); + } + + @Test + public void testPathWithPrefixFilenameMatch() { + assertEquals("path:\"com.android.providers.contacts/databases\" && name:contact*", + LeappUtils.globToLuceneQuery("*/com.android.providers.contacts/databases/contact*")); + } + + @Test + public void testPatternWithTrailingDotWildcard() { + assertEquals("name:contact", LeappUtils.globToLuceneQuery("contact.*")); + } + + @Test + public void testComplexPunctuationInFilename() { + // Hyphens, underscores, and dots all split the string to avoid Lucene operator conflicts (like '-' acting as NOT). + // This maximizes recall, allowing the downstream Regex to do the exact validation. + assertEquals("path:\"logs\" && name:test && name:profile && name:info* && name:db*", + LeappUtils.globToLuceneQuery("*/logs/test-profile_info*.db*")); + } + + @Test + public void testConsecutiveWildcards() { + // Should compress '**' in paths natively through StringUtils.split behavior, + // and compress '**' in names to '*' + assertEquals("path:\"WhatsApp\" && name:db", LeappUtils.globToLuceneQuery("**/WhatsApp/**/*.db")); + } + + @Test + public void testEmptyFilenameJustDirectory() { + // Trailing slash means there's no filename, just a path + assertEquals("path:\"WhatsApp\"", LeappUtils.globToLuceneQuery("*/WhatsApp/")); + } + + @Test + public void testNullInput() { + assertEquals("", LeappUtils.globToLuceneQuery(null)); + } + + @Test + public void testEmptyInput() { + assertEquals("", LeappUtils.globToLuceneQuery("")); + assertEquals("", LeappUtils.globToLuceneQuery(" ")); + } +} diff --git a/iped-parsers/iped-parsers-impl/pom.xml b/iped-parsers/iped-parsers-impl/pom.xml index b6020381b9..79daa21ffa 100644 --- a/iped-parsers/iped-parsers-impl/pom.xml +++ b/iped-parsers/iped-parsers-impl/pom.xml @@ -199,10 +199,10 @@ test - black.ninia - jep - 4.0.3 - + org.ninia + jep + 4.3.1 + net.sf java-dbx diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/android/backup/AndroidBackupParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/android/backup/AndroidBackupParser.java new file mode 100644 index 0000000000..59a98ddc7a --- /dev/null +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/android/backup/AndroidBackupParser.java @@ -0,0 +1,286 @@ +package iped.parsers.android.backup; + +import java.io.Console; +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.bouncycastle.crypto.PBEParametersGenerator; +import org.bouncycastle.crypto.generators.PKCS5S2ParametersGenerator; +import org.bouncycastle.crypto.params.KeyParameter; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import iped.parsers.compress.PackageParser; + +/** + * Parses and extracts content from android backup files done with ADB. This + * work was based on code from + * https://github.com/nelenkov/android-backup-extractor. + * + * @author Patrick Dalla Bernardina + * + */ +public class AndroidBackupParser extends PackageParser { + private static final MediaType AB_MIMETYPE = MediaType.application("x-android-backup"); + + public static final Set SUPPORTED_TYPES = Collections.singleton(AB_MIMETYPE); + + private static final Object MAGIC = "ANDROID BACKUP"; + + private static final String AB_PREFIX = "androidBackup"; + private static final String ENC_METADATA = "encryption"; + + private static final String COMPRESSION_METADATA = "compression"; + + private static final String AB_VERSION_METADATA = "version"; + + private static final int MASTER_KEY_SIZE = 256; + + private static final int PBKDF2_SALT_SIZE = 512; + + private static final String ENCRYPTION_MECHANISM = "AES/CBC/PKCS5Padding"; + + private static final int PBKDF2_KEY_SIZE = 256; + + @Override + public Set getSupportedTypes(ParseContext arg0) { + return SUPPORTED_TYPES; + } + + @Override + public void parse(InputStream in, ContentHandler handler, Metadata metadata, ParseContext context) + throws IOException, SAXException, TikaException { + try { + String magic = readLine(in); + if (!magic.equals(MAGIC)) { + throw new TikaException("Invalid android backup magic."); + } + + int version = Integer.parseInt(readLine(in)); + + if (version < 1 || version > 5) { + throw new TikaException("Invalid android backup version:" + version + "."); + } + metadata.set(AB_PREFIX + ":" + AB_VERSION_METADATA, Integer.toString(version)); + + boolean isCompressed = Integer.parseInt(readLine(in)) == 1; + + String encryption = readLine(in); + metadata.set(AB_PREFIX + ":" + ENC_METADATA, encryption); + + if (encryption.equals("SHA-256")) { + try { + in = unencrypt(in, version, null); + } catch (Exception e) { + throw new TikaException(e.getMessage(), e); + } + if (in == null) { + throw new TikaException("Invalid password or master key checksum."); + } + } + + Inflater inflater; + if (isCompressed) { + inflater = new Inflater(); + in = new InflaterInputStream(in, inflater); + } + + try { + super.parse(in, handler, metadata, context); + }finally { + } + + + } catch (TikaException te) { + throw te; + } catch (Exception e) { + new TikaException(e.getMessage(), e); + } + + } + + private InputStream unencrypt(InputStream in, int version, String password) + throws InvalidKeyException, InvalidAlgorithmParameterException, NoSuchAlgorithmException, IOException, + NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException { + if (Cipher.getMaxAllowedKeyLength("AES") < MASTER_KEY_SIZE) { + System.err.println("WARNING: Maximum allowed key-length seems smaller than needed. " + + "Please check that unlimited strength cryptography is available, see README.md for details"); + } + + if (password == null || "".equals(password)) { + Console console = System.console(); + if (console != null) { + System.err.println("This backup is encrypted, please provide the password"); + password = new String(console.readPassword("Password: ")); + } else { + throw new IllegalArgumentException("Backup encrypted but password not specified"); + } + } + + String userSaltHex = readLine(in); // 5 + byte[] userSalt = hexToByteArray(userSaltHex); + if (userSalt.length != PBKDF2_SALT_SIZE / 8) { + throw new IllegalArgumentException("Invalid salt length: " + userSalt.length); + } + + String ckSaltHex = readLine(in); // 6 + byte[] ckSalt = hexToByteArray(ckSaltHex); + + int rounds = Integer.parseInt(readLine(in)); // 7 + String userIvHex = readLine(in); // 8 + + String masterKeyBlobHex = readLine(in); // 9 + + // decrypt the master key blob + Cipher c = Cipher.getInstance(ENCRYPTION_MECHANISM); + // XXX we don't support non-ASCII passwords + SecretKey userKey = buildPasswordKey(password, userSalt, rounds, false); + byte[] IV = hexToByteArray(userIvHex); + IvParameterSpec ivSpec = new IvParameterSpec(IV); + c.init(Cipher.DECRYPT_MODE, new SecretKeySpec(userKey.getEncoded(), "AES"), ivSpec); + byte[] mkCipher = hexToByteArray(masterKeyBlobHex); + byte[] mkBlob = c.doFinal(mkCipher); + + // first, the master key IV + int offset = 0; + int len = mkBlob[offset++]; + IV = Arrays.copyOfRange(mkBlob, offset, offset + len); + offset += len; + // then the master key itself + len = mkBlob[offset++]; + byte[] mk = Arrays.copyOfRange(mkBlob, offset, offset + len); + offset += len; + // and finally the master key checksum hash + len = mkBlob[offset++]; + byte[] mkChecksum = Arrays.copyOfRange(mkBlob, offset, offset + len); + + // now validate the decrypted master key against the checksum + // first try the algorithm matching the archive version + boolean useUtf = version >= 2; + byte[] calculatedCk = makeKeyChecksum(mk, ckSalt, rounds, useUtf); + System.err.printf("Calculated MK checksum (use UTF-8: %s): %s\n", useUtf, toHex(calculatedCk)); + if (!Arrays.equals(calculatedCk, mkChecksum)) { + System.err.println("Checksum does not match."); + // try the reverse + calculatedCk = makeKeyChecksum(mk, ckSalt, rounds, !useUtf); + System.err.printf("Calculated MK checksum (use UTF-8: %s): %s\n", useUtf, toHex(calculatedCk)); + } + + if (Arrays.equals(calculatedCk, mkChecksum)) { + ivSpec = new IvParameterSpec(IV); + c.init(Cipher.DECRYPT_MODE, new SecretKeySpec(mk, "AES"), ivSpec); + // Only if all of the above worked properly will 'result' be + // assigned + return new CipherInputStream(in, c); + } + return null; + } + + public static String toHex(byte[] bytes) { + StringBuilder buff = new StringBuilder(); + for (byte b : bytes) { + buff.append(String.format("%02X", b)); + } + + return buff.toString(); + } + + public static byte[] makeKeyChecksum(byte[] pwBytes, byte[] salt, int rounds, boolean useUtf8) { + char[] mkAsChar = new char[pwBytes.length]; + for (int i = 0; i < pwBytes.length; i++) { + mkAsChar[i] = (char) pwBytes[i]; + } + + Key checksum = buildCharArrayKey(mkAsChar, salt, rounds, useUtf8); + return checksum.getEncoded(); + } + + public static byte[] hexToByteArray(String digits) { + final int bytes = digits.length() / 2; + if (2 * bytes != digits.length()) { + throw new IllegalArgumentException("Hex string must have an even number of digits"); + } + + byte[] result = new byte[bytes]; + for (int i = 0; i < digits.length(); i += 2) { + result[i / 2] = (byte) Integer.parseInt(digits.substring(i, i + 2), 16); + } + return result; + } + + private String readLine(InputStream in) throws IOException { + StringBuffer buff = new StringBuffer(); + char b = (char) in.read(); + while (b != '\n') { + buff.append(b); + b = (char) in.read(); + } + + return buff.toString(); + } + + public static SecretKey buildCharArrayKey(char[] pwArray, byte[] salt, int rounds, boolean useUtf8) { + // Original code from BackupManagerService + // this produces different results when run with Sun/Oracale Java SE + // which apparently treats password bytes as UTF-8 (16?) + // (the encoding is left unspecified in PKCS#5) + + // try { + // SecretKeyFactory keyFactory = SecretKeyFactory + // .getInstance("PBKDF2WithHmacSHA1"); + // KeySpec ks = new PBEKeySpec(pwArray, salt, rounds, PBKDF2_KEY_SIZE); + // return keyFactory.generateSecret(ks); + // } catch (InvalidKeySpecException e) { + // throw new RuntimeException(e); + // } catch (NoSuchAlgorithmException e) { + // throw new RuntimeException(e); + // } catch (NoSuchProviderException e) { + // throw new RuntimeException(e); + // } + // return null; + + return androidPBKDF2(pwArray, salt, rounds, useUtf8); + } + + public static SecretKey androidPBKDF2(char[] pwArray, byte[] salt, int rounds, boolean useUtf8) { + PBEParametersGenerator generator = new PKCS5S2ParametersGenerator(); + // Android treats password bytes as ASCII, which is obviously + // not the case when an AES key is used as a 'password'. + // Use the same method for compatibility. + + // Android 4.4 however uses all char bytes + // useUtf8 needs to be true for KitKat + byte[] pwBytes = useUtf8 ? PBEParametersGenerator.PKCS5PasswordToUTF8Bytes(pwArray) + : PBEParametersGenerator.PKCS5PasswordToBytes(pwArray); + generator.init(pwBytes, salt, rounds); + KeyParameter params = (KeyParameter) generator.generateDerivedParameters(PBKDF2_KEY_SIZE); + + return new SecretKeySpec(params.getKey(), "AES"); + } + + private static SecretKey buildPasswordKey(String pw, byte[] salt, int rounds, boolean useUtf8) { + return buildCharArrayKey(pw.toCharArray(), salt, rounds, useUtf8); + } +} diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/python/PythonParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/python/PythonParser.java index 439a746ba8..d6f4eb39e5 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/python/PythonParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/python/PythonParser.java @@ -12,6 +12,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.commons.lang3.SystemUtils; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; @@ -48,7 +49,7 @@ public class PythonParser extends AbstractParser { public static final String SEE_MANUAL = Messages.getString("PythonModule.SeeManual"); private static final Map jepPerThread = new HashMap<>(); - private static final Set instancetPerThread = new HashSet<>(); + private static final Set instancePerThread = new HashSet<>(); private static final Map mediaToParserMap = new ConcurrentHashMap<>(); private static final Map mediaTypesToQueueOrder = new ConcurrentHashMap<>(); private static final AtomicBoolean jepNotFoundPrinted = new AtomicBoolean(); @@ -65,14 +66,14 @@ public class PythonParser extends AbstractParser { config.redirectStdout(System.out); SharedInterpreter.setConfig(config); - if (System.getProperty("os.name").toLowerCase().startsWith("windows")) { + if (SystemUtils.IS_OS_WINDOWS) { String ipedRoot = System.getProperty(IConfigurationDirectory.IPED_ROOT); if (ipedRoot != null && new File(ipedRoot).exists()) { - PyConfig pyConfig = new PyConfig(); + PyConfig pyConfig = PyConfig.python(); String pythonHome = ipedRoot + "/python"; - pyConfig.setPythonHome(pythonHome); - pyConfig.setIgnoreEnvironmentFlag(1); - pyConfig.setNoUserSiteDirectory(1); + pyConfig.setHome(pythonHome); + pyConfig.setUseEnvironment(false); + pyConfig.setUserSiteDirectory(false); MainInterpreter.setInitParams(pyConfig); System.load(pythonHome + "/vcruntime140.dll"); System.load(pythonHome + "/python39.dll"); @@ -86,7 +87,7 @@ public class PythonParser extends AbstractParser { } @SuppressWarnings("unchecked") - public PythonParser(){ + public PythonParser() { synchronized (this.getClass()) { if (inited) { @@ -223,10 +224,10 @@ private String getInstanceName() { private String getInstanceMethod(String function) throws JepException { String instanceName = getInstanceName(); - synchronized (instancetPerThread) { - if (!instancetPerThread.contains(instanceName)) { + synchronized (instancePerThread) { + if (!instancePerThread.contains(instanceName)) { loadScript(getJep()); - instancetPerThread.add(instanceName); + instancePerThread.add(instanceName); } } String ret = instanceName + "." + function; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/sqlite/SQLite3DBParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/sqlite/SQLite3DBParser.java index 88808c7e41..dd617e85cf 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/sqlite/SQLite3DBParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/sqlite/SQLite3DBParser.java @@ -43,6 +43,7 @@ import iped.parsers.util.DelegatingConnection; import iped.properties.BasicProps; import iped.search.IItemSearcher; +import iped.utils.DateUtil; import iped.utils.IOUtil; /** @@ -123,11 +124,11 @@ public void close() throws SQLException { return connection; } - private static File exportWalLog(File dbFile, ParseContext context, TemporaryResources tmp) { + public static File exportWalLog(File dbFile, ParseContext context, TemporaryResources tmp) { return exportRelatedFile(dbFile, "-wal", context, tmp); } - private static File exportRollbackJournal(File dbFile, ParseContext context, TemporaryResources tmp) { + public static File exportRollbackJournal(File dbFile, ParseContext context, TemporaryResources tmp) { return exportRelatedFile(dbFile, "-journal", context, tmp); } @@ -156,6 +157,7 @@ private static File exportRelatedFile(File theFile, String suffix, ParseContext File relatedFileTemp = new File(theFile.getAbsolutePath() + suffix); try (InputStream in = relatedItem.getBufferedInputStream()) { Files.copy(in, relatedFileTemp.toPath(), StandardCopyOption.REPLACE_EXISTING); + DateUtil.updatePathTimes(relatedFileTemp.toPath(), relatedItem); } catch (IOException e) { e.printStackTrace(); } diff --git a/iped-utils/src/main/java/iped/utils/DateUtil.java b/iped-utils/src/main/java/iped/utils/DateUtil.java index 15560c05f6..6144be21e9 100644 --- a/iped-utils/src/main/java/iped/utils/DateUtil.java +++ b/iped-utils/src/main/java/iped/utils/DateUtil.java @@ -1,5 +1,10 @@ package iped.utils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; import java.text.DateFormat; import java.text.DateFormatSymbols; import java.text.ParseException; @@ -13,6 +18,8 @@ import org.apache.tika.utils.DateUtils; +import iped.data.IItemReader; + public class DateUtil { public static final TimeZone UTC = TimeZone.getTimeZone("UTC"); @@ -117,4 +124,21 @@ public static Date stringToDate(String date) throws ParseException { return threadLocal.get().parse(date); } + /** + * Updates the file times (creation, modified, access) of the specified path. + */ + public static void updatePathTimes(Path path, IItemReader item) throws IOException { + + // Get the view wrapper for modifying attributes + @SuppressWarnings("null") + BasicFileAttributeView view = Files.getFileAttributeView(path, BasicFileAttributeView.class); + + // Define your new dates using java.time.Instant + FileTime creationTime = item.getCreationDate() != null ? FileTime.from(item.getCreationDate().toInstant()) : null; + FileTime modifiedTime = item.getModDate() != null ? FileTime.from(item.getModDate().toInstant()) : null; + FileTime accessTime = item.getAccessDate() != null ? FileTime.from(item.getAccessDate().toInstant()) : null; + + // Apply changes (Pass null to ignore a specific property) + view.setTimes(modifiedTime, accessTime, creationTime); + } }