From a68557e3cdb7adf9ca49d0529e20a5b95c3c9d0b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 15 Feb 2026 11:08:42 +0000 Subject: [PATCH 1/2] Extend PST import: store contacts (.vcf), calendars (.ics), notes (.txt) - Contacts exported as vCard 3.0 with FN, N, ORG, EMAIL, TEL, ADR, BDAY - Appointments exported as iCalendar 2.0 with VEVENT - Notes (Message items in folders containing 'note') exported as plain text - Same storage structure as .eml: {checksum}-{id}.{ext} in folder hierarchy - readpst fallback counts .vcf/.ics/.txt when present - Update tests to count all extracted file types Co-authored-by: Andrey Oblivantsev --- CONTRIBUTING.md | 9 ++ internal/sync/pst/pst.go | 280 ++++++++++++++++++++++++++++++---- internal/sync/pst/pst_test.go | 46 +++--- 3 files changed, 288 insertions(+), 47 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da614b3..c63a957 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -242,6 +242,15 @@ users/{uuid}/gmail.com/eslider/inbox/a1b2c3d4e5f67890-12345.eml - Deduplication: by content checksum (IMAP/POP3) or message ID (Gmail) - Use `./mails fix-dates` to batch-repair mtime on all existing .eml files +### PST Import Storage + +PST/OST imports use the same structure and naming as .eml files: + +- Emails: `{checksum}-{id}.eml` (RFC 822) +- Contacts: `{checksum}-{id}.vcf` (vCard 3.0) +- Calendars: `{checksum}-{id}.ics` (iCalendar 2.0) +- Notes: `{checksum}-{id}.txt` (plain text, from folder names containing "note") + ## Docker ```bash diff --git a/internal/sync/pst/pst.go b/internal/sync/pst/pst.go index b6f2428..d431f0f 100644 --- a/internal/sync/pst/pst.go +++ b/internal/sync/pst/pst.go @@ -1,6 +1,7 @@ // Package pst implements PST/OST file import. -// Extracts messages from Microsoft Outlook personal storage files -// and saves them as .eml files preserving original dates. +// Extracts messages, contacts, appointments, and notes from Microsoft Outlook +// personal storage files. Emails as .eml, contacts as .vcf, calendars as .ics, +// notes as .txt — stored in the same folder hierarchy as .eml files. package pst import ( @@ -23,6 +24,12 @@ import ( charsets "github.com/emersion/go-message/charset" ) +// MAPI property IDs for common item properties (PidTagSubject, PidTagBody). +const ( + mapiTagSubject = 55 + mapiTagBody = 4096 +) + func init() { // Register extended charsets for go-pst. pst.ExtendCharsets(func(name string, enc encoding.Encoding) { @@ -96,17 +103,17 @@ func importGoPst(pstPath, emailDir string, onProgress ProgressFunc) (int, int, e for iter.Next() { msg := iter.Value() - emlData, date := messageToEML(msg) - if emlData == nil { + data, ext, date := itemToStoredFormat(msg, folderPath) + if data == nil { errCount++ continue } - checksum := contentChecksum(emlData) - filename := fmt.Sprintf("%s-%d.eml", checksum, extracted) + checksum := contentChecksum(data) + filename := fmt.Sprintf("%s-%d.%s", checksum, extracted, ext) path := filepath.Join(dir, filename) - if err := os.WriteFile(path, emlData, 0o644); err != nil { + if err := os.WriteFile(path, data, 0o644); err != nil { log.Printf("WARN: write %s: %v", path, err) errCount++ continue @@ -135,30 +142,40 @@ func importGoPst(pstPath, emailDir string, onProgress ProgressFunc) (int, int, e return extracted, errCount, nil } -// messageToEML converts a PST message to RFC822 .eml format. -func messageToEML(msg *pst.Message) ([]byte, time.Time) { - var subject, from, to, body string - var date time.Time - +// itemToStoredFormat converts a PST item to the appropriate storage format. +// Returns (data, ext, date). ext is "eml", "vcf", "ics", or "txt". +func itemToStoredFormat(msg *pst.Message, folderPath string) ([]byte, string, time.Time) { switch p := msg.Properties.(type) { case *properties.Message: - subject = p.GetSubject() - from = formatSender(p.GetSenderName(), p.GetSenderEmailAddress()) - to = p.GetDisplayTo() - body = p.GetBody() - if ct := p.GetClientSubmitTime(); ct > 0 { - date = time.Unix(ct, 0) - } else if dt := p.GetMessageDeliveryTime(); dt > 0 { - date = time.Unix(dt, 0) + if strings.Contains(folderPath, "note") { + return messageToNoteTxt(p), "txt", messageDate(p.GetClientSubmitTime(), p.GetMessageDeliveryTime()) } + return messageToEML(p), "eml", messageDate(p.GetClientSubmitTime(), p.GetMessageDeliveryTime()) + case *properties.Appointment: + return appointmentToICS(msg, p), "ics", appointmentDate(p) + case *properties.Contact: + return contactToVCF(p), "vcf", contactDate(p) default: - // Skip non-message items (appointments, contacts, etc.). - return nil, time.Time{} + return nil, "", time.Time{} } +} - if date.IsZero() { - date = time.Now() +func messageDate(clientSubmit, messageDelivery int64) time.Time { + if clientSubmit > 0 { + return time.Unix(clientSubmit, 0) + } + if messageDelivery > 0 { + return time.Unix(messageDelivery, 0) } + return time.Now() +} + +func messageToEML(p *properties.Message) []byte { + subject := p.GetSubject() + from := formatSender(p.GetSenderName(), p.GetSenderEmailAddress()) + to := p.GetDisplayTo() + body := p.GetBody() + date := messageDate(p.GetClientSubmitTime(), p.GetMessageDeliveryTime()) dateStr := date.Format(time.RFC1123Z) var sb strings.Builder @@ -173,7 +190,213 @@ func messageToEML(msg *pst.Message) ([]byte, time.Time) { sb.WriteString("\r\n") sb.WriteString(body) - return []byte(sb.String()), date + return []byte(sb.String()) +} + +// messageToNoteTxt converts a sticky-note Message to plain text. +func messageToNoteTxt(p *properties.Message) []byte { + subject := p.GetSubject() + body := p.GetBody() + if subject != "" && body != "" { + return []byte(subject + "\n\n" + body) + } + if subject != "" { + return []byte(subject) + } + return []byte(body) +} + +// readSubjectBody reads Subject and Body from a message's PropertyContext (for Appointment/Contact). +func readSubjectBody(msg *pst.Message) (subject, body string) { + if msg.PropertyContext == nil { + return "", "" + } + if r, err := msg.PropertyContext.GetPropertyReader(mapiTagSubject, msg.LocalDescriptors); err == nil { + subject, _ = r.GetString() + } + if r, err := msg.PropertyContext.GetPropertyReader(mapiTagBody, msg.LocalDescriptors); err == nil { + body, _ = r.GetString() + } + return subject, body +} + +func appointmentDate(p *properties.Appointment) time.Time { + if t := p.GetAppointmentStartWhole(); t > 0 { + return time.Unix(t, 0) + } + if t := p.GetClipStart(); t > 0 { + return time.Unix(t, 0) + } + return time.Now() +} + +// appointmentToICS converts a PST appointment to iCalendar (.ics) format. +func appointmentToICS(msg *pst.Message, p *properties.Appointment) []byte { + subject, body := readSubjectBody(msg) + if subject == "" { + subject = "Untitled" + } + loc := p.GetLocation() + start := p.GetAppointmentStartWhole() + end := p.GetAppointmentEndWhole() + if start == 0 { + start = p.GetClipStart() + } + if end == 0 { + end = p.GetClipEnd() + } + if end <= start { + end = start + 3600 // 1 hour default + } + + startT := time.Unix(start, 0).UTC().Format("20060102T150405Z") + endT := time.Unix(end, 0).UTC().Format("20060102T150405Z") + now := time.Now().UTC().Format("20060102T150405Z") + uid := fmt.Sprintf("pst-%d@imported", start) + + var sb strings.Builder + sb.WriteString("BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//PST Import//EN\r\n") + sb.WriteString("BEGIN:VEVENT\r\n") + sb.WriteString("UID:" + uid + "\r\n") + sb.WriteString("DTSTAMP:" + now + "\r\n") + sb.WriteString("DTSTART:" + startT + "\r\n") + sb.WriteString("DTEND:" + endT + "\r\n") + sb.WriteString("SUMMARY:" + foldLine(escapeICS(subject)) + "\r\n") + if loc != "" { + sb.WriteString("LOCATION:" + foldLine(escapeICS(loc)) + "\r\n") + } + if body != "" { + sb.WriteString("DESCRIPTION:" + foldLine(escapeICS(body)) + "\r\n") + } + sb.WriteString("END:VEVENT\r\nEND:VCALENDAR\r\n") + + return []byte(sb.String()) +} + +func escapeICS(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, ";", "\\;") + s = strings.ReplaceAll(s, ",", "\\,") + s = strings.ReplaceAll(s, "\r\n", "\\n") + s = strings.ReplaceAll(s, "\n", "\\n") + return s +} + +func foldLine(s string) string { + const maxLen = 75 + if len(s) <= maxLen { + return s + } + var sb strings.Builder + for len(s) > maxLen { + sb.WriteString(s[:maxLen]) + sb.WriteString("\r\n ") + s = s[maxLen:] + } + sb.WriteString(s) + return sb.String() +} + +func contactDate(p *properties.Contact) time.Time { + if t := p.GetBirthdayLocal(); t > 0 { + return time.Unix(t, 0) + } + return time.Now() +} + +// contactToVCF converts a PST contact to vCard (.vcf) format. +func contactToVCF(p *properties.Contact) []byte { + fn := contactDisplayName(p) + if fn == "" { + fn = "Unknown" + } + given := p.GetGivenName() + family := p.GetSurname() + org := p.GetCompanyName() + email := p.GetEmail1EmailAddress() + if email == "" { + email = p.GetEmail2EmailAddress() + } + if email == "" { + email = p.GetEmail3EmailAddress() + } + phone := p.GetPrimaryTelephoneNumber() + if phone == "" { + phone = p.GetBusinessTelephoneNumber() + } + if phone == "" { + phone = p.GetHomeTelephoneNumber() + } + addr := p.GetWorkAddressStreet() + if addr == "" { + addr = p.GetHomeAddressStreet() + } + city := p.GetWorkAddressCity() + if city == "" { + city = p.GetHomeAddressCity() + } + region := p.GetWorkAddressState() + if region == "" { + region = p.GetHomeAddressStateOrProvince() + } + postal := p.GetWorkAddressPostalCode() + if postal == "" { + postal = p.GetHomeAddressPostalCode() + } + country := p.GetWorkAddressCountry() + if country == "" { + country = p.GetHomeAddressCountry() + } + + var sb strings.Builder + sb.WriteString("BEGIN:VCARD\r\nVERSION:3.0\r\n") + sb.WriteString("FN:" + vcfEscape(fn) + "\r\n") + if given != "" || family != "" { + sb.WriteString("N:" + vcfEscape(family) + ";" + vcfEscape(given) + ";;;\r\n") + } + if org != "" { + sb.WriteString("ORG:" + vcfEscape(org) + "\r\n") + } + if email != "" { + sb.WriteString("EMAIL:" + vcfEscape(email) + "\r\n") + } + if phone != "" { + sb.WriteString("TEL:" + vcfEscape(phone) + "\r\n") + } + if addr != "" || city != "" || region != "" || postal != "" || country != "" { + sb.WriteString("ADR:;;" + vcfEscape(addr) + ";" + vcfEscape(city) + ";" + vcfEscape(region) + ";" + vcfEscape(postal) + ";" + vcfEscape(country) + "\r\n") + } + if b := p.GetBirthdayLocal(); b > 0 { + sb.WriteString("BDAY:" + time.Unix(b, 0).Format("2006-01-02") + "\r\n") + } + sb.WriteString("END:VCARD\r\n") + + return []byte(sb.String()) +} + +func contactDisplayName(p *properties.Contact) string { + if s := p.GetFileUnder(); s != "" { + return s + } + given := p.GetGivenName() + family := p.GetSurname() + if given != "" || family != "" { + return strings.TrimSpace(given + " " + family) + } + if s := p.GetEmail1DisplayName(); s != "" { + return s + } + if s := p.GetEmail1EmailAddress(); s != "" { + return s + } + return p.GetDisplayNamePrefix() +} + +func vcfEscape(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, ";", "\\;") + s = strings.ReplaceAll(s, ",", "\\,") + return s } func formatSender(name, email string) string { @@ -218,8 +441,11 @@ func importReadpst(pstPath, emailDir string, onProgress ProgressFunc) (int, int, if err != nil { return err } - if !info.IsDir() && strings.ToLower(filepath.Ext(path)) == ".eml" { - count++ + if !info.IsDir() { + ext := strings.ToLower(filepath.Ext(path)) + if ext == ".eml" || ext == ".vcf" || ext == ".ics" || ext == ".txt" { + count++ + } } return nil }) diff --git a/internal/sync/pst/pst_test.go b/internal/sync/pst/pst_test.go index e6d8b75..cc2363d 100644 --- a/internal/sync/pst/pst_test.go +++ b/internal/sync/pst/pst_test.go @@ -74,20 +74,23 @@ func TestImportFromDataFiles(t *testing.T) { t.Logf("file has no extractable messages (may contain only appointments/contacts)") } - // Verify .eml files were written. - var emlCount int + // Verify extracted files were written (.eml, .vcf, .ics, .txt). + var fileCount int filepath.Walk(emailDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if !info.IsDir() && filepath.Ext(path) == ".eml" { - emlCount++ + if !info.IsDir() { + ext := strings.ToLower(filepath.Ext(path)) + if ext == ".eml" || ext == ".vcf" || ext == ".ics" || ext == ".txt" { + fileCount++ + } } return nil }) - if extracted > 0 && emlCount != extracted { - t.Errorf("extracted=%d but found %d .eml files", extracted, emlCount) + if extracted > 0 && fileCount != extracted { + t.Errorf("extracted=%d but found %d files (.eml/.vcf/.ics/.txt)", extracted, fileCount) } t.Logf("extracted=%d errors=%d progressCalls=%d", extracted, errCount, progressCalls) @@ -142,37 +145,40 @@ func TestImportExtractionWorks(t *testing.T) { t.Fatalf("Import: %v", err) } - var emlCount int + var fileCount int filepath.Walk(emailDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if !info.IsDir() && filepath.Ext(path) == ".eml" { - emlCount++ + if !info.IsDir() { + ext := strings.ToLower(filepath.Ext(path)) + if ext == ".eml" || ext == ".vcf" || ext == ".ics" || ext == ".txt" { + fileCount++ + } } return nil }) - if extracted != emlCount { - t.Errorf("extracted=%d but found %d .eml files", extracted, emlCount) + if extracted != fileCount { + t.Errorf("extracted=%d but found %d files", extracted, fileCount) } if extracted == 0 { - t.Fatalf("expected at least one email from %s (errCount=%d)", filepath.Base(pstPath), errCount) + t.Fatalf("expected at least one item from %s (errCount=%d)", filepath.Base(pstPath), errCount) } - // Verify .eml files have RFC822-style headers. - var samplePath string + // Verify at least one .eml file has RFC822-style headers. + var sampleEmlPath string filepath.Walk(emailDir, func(path string, info os.FileInfo, err error) error { - if err != nil || samplePath != "" { + if err != nil || sampleEmlPath != "" { return err } - if !info.IsDir() && filepath.Ext(path) == ".eml" { - samplePath = path + if !info.IsDir() && strings.ToLower(filepath.Ext(path)) == ".eml" { + sampleEmlPath = path } return nil }) - if samplePath != "" { - body, err := os.ReadFile(samplePath) + if sampleEmlPath != "" { + body, err := os.ReadFile(sampleEmlPath) if err != nil { t.Errorf("read sample .eml: %v", err) } else if !containsAll(body, "From:", "Subject:") { @@ -180,7 +186,7 @@ func TestImportExtractionWorks(t *testing.T) { } } - t.Logf("extracted %d emails, %d errors", extracted, errCount) + t.Logf("extracted %d items (eml/vcf/ics/txt), %d errors", extracted, errCount) } func containsAll(b []byte, subs ...string) bool { From 5f20d36d232a1c57d8a4ca764af0ba56731a660f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 15 Feb 2026 11:10:17 +0000 Subject: [PATCH 2/2] Add PST test: verify .vcf, .ics, .txt formats when present Co-authored-by: Andrey Oblivantsev --- internal/sync/pst/pst_test.go | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/internal/sync/pst/pst_test.go b/internal/sync/pst/pst_test.go index cc2363d..4dc10a4 100644 --- a/internal/sync/pst/pst_test.go +++ b/internal/sync/pst/pst_test.go @@ -186,9 +186,59 @@ func TestImportExtractionWorks(t *testing.T) { } } + // Verify .vcf, .ics, .txt formats when present. + verifyNonEmailFormats(t, emailDir) + t.Logf("extracted %d items (eml/vcf/ics/txt), %d errors", extracted, errCount) } +// verifyNonEmailFormats checks that extracted .vcf, .ics, .txt files have valid content. +func verifyNonEmailFormats(t *testing.T, emailDir string) { + var vcfCount, icsCount, txtCount int + filepath.Walk(emailDir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".vcf": + vcfCount++ + body, err := os.ReadFile(path) + if err != nil { + t.Errorf("read .vcf %s: %v", path, err) + return nil + } + if !containsAll(body, "BEGIN:VCARD", "END:VCARD") { + t.Errorf(".vcf %s missing vCard markers", path) + } + case ".ics": + icsCount++ + body, err := os.ReadFile(path) + if err != nil { + t.Errorf("read .ics %s: %v", path, err) + return nil + } + if !containsAll(body, "BEGIN:VCALENDAR", "END:VCALENDAR", "BEGIN:VEVENT", "END:VEVENT") { + t.Errorf(".ics %s missing iCalendar markers", path) + } + case ".txt": + txtCount++ + body, err := os.ReadFile(path) + if err != nil { + t.Errorf("read .txt %s: %v", path, err) + return nil + } + if len(body) == 0 { + t.Errorf(".txt %s is empty", path) + } + } + return nil + }) + if vcfCount+icsCount+txtCount > 0 { + t.Logf("verified formats: %d .vcf, %d .ics, %d .txt", vcfCount, icsCount, txtCount) + } +} + func containsAll(b []byte, subs ...string) bool { s := string(b) for _, sub := range subs {