Fix ANSI (PT_STRING8) text extraction: folder names, subjects, senders, bodies

pkg/ansi_string_test.go

@@ -0,0 +1,108 @@
+// go-pst is a library for reading Personal Storage Table (.pst) files (written in Go/Golang).
+//
+// Copyright 2023 Marten Mooij
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pst_test
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	pst "github.com/mooijtech/go-pst/v6/pkg"
+)
+
+// TestANSIStringProperties guards the ANSI (PT_STRING8) string-extraction fix.
+//
+// The 32-bit (ANSI) sample stores text properties as PropertyTypeString8 rather
+// than the UTF-16 PropertyTypeString used by Unicode PSTs. Two defects used to
+// make every ANSI string come back empty:
+//
+//  1. WalkFolders aborted on the first folder because the folder name
+//     (PidTagDisplayName) was read with GetString, which rejects String8.
+//  2. WriteMessagePackValue wrote String8 values under a "<ID>30" message-pack
+//     key, but the generated property structs key text fields with the Unicode
+//     string type 31 (e.g. Subject is msg:"5531"), so the values never decoded
+//     into the struct.
+//
+// This test reads the sample's one item and asserts that a representative
+// PT_STRING8 property (PidTagSubject, 0x0037) decodes to the expected text.
+func TestANSIStringProperties(t *testing.T) {
+	reader, err := os.Open("../data/32-bit.pst")
+	if err != nil {
+		t.Fatalf("failed to open ANSI sample: %v", err)
+	}
+	defer reader.Close()
+
+	pstFile, err := pst.New(reader)
+	if err != nil {
+		t.Fatalf("failed to open PST file: %v", err)
+	}
+	defer pstFile.Cleanup()
+
+	if pstFile.FormatType != pst.FormatTypeANSI {
+		t.Fatalf("sample is not ANSI: FormatType=%d", pstFile.FormatType)
+	}
+
+	var subjects []string
+
+	err = pstFile.WalkFolders(func(folder *pst.Folder) error {
+		// The folder name itself is a PT_STRING8 property on ANSI; reaching any
+		// sub-folder at all proves defect (1) is fixed.
+		if folder.MessageCount == 0 {
+			return nil
+		}
+		iterator, err := folder.GetMessageIterator()
+		if err != nil {
+			return err
+		}
+		for iterator.Next() {
+			message := iterator.Value()
+			// PidTagSubject (0x0037) is stored as PT_STRING8 in this sample.
+			reader, err := message.PropertyContext.GetPropertyReader(0x0037, message.LocalDescriptors)
+			if err != nil {
+				continue // not every item carries a subject
+			}
+			subject, err := reader.GetStringValue()
+			if err != nil {
+				t.Errorf("GetStringValue on PidTagSubject failed: %v", err)
+				continue
+			}
+			subjects = append(subjects, subject)
+		}
+		return iterator.Err()
+	})
+	if err != nil {
+		t.Fatalf("WalkFolders failed: %v", err)
+	}
+
+	if len(subjects) == 0 {
+		t.Fatal("no subjects read from ANSI sample — String8 extraction is broken")
+	}
+
+	// The sample's item carries this subject (with a leading PidTagSubject
+	// prefix marker that callers strip); assert the decoded text is present.
+	const want = "Updated: Olympus training for new hires"
+	found := false
+	for _, s := range subjects {
+		if strings.Contains(s, want) {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Errorf("expected a subject containing %q, got %q", want, subjects)
+	}
+}

pkg/folder.go

@@ -127,8 +127,20 @@ func (folder *Folder) GetSubFolders() ([]Folder, error) {
 
 			switch {
 			case property.ID == 12289:
-				// TODO - Check if this is String8 on ANSI FormatType.
-				folderName, err := propertyReader.GetString()
+				// PidTagDisplayName (the folder name). ANSI PSTs store it as
+				// PropertyTypeString8 (a codepage string), Unicode PSTs as
+				// PropertyTypeString (UTF-16); branch on the actual type rather
+				// than assuming Unicode (which raised ErrPropertyTypeMismatch on
+				// every ANSI folder and aborted the whole walk). Folder names are
+				// typically ASCII, so the Windows-1252 default decodes them
+				// correctly regardless of the archive's exact codepage.
+				var folderName string
+				var err error
+				if propertyReader.Property.Type == PropertyTypeString8 {
+					folderName, err = propertyReader.GetString8(1252)
+				} else {
+					folderName, err = propertyReader.GetString()
+				}
 
 				if err != nil {
 					return nil, eris.Wrap(err, "failed to get folder name")

pkg/message.go

@@ -229,7 +229,7 @@ func (file *File) GetMessage(identifier Identifier) (*Message, error) {
 		fmt.Printf("Failed to get message class property reader, falling back to properties.Message: %+v\n", eris.New(err.Error()))
 		messageProperties = &properties.Message{}
 	} else {
-		messageClass, err := messageClassPropertyReader.GetString()
+		messageClass, err := messageClassPropertyReader.GetStringValue()
 
 		if err != nil {
 			fmt.Printf("Failed to get message class, falling back to properties.Message: %+v\n", eris.New(err.Error()))

pkg/property_reader.go

@@ -77,13 +77,25 @@ func (propertyReader *PropertyReader) WriteMessagePackValue(writer *msgp.Writer)
 
 		return nil
 	case PropertyTypeString8:
-		value, err := propertyReader.GetString8(65001) // TODO - Get from called, this is UTF-8 for now.
+		// ANSI PSTs (and the occasional String8 property in a Unicode PST)
+		// store text as a code-page string rather than UTF-16. The generated
+		// property structs key every text field by "<ID><Type>" using the
+		// Unicode string type (31) — e.g. Subject is msg:"5531". A String8
+		// value carries type 30, so writing it under its own "<ID>30" key would
+		// never match the struct field and every string would decode empty.
+		// Normalize the key to the Unicode string type so the value lands in
+		// the same field the Unicode path populates. Windows-1252 is the
+		// pragmatic default (a superset of ASCII covering Western European
+		// text); it matches the folder-name decode path.
+		value, err := propertyReader.GetString8(1252)
 
 		if err != nil {
 			return eris.Wrap(err, "failed to get string8")
 		}
 
-		if err := writer.WriteString(key); err != nil {
+		string8Key := fmt.Sprintf("%d%d", propertyReader.Property.ID, PropertyTypeString)
+
+		if err := writer.WriteString(string8Key); err != nil {
 			return eris.Wrap(err, "failed to write key")
 		} else if err := writer.WriteString(value); err != nil {
 			return eris.Wrap(err, "failed to write value")
@@ -183,6 +195,19 @@ func (propertyReader *PropertyReader) GetString() (string, error) {
 	return propertyReader.DecodeString(data)
 }
 
+// GetStringValue returns the string value of the property regardless of whether
+// it is stored as PropertyTypeString (UTF-16, Unicode PSTs) or
+// PropertyTypeString8 (a code-page string, ANSI PSTs). Callers that need a
+// string but don't control the PST format (e.g. reading the message class)
+// should prefer this over GetString, which rejects String8 with
+// ErrPropertyTypeMismatch. String8 is decoded as Windows-1252.
+func (propertyReader *PropertyReader) GetStringValue() (string, error) {
+	if propertyReader.Property.Type == PropertyTypeString8 {
+		return propertyReader.GetString8(1252)
+	}
+	return propertyReader.GetString()
+}
+
 // GetString8 returns the string using the external encoding.
 func (propertyReader *PropertyReader) GetString8(codepageIdentifier int) (string, error) {
 	if propertyReader.Property.Type != PropertyTypeString8 {