Internal: Refactor core classifications system to be based on harm identifiers instead

CONTRIBUTING.md

@@ -56,8 +56,9 @@ Policyserv is written to:
 
 ## Definitions / Architecture
 
-**Filter**: Code which analyzes an event in isolation from other filters, producing a series of classifications and a
-"neutral" or "spammy" determination.
+**Filter**: Code which analyzes an event in isolation from other filters, producing a series of harm identifiers/content
+classifications. The content classifications are used to determine whether an event is "spammy", neutral, or not spammy.
+Content is by default classified as "neutral".
 
 **Set Group**: The precise filters to run on an event for a given stage. All filters in a set group are executed
 concurrently. Can be used to "pre" and "post" process events. For example, within a filter set, the first set group may
@@ -68,11 +69,12 @@ group may inspect the result from the prior set groups to have future events be
 or room into the filters that community would like to run against events. Internally, the set groups which make up the
 filter set are ordered to provide some priority filtering and efficient processing.
 
-**Confidence Vector**: A float value between 0 and 1 to carry confidence information about a classification. Zero is low/no
-confidence that the classification applies while 1 is high/total confidence.
+**Harm** (Identifier): A specific classification of prohibited content. Using a harm identifier implies the content is 
+not allowed (spammy). Most harm identifiers are defined by [MSC4456](https://github.com/matrix-org/matrix-spec-proposals/pull/4456),
+though some are policyserv-specific. 
 
-**Classification**: A measurable and useful datum about an event. The most common being "spam". Events may have multiple
-classifications through confidence vectors.
+**Content Class**: Whether the content is allowed, neutral, or prohibited (spammy). A neutral designation does not mean
+that the content is spam-free, but rather that the content doesn't trip any flags (positive or negative).
 
 ## PR review process
 

README.md

@@ -120,9 +120,6 @@ server are trusted by the server administrator(s).
 
 ### General
 
-* `PS_SPAM_THRESHOLD` (default `0.8`) - A value between 0 and 1 denoting how much "confidence" is needed before an event 
-  is considered spammy. Note that policyserv can currently only generate scores of 0, 0.5, and 1 - a future version will 
-  allow for more precise scores, so it's recommended to keep this value as a decimal.
 * `PS_MODERATION_BOT_USER_ID` (default empty value) - The user ID of the bot account where policyserv can send redaction
   commands to. A device on the user *must* implement the [policyserv to-device protocol](./docs/to_device.md) for this to
   work. Set to an empty value to disable this feature.

ai/openai_omni_moderation.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/matrix-org/policyserv/config"
 	"github.com/matrix-org/policyserv/event"
-	"github.com/matrix-org/policyserv/filter/classification"
+	"github.com/matrix-org/policyserv/harms"
 	"github.com/openai/openai-go/v3"
 	"github.com/openai/openai-go/v3/option"
 )
@@ -35,7 +35,7 @@ func NewOpenAIOmniModeration(cnf *config.InstanceConfig, additionalClientOptions
 	}, nil
 }
 
-func (m *OpenAIOmniModeration) CheckEvent(ctx context.Context, cnf *OpenAIOmniModerationConfig, input *Input) ([]classification.Classification, error) {
+func (m *OpenAIOmniModeration) CheckEvent(ctx context.Context, cnf *OpenAIOmniModerationConfig, input *Input) (*harms.ContentInfo, error) {
 	messages, err := event.RenderToText(input.Event)
 	if err != nil {
 		return nil, err
@@ -53,25 +53,25 @@ func (m *OpenAIOmniModeration) CheckEvent(ctx context.Context, cnf *OpenAIOmniMo
 			log.Printf("[%s | %s] Error checking message: %s", input.Event.EventID(), input.Event.RoomID(), err)
 			if cnf.FailSecure {
 				log.Printf("[%s | %s] Returning spam response to block events and discourage retries", input.Event.EventID(), input.Event.RoomID())
-				return []classification.Classification{classification.Spam, classification.Frequency}, nil
+				return harms.ProhibitedContent(harms.OtherGeneral), nil
 			} else {
 				log.Printf("[%s | %s] Returning neutral response despite error, per config", input.Event.EventID(), input.Event.RoomID())
-				return nil, nil
+				return harms.NeutralContent(), nil
 			}
 		}
 		for _, r := range res.Results {
 			// Note: we compress JSON here because the OpenAI library tends to return *a lot* of redundant detail, including JSON with newlines in it.
 			log.Printf("[%s | %s] Result for sender %s: Flagged=%t Flags=%s Scores=%s", input.Event.EventID(), input.Event.RoomID(), input.Event.SenderID(), r.Flagged, compressJsonResponse(r.Categories), compressJsonResponse(r.CategoryScores))
 			if r.Flagged {
-				flags := []classification.Classification{classification.Spam}
+				harmIds := []harms.Harm{harms.SpamGeneral}
 				if r.Categories.SexualMinors {
-					flags = append(flags, classification.CSAM)
+					harmIds = append(harmIds, harms.ChildSafetyCSAM)
 				}
-				return flags, nil
+				return harms.ProhibitedContent(harmIds...), nil
 			}
 		}
 	}
-	return nil, nil
+	return harms.NeutralContent(), nil
 }
 
 type compressible interface {

ai/openai_omni_moderation_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/matrix-org/policyserv/config"
-	"github.com/matrix-org/policyserv/filter/classification"
+	"github.com/matrix-org/policyserv/harms"
 	"github.com/matrix-org/policyserv/test"
 	"github.com/openai/openai-go/v3/option"
 	"github.com/stretchr/testify/assert"
@@ -38,35 +38,32 @@ func TestOpenAIOmniModeration(t *testing.T) {
 	spammyEvent1 := test.MustMakeKeywordEvent(test.KeywordSpammy)
 	ret, err := provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: true}, &Input{Event: spammyEvent1})
 	assert.NoError(t, err)
-	assert.Equal(t, []classification.Classification{
-		classification.Spam,
-	}, ret)
+	test.AssertEqualContentInfo(t, harms.ProhibitedContent(harms.SpamGeneral), ret)
 
 	spammyEvent2 := test.MustMakeKeywordEvent(test.KeywordSpammyCSAM)
 	ret, err = provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: true}, &Input{Event: spammyEvent2})
 	assert.NoError(t, err)
-	assert.Equal(t, []classification.Classification{
-		classification.Spam,
-		classification.CSAM, // should have been detected
-	}, ret)
+	test.AssertEqualContentInfo(t, harms.ProhibitedContent(
+		harms.SpamGeneral,
+		harms.ChildSafetyCSAM, // should have been detected
+	), ret)
 
 	neutralEvent := test.MustMakeKeywordEvent(test.KeywordNeutral)
 	ret, err = provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: true}, &Input{Event: neutralEvent})
 	assert.NoError(t, err)
-	assert.Nil(t, ret) // no classifications
+	test.AssertEqualContentInfo(t, harms.NeutralContent(), ret) // no classifications
 
 	failEvent := test.MustMakeKeywordEvent(test.KeywordIntentionalFail)
 
 	// First test that when FailSecure: true we return a spam classification
 	ret, err = provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: true}, &Input{Event: failEvent})
 	assert.NoError(t, err)
-	assert.Equal(t, []classification.Classification{
-		classification.Spam,
-		classification.Frequency, // also added by the provider
-	}, ret)
+	test.AssertEqualContentInfo(t, harms.ProhibitedContent(
+		harms.OtherGeneral, // added by the provider
+	), ret)
 
 	// Now when FailSecure: false, we should get no classifications (but also no errors)
 	ret, err = provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: false}, &Input{Event: failEvent})
 	assert.NoError(t, err)
-	assert.Nil(t, ret)
+	test.AssertEqualContentInfo(t, harms.NeutralContent(), ret)
 }

ai/types.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/policyserv/filter/classification"
+	"github.com/matrix-org/policyserv/harms"
 	"github.com/matrix-org/policyserv/media"
 )
 
@@ -14,5 +14,5 @@ type Input struct {
 }
 
 type Provider[ConfigT any] interface {
-	CheckEvent(ctx context.Context, cnf ConfigT, input *Input) ([]classification.Classification, error)
+	CheckEvent(ctx context.Context, cnf ConfigT, input *Input) (*harms.ContentInfo, error)
 }

api/errors.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"net/http"
 
+	"github.com/matrix-org/policyserv/harms"
 	"github.com/matrix-org/policyserv/homeserver"
 	"github.com/matrix-org/policyserv/metrics"
 )
@@ -12,10 +13,10 @@ type errorResponder struct {
 	action string
 	w      http.ResponseWriter
 	r      *http.Request
-	harms  []string
+	harms  []harms.Harm
 }
 
-func (e *errorResponder) addHarm(harm string) *errorResponder {
+func (e *errorResponder) addHarm(harm harms.Harm) *errorResponder {
 	e.harms = append(e.harms, harm)
 	return e
 }
@@ -49,6 +50,6 @@ func newErrorResponder(action string, w http.ResponseWriter, r *http.Request) *e
 		action: action,
 		w:      w,
 		r:      r,
-		harms:  make([]string, 0),
+		harms:  make([]harms.Harm, 0),
 	}
 }

api/http_server_api_check.go

@@ -5,7 +5,7 @@ import (
 	"log"
 	"net/http"
 
-	"github.com/matrix-org/policyserv/filter/classification"
+	"github.com/matrix-org/policyserv/harms"
 	"github.com/matrix-org/policyserv/metrics"
 	"github.com/matrix-org/policyserv/queue"
 	"github.com/matrix-org/policyserv/storage"
@@ -36,16 +36,15 @@ func httpCheckTextCommunityApi(api *Api, community *storage.StoredCommunity, w h
 	}
 	textToCheck := string(b)
 
-	vecs, err := set.CheckText(r.Context(), textToCheck)
+	info, err := set.CheckText(r.Context(), textToCheck)
 	if err != nil {
 		errs.err(http.StatusInternalServerError, "M_UNKNOWN", err)
 		return
 	}
 
-	if set.IsSpamResponse(r.Context(), vecs) {
-		errs.addHarm("org.matrix.msc4387.spam")
-		if vecs.GetVector(classification.CSAM) > 0.5 {
-			errs.addHarm("org.matrix.msc4387.child_safety.csam")
+	if info.Class() == harms.ContentClassProhibited {
+		for _, h := range info.Harms() {
+			errs.addHarm(h)
 		}
 		errs.text(http.StatusBadRequest, "ORG.MATRIX.MSC4387_SAFETY", "Text is probably spammy")
 	} else {
@@ -87,7 +86,7 @@ func httpCheckEventIdCommunityApi(api *Api, community *storage.StoredCommunity,
 		return
 	}
 	if event != nil {
-		renderEventResult(event.IsProbablySpam, w, r, errs)
+		renderEventResult(event.ContentInfo, w, r, errs)
 		return
 	}
 
@@ -122,11 +121,14 @@ func httpCheckEventIdCommunityApi(api *Api, community *storage.StoredCommunity,
 		errs.err(http.StatusInternalServerError, "M_UNKNOWN", res.Err)
 		return
 	}
-	renderEventResult(res.IsProbablySpam, w, r, errs)
+	renderEventResult(res.ContentInfo, w, r, errs)
 }
 
-func renderEventResult(isProbablySpam bool, w http.ResponseWriter, r *http.Request, errs *errorResponder) {
-	if isProbablySpam {
+func renderEventResult(info *harms.ContentInfo, w http.ResponseWriter, r *http.Request, errs *errorResponder) {
+	if info.Class() == harms.ContentClassProhibited {
+		for _, h := range info.Harms() {
+			errs.addHarm(h)
+		}
 		errs.text(http.StatusBadRequest, "M_FORBIDDEN", "This message is not allowed by the policy server")
 	} else {
 		err := respondJson("httpCheckEventIdCommunityApi", r, w, map[string]any{})

api/http_server_api_check_test.go

@@ -11,8 +11,7 @@ import (
 	"testing"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/policyserv/filter/classification"
-	"github.com/matrix-org/policyserv/filter/confidence"
+	"github.com/matrix-org/policyserv/harms"
 	"github.com/matrix-org/policyserv/homeserver"
 	"github.com/matrix-org/policyserv/internal"
 	"github.com/matrix-org/policyserv/storage"
@@ -51,7 +50,7 @@ func TestHttpCheckTextCommunityApi(t *testing.T) {
 	httpCheckTextCommunityApi(api, serverCommunity, w, r)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	test.AssertApiError(t, w, "ORG.MATRIX.MSC4387_SAFETY", "Text is probably spammy")
-	test.AssertApiErrorHarms(t, w, []string{"org.matrix.msc4387.spam"})
+	test.AssertApiErrorHarms(t, w, []string{string(harms.SpamGeneral)})
 
 	// Then test a non-match
 	w = httptest.NewRecorder()
@@ -81,15 +80,15 @@ func TestHttpCheckEventIdCommunityApiWithKnownEvent(t *testing.T) {
 
 	// For this test, cache a couple event results to ensure we can avoid federation calls
 	err := api.storage.UpsertEventResult(context.Background(), &storage.StoredEventResult{
-		EventId:           "$spam",
-		IsProbablySpam:    true,
-		ConfidenceVectors: confidence.Vectors{classification.Spam: 1.0},
+		EventId:        "$spam",
+		IsProbablySpam: true,
+		ContentInfo:    harms.ProhibitedContent(harms.SpamGeneral),
 	})
 	assert.NoError(t, err)
 	err = api.storage.UpsertEventResult(context.Background(), &storage.StoredEventResult{
-		EventId:           "$neutral",
-		IsProbablySpam:    false,
-		ConfidenceVectors: confidence.NewConfidenceVectors(),
+		EventId:        "$neutral",
+		IsProbablySpam: false,
+		ContentInfo:    harms.NeutralContent(),
 	})
 	assert.NoError(t, err)
 

community/manager.go

@@ -11,6 +11,7 @@ import (
 	"github.com/matrix-org/policyserv/config"
 	"github.com/matrix-org/policyserv/content"
 	"github.com/matrix-org/policyserv/filter"
+	"github.com/matrix-org/policyserv/harms"
 	"github.com/matrix-org/policyserv/internal"
 	"github.com/matrix-org/policyserv/notifiers"
 	"github.com/matrix-org/policyserv/pubsub"
@@ -97,7 +98,7 @@ func (m *Manager) GetFilterSetForCommunityId(ctx context.Context, communityId st
 	prefilters := []string{filter.ProtectLocalUserFilterName}
 	hellbanPrefilters := make([]string, 0) // these run after the prefilters, but before the other filters
 	filters := make([]string, 0)
-	postfilters := make([]string, 0)
+	postfilterSilences := make([]string, 0)
 	if len(internal.Dereference(communityConfig.KeywordFilterKeywords)) > 0 {
 		filters = append(filters, filter.KeywordFilterName)
 	}
@@ -139,7 +140,7 @@ func (m *Manager) GetFilterSetForCommunityId(ctx context.Context, communityId st
 	}
 	if internal.Dereference(communityConfig.HellbanPostfilterMinutes) > 0 {
 		hellbanPrefilters = append(hellbanPrefilters, filter.HellbanPrefilterName)
-		postfilters = append(postfilters, filter.HellbanPostfilterName)
+		postfilterSilences = append(postfilterSilences, filter.HellbanPostfilterName)
 	}
 	if m.instanceConfig.OpenAIApiKey != "" {
 		// Access to this filter is gated by further instance config (namely, the room IDs allowed to use it)
@@ -180,34 +181,31 @@ func (m *Manager) GetFilterSetForCommunityId(ctx context.Context, communityId st
 		InstanceConfig:  m.instanceConfig,
 		Groups: []*filter.SetGroupConfig{{
 			// The first set group replaces the concept of "prefilters". We want this group to capture all events,
-			// so we set the min and max to cover the full range.
-			EnabledNames:           prefilters,
-			MinimumSpamVectorValue: 0.0,
-			MaximumSpamVectorValue: 1.0,
+			// so we set the classes appropriately.
+			EnabledNames: prefilters,
+			// Micro optimization: Put neutral first so we can skip CPU cycles in a loop in the general case. We also
+			// probably don't need to specify allowed and prohibited here because we have no code which pushes such
+			// classes right away, but for safety we might as well.
+			RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral, harms.ContentClassAllowed, harms.ContentClassProhibited},
 		}, {
 			// This set group is similar to prefilters, but only contains the hellban prefilters. This is to prevent
 			// users being denied abilities that are intended to be granted to them via other prefilters (like an
 			// ability to leave the room).
 			EnabledNames: hellbanPrefilters,
-			// We want the min/max values to catch "maybe spam" from the prefilters level, but explicitly not events
-			// that were flagged as not-spam by the same layer. This is why the minimum is not quite zero.
-			MinimumSpamVectorValue: 0.1,
-			MaximumSpamVectorValue: 1.0,
+			// We want to capture "maybe spam", but not events that were already flagged as (not) spam.
+			RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral},
 		}, {
 			// The third set group is what was previously the middle layer for filters. This is where the bulk of
 			// the work happens. We only want it to run if the prefilters didn't already declare an event spammy or
 			// neutral though, so we narrow the min/max range a bit.
 			EnabledNames: filters,
-			// We want the min/max to be slightly less than the extremes to avoid doing work
-			// when the event is already flagged as (not) spam.
-			MinimumSpamVectorValue: 0.3,
-			MaximumSpamVectorValue: 0.7,
+			// Skip this group for events that are already (not) spam.
+			RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral},
 		}, {
-			// The last set group replaces "postfilters", and should be run regardless of whether the previous groups
-			// flagged the event as spam. As such, we set the min/max to the widest possible range.
-			EnabledNames:           postfilters,
-			MinimumSpamVectorValue: 0.0,
-			MaximumSpamVectorValue: 1.0,
+			// The last set group is for telling the hellban postfilter if any previous filter flagged an event as
+			// spammy so it can put a silence in place.
+			EnabledNames: postfilterSilences,
+			RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited},
 		}},
 	}
 	filterSet, err := filter.NewSet(setConfig, m.storage, m.pubsubClient, m.notifier, scanner)